You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by sy...@apache.org on 2016/05/11 04:57:33 UTC
[01/50] [abbrv] hbase git commit: HBASE-15607 Remove PB references
from Admin for 2.0 (Ram)
Repository: hbase
Updated Branches:
refs/heads/hbase-12439 a395922ad -> a11091c49
http://git-wip-us.apache.org/repos/asf/hbase/blob/c06a976a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromClient.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromClient.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromClient.java
index 1b61147..8317376 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromClient.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromClient.java
@@ -36,7 +36,7 @@ import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.TableNotFoundException;
import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
-import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
import org.apache.hadoop.hbase.snapshot.SnapshotCreationException;
import org.apache.hadoop.hbase.snapshot.SnapshotDoesNotExistException;
@@ -212,18 +212,13 @@ public class TestSnapshotFromClient {
final String SNAPSHOT_NAME = "offlineTableSnapshot";
byte[] snapshot = Bytes.toBytes(SNAPSHOT_NAME);
- SnapshotDescription desc = SnapshotDescription.newBuilder()
- .setType(SnapshotDescription.Type.DISABLED)
- .setTable(STRING_TABLE_NAME)
- .setName(SNAPSHOT_NAME)
- .setVersion(SnapshotManifestV1.DESCRIPTOR_VERSION)
- .build();
- admin.snapshot(desc);
+ admin.snapshot(new SnapshotDescription(SNAPSHOT_NAME, STRING_TABLE_NAME,
+ SnapshotType.DISABLED, null, -1, SnapshotManifestV1.DESCRIPTOR_VERSION));
LOG.debug("Snapshot completed.");
// make sure we have the snapshot
- List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin,
- snapshot, TABLE_NAME);
+ List<SnapshotDescription> snapshots =
+ SnapshotTestingUtils.assertOneSnapshotThatMatches(admin, snapshot, TABLE_NAME);
// make sure its a valid snapshot
FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
@@ -231,9 +226,9 @@ public class TestSnapshotFromClient {
LOG.debug("FS state after snapshot:");
FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
-
- SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, TEST_FAM, rootDir,
- admin, fs);
+ SnapshotTestingUtils.confirmSnapshotValid(
+ ProtobufUtil.createHBaseProtosSnapshotDesc(snapshots.get(0)), TABLE_NAME, TEST_FAM,
+ rootDir, admin, fs);
admin.deleteSnapshot(snapshot);
snapshots = admin.listSnapshots();
@@ -292,8 +287,8 @@ public class TestSnapshotFromClient {
LOG.debug("Snapshot completed.");
// make sure we have the snapshot
- List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin,
- snapshot, TABLE_NAME);
+ List<SnapshotDescription> snapshots =
+ SnapshotTestingUtils.assertOneSnapshotThatMatches(admin, snapshot, TABLE_NAME);
// make sure its a valid snapshot
FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
@@ -304,8 +299,9 @@ public class TestSnapshotFromClient {
List<byte[]> emptyCfs = Lists.newArrayList(TEST_FAM); // no file in the region
List<byte[]> nonEmptyCfs = Lists.newArrayList();
- SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, nonEmptyCfs, emptyCfs,
- rootDir, admin, fs);
+ SnapshotTestingUtils.confirmSnapshotValid(
+ ProtobufUtil.createHBaseProtosSnapshotDesc(snapshots.get(0)), TABLE_NAME, nonEmptyCfs,
+ emptyCfs, rootDir, admin, fs);
admin.deleteSnapshot(snapshot);
snapshots = admin.listSnapshots();
@@ -375,7 +371,8 @@ public class TestSnapshotFromClient {
admin.snapshot(Bytes.toBytes(table2Snapshot1), TABLE_NAME);
LOG.debug(table2Snapshot1 + " completed.");
- List<SnapshotDescription> listTableSnapshots = admin.listTableSnapshots("test.*", "Table1.*");
+ List<SnapshotDescription> listTableSnapshots =
+ admin.listTableSnapshots("test.*", "Table1.*");
List<String> listTableSnapshotNames = new ArrayList<String>();
assertEquals(2, listTableSnapshots.size());
for (SnapshotDescription s : listTableSnapshots) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/c06a976a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSplitOrMergeStatus.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSplitOrMergeStatus.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSplitOrMergeStatus.java
index e1ce63b..c5ca0b5 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSplitOrMergeStatus.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSplitOrMergeStatus.java
@@ -78,14 +78,14 @@ public class TestSplitOrMergeStatus {
Admin admin = TEST_UTIL.getAdmin();
initSwitchStatus(admin);
boolean[] results = admin.setSplitOrMergeEnabled(false, false,
- true, Admin.MasterSwitchType.SPLIT);
+ true, MasterSwitchType.SPLIT);
assertEquals(results.length, 1);
assertTrue(results[0]);
admin.split(t.getName());
int count = waitOnSplitOrMerge(t).size();
assertTrue(orignalCount == count);
- results = admin.setSplitOrMergeEnabled(true, false, true, Admin.MasterSwitchType.SPLIT);
+ results = admin.setSplitOrMergeEnabled(true, false, true, MasterSwitchType.SPLIT);
assertEquals(results.length, 1);
assertFalse(results[0]);
admin.split(t.getName());
@@ -111,7 +111,7 @@ public class TestSplitOrMergeStatus {
waitForMergable(admin, name);
int orignalCount = locator.getAllRegionLocations().size();
boolean[] results = admin.setSplitOrMergeEnabled(false, false,
- true, Admin.MasterSwitchType.MERGE);
+ true, MasterSwitchType.MERGE);
assertEquals(results.length, 1);
assertTrue(results[0]);
List<HRegionInfo> regions = admin.getTableRegions(t.getName());
@@ -122,7 +122,7 @@ public class TestSplitOrMergeStatus {
assertTrue(orignalCount == count);
waitForMergable(admin, name);
- results = admin.setSplitOrMergeEnabled(true, false, true, Admin.MasterSwitchType.MERGE);
+ results = admin.setSplitOrMergeEnabled(true, false, true, MasterSwitchType.MERGE);
assertEquals(results.length, 1);
assertFalse(results[0]);
admin.mergeRegions(regions.get(0).getEncodedNameAsBytes(),
@@ -136,12 +136,12 @@ public class TestSplitOrMergeStatus {
public void testMultiSwitches() throws IOException {
Admin admin = TEST_UTIL.getAdmin();
boolean[] switches = admin.setSplitOrMergeEnabled(false, false, true,
- Admin.MasterSwitchType.SPLIT, Admin.MasterSwitchType.MERGE);
+ MasterSwitchType.SPLIT, MasterSwitchType.MERGE);
for (boolean s : switches){
assertTrue(s);
}
- assertFalse(admin.isSplitOrMergeEnabled(Admin.MasterSwitchType.SPLIT));
- assertFalse(admin.isSplitOrMergeEnabled(Admin.MasterSwitchType.MERGE));
+ assertFalse(admin.isSplitOrMergeEnabled(MasterSwitchType.SPLIT));
+ assertFalse(admin.isSplitOrMergeEnabled(MasterSwitchType.MERGE));
admin.close();
}
@@ -149,10 +149,10 @@ public class TestSplitOrMergeStatus {
public void testSwitchLock() throws IOException {
Admin admin = TEST_UTIL.getAdmin();
admin.setSplitOrMergeEnabled(false, false, false,
- Admin.MasterSwitchType.SPLIT, Admin.MasterSwitchType.MERGE);
+ MasterSwitchType.SPLIT, MasterSwitchType.MERGE);
try {
admin.setSplitOrMergeEnabled(false, false, true,
- Admin.MasterSwitchType.SPLIT, Admin.MasterSwitchType.MERGE);
+ MasterSwitchType.SPLIT, MasterSwitchType.MERGE);
fail();
} catch (IOException e) {
LOG.info("", e);
@@ -160,7 +160,7 @@ public class TestSplitOrMergeStatus {
admin.releaseSplitOrMergeLockAndRollback();
try {
admin.setSplitOrMergeEnabled(true, false, true,
- Admin.MasterSwitchType.SPLIT, Admin.MasterSwitchType.MERGE);
+ MasterSwitchType.SPLIT, MasterSwitchType.MERGE);
} catch (IOException e) {
fail();
}
@@ -168,14 +168,14 @@ public class TestSplitOrMergeStatus {
}
private void initSwitchStatus(Admin admin) throws IOException {
- if (!admin.isSplitOrMergeEnabled(Admin.MasterSwitchType.SPLIT)) {
- admin.setSplitOrMergeEnabled(true, false, true, Admin.MasterSwitchType.SPLIT);
+ if (!admin.isSplitOrMergeEnabled(MasterSwitchType.SPLIT)) {
+ admin.setSplitOrMergeEnabled(true, false, true, MasterSwitchType.SPLIT);
}
- if (!admin.isSplitOrMergeEnabled(Admin.MasterSwitchType.MERGE)) {
- admin.setSplitOrMergeEnabled(true, false, true, Admin.MasterSwitchType.MERGE);
+ if (!admin.isSplitOrMergeEnabled(MasterSwitchType.MERGE)) {
+ admin.setSplitOrMergeEnabled(true, false, true, MasterSwitchType.MERGE);
}
- assertTrue(admin.isSplitOrMergeEnabled(Admin.MasterSwitchType.SPLIT));
- assertTrue(admin.isSplitOrMergeEnabled(Admin.MasterSwitchType.MERGE));
+ assertTrue(admin.isSplitOrMergeEnabled(MasterSwitchType.SPLIT));
+ assertTrue(admin.isSplitOrMergeEnabled(MasterSwitchType.MERGE));
}
private void waitForMergable(Admin admin, TableName t) throws InterruptedException, IOException {
http://git-wip-us.apache.org/repos/asf/hbase/blob/c06a976a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java
index 65fc0c3..5257cee 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java
@@ -48,6 +48,7 @@ import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.client.MasterSwitchType;
import org.apache.hadoop.hbase.client.RegionLocator;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.master.AssignmentManager;
@@ -348,13 +349,13 @@ public class TestMasterObserver {
@Override
public boolean preSetSplitOrMergeEnabled(final ObserverContext<MasterCoprocessorEnvironment> ctx,
- final boolean newValue, final Admin.MasterSwitchType switchType) throws IOException {
+ final boolean newValue, final MasterSwitchType switchType) throws IOException {
return false;
}
@Override
public void postSetSplitOrMergeEnabled(final ObserverContext<MasterCoprocessorEnvironment> ctx,
- final boolean newValue, final Admin.MasterSwitchType switchType) throws IOException {
+ final boolean newValue, final MasterSwitchType switchType) throws IOException {
}
@Override
http://git-wip-us.apache.org/repos/asf/hbase/blob/c06a976a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java
index cff8db0..4f2385f 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java
@@ -69,6 +69,7 @@ import org.apache.hadoop.hbase.SplitLogCounters;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.Waiter;
import org.apache.hadoop.hbase.client.ClusterConnection;
+import org.apache.hadoop.hbase.client.CompactionState;
import org.apache.hadoop.hbase.client.ConnectionUtils;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
@@ -87,7 +88,6 @@ import org.apache.hadoop.hbase.exceptions.RegionInRecoveryException;
import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException;
import org.apache.hadoop.hbase.master.SplitLogManager.TaskBatch;
import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
-import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoResponse.CompactionState;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.HRegionServer;
import org.apache.hadoop.hbase.regionserver.Region;
@@ -1421,7 +1421,8 @@ public class TestDistributedLogSplitting {
TEST_UTIL.waitFor(30000, 200, new Waiter.Predicate<Exception>() {
@Override
public boolean evaluate() throws Exception {
- return (TEST_UTIL.getHBaseAdmin().getCompactionState(tableName) == CompactionState.NONE);
+ return (TEST_UTIL.getHBaseAdmin()
+ .getCompactionState(tableName) == CompactionState.NONE);
}
});
http://git-wip-us.apache.org/repos/asf/hbase/blob/c06a976a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestWarmupRegion.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestWarmupRegion.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestWarmupRegion.java
index 53ee92b..cb7337e 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestWarmupRegion.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestWarmupRegion.java
@@ -31,7 +31,8 @@ import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.Waiter;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Table;
-import org.apache.hadoop.hbase.protobuf.generated.AdminProtos;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.CompactionState;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.HRegionServer;
import org.apache.hadoop.hbase.testclassification.LargeTests;
@@ -106,7 +107,7 @@ public class TestWarmupRegion {
@Override
public boolean evaluate() throws IOException {
return TEST_UTIL.getHBaseAdmin().getCompactionState(TABLENAME) ==
- AdminProtos.GetRegionInfoResponse.CompactionState.NONE;
+ CompactionState.NONE;
}
});
http://git-wip-us.apache.org/repos/asf/hbase/blob/c06a976a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCloneSnapshotProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCloneSnapshotProcedure.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCloneSnapshotProcedure.java
index aeafbf8..96f78f7 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCloneSnapshotProcedure.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCloneSnapshotProcedure.java
@@ -33,7 +33,9 @@ import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
-import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos;
+import org.apache.hadoop.hbase.client.SnapshotDescription;
import org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos.CloneSnapshotState;
import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
import org.apache.hadoop.hbase.testclassification.MasterTests;
@@ -59,7 +61,7 @@ public class TestCloneSnapshotProcedure {
private static long nonceGroup = HConstants.NO_NONCE;
private static long nonce = HConstants.NO_NONCE;
- private static SnapshotDescription snapshot = null;
+ private static HBaseProtos.SnapshotDescription snapshot = null;
private static void setupConf(Configuration conf) {
conf.setInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS, 1);
@@ -99,7 +101,7 @@ public class TestCloneSnapshotProcedure {
assertTrue("expected executor to be running", procExec.isRunning());
}
- private SnapshotDescription getSnapshot() throws Exception {
+ private HBaseProtos.SnapshotDescription getSnapshot() throws Exception {
if (snapshot == null) {
final TableName snapshotTableName = TableName.valueOf("testCloneSnapshot");
long tid = System.currentTimeMillis();
@@ -116,7 +118,7 @@ public class TestCloneSnapshotProcedure {
admin.enableTable(snapshotTableName);
List<SnapshotDescription> snapshotList = admin.listSnapshots();
- snapshot = snapshotList.get(0);
+ snapshot = ProtobufUtil.createHBaseProtosSnapshotDesc(snapshotList.get(0));
}
return snapshot;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/c06a976a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestRestoreSnapshotProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestRestoreSnapshotProcedure.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestRestoreSnapshotProcedure.java
index 44d6988..733dcb9 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestRestoreSnapshotProcedure.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestRestoreSnapshotProcedure.java
@@ -35,7 +35,9 @@ import org.apache.hadoop.hbase.TableNotFoundException;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
-import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.client.SnapshotDescription;
+import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos;
import org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos.RestoreSnapshotState;
import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
import org.apache.hadoop.hbase.testclassification.MasterTests;
@@ -73,7 +75,7 @@ public class TestRestoreSnapshotProcedure {
private static long nonceGroup = HConstants.NO_NONCE;
private static long nonce = HConstants.NO_NONCE;
- private SnapshotDescription snapshot = null;
+ private HBaseProtos.SnapshotDescription snapshot = null;
private HTableDescriptor snapshotHTD = null;
private static void setupConf(Configuration conf) {
@@ -141,7 +143,7 @@ public class TestRestoreSnapshotProcedure {
admin.snapshot(snapshotName, snapshotTableName);
List<SnapshotDescription> snapshotList = admin.listSnapshots();
- snapshot = snapshotList.get(0);
+ snapshot = ProtobufUtil.createHBaseProtosSnapshotDesc(snapshotList.get(0));
// modify the table
HColumnDescriptor columnFamilyDescriptor3 = new HColumnDescriptor(CF3);
http://git-wip-us.apache.org/repos/asf/hbase/blob/c06a976a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestMobCompactor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestMobCompactor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestMobCompactor.java
index 9922aff..c0ad2dd 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestMobCompactor.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestMobCompactor.java
@@ -51,6 +51,8 @@ import org.apache.hadoop.hbase.NamespaceDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.BufferedMutator;
+import org.apache.hadoop.hbase.client.CompactType;
+import org.apache.hadoop.hbase.client.CompactionState;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.Delete;
@@ -68,7 +70,6 @@ import org.apache.hadoop.hbase.io.hfile.CacheConfig;
import org.apache.hadoop.hbase.io.hfile.HFile;
import org.apache.hadoop.hbase.mob.MobConstants;
import org.apache.hadoop.hbase.mob.MobUtils;
-import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoResponse.CompactionState;
import org.apache.hadoop.hbase.regionserver.BloomType;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.StoreFile;
@@ -356,7 +357,7 @@ public class TestMobCompactor {
countFiles(tableName, false, family2));
// do the major mob compaction, it will force all files to compaction
- admin.majorCompact(tableName, hcd1.getName(), Admin.CompactType.MOB);
+ admin.majorCompact(tableName, hcd1.getName(), CompactType.MOB);
waitUntilMobCompactionFinished(tableName);
assertEquals("After compaction: mob rows count", regionNum * (rowNumPerRegion - delRowNum),
@@ -399,7 +400,7 @@ public class TestMobCompactor {
Cell cell = result.getColumnLatestCell(hcd1.getName(), Bytes.toBytes(qf1));
assertEquals("Before compaction: mob value of k0", newValue0,
Bytes.toString(CellUtil.cloneValue(cell)));
- admin.majorCompact(tableName, hcd1.getName(), Admin.CompactType.MOB);
+ admin.majorCompact(tableName, hcd1.getName(), CompactType.MOB);
waitUntilMobCompactionFinished(tableName);
// read the latest cell of key0, the cell seqId in bulk loaded file is not reset in the
// scanner. The cell that has "new" value is still visible.
@@ -449,7 +450,7 @@ public class TestMobCompactor {
loadData(admin, bufMut, tableName, new Put[] { put1 }); // now two mob files
admin.majorCompact(tableName);
waitUntilCompactionFinished(tableName);
- admin.majorCompact(tableName, hcd1.getName(), Admin.CompactType.MOB);
+ admin.majorCompact(tableName, hcd1.getName(), CompactType.MOB);
waitUntilMobCompactionFinished(tableName);
// read the latest cell of key1.
Get get = new Get(key1);
@@ -475,12 +476,12 @@ public class TestMobCompactor {
private void waitUntilMobCompactionFinished(TableName tableName) throws IOException,
InterruptedException {
long finished = EnvironmentEdgeManager.currentTime() + 60000;
- CompactionState state = admin.getCompactionState(tableName, Admin.CompactType.MOB);
+ CompactionState state = admin.getCompactionState(tableName, CompactType.MOB);
while (EnvironmentEdgeManager.currentTime() < finished) {
if (state == CompactionState.NONE) {
break;
}
- state = admin.getCompactionState(tableName, Admin.CompactType.MOB);
+ state = admin.getCompactionState(tableName, CompactType.MOB);
Thread.sleep(10);
}
assertEquals(CompactionState.NONE, state);
http://git-wip-us.apache.org/repos/asf/hbase/blob/c06a976a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionState.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionState.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionState.java
index 4715d53..d6302b3 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionState.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionState.java
@@ -31,9 +31,9 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.CompactionState;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Table;
-import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoResponse.CompactionState;
import org.apache.hadoop.hbase.testclassification.LargeTests;
import org.apache.hadoop.hbase.testclassification.VerySlowRegionServerTests;
import org.apache.hadoop.hbase.util.Bytes;
@@ -164,7 +164,7 @@ public class TestCompactionState {
// otherwise, the compaction should have already been done
if (expectedState != state) {
for (Region region: regions) {
- state = region.getCompactionState();
+ state = CompactionState.valueOf(region.getCompactionState().toString());
assertEquals(CompactionState.NONE, state);
}
} else {
http://git-wip-us.apache.org/repos/asf/hbase/blob/c06a976a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestTags.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestTags.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestTags.java
index 3a9ace2..06dbc37 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestTags.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestTags.java
@@ -40,6 +40,7 @@ import org.apache.hadoop.hbase.TagUtil;
import org.apache.hadoop.hbase.ArrayBackedTag;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.Append;
+import org.apache.hadoop.hbase.client.CompactionState;
import org.apache.hadoop.hbase.client.Durability;
import org.apache.hadoop.hbase.client.Increment;
import org.apache.hadoop.hbase.client.Mutation;
@@ -53,7 +54,6 @@ import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
import org.apache.hadoop.hbase.coprocessor.ObserverContext;
import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
-import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoResponse.CompactionState;
import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.testclassification.RegionServerTests;
http://git-wip-us.apache.org/repos/asf/hbase/blob/c06a976a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
index 37c42a0..92d7806 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
@@ -62,6 +62,7 @@ import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Increment;
+import org.apache.hadoop.hbase.client.MasterSwitchType;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.RegionLocator;
import org.apache.hadoop.hbase.client.Result;
@@ -710,7 +711,7 @@ public class TestAccessController extends SecureTestUtil {
@Override
public Object run() throws Exception {
ACCESS_CONTROLLER.preSetSplitOrMergeEnabled(ObserverContext.createAndPrepare(CP_ENV, null),
- true, Admin.MasterSwitchType.MERGE);
+ true, MasterSwitchType.MERGE);
return null;
}
};
http://git-wip-us.apache.org/repos/asf/hbase/blob/c06a976a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java
index 666eea3..6615a8f 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java
@@ -48,6 +48,7 @@ import org.apache.hadoop.hbase.TableDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.TableNotEnabledException;
import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.SnapshotType;
import org.apache.hadoop.hbase.client.BufferedMutator;
import org.apache.hadoop.hbase.client.Durability;
import org.apache.hadoop.hbase.client.Put;
@@ -59,7 +60,8 @@ import org.apache.hadoop.hbase.master.HMaster;
import org.apache.hadoop.hbase.master.MasterFileSystem;
import org.apache.hadoop.hbase.mob.MobUtils;
import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
-import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos;
+import org.apache.hadoop.hbase.client.SnapshotDescription;
import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsSnapshotDoneRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsSnapshotDoneResponse;
@@ -120,7 +122,7 @@ public class SnapshotTestingUtils {
* Make sure that there is only one snapshot returned from the master
*/
public static void assertOneSnapshotThatMatches(Admin admin,
- SnapshotDescription snapshot) throws IOException {
+ HBaseProtos.SnapshotDescription snapshot) throws IOException {
assertOneSnapshotThatMatches(admin, snapshot.getName(),
TableName.valueOf(snapshot.getTable()));
}
@@ -153,7 +155,7 @@ public class SnapshotTestingUtils {
}
public static void confirmSnapshotValid(HBaseTestingUtility testUtil,
- SnapshotDescription snapshotDescriptor, TableName tableName, byte[] family)
+ HBaseProtos.SnapshotDescription snapshotDescriptor, TableName tableName, byte[] family)
throws IOException {
MasterFileSystem mfs = testUtil.getHBaseCluster().getMaster().getMasterFileSystem();
confirmSnapshotValid(snapshotDescriptor, tableName, family,
@@ -165,7 +167,7 @@ public class SnapshotTestingUtils {
* be in the snapshot.
*/
public static void confirmSnapshotValid(
- SnapshotDescription snapshotDescriptor, TableName tableName,
+ HBaseProtos.SnapshotDescription snapshotDescriptor, TableName tableName,
byte[] testFamily, Path rootDir, Admin admin, FileSystem fs)
throws IOException {
ArrayList nonEmptyTestFamilies = new ArrayList(1);
@@ -178,7 +180,7 @@ public class SnapshotTestingUtils {
* Confirm that the snapshot has no references files but only metadata.
*/
public static void confirmEmptySnapshotValid(
- SnapshotDescription snapshotDescriptor, TableName tableName,
+ HBaseProtos.SnapshotDescription snapshotDescriptor, TableName tableName,
byte[] testFamily, Path rootDir, Admin admin, FileSystem fs)
throws IOException {
ArrayList emptyTestFamilies = new ArrayList(1);
@@ -194,7 +196,7 @@ public class SnapshotTestingUtils {
* by the MasterSnapshotVerifier, at the end of the snapshot operation.
*/
public static void confirmSnapshotValid(
- SnapshotDescription snapshotDescriptor, TableName tableName,
+ HBaseProtos.SnapshotDescription snapshotDescriptor, TableName tableName,
List<byte[]> nonEmptyTestFamilies, List<byte[]> emptyTestFamilies,
Path rootDir, Admin admin, FileSystem fs) throws IOException {
final Configuration conf = admin.getConfiguration();
@@ -204,7 +206,7 @@ public class SnapshotTestingUtils {
snapshotDescriptor, rootDir);
assertTrue(fs.exists(snapshotDir));
- SnapshotDescription desc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
+ HBaseProtos.SnapshotDescription desc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
// Extract regions and families with store files
final Set<byte[]> snapshotFamilies = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);
@@ -265,7 +267,7 @@ public class SnapshotTestingUtils {
* @throws ServiceException if the snapshot fails
*/
public static void waitForSnapshotToComplete(HMaster master,
- SnapshotDescription snapshot, long sleep) throws ServiceException {
+ HBaseProtos.SnapshotDescription snapshot, long sleep) throws ServiceException {
final IsSnapshotDoneRequest request = IsSnapshotDoneRequest.newBuilder()
.setSnapshot(snapshot).build();
IsSnapshotDoneResponse done = IsSnapshotDoneResponse.newBuilder()
@@ -286,12 +288,13 @@ public class SnapshotTestingUtils {
*/
public static void snapshot(Admin admin,
final String snapshotName, final String tableName,
- SnapshotDescription.Type type, int numTries) throws IOException {
+ HBaseProtos.SnapshotDescription.Type type, int numTries) throws IOException {
int tries = 0;
CorruptedSnapshotException lastEx = null;
while (tries++ < numTries) {
try {
- admin.snapshot(snapshotName, TableName.valueOf(tableName), type);
+ admin.snapshot(new SnapshotDescription(snapshotName, tableName,
+ SnapshotType.valueOf(type.toString())));
return;
} catch (CorruptedSnapshotException cse) {
LOG.warn("Got CorruptedSnapshotException", cse);
@@ -393,13 +396,14 @@ public class SnapshotTestingUtils {
}
admin.snapshot(snapshotNameString, tableName);
- List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertExistsMatchingSnapshot(admin,
- snapshotNameString, tableName);
+ List<SnapshotDescription> snapshots =
+ SnapshotTestingUtils.assertExistsMatchingSnapshot(admin, snapshotNameString, tableName);
if (snapshots == null || snapshots.size() != 1) {
Assert.fail("Incorrect number of snapshots for table " + tableName);
}
- SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), tableName, nonEmptyFamilyNames,
+ SnapshotTestingUtils.confirmSnapshotValid(
+ ProtobufUtil.createHBaseProtosSnapshotDesc(snapshots.get(0)), tableName, nonEmptyFamilyNames,
emptyFamilyNames, rootDir, admin, fs);
}
@@ -418,7 +422,8 @@ public class SnapshotTestingUtils {
Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName,
mfs.getRootDir());
- SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
+ HBaseProtos.SnapshotDescription snapshotDesc =
+ SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
final TableName table = TableName.valueOf(snapshotDesc.getTable());
final ArrayList corruptedFiles = new ArrayList();
@@ -467,7 +472,7 @@ public class SnapshotTestingUtils {
public static class SnapshotBuilder {
private final RegionData[] tableRegions;
- private final SnapshotDescription desc;
+ private final HBaseProtos.SnapshotDescription desc;
private final HTableDescriptor htd;
private final Configuration conf;
private final FileSystem fs;
@@ -477,7 +482,7 @@ public class SnapshotTestingUtils {
public SnapshotBuilder(final Configuration conf, final FileSystem fs,
final Path rootDir, final HTableDescriptor htd,
- final SnapshotDescription desc, final RegionData[] tableRegions)
+ final HBaseProtos.SnapshotDescription desc, final RegionData[] tableRegions)
throws IOException {
this.fs = fs;
this.conf = conf;
@@ -495,7 +500,7 @@ public class SnapshotTestingUtils {
return this.htd;
}
- public SnapshotDescription getSnapshotDescription() {
+ public HBaseProtos.SnapshotDescription getSnapshotDescription() {
return this.desc;
}
@@ -519,7 +524,7 @@ public class SnapshotTestingUtils {
.build());
}
- private Path[] addRegion(final SnapshotDescription desc) throws IOException {
+ private Path[] addRegion(final HBaseProtos.SnapshotDescription desc) throws IOException {
if (this.snapshotted == tableRegions.length) {
throw new UnsupportedOperationException("No more regions in the table");
}
@@ -648,7 +653,7 @@ public class SnapshotTestingUtils {
HTableDescriptor htd = createHtd(tableName);
RegionData[] regions = createTable(htd, numRegions);
- SnapshotDescription desc = SnapshotDescription.newBuilder()
+ HBaseProtos.SnapshotDescription desc = HBaseProtos.SnapshotDescription.newBuilder()
.setTable(htd.getNameAsString())
.setName(snapshotName)
.setVersion(version)
http://git-wip-us.apache.org/repos/asf/hbase/blob/c06a976a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java
index 4b988a6..0a933c6 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java
@@ -44,7 +44,9 @@ import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.master.HMaster;
import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
-import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos;
+import org.apache.hadoop.hbase.client.SnapshotDescription;
import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
import org.apache.hadoop.hbase.testclassification.LargeTests;
import org.apache.hadoop.hbase.testclassification.RegionServerTests;
@@ -147,18 +149,20 @@ public class TestFlushSnapshotFromClient {
// take a snapshot of the enabled table
String snapshotString = "offlineTableSnapshot";
byte[] snapshot = Bytes.toBytes(snapshotString);
- admin.snapshot(snapshotString, TABLE_NAME, SnapshotDescription.Type.FLUSH);
+ admin.snapshot(snapshotString, TABLE_NAME,
+ ProtobufUtil.createSnapshotType(HBaseProtos.SnapshotDescription.Type.FLUSH));
LOG.debug("Snapshot completed.");
// make sure we have the snapshot
- List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin,
- snapshot, TABLE_NAME);
+ List<SnapshotDescription> snapshots =
+ SnapshotTestingUtils.assertOneSnapshotThatMatches(admin, snapshot, TABLE_NAME);
// make sure its a valid snapshot
LOG.debug("FS state after snapshot:");
UTIL.getHBaseCluster().getMaster().getMasterFileSystem().logFileSystemState(LOG);
- SnapshotTestingUtils.confirmSnapshotValid(UTIL, snapshots.get(0), TABLE_NAME, TEST_FAM);
+ SnapshotTestingUtils.confirmSnapshotValid(UTIL,
+ ProtobufUtil.createHBaseProtosSnapshotDesc(snapshots.get(0)), TABLE_NAME, TEST_FAM);
}
/**
@@ -181,18 +185,20 @@ public class TestFlushSnapshotFromClient {
// take a snapshot of the enabled table
String snapshotString = "skipFlushTableSnapshot";
byte[] snapshot = Bytes.toBytes(snapshotString);
- admin.snapshot(snapshotString, TABLE_NAME, SnapshotDescription.Type.SKIPFLUSH);
+ admin.snapshot(snapshotString, TABLE_NAME,
+ ProtobufUtil.createSnapshotType(HBaseProtos.SnapshotDescription.Type.SKIPFLUSH));
LOG.debug("Snapshot completed.");
// make sure we have the snapshot
- List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin,
- snapshot, TABLE_NAME);
+ List<SnapshotDescription> snapshots =
+ SnapshotTestingUtils.assertOneSnapshotThatMatches(admin, snapshot, TABLE_NAME);
// make sure its a valid snapshot
LOG.debug("FS state after snapshot:");
UTIL.getHBaseCluster().getMaster().getMasterFileSystem().logFileSystemState(LOG);
- SnapshotTestingUtils.confirmSnapshotValid(UTIL, snapshots.get(0), TABLE_NAME, TEST_FAM);
+ SnapshotTestingUtils.confirmSnapshotValid(UTIL,
+ ProtobufUtil.createHBaseProtosSnapshotDesc(snapshots.get(0)), TABLE_NAME, TEST_FAM);
admin.deleteSnapshot(snapshot);
snapshots = admin.listSnapshots();
@@ -234,7 +240,8 @@ public class TestFlushSnapshotFromClient {
LOG.debug("FS state after snapshot:");
UTIL.getHBaseCluster().getMaster().getMasterFileSystem().logFileSystemState(LOG);
- SnapshotTestingUtils.confirmSnapshotValid(UTIL, snapshots.get(0), TABLE_NAME, TEST_FAM);
+ SnapshotTestingUtils.confirmSnapshotValid(UTIL,
+ ProtobufUtil.createHBaseProtosSnapshotDesc(snapshots.get(0)), TABLE_NAME, TEST_FAM);
}
@Test
@@ -258,7 +265,8 @@ public class TestFlushSnapshotFromClient {
// snapshot the non-existant table
try {
- admin.snapshot("fail", tableName, SnapshotDescription.Type.FLUSH);
+ admin.snapshot("fail", tableName,
+ ProtobufUtil.createSnapshotType(HBaseProtos.SnapshotDescription.Type.FLUSH));
fail("Snapshot succeeded even though there is not table.");
} catch (SnapshotCreationException e) {
LOG.info("Correctly failed to snapshot a non-existant table:" + e.getMessage());
@@ -267,13 +275,14 @@ public class TestFlushSnapshotFromClient {
@Test
public void testAsyncFlushSnapshot() throws Exception {
- SnapshotDescription snapshot = SnapshotDescription.newBuilder().setName("asyncSnapshot")
- .setTable(TABLE_NAME.getNameAsString())
- .setType(SnapshotDescription.Type.FLUSH)
- .build();
+ HBaseProtos.SnapshotDescription snapshot = HBaseProtos.SnapshotDescription.newBuilder()
+ .setName("asyncSnapshot").setTable(TABLE_NAME.getNameAsString())
+ .setType(HBaseProtos.SnapshotDescription.Type.FLUSH).build();
// take the snapshot async
- admin.takeSnapshotAsync(snapshot);
+ admin.takeSnapshotAsync(
+ new SnapshotDescription("asyncSnapshot", TABLE_NAME.getNameAsString(),
+ ProtobufUtil.createSnapshotType(HBaseProtos.SnapshotDescription.Type.FLUSH)));
// constantly loop, looking for the snapshot to complete
HMaster master = UTIL.getMiniHBaseCluster().getMaster();
@@ -295,7 +304,8 @@ public class TestFlushSnapshotFromClient {
// Take a snapshot
String snapshotBeforeMergeName = "snapshotBeforeMerge";
- admin.snapshot(snapshotBeforeMergeName, TABLE_NAME, SnapshotDescription.Type.FLUSH);
+ admin.snapshot(snapshotBeforeMergeName, TABLE_NAME,
+ ProtobufUtil.createSnapshotType(HBaseProtos.SnapshotDescription.Type.FLUSH));
// Clone the table
TableName cloneBeforeMergeName = TableName.valueOf("cloneBeforeMerge");
@@ -364,7 +374,7 @@ public class TestFlushSnapshotFromClient {
// Take a snapshot
String snapshotName = "snapshotAfterMerge";
SnapshotTestingUtils.snapshot(admin, snapshotName, TABLE_NAME.getNameAsString(),
- SnapshotDescription.Type.FLUSH, 3);
+ HBaseProtos.SnapshotDescription.Type.FLUSH, 3);
// Clone the table
TableName cloneName = TableName.valueOf("cloneMerge");
@@ -425,14 +435,16 @@ public class TestFlushSnapshotFromClient {
@Override
public void run() {
try {
- LOG.info("Submitting snapshot request: " + ClientSnapshotDescriptionUtils.toString(ss));
+ LOG.info("Submitting snapshot request: " + ClientSnapshotDescriptionUtils
+ .toString(ProtobufUtil.createHBaseProtosSnapshotDesc(ss)));
admin.takeSnapshotAsync(ss);
} catch (Exception e) {
LOG.info("Exception during snapshot request: " + ClientSnapshotDescriptionUtils.toString(
- ss)
+ ProtobufUtil.createHBaseProtosSnapshotDesc(ss))
+ ". This is ok, we expect some", e);
}
- LOG.info("Submitted snapshot request: " + ClientSnapshotDescriptionUtils.toString(ss));
+ LOG.info("Submitted snapshot request: " + ClientSnapshotDescriptionUtils
+ .toString(ProtobufUtil.createHBaseProtosSnapshotDesc(ss)));
toBeSubmitted.countDown();
}
};
@@ -440,11 +452,15 @@ public class TestFlushSnapshotFromClient {
// build descriptions
SnapshotDescription[] descs = new SnapshotDescription[ssNum];
for (int i = 0; i < ssNum; i++) {
- SnapshotDescription.Builder builder = SnapshotDescription.newBuilder();
- builder.setTable(((i % 2) == 0 ? TABLE_NAME : TABLE2_NAME).getNameAsString());
- builder.setName("ss"+i);
- builder.setType(SnapshotDescription.Type.FLUSH);
- descs[i] = builder.build();
+ HBaseProtos.SnapshotDescription.Builder builder =
+ HBaseProtos.SnapshotDescription.newBuilder();
+ if(i %2 ==0) {
+ descs[i] = new SnapshotDescription("ss" + i, TABLE_NAME.getNameAsString(),
+ ProtobufUtil.createSnapshotType(HBaseProtos.SnapshotDescription.Type.FLUSH));
+ } else {
+ descs[i] = new SnapshotDescription("ss" + i, TABLE2_NAME.getNameAsString(),
+ ProtobufUtil.createSnapshotType(HBaseProtos.SnapshotDescription.Type.FLUSH));
+ }
}
// kick each off its own thread
http://git-wip-us.apache.org/repos/asf/hbase/blob/c06a976a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreFlushSnapshotFromClient.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreFlushSnapshotFromClient.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreFlushSnapshotFromClient.java
index 877ee21..04fce5c 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreFlushSnapshotFromClient.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreFlushSnapshotFromClient.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
+import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
import org.apache.hadoop.hbase.regionserver.snapshot.RegionServerSnapshotManager;
import org.apache.hadoop.hbase.testclassification.LargeTests;
@@ -116,7 +117,7 @@ public class TestRestoreFlushSnapshotFromClient {
// take a snapshot
admin.snapshot(Bytes.toString(snapshotName0), tableName,
- SnapshotDescription.Type.FLUSH);
+ ProtobufUtil.createSnapshotType(SnapshotDescription.Type.FLUSH));
LOG.info("=== after snapshot with 500 rows");
logFSTree();
@@ -129,7 +130,7 @@ public class TestRestoreFlushSnapshotFromClient {
// take a snapshot of the updated table
admin.snapshot(Bytes.toString(snapshotName1), tableName,
- SnapshotDescription.Type.FLUSH);
+ ProtobufUtil.createSnapshotType(SnapshotDescription.Type.FLUSH));
LOG.info("=== after snapshot with 1000 rows");
logFSTree();
table.close();
@@ -194,7 +195,8 @@ public class TestRestoreFlushSnapshotFromClient {
TableName clonedTableName = TableName.valueOf("clonedtb-" + System.currentTimeMillis());
admin.cloneSnapshot(snapshotName0, clonedTableName);
verifyRowCount(UTIL, clonedTableName, snapshot0Rows);
- admin.snapshot(Bytes.toString(snapshotName2), clonedTableName, SnapshotDescription.Type.FLUSH);
+ admin.snapshot(Bytes.toString(snapshotName2), clonedTableName,
+ ProtobufUtil.createSnapshotType(SnapshotDescription.Type.FLUSH));
UTIL.deleteTable(clonedTableName);
admin.cloneSnapshot(snapshotName2, clonedTableName);
http://git-wip-us.apache.org/repos/asf/hbase/blob/c06a976a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckOneRS.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckOneRS.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckOneRS.java
index dbb23a5..d5869ed 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckOneRS.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckOneRS.java
@@ -33,12 +33,12 @@ import org.apache.hadoop.hbase.MetaTableAccessor;
import org.apache.hadoop.hbase.MiniHBaseCluster;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.ClusterConnection;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.MasterSwitchType;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.RegionLocator;
import org.apache.hadoop.hbase.client.Result;
@@ -1856,9 +1856,9 @@ public class TestHBaseFsckOneRS extends BaseTestHBaseFsck {
@Test
public void testSplitOrMergeStatWhenHBCKAbort() throws Exception {
admin.setSplitOrMergeEnabled(true, false, true,
- Admin.MasterSwitchType.SPLIT, Admin.MasterSwitchType.MERGE);
- boolean oldSplit = admin.isSplitOrMergeEnabled(Admin.MasterSwitchType.SPLIT);
- boolean oldMerge = admin.isSplitOrMergeEnabled(Admin.MasterSwitchType.MERGE);
+ MasterSwitchType.SPLIT, MasterSwitchType.MERGE);
+ boolean oldSplit = admin.isSplitOrMergeEnabled(MasterSwitchType.SPLIT);
+ boolean oldMerge = admin.isSplitOrMergeEnabled(MasterSwitchType.MERGE);
assertTrue(oldSplit);
assertTrue(oldMerge);
@@ -1880,8 +1880,8 @@ public class TestHBaseFsckOneRS extends BaseTestHBaseFsck {
spiedHbck.onlineHbck();
spiedHbck.close();
- boolean split = admin.isSplitOrMergeEnabled(Admin.MasterSwitchType.SPLIT);
- boolean merge = admin.isSplitOrMergeEnabled(Admin.MasterSwitchType.MERGE);
+ boolean split = admin.isSplitOrMergeEnabled(MasterSwitchType.SPLIT);
+ boolean merge = admin.isSplitOrMergeEnabled(MasterSwitchType.MERGE);
assertFalse(split);
assertFalse(merge);
@@ -1892,8 +1892,8 @@ public class TestHBaseFsckOneRS extends BaseTestHBaseFsck {
hbck.onlineHbck();
hbck.close();
- split = admin.isSplitOrMergeEnabled(Admin.MasterSwitchType.SPLIT);
- merge = admin.isSplitOrMergeEnabled(Admin.MasterSwitchType.MERGE);
+ split = admin.isSplitOrMergeEnabled(MasterSwitchType.SPLIT);
+ merge = admin.isSplitOrMergeEnabled(MasterSwitchType.MERGE);
assertTrue(split);
assertTrue(merge);
[45/50] [abbrv] hbase git commit: HBASE-15236 Inconsistent cell reads
over multiple bulk-loaded HFiles. In KeyValueHeap,
if two cells are same i.e. have same key and timestamp,
then instead of directly using seq id to determine newer one, we should use S
Posted by sy...@apache.org.
HBASE-15236 Inconsistent cell reads over multiple bulk-loaded HFiles. In KeyValueHeap, if two cells are same i.e. have same key and timestamp, then instead of directly using seq id to determine newer one, we should use StoreFile.Comparater.SEQ_ID because that's what is used to determine order of hfiles. In this patch, we assign each scanner an order based on it's index in storefiles list, which is then used in KeyValueHeap to disambiguate between same cells. Changes the getSequenceId() in KeyValueScanner class to getScannerOrder(). Testing: Adds unit test to TestKeyValueHeap. Manual testing: Three cases (Tables t, t2, t3 in the jira description), single region, 2 hfiles with same seq id, timestamps and duplicate KVs. Made sure that returned kv was same for get and scan. (Apekshit)
Change-Id: I22600c91c0a51fb63eb17db73472839d2f13957c
Signed-off-by: stack <st...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/11740570
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/11740570
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/11740570
Branch: refs/heads/hbase-12439
Commit: 11740570c1440254a76fae67d318c6a852cb56b8
Parents: a6e2967
Author: Apekshit <ap...@gmail.com>
Authored: Tue Feb 23 00:31:18 2016 -0800
Committer: stack <st...@apache.org>
Committed: Mon May 9 16:57:06 2016 -0700
----------------------------------------------------------------------
.../hadoop/hbase/regionserver/KeyValueHeap.java | 20 +-
.../hbase/regionserver/KeyValueScanner.java | 12 +-
.../hbase/regionserver/MemStoreScanner.java | 7 +-
.../hbase/regionserver/SegmentScanner.java | 35 +--
.../hadoop/hbase/regionserver/StoreFile.java | 6 +-
.../hbase/regionserver/StoreFileReader.java | 43 +--
.../hbase/regionserver/StoreFileScanner.java | 38 ++-
.../hadoop/hbase/regionserver/StoreScanner.java | 5 +-
.../hbase/util/CollectionBackedScanner.java | 5 +-
.../hbase/regionserver/TestKeyValueHeap.java | 269 +++++++------------
.../hbase/regionserver/TestStoreFile.java | 2 +-
.../regionserver/compactions/TestCompactor.java | 2 +-
.../compactions/TestStripeCompactionPolicy.java | 3 +-
13 files changed, 212 insertions(+), 235 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/11740570/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueHeap.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueHeap.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueHeap.java
index 89fc8fb..9ece14b 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueHeap.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueHeap.java
@@ -189,17 +189,10 @@ public class KeyValueHeap extends NonReversedNonLazyKeyValueScanner
if (comparison != 0) {
return comparison;
} else {
- // Since both the keys are exactly the same, we break the tie in favor
- // of the key which came latest.
- long leftSequenceID = left.getSequenceID();
- long rightSequenceID = right.getSequenceID();
- if (leftSequenceID > rightSequenceID) {
- return -1;
- } else if (leftSequenceID < rightSequenceID) {
- return 1;
- } else {
- return 0;
- }
+ // Since both the keys are exactly the same, we break the tie in favor of higher ordered
+ // scanner since it'll have newer data. Since higher value should come first, we reverse
+ // sort here.
+ return Long.compare(right.getScannerOrder(), left.getScannerOrder());
}
}
/**
@@ -406,8 +399,11 @@ public class KeyValueHeap extends NonReversedNonLazyKeyValueScanner
return this.heap;
}
+ /**
+ * @see KeyValueScanner#getScannerOrder()
+ */
@Override
- public long getSequenceID() {
+ public long getScannerOrder() {
return 0;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/11740570/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueScanner.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueScanner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueScanner.java
index ed86a83..44b081b 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueScanner.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/KeyValueScanner.java
@@ -70,13 +70,13 @@ public interface KeyValueScanner extends Shipper, Closeable {
boolean reseek(Cell key) throws IOException;
/**
- * Get the sequence id associated with this KeyValueScanner. This is required
- * for comparing multiple files to find out which one has the latest data.
- * The default implementation for this would be to return 0. A file having
- * lower sequence id will be considered to be the older one.
+ * Get the order of this KeyValueScanner. This is only relevant for StoreFileScanners and
+ * MemStoreScanners (other scanners simply return 0). This is required for comparing multiple
+ * files to find out which one has the latest data. StoreFileScanners are ordered from 0
+ * (oldest) to newest in increasing order. MemStoreScanner gets LONG.max since it always
+ * contains freshest data.
*/
- // TODO: Implement SequenceId Interface instead.
- long getSequenceID();
+ long getScannerOrder();
/**
* Close the KeyValue scanner.
http://git-wip-us.apache.org/repos/asf/hbase/blob/11740570/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreScanner.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreScanner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreScanner.java
index dfcec25..01a7ff3 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreScanner.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreScanner.java
@@ -186,11 +186,12 @@ public class MemStoreScanner extends NonLazyKeyValueScanner {
}
/**
- * MemStoreScanner returns max value as sequence id because it will
- * always have the latest data among all files.
+ * MemStoreScanner returns Long.MAX_VALUE because it will always have the latest data among all
+ * scanners.
+ * @see KeyValueScanner#getScannerOrder()
*/
@Override
- public synchronized long getSequenceID() {
+ public long getScannerOrder() {
return Long.MAX_VALUE;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/11740570/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SegmentScanner.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SegmentScanner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SegmentScanner.java
index b5aabb8..45f72d83 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SegmentScanner.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SegmentScanner.java
@@ -33,7 +33,12 @@ import org.apache.hadoop.hbase.client.Scan;
@InterfaceAudience.Private
public class SegmentScanner implements KeyValueScanner {
- private long sequenceID = Long.MAX_VALUE;
+ /**
+ * Order of this scanner relative to other scanners. See
+ * {@link KeyValueScanner#getScannerOrder()}.
+ */
+ private long scannerOrder;
+ private static final long DEFAULT_SCANNER_ORDER = Long.MAX_VALUE;
// the observed structure
private final Segment segment;
@@ -52,6 +57,13 @@ public class SegmentScanner implements KeyValueScanner {
private Cell last = null;
protected SegmentScanner(Segment segment, long readPoint) {
+ this(segment, readPoint, DEFAULT_SCANNER_ORDER);
+ }
+
+ /**
+ * @param scannerOrder see {@link KeyValueScanner#getScannerOrder()}.
+ */
+ protected SegmentScanner(Segment segment, long readPoint, long scannerOrder) {
this.segment = segment;
this.readPoint = readPoint;
iter = segment.iterator();
@@ -59,6 +71,7 @@ public class SegmentScanner implements KeyValueScanner {
current = getNext();
//increase the reference count so the underlying structure will not be de-allocated
this.segment.incScannerCount();
+ this.scannerOrder = scannerOrder;
}
/**
@@ -208,14 +221,11 @@ public class SegmentScanner implements KeyValueScanner {
}
/**
- * Get the sequence id associated with this KeyValueScanner. This is required
- * for comparing multiple files (or memstore segments) scanners to find out
- * which one has the latest data.
- *
+ * @see KeyValueScanner#getScannerOrder()
*/
@Override
- public long getSequenceID() {
- return sequenceID;
+ public long getScannerOrder() {
+ return scannerOrder;
}
/**
@@ -297,15 +307,6 @@ public class SegmentScanner implements KeyValueScanner {
}
/**
- * Set the sequence id of the scanner.
- * This is used to determine an order between memory segment scanners.
- * @param x a unique sequence id
- */
- public void setSequenceID(long x) {
- sequenceID = x;
- }
-
- /**
* Returns whether the given scan should seek in this segment
* @return whether the given scan should seek in this segment
*/
@@ -321,7 +322,7 @@ public class SegmentScanner implements KeyValueScanner {
@Override
public String toString() {
String res = "Store segment scanner of type "+this.getClass().getName()+"; ";
- res += "sequence id "+getSequenceID()+"; ";
+ res += "Scanner order " + getScannerOrder() + "; ";
res += getSegment().toString();
return res;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/11740570/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
index b14c6b6..46f0cb8 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
@@ -157,7 +157,8 @@ public class StoreFile {
Bytes.toBytes("BULKLOAD_TIMESTAMP");
/**
- * Map of the metadata entries in the corresponding HFile
+ * Map of the metadata entries in the corresponding HFile. Populated when Reader is opened
+ * after which it is not modified again.
*/
private Map<byte[], byte[]> metadataMap;
@@ -237,6 +238,7 @@ public class StoreFile {
this.fileInfo = other.fileInfo;
this.cacheConf = other.cacheConf;
this.cfBloomType = other.cfBloomType;
+ this.metadataMap = other.metadataMap;
}
/**
@@ -370,7 +372,7 @@ public class StoreFile {
if (startPos != -1) {
bulkLoadedHFile = true;
}
- return bulkLoadedHFile || metadataMap.containsKey(BULKLOAD_TIME_KEY);
+ return bulkLoadedHFile || (metadataMap != null && metadataMap.containsKey(BULKLOAD_TIME_KEY));
}
@VisibleForTesting
http://git-wip-us.apache.org/repos/asf/hbase/blob/11740570/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileReader.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileReader.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileReader.java
index 4e7e829..e890381 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileReader.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileReader.java
@@ -114,19 +114,27 @@ public class StoreFileReader {
}
/**
- * Get a scanner to scan over this StoreFile. Do not use
- * this overload if using this scanner for compactions.
+ * Uses {@link #getStoreFileScanner(boolean, boolean, boolean, long, long)} by setting
+ * {@code isCompaction} to false, {@code readPt} to 0 and {@code scannerOrder} to 0.
+ * Do not use this overload if using this scanner for compactions.
*
- * @param cacheBlocks should this scanner cache blocks?
- * @param pread use pread (for highly concurrent small readers)
- * @return a scanner
+ * @see #getStoreFileScanner(boolean, boolean, boolean, long, long)
+ */
+ public StoreFileScanner getStoreFileScanner(boolean cacheBlocks, boolean pread) {
+ // 0 is passed as readpoint because this method is only used by test
+ // where StoreFile is directly operated upon
+ return getStoreFileScanner(cacheBlocks, pread, false, 0, 0);
+ }
+
+ /**
+ * Uses {@link #getStoreFileScanner(boolean, boolean, boolean, long, long)} by setting
+ * {@code scannerOrder} to 0.
+ *
+ * @see #getStoreFileScanner(boolean, boolean, boolean, long, long)
*/
- public StoreFileScanner getStoreFileScanner(boolean cacheBlocks,
- boolean pread) {
- return getStoreFileScanner(cacheBlocks, pread, false,
- // 0 is passed as readpoint because this method is only used by test
- // where StoreFile is directly operated upon
- 0);
+ public StoreFileScanner getStoreFileScanner(
+ boolean cacheBlocks, boolean pread, boolean isCompaction, long readPt) {
+ return getStoreFileScanner(cacheBlocks, pread, isCompaction, readPt, 0);
}
/**
@@ -135,16 +143,17 @@ public class StoreFileReader {
* @param cacheBlocks should this scanner cache blocks?
* @param pread use pread (for highly concurrent small readers)
* @param isCompaction is scanner being used for compaction?
+ * @param scannerOrder Order of this scanner relative to other scanners. See
+ * {@link KeyValueScanner#getScannerOrder()}.
* @return a scanner
*/
- public StoreFileScanner getStoreFileScanner(boolean cacheBlocks,
- boolean pread,
- boolean isCompaction, long readPt) {
+ public StoreFileScanner getStoreFileScanner(
+ boolean cacheBlocks, boolean pread, boolean isCompaction, long readPt, long scannerOrder) {
// Increment the ref count
refCount.incrementAndGet();
- return new StoreFileScanner(this,
- getScanner(cacheBlocks, pread, isCompaction),
- !isCompaction, reader.hasMVCCInfo(), readPt);
+ return new StoreFileScanner(
+ this, getScanner(cacheBlocks, pread, isCompaction), !isCompaction, reader.hasMVCCInfo(),
+ readPt, scannerOrder);
}
/**
http://git-wip-us.apache.org/repos/asf/hbase/blob/11740570/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java
index e7f8f88..abade0e 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java
@@ -23,6 +23,7 @@ import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.Collections;
import java.util.List;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
@@ -65,17 +66,37 @@ public class StoreFileScanner implements KeyValueScanner {
private long readPt;
+ // Order of this scanner relative to other scanners when duplicate key-value is found.
+ // Higher values means scanner has newer data.
+ private long scannerOrder;
+
/**
* Implements a {@link KeyValueScanner} on top of the specified {@link HFileScanner}
- * @param hfs HFile scanner
+ * @param useMVCC If true, scanner will filter out updates with MVCC larger than {@code readPt}.
+ * @param readPt MVCC value to use to filter out the updates newer than this scanner.
+ * @param hasMVCC Set to true if underlying store file reader has MVCC info.
*/
public StoreFileScanner(StoreFileReader reader, HFileScanner hfs, boolean useMVCC,
boolean hasMVCC, long readPt) {
+ this (reader, hfs, useMVCC, hasMVCC, readPt, 0);
+ }
+
+ /**
+ * Implements a {@link KeyValueScanner} on top of the specified {@link HFileScanner}
+ * @param useMVCC If true, scanner will filter out updates with MVCC larger than {@code readPt}.
+ * @param readPt MVCC value to use to filter out the updates newer than this scanner.
+ * @param hasMVCC Set to true if underlying store file reader has MVCC info.
+ * @param scannerOrder Order of the scanner relative to other scanners.
+ * See {@link KeyValueScanner#getScannerOrder()}.
+ */
+ public StoreFileScanner(StoreFileReader reader, HFileScanner hfs, boolean useMVCC,
+ boolean hasMVCC, long readPt, long scannerOrder) {
this.readPt = readPt;
this.reader = reader;
this.hfs = hfs;
this.enforceMVCC = useMVCC;
this.hasMVCCInfo = hasMVCC;
+ this.scannerOrder = scannerOrder;
}
boolean isPrimaryReplica() {
@@ -115,11 +136,13 @@ public class StoreFileScanner implements KeyValueScanner {
ScanQueryMatcher matcher, long readPt, boolean isPrimaryReplica) throws IOException {
List<StoreFileScanner> scanners = new ArrayList<StoreFileScanner>(
files.size());
- for (StoreFile file : files) {
- StoreFileReader r = file.createReader(canUseDrop);
+ List<StoreFile> sorted_files = new ArrayList<>(files);
+ Collections.sort(sorted_files, StoreFile.Comparators.SEQ_ID);
+ for (int i = 0; i < sorted_files.size(); i++) {
+ StoreFileReader r = sorted_files.get(i).createReader();
r.setReplicaStoreFile(isPrimaryReplica);
StoreFileScanner scanner = r.getStoreFileScanner(cacheBlocks, usePread,
- isCompaction, readPt);
+ isCompaction, readPt, i);
scanner.setScanQueryMatcher(matcher);
scanners.add(scanner);
}
@@ -303,9 +326,12 @@ public class StoreFileScanner implements KeyValueScanner {
return s.next();
}
+ /**
+ * @see KeyValueScanner#getScannerOrder()
+ */
@Override
- public long getSequenceID() {
- return reader.getSequenceID();
+ public long getScannerOrder() {
+ return scannerOrder;
}
/**
http://git-wip-us.apache.org/repos/asf/hbase/blob/11740570/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
index ecae787..7ebae94 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
@@ -887,8 +887,11 @@ public class StoreScanner extends NonReversedNonLazyKeyValueScanner
return false;
}
+ /**
+ * @see KeyValueScanner#getScannerOrder()
+ */
@Override
- public long getSequenceID() {
+ public long getScannerOrder() {
return 0;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/11740570/hbase-server/src/main/java/org/apache/hadoop/hbase/util/CollectionBackedScanner.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/CollectionBackedScanner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/CollectionBackedScanner.java
index 4720880..3f05969 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/CollectionBackedScanner.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/CollectionBackedScanner.java
@@ -116,8 +116,11 @@ public class CollectionBackedScanner extends NonReversedNonLazyKeyValueScanner {
return false;
}
+ /**
+ * @see org.apache.hadoop.hbase.regionserver.KeyValueScanner#getScannerOrder()
+ */
@Override
- public long getSequenceID() {
+ public long getScannerOrder() {
return 0;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/11740570/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestKeyValueHeap.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestKeyValueHeap.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestKeyValueHeap.java
index aff40c1..b030c74 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestKeyValueHeap.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestKeyValueHeap.java
@@ -21,7 +21,7 @@ package org.apache.hadoop.hbase.regionserver;
import java.io.IOException;
import java.util.ArrayList;
-import java.util.Iterator;
+import java.util.Arrays;
import java.util.List;
import org.apache.hadoop.hbase.Cell;
@@ -38,178 +38,105 @@ import org.junit.experimental.categories.Category;
@Category({RegionServerTests.class, SmallTests.class})
public class TestKeyValueHeap extends HBaseTestCase {
- private static final boolean PRINT = false;
-
- List<KeyValueScanner> scanners = new ArrayList<KeyValueScanner>();
-
- private byte[] row1;
- private byte[] fam1;
- private byte[] col1;
- private byte[] data;
+ private byte[] row1 = Bytes.toBytes("row1");
+ private byte[] fam1 = Bytes.toBytes("fam1");
+ private byte[] col1 = Bytes.toBytes("col1");
+ private byte[] data = Bytes.toBytes("data");
+
+ private byte[] row2 = Bytes.toBytes("row2");
+ private byte[] fam2 = Bytes.toBytes("fam2");
+ private byte[] col2 = Bytes.toBytes("col2");
+
+ private byte[] col3 = Bytes.toBytes("col3");
+ private byte[] col4 = Bytes.toBytes("col4");
+ private byte[] col5 = Bytes.toBytes("col5");
+
+ // Variable name encoding. kv<row#><fam#><col#>
+ Cell kv111 = new KeyValue(row1, fam1, col1, data);
+ Cell kv112 = new KeyValue(row1, fam1, col2, data);
+ Cell kv113 = new KeyValue(row1, fam1, col3, data);
+ Cell kv114 = new KeyValue(row1, fam1, col4, data);
+ Cell kv115 = new KeyValue(row1, fam1, col5, data);
+ Cell kv121 = new KeyValue(row1, fam2, col1, data);
+ Cell kv122 = new KeyValue(row1, fam2, col2, data);
+ Cell kv211 = new KeyValue(row2, fam1, col1, data);
+ Cell kv212 = new KeyValue(row2, fam1, col2, data);
+ Cell kv213 = new KeyValue(row2, fam1, col3, data);
+
+ TestScanner s1 = new TestScanner(Arrays.asList(kv115, kv211, kv212));
+ TestScanner s2 = new TestScanner(Arrays.asList(kv111, kv112));
+ TestScanner s3 = new TestScanner(Arrays.asList(kv113, kv114, kv121, kv122, kv213));
+
+ List<KeyValueScanner> scanners = new ArrayList<KeyValueScanner>(Arrays.asList(s1, s2, s3));
+
+ /*
+ * Uses {@code scanners} to build a KeyValueHeap, iterates over it and asserts that returned
+ * Cells are same as {@code expected}.
+ * @return List of Cells returned from scanners.
+ */
+ public List<Cell> assertCells(List<Cell> expected, List<KeyValueScanner> scanners)
+ throws IOException {
+ //Creating KeyValueHeap
+ KeyValueHeap kvh = new KeyValueHeap(scanners, CellComparator.COMPARATOR);
- private byte[] row2;
- private byte[] fam2;
- private byte[] col2;
+ List<Cell> actual = new ArrayList<>();
+ while(kvh.peek() != null){
+ actual.add(kvh.next());
+ }
- private byte[] col3;
- private byte[] col4;
- private byte[] col5;
+ assertEquals(expected, actual);
+ return actual;
+ }
@Before
public void setUp() throws Exception {
super.setUp();
- data = Bytes.toBytes("data");
- row1 = Bytes.toBytes("row1");
- fam1 = Bytes.toBytes("fam1");
- col1 = Bytes.toBytes("col1");
- row2 = Bytes.toBytes("row2");
- fam2 = Bytes.toBytes("fam2");
- col2 = Bytes.toBytes("col2");
- col3 = Bytes.toBytes("col3");
- col4 = Bytes.toBytes("col4");
- col5 = Bytes.toBytes("col5");
}
@Test
public void testSorted() throws IOException{
//Cases that need to be checked are:
- //1. The "smallest" KeyValue is in the same scanners as current
+ //1. The "smallest" Cell is in the same scanners as current
//2. Current scanner gets empty
- List<Cell> l1 = new ArrayList<Cell>();
- l1.add(new KeyValue(row1, fam1, col5, data));
- l1.add(new KeyValue(row2, fam1, col1, data));
- l1.add(new KeyValue(row2, fam1, col2, data));
- scanners.add(new Scanner(l1));
-
- List<Cell> l2 = new ArrayList<Cell>();
- l2.add(new KeyValue(row1, fam1, col1, data));
- l2.add(new KeyValue(row1, fam1, col2, data));
- scanners.add(new Scanner(l2));
-
- List<Cell> l3 = new ArrayList<Cell>();
- l3.add(new KeyValue(row1, fam1, col3, data));
- l3.add(new KeyValue(row1, fam1, col4, data));
- l3.add(new KeyValue(row1, fam2, col1, data));
- l3.add(new KeyValue(row1, fam2, col2, data));
- l3.add(new KeyValue(row2, fam1, col3, data));
- scanners.add(new Scanner(l3));
-
- List<KeyValue> expected = new ArrayList<KeyValue>();
- expected.add(new KeyValue(row1, fam1, col1, data));
- expected.add(new KeyValue(row1, fam1, col2, data));
- expected.add(new KeyValue(row1, fam1, col3, data));
- expected.add(new KeyValue(row1, fam1, col4, data));
- expected.add(new KeyValue(row1, fam1, col5, data));
- expected.add(new KeyValue(row1, fam2, col1, data));
- expected.add(new KeyValue(row1, fam2, col2, data));
- expected.add(new KeyValue(row2, fam1, col1, data));
- expected.add(new KeyValue(row2, fam1, col2, data));
- expected.add(new KeyValue(row2, fam1, col3, data));
+ List<Cell> expected = Arrays.asList(
+ kv111, kv112, kv113, kv114, kv115, kv121, kv122, kv211, kv212, kv213);
- //Creating KeyValueHeap
- KeyValueHeap kvh =
- new KeyValueHeap(scanners, CellComparator.COMPARATOR);
-
- List<Cell> actual = new ArrayList<Cell>();
- while(kvh.peek() != null){
- actual.add(kvh.next());
- }
-
- assertEquals(expected.size(), actual.size());
- for(int i=0; i<expected.size(); i++){
- assertEquals(expected.get(i), actual.get(i));
- if(PRINT){
- System.out.println("expected " +expected.get(i)+
- "\nactual " +actual.get(i) +"\n");
- }
- }
+ List<Cell> actual = assertCells(expected, scanners);
//Check if result is sorted according to Comparator
for(int i=0; i<actual.size()-1; i++){
int ret = CellComparator.COMPARATOR.compare(actual.get(i), actual.get(i+1));
assertTrue(ret < 0);
}
-
}
@Test
public void testSeek() throws IOException {
//Cases:
- //1. Seek KeyValue that is not in scanner
+ //1. Seek Cell that is not in scanner
//2. Check that smallest that is returned from a seek is correct
- List<Cell> l1 = new ArrayList<Cell>();
- l1.add(new KeyValue(row1, fam1, col5, data));
- l1.add(new KeyValue(row2, fam1, col1, data));
- l1.add(new KeyValue(row2, fam1, col2, data));
- scanners.add(new Scanner(l1));
-
- List<Cell> l2 = new ArrayList<Cell>();
- l2.add(new KeyValue(row1, fam1, col1, data));
- l2.add(new KeyValue(row1, fam1, col2, data));
- scanners.add(new Scanner(l2));
-
- List<Cell> l3 = new ArrayList<Cell>();
- l3.add(new KeyValue(row1, fam1, col3, data));
- l3.add(new KeyValue(row1, fam1, col4, data));
- l3.add(new KeyValue(row1, fam2, col1, data));
- l3.add(new KeyValue(row1, fam2, col2, data));
- l3.add(new KeyValue(row2, fam1, col3, data));
- scanners.add(new Scanner(l3));
-
- List<KeyValue> expected = new ArrayList<KeyValue>();
- expected.add(new KeyValue(row2, fam1, col1, data));
+ List<Cell> expected = Arrays.asList(kv211);
//Creating KeyValueHeap
KeyValueHeap kvh =
new KeyValueHeap(scanners, CellComparator.COMPARATOR);
- KeyValue seekKv = new KeyValue(row2, fam1, null, null);
+ Cell seekKv = new KeyValue(row2, fam1, null, null);
kvh.seek(seekKv);
- List<Cell> actual = new ArrayList<Cell>();
- actual.add(kvh.peek());
-
- assertEquals(expected.size(), actual.size());
- for(int i=0; i<expected.size(); i++){
- assertEquals(expected.get(i), actual.get(i));
- if(PRINT){
- System.out.println("expected " +expected.get(i)+
- "\nactual " +actual.get(i) +"\n");
- }
- }
+ List<Cell> actual = Arrays.asList(kvh.peek());
+ assertEquals("Expected = " + Arrays.toString(expected.toArray())
+ + "\n Actual = " + Arrays.toString(actual.toArray()), expected, actual);
}
@Test
public void testScannerLeak() throws IOException {
// Test for unclosed scanners (HBASE-1927)
- List<Cell> l1 = new ArrayList<Cell>();
- l1.add(new KeyValue(row1, fam1, col5, data));
- l1.add(new KeyValue(row2, fam1, col1, data));
- l1.add(new KeyValue(row2, fam1, col2, data));
- Scanner s1 = new Scanner(l1);
- scanners.add(s1);
-
- List<Cell> l2 = new ArrayList<Cell>();
- l2.add(new KeyValue(row1, fam1, col1, data));
- l2.add(new KeyValue(row1, fam1, col2, data));
- Scanner s2 = new Scanner(l2);
- scanners.add(s2);
-
- List<Cell> l3 = new ArrayList<Cell>();
- l3.add(new KeyValue(row1, fam1, col3, data));
- l3.add(new KeyValue(row1, fam1, col4, data));
- l3.add(new KeyValue(row1, fam2, col1, data));
- l3.add(new KeyValue(row1, fam2, col2, data));
- l3.add(new KeyValue(row2, fam1, col3, data));
- Scanner s3 = new Scanner(l3);
- scanners.add(s3);
-
- List<Cell> l4 = new ArrayList<Cell>();
- Scanner s4 = new Scanner(l4);
+ TestScanner s4 = new TestScanner(new ArrayList<Cell>());
scanners.add(s4);
//Creating KeyValueHeap
@@ -225,7 +152,7 @@ public class TestKeyValueHeap extends HBaseTestCase {
assertTrue(kvh.scannersForDelayedClose.contains(s4));
kvh.close();
for(KeyValueScanner scanner : scanners) {
- assertTrue(((Scanner)scanner).isClosed());
+ assertTrue(((TestScanner)scanner).isClosed());
}
}
@@ -233,38 +160,19 @@ public class TestKeyValueHeap extends HBaseTestCase {
public void testScannerException() throws IOException {
// Test for NPE issue when exception happens in scanners (HBASE-13835)
- List<Cell> l1 = new ArrayList<Cell>();
- l1.add(new KeyValue(row1, fam1, col5, data));
- l1.add(new KeyValue(row2, fam1, col1, data));
- l1.add(new KeyValue(row2, fam1, col2, data));
- SeekScanner s1 = new SeekScanner(l1);
- scanners.add(s1);
-
- List<Cell> l2 = new ArrayList<Cell>();
- l2.add(new KeyValue(row1, fam1, col1, data));
- l2.add(new KeyValue(row1, fam1, col2, data));
- SeekScanner s2 = new SeekScanner(l2);
- scanners.add(s2);
-
- List<Cell> l3 = new ArrayList<Cell>();
- l3.add(new KeyValue(row1, fam1, col3, data));
- l3.add(new KeyValue(row1, fam1, col4, data));
- l3.add(new KeyValue(row1, fam2, col1, data));
- l3.add(new KeyValue(row1, fam2, col2, data));
- l3.add(new KeyValue(row2, fam1, col3, data));
- SeekScanner s3 = new SeekScanner(l3);
- scanners.add(s3);
-
- List<Cell> l4 = new ArrayList<Cell>();
- SeekScanner s4 = new SeekScanner(l4);
- scanners.add(s4);
+ TestScanner s1 = new SeekTestScanner(Arrays.asList(kv115, kv211, kv212));
+ TestScanner s2 = new SeekTestScanner(Arrays.asList(kv111, kv112));
+ TestScanner s3 = new SeekTestScanner(Arrays.asList(kv113, kv114, kv121, kv122, kv213));
+ TestScanner s4 = new SeekTestScanner(new ArrayList<Cell>());
+
+ List<KeyValueScanner> scanners = new ArrayList<KeyValueScanner>(Arrays.asList(s1, s2, s3, s4));
// Creating KeyValueHeap
KeyValueHeap kvh = new KeyValueHeap(scanners, CellComparator.COMPARATOR);
try {
for (KeyValueScanner scanner : scanners) {
- ((SeekScanner) scanner).setRealSeekDone(false);
+ ((SeekTestScanner) scanner).setRealSeekDone(false);
}
while (kvh.next() != null);
// The pollRealKV should throw IOE.
@@ -276,20 +184,47 @@ public class TestKeyValueHeap extends HBaseTestCase {
// It implies there is no NPE thrown from kvh.close() if getting here
for (KeyValueScanner scanner : scanners) {
// Verify that close is called and only called once for each scanner
- assertTrue(((SeekScanner) scanner).isClosed());
- assertEquals(((SeekScanner) scanner).getClosedNum(), 1);
+ assertTrue(((SeekTestScanner) scanner).isClosed());
+ assertEquals(((SeekTestScanner) scanner).getClosedNum(), 1);
+ }
+ }
+
+ @Test
+ public void testPriorityId() throws IOException {
+ Cell kv113A = new KeyValue(row1, fam1, col3, Bytes.toBytes("aaa"));
+ Cell kv113B = new KeyValue(row1, fam1, col3, Bytes.toBytes("bbb"));
+ {
+ TestScanner scan1 = new TestScanner(Arrays.asList(kv111, kv112, kv113A), 1);
+ TestScanner scan2 = new TestScanner(Arrays.asList(kv113B), 2);
+ List<Cell> expected = Arrays.asList(kv111, kv112, kv113B, kv113A);
+ assertCells(expected, new ArrayList<KeyValueScanner>(Arrays.asList(scan1, scan2)));
+ }
+ {
+ TestScanner scan1 = new TestScanner(Arrays.asList(kv111, kv112, kv113A), 2);
+ TestScanner scan2 = new TestScanner(Arrays.asList(kv113B), 1);
+ List<Cell> expected = Arrays.asList(kv111, kv112, kv113A, kv113B);
+ assertCells(expected, new ArrayList<KeyValueScanner>(Arrays.asList(scan1, scan2)));
}
}
- private static class Scanner extends CollectionBackedScanner {
- private Iterator<Cell> iter;
- private Cell current;
+ private static class TestScanner extends CollectionBackedScanner {
private boolean closed = false;
+ private long scannerOrder = 0;
- public Scanner(List<Cell> list) {
+ public TestScanner(List<Cell> list) {
super(list);
}
+ public TestScanner(List<Cell> list, long scannerOrder) {
+ this(list);
+ this.scannerOrder = scannerOrder;
+ }
+
+ @Override
+ public long getScannerOrder() {
+ return scannerOrder;
+ }
+
@Override
public void close(){
closed = true;
@@ -300,11 +235,11 @@ public class TestKeyValueHeap extends HBaseTestCase {
}
}
- private static class SeekScanner extends Scanner {
+ private static class SeekTestScanner extends TestScanner {
private int closedNum = 0;
private boolean realSeekDone = true;
- public SeekScanner(List<Cell> list) {
+ public SeekTestScanner(List<Cell> list) {
super(list);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/11740570/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java
index d8acd44..ab0c173 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java
@@ -211,7 +211,7 @@ public class TestStoreFile extends HBaseTestCase {
when(hcd.getName()).thenReturn(cf);
when(store.getFamily()).thenReturn(hcd);
StoreFileScanner scanner =
- new StoreFileScanner(reader, mock(HFileScanner.class), false, false, 0);
+ new StoreFileScanner(reader, mock(HFileScanner.class), false, false, 0, 0);
Scan scan = new Scan();
scan.setColumnFamilyTimeRange(cf, 0, 1);
assertFalse(scanner.shouldUseScanner(scan, store, 0));
http://git-wip-us.apache.org/repos/asf/hbase/blob/11740570/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/TestCompactor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/TestCompactor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/TestCompactor.java
index 7707116..670a8d3 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/TestCompactor.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/TestCompactor.java
@@ -64,7 +64,7 @@ public class TestCompactor {
when(r.length()).thenReturn(1L);
when(r.getBloomFilterType()).thenReturn(BloomType.NONE);
when(r.getHFileReader()).thenReturn(mock(HFile.Reader.class));
- when(r.getStoreFileScanner(anyBoolean(), anyBoolean(), anyBoolean(), anyLong()))
+ when(r.getStoreFileScanner(anyBoolean(), anyBoolean(), anyBoolean(), anyLong(), anyLong()))
.thenReturn(mock(StoreFileScanner.class));
when(sf.getReader()).thenReturn(r);
when(sf.createReader()).thenReturn(r);
http://git-wip-us.apache.org/repos/asf/hbase/blob/11740570/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/TestStripeCompactionPolicy.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/TestStripeCompactionPolicy.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/TestStripeCompactionPolicy.java
index 160deb3..d8770e0 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/TestStripeCompactionPolicy.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/TestStripeCompactionPolicy.java
@@ -750,7 +750,8 @@ public class TestStripeCompactionPolicy {
when(r.length()).thenReturn(size);
when(r.getBloomFilterType()).thenReturn(BloomType.NONE);
when(r.getHFileReader()).thenReturn(mock(HFile.Reader.class));
- when(r.getStoreFileScanner(anyBoolean(), anyBoolean(), anyBoolean(), anyLong())).thenReturn(
+ when(r.getStoreFileScanner(anyBoolean(), anyBoolean(), anyBoolean(), anyLong(), anyLong()))
+ .thenReturn(
mock(StoreFileScanner.class));
when(sf.getReader()).thenReturn(r);
when(sf.createReader(anyBoolean())).thenReturn(r);
[08/50] [abbrv] hbase git commit: HBASE-15528 Clean up outdated
entries in hbase-default.xml
Posted by sy...@apache.org.
HBASE-15528 Clean up outdated entries in hbase-default.xml
Signed-off-by: Enis Soztutar <en...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/0e370633
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/0e370633
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/0e370633
Branch: refs/heads/hbase-12439
Commit: 0e3706334519d18ade5089e883173de41f6202e8
Parents: 6abe187
Author: Junegunn Choi <ju...@gmail.com>
Authored: Fri Mar 25 17:23:04 2016 +0900
Committer: Enis Soztutar <en...@apache.org>
Committed: Tue May 3 19:09:19 2016 -0700
----------------------------------------------------------------------
.../src/main/resources/hbase-default.xml | 45 +-----------
src/main/asciidoc/_chapters/hbase-default.adoc | 77 +-------------------
2 files changed, 3 insertions(+), 119 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/0e370633/hbase-common/src/main/resources/hbase-default.xml
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/resources/hbase-default.xml b/hbase-common/src/main/resources/hbase-default.xml
index e19cbf8..62a6b62 100644
--- a/hbase-common/src/main/resources/hbase-default.xml
+++ b/hbase-common/src/main/resources/hbase-default.xml
@@ -155,12 +155,6 @@ possible configurations would overwhelm and obscure the important.
hbase-site.xml.</description>
</property>
<property>
- <name>hbase.master.catalog.timeout</name>
- <value>600000</value>
- <description>Timeout value for the Catalog Janitor from the master to
- META.</description>
- </property>
- <property>
<name>hbase.master.infoserver.redirect</name>
<value>true</value>
<description>Whether or not the Master listens to the Master web
@@ -311,11 +305,6 @@ possible configurations would overwhelm and obscure the important.
Default 1 hour. Set it to 0 to disable automatic flushing.</description>
</property>
<property>
- <name>hbase.regionserver.catalog.timeout</name>
- <value>600000</value>
- <description>Timeout value for the Catalog Janitor from the regionserver to META.</description>
- </property>
- <property>
<name>hbase.regionserver.dns.interface</name>
<value>default</value>
<description>The name of the Network Interface from which a region server
@@ -373,14 +362,6 @@ possible configurations would overwhelm and obscure the important.
relative path, so they will all go under this directory unless changed.</description>
</property>
<property>
- <name>zookeeper.znode.rootserver</name>
- <value>root-region-server</value>
- <description>Path to ZNode holding root region location. This is written by
- the master and read by clients and region servers. If a relative path is
- given, the parent folder will be ${zookeeper.znode.parent}. By default,
- this means the root location is stored at /hbase/root-region-server.</description>
- </property>
- <property>
<name>zookeeper.znode.acl.parent</name>
<value>acl</value>
<description>Root ZNode for access control lists.</description>
@@ -876,7 +857,7 @@ possible configurations would overwhelm and obscure the important.
megabytes of BucketCache. Default: 0.0</description>
</property>
<property>
- <name>hbase.bucketcache.sizes</name>
+ <name>hbase.bucketcache.bucket.sizes</name>
<value></value>
<description>A comma-separated list of sizes for buckets for the bucketcache.
Can be multiple sizes. List block sizes in order from smallest to largest.
@@ -1193,15 +1174,6 @@ possible configurations would overwhelm and obscure the important.
hbase.thrift.maxQueuedRequests threads.</description>
</property>
<property>
- <name>hbase.thrift.htablepool.size.max</name>
- <value>1000</value>
- <description>The upper bound for the table pool used in the Thrift gateways server.
- Since this is per table name, we assume a single table and so with 1000 default
- worker threads max this is set to a matching number. For other workloads this number
- can be adjusted as needed.
- </description>
- </property>
- <property>
<name>hbase.regionserver.thrift.framed</name>
<value>false</value>
<description>Use Thrift TFramedTransport on the server side.
@@ -1240,21 +1212,6 @@ possible configurations would overwhelm and obscure the important.
files when hbase.data.umask.enable is true</description>
</property>
<property>
- <name>hbase.metrics.showTableName</name>
- <value>true</value>
- <description>Whether to include the prefix "tbl.tablename" in per-column family metrics.
- If true, for each metric M, per-cf metrics will be reported for tbl.T.cf.CF.M, if false,
- per-cf metrics will be aggregated by column-family across tables, and reported for cf.CF.M.
- In both cases, the aggregated metric M across tables and cfs will be reported.</description>
- </property>
- <property>
- <name>hbase.metrics.exposeOperationTimes</name>
- <value>true</value>
- <description>Whether to report metrics about time taken performing an
- operation on the region server. Get, Put, Delete, Increment, and Append can all
- have their times exposed through Hadoop metrics per CF and per region.</description>
- </property>
- <property>
<name>hbase.snapshot.enabled</name>
<value>true</value>
<description>Set to true to allow snapshots to be taken / restored / cloned.</description>
http://git-wip-us.apache.org/repos/asf/hbase/blob/0e370633/src/main/asciidoc/_chapters/hbase-default.adoc
----------------------------------------------------------------------
diff --git a/src/main/asciidoc/_chapters/hbase-default.adoc b/src/main/asciidoc/_chapters/hbase-default.adoc
index 26929a3..df750e0 100644
--- a/src/main/asciidoc/_chapters/hbase-default.adoc
+++ b/src/main/asciidoc/_chapters/hbase-default.adoc
@@ -173,17 +173,6 @@ A comma-separated list of BaseHFileCleanerDelegate invoked by
`org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner`
-[[hbase.master.catalog.timeout]]
-*`hbase.master.catalog.timeout`*::
-+
-.Description
-Timeout value for the Catalog Janitor from the master to
- META.
-+
-.Default
-`600000`
-
-
[[hbase.master.infoserver.redirect]]
*`hbase.master.infoserver.redirect`*::
+
@@ -442,16 +431,6 @@ Maximum size of all memstores in a region server before flushes are forced.
`3600000`
-[[hbase.regionserver.catalog.timeout]]
-*`hbase.regionserver.catalog.timeout`*::
-+
-.Description
-Timeout value for the Catalog Janitor from the regionserver to META.
-+
-.Default
-`600000`
-
-
[[hbase.regionserver.dns.interface]]
*`hbase.regionserver.dns.interface`*::
+
@@ -522,19 +501,6 @@ Root ZNode for HBase in ZooKeeper. All of HBase's ZooKeeper
`/hbase`
-[[zookeeper.znode.rootserver]]
-*`zookeeper.znode.rootserver`*::
-+
-.Description
-Path to ZNode holding root region location. This is written by
- the master and read by clients and region servers. If a relative path is
- given, the parent folder will be ${zookeeper.znode.parent}. By default,
- this means the root location is stored at /hbase/root-region-server.
-+
-.Default
-`root-region-server`
-
-
[[zookeeper.znode.acl.parent]]
*`zookeeper.znode.acl.parent`*::
+
@@ -1280,8 +1246,8 @@ Used along with bucket cache, this is a float that EITHER represents a percentag
`0` when specified as a float
-[[hbase.bucketcache.sizes]]
-*`hbase.bucketcache.sizes`*::
+[[hbase.bucketcache.bucket.sizes]]
+*`hbase.bucketcache.bucket.sizes`*::
+
.Description
A comma-separated list of sizes for buckets for the bucketcache
@@ -1691,20 +1657,6 @@ The maximum number of pending Thrift connections waiting in the queue. If
`1000`
-[[hbase.thrift.htablepool.size.max]]
-*`hbase.thrift.htablepool.size.max`*::
-+
-.Description
-The upper bound for the table pool used in the Thrift gateways server.
- Since this is per table name, we assume a single table and so with 1000 default
- worker threads max this is set to a matching number. For other workloads this number
- can be adjusted as needed.
-
-+
-.Default
-`1000`
-
-
[[hbase.regionserver.thrift.framed]]
*`hbase.regionserver.thrift.framed`*::
+
@@ -1761,31 +1713,6 @@ File permissions that should be used to write data
`000`
-[[hbase.metrics.showTableName]]
-*`hbase.metrics.showTableName`*::
-+
-.Description
-Whether to include the prefix "tbl.tablename" in per-column family metrics.
- If true, for each metric M, per-cf metrics will be reported for tbl.T.cf.CF.M, if false,
- per-cf metrics will be aggregated by column-family across tables, and reported for cf.CF.M.
- In both cases, the aggregated metric M across tables and cfs will be reported.
-+
-.Default
-`true`
-
-
-[[hbase.metrics.exposeOperationTimes]]
-*`hbase.metrics.exposeOperationTimes`*::
-+
-.Description
-Whether to report metrics about time taken performing an
- operation on the region server. Get, Put, Delete, Increment, and Append can all
- have their times exposed through Hadoop metrics per CF and per region.
-+
-.Default
-`true`
-
-
[[hbase.snapshot.enabled]]
*`hbase.snapshot.enabled`*::
+
[28/50] [abbrv] hbase git commit: HBASE-15773 Improvements to
CellCounter job
Posted by sy...@apache.org.
HBASE-15773 Improvements to CellCounter job
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/86ca09e0
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/86ca09e0
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/86ca09e0
Branch: refs/heads/hbase-12439
Commit: 86ca09e0e581b897582124938004e948fe38df3b
Parents: d90f057
Author: Gary Helmling <ga...@apache.org>
Authored: Thu May 5 12:40:47 2016 -0700
Committer: Gary Helmling <ga...@apache.org>
Committed: Fri May 6 11:08:18 2016 -0700
----------------------------------------------------------------------
.../hadoop/hbase/mapreduce/CellCounter.java | 117 ++++++++++++-------
.../hbase/mapreduce/TableInputFormat.java | 88 ++++++++------
2 files changed, 125 insertions(+), 80 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/86ca09e0/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/CellCounter.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/CellCounter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/CellCounter.java
index aaa32bd..73f9b93 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/CellCounter.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/CellCounter.java
@@ -92,7 +92,30 @@ public class CellCounter extends Configured implements Tool {
* Counter enumeration to count the actual rows.
*/
public static enum Counters {
- ROWS
+ ROWS,
+ CELLS
+ }
+
+ private Configuration conf;
+ private String separator;
+
+ // state of current row, family, column needs to persist across map() invocations
+ // in order to properly handle scanner batching, where a single qualifier may have too
+ // many versions for a single map() call
+ private byte[] lastRow;
+ private String currentRowKey;
+ byte[] currentFamily = null;
+ String currentFamilyName = null;
+ byte[] currentQualifier = null;
+ // family + qualifier
+ String currentQualifierName = null;
+ // rowkey + family + qualifier
+ String currentRowQualifierName = null;
+
+ @Override
+ protected void setup(Context context) throws IOException, InterruptedException {
+ conf = context.getConfiguration();
+ separator = conf.get("ReportSeparator",":");
}
/**
@@ -112,49 +135,45 @@ public class CellCounter extends Configured implements Tool {
throws IOException {
Preconditions.checkState(values != null,
"values passed to the map is null");
- String currentFamilyName = null;
- String currentQualifierName = null;
- String currentRowKey = null;
- Configuration config = context.getConfiguration();
- String separator = config.get("ReportSeparator",":");
+
try {
- context.getCounter(Counters.ROWS).increment(1);
- context.write(new Text("Total ROWS"), new IntWritable(1));
- if (values != null && !values.isEmpty()) {
+ byte[] currentRow = values.getRow();
+ if (lastRow == null || !Bytes.equals(lastRow, currentRow)) {
+ lastRow = currentRow;
+ currentRowKey = Bytes.toStringBinary(currentRow);
+ currentFamily = null;
+ currentQualifier = null;
+ context.getCounter(Counters.ROWS).increment(1);
+ context.write(new Text("Total ROWS"), new IntWritable(1));
+ }
+ if (!values.isEmpty()) {
+ int cellCount = 0;
for (Cell value : values.listCells()) {
- currentRowKey = Bytes.toStringBinary(CellUtil.cloneRow(value));
- String thisRowFamilyName = Bytes.toStringBinary(CellUtil.cloneFamily(value));
- if (!thisRowFamilyName.equals(currentFamilyName)) {
- currentFamilyName = thisRowFamilyName;
- context.getCounter("CF", thisRowFamilyName).increment(1);
- if (1 == context.getCounter("CF", thisRowFamilyName).getValue()) {
+ cellCount++;
+ if (currentFamily == null || !CellUtil.matchingFamily(value, currentFamily)) {
+ currentFamily = CellUtil.cloneFamily(value);
+ currentFamilyName = Bytes.toStringBinary(currentFamily);
+ currentQualifier = null;
+ context.getCounter("CF", currentFamilyName).increment(1);
+ if (1 == context.getCounter("CF", currentFamilyName).getValue()) {
context.write(new Text("Total Families Across all Rows"), new IntWritable(1));
- context.write(new Text(thisRowFamilyName), new IntWritable(1));
+ context.write(new Text(currentFamily), new IntWritable(1));
}
}
- String thisRowQualifierName = thisRowFamilyName + separator
- + Bytes.toStringBinary(CellUtil.cloneQualifier(value));
- if (!thisRowQualifierName.equals(currentQualifierName)) {
- currentQualifierName = thisRowQualifierName;
- context.getCounter("CFQL", thisRowQualifierName).increment(1);
+ if (currentQualifier == null || !CellUtil.matchingQualifier(value, currentQualifier)) {
+ currentQualifier = CellUtil.cloneQualifier(value);
+ currentQualifierName = currentFamilyName + separator +
+ Bytes.toStringBinary(currentQualifier);
+ currentRowQualifierName = currentRowKey + separator + currentQualifierName;
+
context.write(new Text("Total Qualifiers across all Rows"),
new IntWritable(1));
- context.write(new Text(thisRowQualifierName), new IntWritable(1));
- // Intialize versions
- context.getCounter("QL_VERSIONS", currentRowKey + separator +
- thisRowQualifierName).increment(1);
- context.write(new Text(currentRowKey + separator
- + thisRowQualifierName + "_Versions"), new IntWritable(1));
-
- } else {
- // Increment versions
- currentQualifierName = thisRowQualifierName;
- context.getCounter("QL_VERSIONS", currentRowKey + separator +
- thisRowQualifierName).increment(1);
- context.write(new Text(currentRowKey + separator
- + thisRowQualifierName + "_Versions"), new IntWritable(1));
+ context.write(new Text(currentQualifierName), new IntWritable(1));
}
+ // Increment versions
+ context.write(new Text(currentRowQualifierName + "_Versions"), new IntWritable(1));
}
+ context.getCounter(Counters.CELLS).increment(cellCount);
}
} catch (InterruptedException e) {
e.printStackTrace();
@@ -208,15 +227,16 @@ public class CellCounter extends Configured implements Tool {
return job;
}
- private static Scan getConfiguredScanForJob(Configuration conf, String[] args) throws IOException {
- Scan s = new Scan();
+ private static Scan getConfiguredScanForJob(Configuration conf, String[] args)
+ throws IOException {
+ // create scan with any properties set from TableInputFormat
+ Scan s = TableInputFormat.createScanFromConfiguration(conf);
// Set Scan Versions
- s.setMaxVersions(Integer.MAX_VALUE);
- s.setCacheBlocks(false);
- // Set Scan Column Family
- if (conf.get(TableInputFormat.SCAN_COLUMN_FAMILY) != null) {
- s.addFamily(Bytes.toBytes(conf.get(TableInputFormat.SCAN_COLUMN_FAMILY)));
+ if (conf.get(TableInputFormat.SCAN_MAXVERSIONS) == null) {
+ // default to all versions unless explicitly set
+ s.setMaxVersions(Integer.MAX_VALUE);
}
+ s.setCacheBlocks(false);
// Set RowFilter or Prefix Filter if applicable.
Filter rowFilter = getRowFilter(args);
if (rowFilter!= null) {
@@ -277,9 +297,18 @@ public class CellCounter extends Configured implements Tool {
System.err.println(" <tablename> <outputDir> <reportSeparator> [^[regex pattern] or " +
"[Prefix] for row filter]] --starttime=[starttime] --endtime=[endtime]");
System.err.println(" Note: -D properties will be applied to the conf used. ");
- System.err.println(" Additionally, the following SCAN properties can be specified");
- System.err.println(" to get fine grained control on what is counted..");
+ System.err.println(" Additionally, all of the SCAN properties from TableInputFormat");
+ System.err.println(" can be specified to get fine grained control on what is counted..");
+ System.err.println(" -D " + TableInputFormat.SCAN_ROW_START + "=<rowkey>");
+ System.err.println(" -D " + TableInputFormat.SCAN_ROW_STOP + "=<rowkey>");
+ System.err.println(" -D " + TableInputFormat.SCAN_COLUMNS + "=\"<col1> <col2>...\"");
System.err.println(" -D " + TableInputFormat.SCAN_COLUMN_FAMILY + "=<familyName>");
+ System.err.println(" -D " + TableInputFormat.SCAN_TIMESTAMP + "=<timestamp>");
+ System.err.println(" -D " + TableInputFormat.SCAN_TIMERANGE_START + "=<timestamp>");
+ System.err.println(" -D " + TableInputFormat.SCAN_TIMERANGE_END + "=<timestamp>");
+ System.err.println(" -D " + TableInputFormat.SCAN_MAXVERSIONS + "=<count>");
+ System.err.println(" -D " + TableInputFormat.SCAN_CACHEDROWS + "=<count>");
+ System.err.println(" -D " + TableInputFormat.SCAN_BATCHSIZE + "=<count>");
System.err.println(" <reportSeparator> parameter can be used to override the default report separator " +
"string : used to separate the rowId/column family name and qualifier name.");
System.err.println(" [^[regex pattern] or [Prefix] parameter can be used to limit the cell counter count " +
http://git-wip-us.apache.org/repos/asf/hbase/blob/86ca09e0/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableInputFormat.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableInputFormat.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableInputFormat.java
index ebeb158..be20d90 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableInputFormat.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableInputFormat.java
@@ -126,54 +126,70 @@ implements Configurable {
}
} else {
try {
- scan = new Scan();
+ scan = createScanFromConfiguration(conf);
+ } catch (Exception e) {
+ LOG.error(StringUtils.stringifyException(e));
+ }
+ }
- if (conf.get(SCAN_ROW_START) != null) {
- scan.setStartRow(Bytes.toBytesBinary(conf.get(SCAN_ROW_START)));
- }
+ setScan(scan);
+ }
- if (conf.get(SCAN_ROW_STOP) != null) {
- scan.setStopRow(Bytes.toBytesBinary(conf.get(SCAN_ROW_STOP)));
- }
+ /**
+ * Sets up a {@link Scan} instance, applying settings from the configuration property
+ * constants defined in {@code TableInputFormat}. This allows specifying things such as:
+ * <ul>
+ * <li>start and stop rows</li>
+ * <li>column qualifiers or families</li>
+ * <li>timestamps or timerange</li>
+ * <li>scanner caching and batch size</li>
+ * </ul>
+ */
+ public static Scan createScanFromConfiguration(Configuration conf) throws IOException {
+ Scan scan = new Scan();
- if (conf.get(SCAN_COLUMNS) != null) {
- addColumns(scan, conf.get(SCAN_COLUMNS));
- }
+ if (conf.get(SCAN_ROW_START) != null) {
+ scan.setStartRow(Bytes.toBytesBinary(conf.get(SCAN_ROW_START)));
+ }
- if (conf.get(SCAN_COLUMN_FAMILY) != null) {
- scan.addFamily(Bytes.toBytes(conf.get(SCAN_COLUMN_FAMILY)));
- }
+ if (conf.get(SCAN_ROW_STOP) != null) {
+ scan.setStopRow(Bytes.toBytesBinary(conf.get(SCAN_ROW_STOP)));
+ }
- if (conf.get(SCAN_TIMESTAMP) != null) {
- scan.setTimeStamp(Long.parseLong(conf.get(SCAN_TIMESTAMP)));
- }
+ if (conf.get(SCAN_COLUMNS) != null) {
+ addColumns(scan, conf.get(SCAN_COLUMNS));
+ }
- if (conf.get(SCAN_TIMERANGE_START) != null && conf.get(SCAN_TIMERANGE_END) != null) {
- scan.setTimeRange(
- Long.parseLong(conf.get(SCAN_TIMERANGE_START)),
- Long.parseLong(conf.get(SCAN_TIMERANGE_END)));
- }
+ if (conf.get(SCAN_COLUMN_FAMILY) != null) {
+ scan.addFamily(Bytes.toBytes(conf.get(SCAN_COLUMN_FAMILY)));
+ }
- if (conf.get(SCAN_MAXVERSIONS) != null) {
- scan.setMaxVersions(Integer.parseInt(conf.get(SCAN_MAXVERSIONS)));
- }
+ if (conf.get(SCAN_TIMESTAMP) != null) {
+ scan.setTimeStamp(Long.parseLong(conf.get(SCAN_TIMESTAMP)));
+ }
- if (conf.get(SCAN_CACHEDROWS) != null) {
- scan.setCaching(Integer.parseInt(conf.get(SCAN_CACHEDROWS)));
- }
+ if (conf.get(SCAN_TIMERANGE_START) != null && conf.get(SCAN_TIMERANGE_END) != null) {
+ scan.setTimeRange(
+ Long.parseLong(conf.get(SCAN_TIMERANGE_START)),
+ Long.parseLong(conf.get(SCAN_TIMERANGE_END)));
+ }
- if (conf.get(SCAN_BATCHSIZE) != null) {
- scan.setBatch(Integer.parseInt(conf.get(SCAN_BATCHSIZE)));
- }
+ if (conf.get(SCAN_MAXVERSIONS) != null) {
+ scan.setMaxVersions(Integer.parseInt(conf.get(SCAN_MAXVERSIONS)));
+ }
- // false by default, full table scans generate too much BC churn
- scan.setCacheBlocks((conf.getBoolean(SCAN_CACHEBLOCKS, false)));
- } catch (Exception e) {
- LOG.error(StringUtils.stringifyException(e));
- }
+ if (conf.get(SCAN_CACHEDROWS) != null) {
+ scan.setCaching(Integer.parseInt(conf.get(SCAN_CACHEDROWS)));
}
- setScan(scan);
+ if (conf.get(SCAN_BATCHSIZE) != null) {
+ scan.setBatch(Integer.parseInt(conf.get(SCAN_BATCHSIZE)));
+ }
+
+ // false by default, full table scans generate too much BC churn
+ scan.setCacheBlocks((conf.getBoolean(SCAN_CACHEBLOCKS, false)));
+
+ return scan;
}
@Override
[23/50] [abbrv] hbase git commit: HBASE-15743 Add Transparent Data
Encryption support for FanOutOneBlockAsyncDFSOutput
Posted by sy...@apache.org.
HBASE-15743 Add Transparent Data Encryption support for FanOutOneBlockAsyncDFSOutput
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/0d252918
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/0d252918
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/0d252918
Branch: refs/heads/hbase-12439
Commit: 0d252918f65aca8dba070f2da1d32de8b2fd1c1c
Parents: 532b914
Author: zhangduo <zh...@apache.org>
Authored: Thu May 5 10:06:55 2016 +0800
Committer: zhangduo <zh...@apache.org>
Committed: Fri May 6 08:55:58 2016 +0800
----------------------------------------------------------------------
.../asyncfs/FanOutOneBlockAsyncDFSOutput.java | 24 +-
.../FanOutOneBlockAsyncDFSOutputHelper.java | 9 +-
.../FanOutOneBlockAsyncDFSOutputSaslHelper.java | 239 +++++++++++++------
.../TestSaslFanOutOneBlockAsyncDFSOutput.java | 75 +++++-
4 files changed, 264 insertions(+), 83 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/0d252918/hbase-server/src/main/java/org/apache/hadoop/hbase/io/asyncfs/FanOutOneBlockAsyncDFSOutput.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/asyncfs/FanOutOneBlockAsyncDFSOutput.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/asyncfs/FanOutOneBlockAsyncDFSOutput.java
index 7d6a676..8dd7f5e 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/asyncfs/FanOutOneBlockAsyncDFSOutput.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/asyncfs/FanOutOneBlockAsyncDFSOutput.java
@@ -43,6 +43,7 @@ import io.netty.util.concurrent.FutureListener;
import io.netty.util.concurrent.Promise;
import java.io.IOException;
+import java.nio.ByteBuffer;
import java.nio.channels.CompletionHandler;
import java.util.ArrayDeque;
import java.util.Collection;
@@ -58,6 +59,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.CancelOnClose;
+import org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.CryptoCodec;
import org.apache.hadoop.hbase.util.CancelableProgressable;
import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hdfs.DFSClient;
@@ -119,6 +121,8 @@ public class FanOutOneBlockAsyncDFSOutput implements AsyncFSOutput {
private final LocatedBlock locatedBlock;
+ private final CryptoCodec cryptoCodec;
+
private final EventLoop eventLoop;
private final List<Channel> datanodeList;
@@ -317,8 +321,8 @@ public class FanOutOneBlockAsyncDFSOutput implements AsyncFSOutput {
FanOutOneBlockAsyncDFSOutput(Configuration conf, FSUtils fsUtils, DistributedFileSystem dfs,
DFSClient client, ClientProtocol namenode, String clientName, String src, long fileId,
- LocatedBlock locatedBlock, EventLoop eventLoop, List<Channel> datanodeList,
- DataChecksum summer, ByteBufAllocator alloc) {
+ LocatedBlock locatedBlock, CryptoCodec cryptoCodec, EventLoop eventLoop,
+ List<Channel> datanodeList, DataChecksum summer, ByteBufAllocator alloc) {
this.conf = conf;
this.fsUtils = fsUtils;
this.dfs = dfs;
@@ -328,6 +332,7 @@ public class FanOutOneBlockAsyncDFSOutput implements AsyncFSOutput {
this.clientName = clientName;
this.src = src;
this.locatedBlock = locatedBlock;
+ this.cryptoCodec = cryptoCodec;
this.eventLoop = eventLoop;
this.datanodeList = datanodeList;
this.summer = summer;
@@ -342,16 +347,27 @@ public class FanOutOneBlockAsyncDFSOutput implements AsyncFSOutput {
write(b, 0, b.length);
}
+ private void write0(byte[] b, final int off, final int len) {
+ buf.ensureWritable(len);
+ if (cryptoCodec == null) {
+ buf.writeBytes(b, off, len);
+ } else {
+ ByteBuffer inBuffer = ByteBuffer.wrap(b, off, len);
+ cryptoCodec.encrypt(inBuffer, buf.nioBuffer(buf.writerIndex(), len));
+ buf.writerIndex(buf.writerIndex() + len);
+ }
+ }
+
@Override
public void write(final byte[] b, final int off, final int len) {
if (eventLoop.inEventLoop()) {
- buf.ensureWritable(len).writeBytes(b, off, len);
+ write0(b, off, len);
} else {
eventLoop.submit(new Runnable() {
@Override
public void run() {
- buf.ensureWritable(len).writeBytes(b, off, len);
+ write0(b, off, len);
}
}).syncUninterruptibly();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/0d252918/hbase-server/src/main/java/org/apache/hadoop/hbase/io/asyncfs/FanOutOneBlockAsyncDFSOutputHelper.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/asyncfs/FanOutOneBlockAsyncDFSOutputHelper.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/asyncfs/FanOutOneBlockAsyncDFSOutputHelper.java
index 4f9058c..7b680e1 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/asyncfs/FanOutOneBlockAsyncDFSOutputHelper.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/asyncfs/FanOutOneBlockAsyncDFSOutputHelper.java
@@ -21,6 +21,7 @@ import static io.netty.channel.ChannelOption.CONNECT_TIMEOUT_MILLIS;
import static io.netty.handler.timeout.IdleState.READER_IDLE;
import static org.apache.hadoop.fs.CreateFlag.CREATE;
import static org.apache.hadoop.fs.CreateFlag.OVERWRITE;
+import static org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createCryptoCodec;
import static org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.trySaslNegotiate;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_SOCKET_TIMEOUT_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_USE_DN_HOSTNAME;
@@ -73,6 +74,7 @@ import org.apache.hadoop.fs.UnresolvedLinkException;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.client.ConnectionUtils;
+import org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.CryptoCodec;
import org.apache.hadoop.hbase.util.CancelableProgressable;
import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hdfs.DFSClient;
@@ -672,9 +674,12 @@ public final class FanOutOneBlockAsyncDFSOutputHelper {
// layer should retry itself if needed.
datanodeList.add(future.syncUninterruptibly().getNow());
}
+ CryptoCodec cryptocodec = createCryptoCodec(conf, stat, client);
+ FanOutOneBlockAsyncDFSOutput output = new FanOutOneBlockAsyncDFSOutput(conf, fsUtils, dfs,
+ client, namenode, clientName, src, stat.getFileId(), locatedBlock, cryptocodec, eventLoop,
+ datanodeList, summer, ALLOC);
succ = true;
- return new FanOutOneBlockAsyncDFSOutput(conf, fsUtils, dfs, client, namenode, clientName,
- src, stat.getFileId(), locatedBlock, eventLoop, datanodeList, summer, ALLOC);
+ return output;
} finally {
if (!succ) {
if (futureList != null) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/0d252918/hbase-server/src/main/java/org/apache/hadoop/hbase/io/asyncfs/FanOutOneBlockAsyncDFSOutputSaslHelper.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/asyncfs/FanOutOneBlockAsyncDFSOutputSaslHelper.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/asyncfs/FanOutOneBlockAsyncDFSOutputSaslHelper.java
index 22c4e04..33e8841 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/asyncfs/FanOutOneBlockAsyncDFSOutputSaslHelper.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/asyncfs/FanOutOneBlockAsyncDFSOutputSaslHelper.java
@@ -21,6 +21,7 @@ import static io.netty.handler.timeout.IdleState.READER_IDLE;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Charsets;
+import com.google.common.base.Throwables;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
@@ -79,6 +80,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hdfs.DFSClient;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
+import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.protocol.datatransfer.InvalidEncryptionKeyException;
import org.apache.hadoop.hdfs.protocol.datatransfer.TrustedChannelResolver;
import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.DataTransferEncryptorMessageProto;
@@ -111,7 +113,7 @@ public final class FanOutOneBlockAsyncDFSOutputSaslHelper {
@VisibleForTesting
static final String DFS_ENCRYPT_DATA_TRANSFER_CIPHER_SUITES_KEY =
- "dfs.encrypt.data.transfer.cipher.suites";
+ "dfs.encrypt.data.transfer.cipher.suites";
@VisibleForTesting
static final String AES_CTR_NOPADDING = "AES/CTR/NoPadding";
@@ -129,7 +131,7 @@ public final class FanOutOneBlockAsyncDFSOutputSaslHelper {
private static final SaslAdaptor SASL_ADAPTOR;
- private interface CipherHelper {
+ private interface CipherOptionHelper {
List<Object> getCipherOptions(Configuration conf) throws IOException;
@@ -150,9 +152,19 @@ public final class FanOutOneBlockAsyncDFSOutputSaslHelper {
byte[] getOutIv(Object cipherOption);
}
- private static final CipherHelper CIPHER_HELPER;
+ private static final CipherOptionHelper CIPHER_OPTION_HELPER;
- private static final class CryptoCodec {
+ private interface TransparentCryptoHelper {
+
+ Object getFileEncryptionInfo(HdfsFileStatus stat);
+
+ CryptoCodec createCryptoCodec(Configuration conf, Object feInfo, DFSClient client)
+ throws IOException;
+ }
+
+ private static final TransparentCryptoHelper TRANSPARENT_CRYPTO_HELPER;
+
+ static final class CryptoCodec {
private static final Method CREATE_CODEC;
@@ -215,23 +227,34 @@ public final class FanOutOneBlockAsyncDFSOutputSaslHelper {
private final Object decryptor;
public CryptoCodec(Configuration conf, Object cipherOption) {
- Object codec;
try {
- codec = CREATE_CODEC.invoke(null, conf, CIPHER_HELPER.getCipherSuite(cipherOption));
+ Object codec = CREATE_CODEC.invoke(null, conf,
+ CIPHER_OPTION_HELPER.getCipherSuite(cipherOption));
encryptor = CREATE_ENCRYPTOR.invoke(codec);
- byte[] encKey = CIPHER_HELPER.getInKey(cipherOption);
- byte[] encIv = CIPHER_HELPER.getInIv(cipherOption);
+ byte[] encKey = CIPHER_OPTION_HELPER.getInKey(cipherOption);
+ byte[] encIv = CIPHER_OPTION_HELPER.getInIv(cipherOption);
INIT_ENCRYPTOR.invoke(encryptor, encKey, Arrays.copyOf(encIv, encIv.length));
decryptor = CREATE_DECRYPTOR.invoke(codec);
- byte[] decKey = CIPHER_HELPER.getOutKey(cipherOption);
- byte[] decIv = CIPHER_HELPER.getOutIv(cipherOption);
+ byte[] decKey = CIPHER_OPTION_HELPER.getOutKey(cipherOption);
+ byte[] decIv = CIPHER_OPTION_HELPER.getOutIv(cipherOption);
INIT_DECRYPTOR.invoke(decryptor, decKey, Arrays.copyOf(decIv, decIv.length));
} catch (IllegalAccessException | InvocationTargetException e) {
throw new RuntimeException(e);
}
}
+ public CryptoCodec(Configuration conf, Object cipherSuite, byte[] encKey, byte[] encIv) {
+ try {
+ Object codec = CREATE_CODEC.invoke(null, conf, cipherSuite);
+ encryptor = CREATE_ENCRYPTOR.invoke(codec);
+ INIT_ENCRYPTOR.invoke(encryptor, encKey, encIv);
+ decryptor = null;
+ } catch (IllegalAccessException | InvocationTargetException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
public void encrypt(ByteBuffer inBuffer, ByteBuffer outBuffer) {
try {
ENCRYPT.invoke(encryptor, inBuffer, outBuffer);
@@ -251,17 +274,17 @@ public final class FanOutOneBlockAsyncDFSOutputSaslHelper {
private static SaslAdaptor createSaslAdaptor27(Class<?> saslDataTransferClientClass)
throws NoSuchFieldException, NoSuchMethodException {
- final Field saslPropsResolverField =
- saslDataTransferClientClass.getDeclaredField("saslPropsResolver");
+ final Field saslPropsResolverField = saslDataTransferClientClass
+ .getDeclaredField("saslPropsResolver");
saslPropsResolverField.setAccessible(true);
- final Field trustedChannelResolverField =
- saslDataTransferClientClass.getDeclaredField("trustedChannelResolver");
+ final Field trustedChannelResolverField = saslDataTransferClientClass
+ .getDeclaredField("trustedChannelResolver");
trustedChannelResolverField.setAccessible(true);
- final Field fallbackToSimpleAuthField =
- saslDataTransferClientClass.getDeclaredField("fallbackToSimpleAuth");
+ final Field fallbackToSimpleAuthField = saslDataTransferClientClass
+ .getDeclaredField("fallbackToSimpleAuth");
fallbackToSimpleAuthField.setAccessible(true);
- final Method getSaslDataTransferClientMethod =
- DFSClient.class.getMethod("getSaslDataTransferClient");
+ final Method getSaslDataTransferClientMethod = DFSClient.class
+ .getMethod("getSaslDataTransferClient");
final Method newDataEncryptionKeyMethod = DFSClient.class.getMethod("newDataEncryptionKey");
return new SaslAdaptor() {
@@ -288,8 +311,8 @@ public final class FanOutOneBlockAsyncDFSOutputSaslHelper {
@Override
public AtomicBoolean getFallbackToSimpleAuth(DFSClient client) {
try {
- return (AtomicBoolean) fallbackToSimpleAuthField.get(getSaslDataTransferClientMethod
- .invoke(client));
+ return (AtomicBoolean) fallbackToSimpleAuthField
+ .get(getSaslDataTransferClientMethod.invoke(client));
} catch (IllegalAccessException | InvocationTargetException e) {
throw new RuntimeException(e);
}
@@ -308,8 +331,8 @@ public final class FanOutOneBlockAsyncDFSOutputSaslHelper {
private static SaslAdaptor createSaslAdaptor25() {
try {
- final Field trustedChannelResolverField =
- DFSClient.class.getDeclaredField("trustedChannelResolver");
+ final Field trustedChannelResolverField = DFSClient.class
+ .getDeclaredField("trustedChannelResolver");
trustedChannelResolverField.setAccessible(true);
final Method getDataEncryptionKeyMethod = DFSClient.class.getMethod("getDataEncryptionKey");
return new SaslAdaptor() {
@@ -351,8 +374,8 @@ public final class FanOutOneBlockAsyncDFSOutputSaslHelper {
private static SaslAdaptor createSaslAdaptor() {
Class<?> saslDataTransferClientClass = null;
try {
- saslDataTransferClientClass =
- Class.forName("org.apache.hadoop.hdfs.protocol.datatransfer.sasl.SaslDataTransferClient");
+ saslDataTransferClientClass = Class
+ .forName("org.apache.hadoop.hdfs.protocol.datatransfer.sasl.SaslDataTransferClient");
} catch (ClassNotFoundException e) {
LOG.warn("No SaslDataTransferClient class found, should be hadoop 2.5-");
}
@@ -364,8 +387,8 @@ public final class FanOutOneBlockAsyncDFSOutputSaslHelper {
}
}
- private static CipherHelper createCipherHelper25() {
- return new CipherHelper() {
+ private static CipherOptionHelper createCipherHelper25() {
+ return new CipherOptionHelper() {
@Override
public byte[] getOutKey(Object cipherOption) {
@@ -410,18 +433,17 @@ public final class FanOutOneBlockAsyncDFSOutputSaslHelper {
};
}
- private static CipherHelper createCipherHelper27(Class<?> cipherOptionClass)
+ private static CipherOptionHelper createCipherHelper27(Class<?> cipherOptionClass)
throws ClassNotFoundException, NoSuchMethodException {
@SuppressWarnings("rawtypes")
- Class<? extends Enum> cipherSuiteClass =
- Class.forName("org.apache.hadoop.crypto.CipherSuite").asSubclass(Enum.class);
+ Class<? extends Enum> cipherSuiteClass = Class.forName("org.apache.hadoop.crypto.CipherSuite")
+ .asSubclass(Enum.class);
@SuppressWarnings("unchecked")
final Enum<?> aesCipherSuite = Enum.valueOf(cipherSuiteClass, "AES_CTR_NOPADDING");
- final Constructor<?> cipherOptionConstructor =
- cipherOptionClass.getConstructor(cipherSuiteClass);
- final Constructor<?> cipherOptionWithKeyAndIvConstructor =
- cipherOptionClass.getConstructor(cipherSuiteClass, byte[].class, byte[].class,
- byte[].class, byte[].class);
+ final Constructor<?> cipherOptionConstructor = cipherOptionClass
+ .getConstructor(cipherSuiteClass);
+ final Constructor<?> cipherOptionWithKeyAndIvConstructor = cipherOptionClass
+ .getConstructor(cipherSuiteClass, byte[].class, byte[].class, byte[].class, byte[].class);
final Method getCipherSuiteMethod = cipherOptionClass.getMethod("getCipherSuite");
final Method getInKeyMethod = cipherOptionClass.getMethod("getInKey");
@@ -429,16 +451,15 @@ public final class FanOutOneBlockAsyncDFSOutputSaslHelper {
final Method getOutKeyMethod = cipherOptionClass.getMethod("getOutKey");
final Method getOutIvMethod = cipherOptionClass.getMethod("getOutIv");
- final Method convertCipherOptionsMethod =
- PBHelper.class.getMethod("convertCipherOptions", List.class);
- final Method convertCipherOptionProtosMethod =
- PBHelper.class.getMethod("convertCipherOptionProtos", List.class);
- final Method addAllCipherOptionMethod =
- DataTransferEncryptorMessageProto.Builder.class.getMethod("addAllCipherOption",
- Iterable.class);
- final Method getCipherOptionListMethod =
- DataTransferEncryptorMessageProto.class.getMethod("getCipherOptionList");
- return new CipherHelper() {
+ final Method convertCipherOptionsMethod = PBHelper.class.getMethod("convertCipherOptions",
+ List.class);
+ final Method convertCipherOptionProtosMethod = PBHelper.class
+ .getMethod("convertCipherOptionProtos", List.class);
+ final Method addAllCipherOptionMethod = DataTransferEncryptorMessageProto.Builder.class
+ .getMethod("addAllCipherOption", Iterable.class);
+ final Method getCipherOptionListMethod = DataTransferEncryptorMessageProto.class
+ .getMethod("getCipherOptionList");
+ return new CipherOptionHelper() {
@Override
public byte[] getOutKey(Object cipherOption) {
@@ -532,9 +553,8 @@ public final class FanOutOneBlockAsyncDFSOutputSaslHelper {
boolean isNegotiatedQopPrivacy, SaslClient saslClient) throws IOException {
List<Object> cipherOptions;
try {
- cipherOptions =
- (List<Object>) convertCipherOptionProtosMethod.invoke(null,
- getCipherOptionListMethod.invoke(proto));
+ cipherOptions = (List<Object>) convertCipherOptionProtosMethod.invoke(null,
+ getCipherOptionListMethod.invoke(proto));
} catch (IllegalAccessException | InvocationTargetException e) {
throw new RuntimeException(e);
}
@@ -557,7 +577,7 @@ public final class FanOutOneBlockAsyncDFSOutputSaslHelper {
};
}
- private static CipherHelper createCipherHelper() {
+ private static CipherOptionHelper createCipherHelper() {
Class<?> cipherOptionClass;
try {
cipherOptionClass = Class.forName("org.apache.hadoop.crypto.CipherOption");
@@ -572,9 +592,79 @@ public final class FanOutOneBlockAsyncDFSOutputSaslHelper {
}
}
+ private static TransparentCryptoHelper createTransparentCryptoHelper25() {
+ return new TransparentCryptoHelper() {
+
+ @Override
+ public Object getFileEncryptionInfo(HdfsFileStatus stat) {
+ return null;
+ }
+
+ @Override
+ public CryptoCodec createCryptoCodec(Configuration conf, Object feInfo, DFSClient client) {
+ throw new UnsupportedOperationException();
+ }
+ };
+ }
+
+ private static TransparentCryptoHelper createTransparentCryptoHelper27(Class<?> feInfoClass)
+ throws NoSuchMethodException, ClassNotFoundException {
+ final Method getFileEncryptionInfoMethod = HdfsFileStatus.class
+ .getMethod("getFileEncryptionInfo");
+ final Method decryptEncryptedDataEncryptionKeyMethod = DFSClient.class
+ .getDeclaredMethod("decryptEncryptedDataEncryptionKey", feInfoClass);
+ decryptEncryptedDataEncryptionKeyMethod.setAccessible(true);
+ final Method getCipherSuiteMethod = feInfoClass.getMethod("getCipherSuite");
+ Class<?> keyVersionClass = Class.forName("org.apache.hadoop.crypto.key.KeyProvider$KeyVersion");
+ final Method getMaterialMethod = keyVersionClass.getMethod("getMaterial");
+ final Method getIVMethod = feInfoClass.getMethod("getIV");
+ return new TransparentCryptoHelper() {
+
+ @Override
+ public Object getFileEncryptionInfo(HdfsFileStatus stat) {
+ try {
+ return getFileEncryptionInfoMethod.invoke(stat);
+ } catch (IllegalAccessException | InvocationTargetException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public CryptoCodec createCryptoCodec(Configuration conf, Object feInfo, DFSClient client)
+ throws IOException {
+ try {
+ Object decrypted = decryptEncryptedDataEncryptionKeyMethod.invoke(client, feInfo);
+ return new CryptoCodec(conf, getCipherSuiteMethod.invoke(feInfo),
+ (byte[]) getMaterialMethod.invoke(decrypted), (byte[]) getIVMethod.invoke(feInfo));
+ } catch (InvocationTargetException e) {
+ Throwables.propagateIfPossible(e.getTargetException(), IOException.class);
+ throw new RuntimeException(e.getTargetException());
+ } catch (IllegalAccessException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ };
+ }
+
+ private static TransparentCryptoHelper createTransparentCryptoHelper() {
+ Class<?> feInfoClass;
+ try {
+ feInfoClass = Class.forName("org.apache.hadoop.fs.FileEncryptionInfo");
+ } catch (ClassNotFoundException e) {
+ LOG.warn("No FileEncryptionInfo class found, should be hadoop 2.5-");
+ return createTransparentCryptoHelper25();
+ }
+ try {
+ return createTransparentCryptoHelper27(feInfoClass);
+ } catch (NoSuchMethodException | ClassNotFoundException e) {
+ throw new Error(e);
+ }
+ }
+
static {
SASL_ADAPTOR = createSaslAdaptor();
- CIPHER_HELPER = createCipherHelper();
+ CIPHER_OPTION_HELPER = createCipherHelper();
+ TRANSPARENT_CRYPTO_HELPER = createTransparentCryptoHelper();
}
/**
@@ -643,9 +733,8 @@ public final class FanOutOneBlockAsyncDFSOutputSaslHelper {
Map<String, String> saslProps, int timeoutMs, Promise<Void> promise) throws SaslException {
this.conf = conf;
this.saslProps = saslProps;
- this.saslClient =
- Sasl.createSaslClient(new String[] { MECHANISM }, username, PROTOCOL, SERVER_NAME,
- saslProps, new SaslClientCallbackHandler(username, password));
+ this.saslClient = Sasl.createSaslClient(new String[] { MECHANISM }, username, PROTOCOL,
+ SERVER_NAME, saslProps, new SaslClientCallbackHandler(username, password));
this.timeoutMs = timeoutMs;
this.promise = promise;
}
@@ -656,14 +745,14 @@ public final class FanOutOneBlockAsyncDFSOutputSaslHelper {
private void sendSaslMessage(ChannelHandlerContext ctx, byte[] payload, List<Object> options)
throws IOException {
- DataTransferEncryptorMessageProto.Builder builder =
- DataTransferEncryptorMessageProto.newBuilder();
+ DataTransferEncryptorMessageProto.Builder builder = DataTransferEncryptorMessageProto
+ .newBuilder();
builder.setStatus(DataTransferEncryptorStatus.SUCCESS);
if (payload != null) {
builder.setPayload(ByteString.copyFrom(payload));
}
if (options != null) {
- CIPHER_HELPER.addCipherOptions(builder, options);
+ CIPHER_OPTION_HELPER.addCipherOptions(builder, options);
}
DataTransferEncryptorMessageProto proto = builder.build();
int size = proto.getSerializedSize();
@@ -704,8 +793,8 @@ public final class FanOutOneBlockAsyncDFSOutputSaslHelper {
}
private boolean requestedQopContainsPrivacy() {
- Set<String> requestedQop =
- ImmutableSet.copyOf(Arrays.asList(saslProps.get(Sasl.QOP).split(",")));
+ Set<String> requestedQop = ImmutableSet
+ .copyOf(Arrays.asList(saslProps.get(Sasl.QOP).split(",")));
return requestedQop.contains("auth-conf");
}
@@ -713,15 +802,16 @@ public final class FanOutOneBlockAsyncDFSOutputSaslHelper {
if (!saslClient.isComplete()) {
throw new IOException("Failed to complete SASL handshake");
}
- Set<String> requestedQop =
- ImmutableSet.copyOf(Arrays.asList(saslProps.get(Sasl.QOP).split(",")));
+ Set<String> requestedQop = ImmutableSet
+ .copyOf(Arrays.asList(saslProps.get(Sasl.QOP).split(",")));
String negotiatedQop = getNegotiatedQop();
- LOG.debug("Verifying QOP, requested QOP = " + requestedQop + ", negotiated QOP = "
- + negotiatedQop);
+ LOG.debug(
+ "Verifying QOP, requested QOP = " + requestedQop + ", negotiated QOP = " + negotiatedQop);
if (!requestedQop.contains(negotiatedQop)) {
throw new IOException(String.format("SASL handshake completed, but "
+ "channel does not have acceptable quality of protection, "
- + "requested = %s, negotiated = %s", requestedQop, negotiatedQop));
+ + "requested = %s, negotiated = %s",
+ requestedQop, negotiatedQop));
}
}
@@ -741,7 +831,7 @@ public final class FanOutOneBlockAsyncDFSOutputSaslHelper {
case 1: {
List<Object> cipherOptions = null;
if (requestedQopContainsPrivacy()) {
- cipherOptions = CIPHER_HELPER.getCipherOptions(conf);
+ cipherOptions = CIPHER_OPTION_HELPER.getCipherOptions(conf);
}
sendSaslMessage(ctx, response, cipherOptions);
ctx.flush();
@@ -752,7 +842,7 @@ public final class FanOutOneBlockAsyncDFSOutputSaslHelper {
assert response == null;
checkSaslComplete();
Object cipherOption =
- CIPHER_HELPER.getCipherOption(proto, isNegotiatedQopPrivacy(), saslClient);
+ CIPHER_OPTION_HELPER.getCipherOption(proto, isNegotiatedQopPrivacy(), saslClient);
ChannelPipeline p = ctx.pipeline();
while (p.first() != null) {
p.removeFirst();
@@ -762,8 +852,9 @@ public final class FanOutOneBlockAsyncDFSOutputSaslHelper {
p.addLast(new EncryptHandler(codec), new DecryptHandler(codec));
} else {
if (useWrap()) {
- p.addLast(new SaslWrapHandler(saslClient), new LengthFieldBasedFrameDecoder(
- Integer.MAX_VALUE, 0, 4), new SaslUnwrapHandler(saslClient));
+ p.addLast(new SaslWrapHandler(saslClient),
+ new LengthFieldBasedFrameDecoder(Integer.MAX_VALUE, 0, 4),
+ new SaslUnwrapHandler(saslClient));
}
}
promise.trySuccess(null);
@@ -992,8 +1083,8 @@ public final class FanOutOneBlockAsyncDFSOutputSaslHelper {
}
if (encryptionKey != null) {
if (LOG.isDebugEnabled()) {
- LOG.debug("SASL client doing encrypted handshake for addr = " + addr + ", datanodeId = "
- + dnInfo);
+ LOG.debug(
+ "SASL client doing encrypted handshake for addr = " + addr + ", datanodeId = " + dnInfo);
}
doSaslNegotiation(conf, channel, timeoutMs, getUserNameFromEncryptionKey(encryptionKey),
encryptionKeyToPassword(encryptionKey.encryptionKey),
@@ -1018,8 +1109,8 @@ public final class FanOutOneBlockAsyncDFSOutputSaslHelper {
saslPromise.trySuccess(null);
} else if (saslPropsResolver != null) {
if (LOG.isDebugEnabled()) {
- LOG.debug("SASL client doing general handshake for addr = " + addr + ", datanodeId = "
- + dnInfo);
+ LOG.debug(
+ "SASL client doing general handshake for addr = " + addr + ", datanodeId = " + dnInfo);
}
doSaslNegotiation(conf, channel, timeoutMs, buildUsername(accessToken),
buildClientPassword(accessToken), saslPropsResolver.getClientProperties(addr), saslPromise);
@@ -1035,4 +1126,12 @@ public final class FanOutOneBlockAsyncDFSOutputSaslHelper {
}
}
+ static CryptoCodec createCryptoCodec(Configuration conf, HdfsFileStatus stat, DFSClient client)
+ throws IOException {
+ Object feInfo = TRANSPARENT_CRYPTO_HELPER.getFileEncryptionInfo(stat);
+ if (feInfo == null) {
+ return null;
+ }
+ return TRANSPARENT_CRYPTO_HELPER.createCryptoCodec(conf, feInfo, client);
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/0d252918/hbase-server/src/test/java/org/apache/hadoop/hbase/io/asyncfs/TestSaslFanOutOneBlockAsyncDFSOutput.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/asyncfs/TestSaslFanOutOneBlockAsyncDFSOutput.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/asyncfs/TestSaslFanOutOneBlockAsyncDFSOutput.java
index 545a39e..4637a01 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/asyncfs/TestSaslFanOutOneBlockAsyncDFSOutput.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/asyncfs/TestSaslFanOutOneBlockAsyncDFSOutput.java
@@ -29,6 +29,8 @@ import io.netty.channel.nio.NioEventLoopGroup;
import java.io.File;
import java.io.IOException;
+import java.lang.reflect.Method;
+import java.net.URI;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
@@ -42,7 +44,7 @@ import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.http.ssl.KeyStoreTestUtil;
import org.apache.hadoop.hbase.security.HBaseKerberosUtils;
import org.apache.hadoop.hbase.security.token.TestGenerateDelegationToken;
-import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.apache.hadoop.hbase.testclassification.LargeTests;
import org.apache.hadoop.hbase.testclassification.MiscTests;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DistributedFileSystem;
@@ -63,7 +65,7 @@ import org.junit.runners.Parameterized.Parameter;
import org.junit.runners.Parameterized.Parameters;
@RunWith(Parameterized.class)
-@Category({ MiscTests.class, MediumTests.class })
+@Category({ MiscTests.class, LargeTests.class })
public class TestSaslFanOutOneBlockAsyncDFSOutput {
private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
@@ -74,8 +76,8 @@ public class TestSaslFanOutOneBlockAsyncDFSOutput {
private static int READ_TIMEOUT_MS = 200000;
- private static final File KEYTAB_FILE = new File(
- TEST_UTIL.getDataTestDir("keytab").toUri().getPath());
+ private static final File KEYTAB_FILE =
+ new File(TEST_UTIL.getDataTestDir("keytab").toUri().getPath());
private static MiniKdc KDC;
@@ -86,6 +88,11 @@ public class TestSaslFanOutOneBlockAsyncDFSOutput {
private static String PRINCIPAL;
private static String HTTP_PRINCIPAL;
+
+ private static String TEST_KEY_NAME = "test_key";
+
+ private static boolean TEST_TRANSPARENT_ENCRYPTION = true;
+
@Rule
public TestName name = new TestName();
@@ -98,13 +105,20 @@ public class TestSaslFanOutOneBlockAsyncDFSOutput {
@Parameter(2)
public String cipherSuite;
- @Parameters(name = "{index}: protection={0}, encryption={1}, cipherSuite={2}")
+ @Parameter(3)
+ public boolean useTransparentEncryption;
+
+ @Parameters(
+ name = "{index}: protection={0}, encryption={1}, cipherSuite={2}, transparent_enc={3}")
public static Iterable<Object[]> data() {
List<Object[]> params = new ArrayList<>();
for (String protection : Arrays.asList("authentication", "integrity", "privacy")) {
for (String encryptionAlgorithm : Arrays.asList("", "3des", "rc4")) {
for (String cipherSuite : Arrays.asList("", AES_CTR_NOPADDING)) {
- params.add(new Object[] { protection, encryptionAlgorithm, cipherSuite });
+ for (boolean useTransparentEncryption : Arrays.asList(false, true)) {
+ params.add(new Object[] { protection, encryptionAlgorithm, cipherSuite,
+ useTransparentEncryption });
+ }
}
}
}
@@ -132,6 +146,35 @@ public class TestSaslFanOutOneBlockAsyncDFSOutput {
conf.setBoolean("ignore.secure.ports.for.testing", true);
}
+ private static void setUpKeyProvider(Configuration conf) throws Exception {
+ Class<?> keyProviderFactoryClass;
+ try {
+ keyProviderFactoryClass = Class.forName("org.apache.hadoop.crypto.key.KeyProviderFactory");
+ } catch (ClassNotFoundException e) {
+ // should be hadoop 2.5-, give up
+ TEST_TRANSPARENT_ENCRYPTION = false;
+ return;
+ }
+
+ URI keyProviderUri =
+ new URI("jceks://file" + TEST_UTIL.getDataTestDir("test.jks").toUri().toString());
+ conf.set("dfs.encryption.key.provider.uri", keyProviderUri.toString());
+ Method getKeyProviderMethod =
+ keyProviderFactoryClass.getMethod("get", URI.class, Configuration.class);
+ Object keyProvider = getKeyProviderMethod.invoke(null, keyProviderUri, conf);
+ Class<?> keyProviderClass = Class.forName("org.apache.hadoop.crypto.key.KeyProvider");
+ Class<?> keyProviderOptionsClass =
+ Class.forName("org.apache.hadoop.crypto.key.KeyProvider$Options");
+ Method createKeyMethod =
+ keyProviderClass.getMethod("createKey", String.class, keyProviderOptionsClass);
+ Object options = keyProviderOptionsClass.getConstructor(Configuration.class).newInstance(conf);
+ createKeyMethod.invoke(keyProvider, TEST_KEY_NAME, options);
+ Method flushMethod = keyProviderClass.getMethod("flush");
+ flushMethod.invoke(keyProvider);
+ Method closeMethod = keyProviderClass.getMethod("close");
+ closeMethod.invoke(keyProvider);
+ }
+
@BeforeClass
public static void setUpBeforeClass() throws Exception {
EVENT_LOOP_GROUP = new NioEventLoopGroup();
@@ -144,6 +187,8 @@ public class TestSaslFanOutOneBlockAsyncDFSOutput {
PRINCIPAL = USERNAME + "/" + HOST;
HTTP_PRINCIPAL = "HTTP/" + HOST;
KDC.createPrincipal(KEYTAB_FILE, PRINCIPAL, HTTP_PRINCIPAL);
+
+ setUpKeyProvider(TEST_UTIL.getConfiguration());
setHdfsSecuredConfiguration(TEST_UTIL.getConfiguration());
HBaseKerberosUtils.setKeytabFileForTesting(KEYTAB_FILE.getAbsolutePath());
HBaseKerberosUtils.setPrincipalForTesting(PRINCIPAL + "@" + KDC.getRealm());
@@ -161,6 +206,17 @@ public class TestSaslFanOutOneBlockAsyncDFSOutput {
}
}
+ private Path testDirOnTestFs;
+
+ private void createEncryptionZone() throws Exception {
+ if (!TEST_TRANSPARENT_ENCRYPTION) {
+ return;
+ }
+ Method method =
+ DistributedFileSystem.class.getMethod("createEncryptionZone", Path.class, String.class);
+ method.invoke(FS, testDirOnTestFs, TEST_KEY_NAME);
+ }
+
@Before
public void setUp() throws Exception {
TEST_UTIL.getConfiguration().set("dfs.data.transfer.protection", protection);
@@ -182,6 +238,11 @@ public class TestSaslFanOutOneBlockAsyncDFSOutput {
TEST_UTIL.startMiniDFSCluster(3);
FS = TEST_UTIL.getDFSCluster().getFileSystem();
+ testDirOnTestFs = new Path("/" + name.getMethodName().replaceAll("[^0-9a-zA-Z]", "_"));
+ FS.mkdirs(testDirOnTestFs);
+ if (useTransparentEncryption) {
+ createEncryptionZone();
+ }
}
@After
@@ -190,7 +251,7 @@ public class TestSaslFanOutOneBlockAsyncDFSOutput {
}
private Path getTestFile() {
- return new Path("/" + name.getMethodName().replaceAll("[^0-9a-zA-Z]", "_"));
+ return new Path(testDirOnTestFs, "test");
}
@Test
[37/50] [abbrv] hbase git commit: HBASE-11625 - Verifies data before
building HFileBlock. - Adds HFileBlock.Header class which contains
information about location of fields. Testing: Adds CorruptedFSReaderImpl to
TestChecksum.
Posted by sy...@apache.org.
HBASE-11625 - Verifies data before building HFileBlock. - Adds HFileBlock.Header class which contains information about location of fields. Testing: Adds CorruptedFSReaderImpl to TestChecksum.
Change-Id: I107e9636b28abb6b15ec329e885f1e31b1b1b988
Signed-off-by: stack <st...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/48b77dbc
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/48b77dbc
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/48b77dbc
Branch: refs/heads/hbase-12439
Commit: 48b77dbcbccf9683f4da160dedb218bef9ae8bca
Parents: 9ee0cbb
Author: Apekshit <ap...@gmail.com>
Authored: Thu May 5 17:05:17 2016 -0700
Committer: stack <st...@apache.org>
Committed: Sun May 8 17:33:04 2016 -0700
----------------------------------------------------------------------
.../hadoop/hbase/io/hfile/ChecksumUtil.java | 53 ++++++-------
.../hadoop/hbase/io/hfile/HFileBlock.java | 81 +++++++++++++-------
.../hadoop/hbase/io/hfile/TestChecksum.java | 64 ++++++++++++----
3 files changed, 126 insertions(+), 72 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/48b77dbc/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/ChecksumUtil.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/ChecksumUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/ChecksumUtil.java
index b0b1714..a47cc12 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/ChecksumUtil.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/ChecksumUtil.java
@@ -78,46 +78,35 @@ public class ChecksumUtil {
}
/**
- * Validates that the data in the specified HFileBlock matches the
- * checksum. Generates the checksum for the data and
- * then validate that it matches the value stored in the header.
- * If there is a checksum mismatch, then return false. Otherwise
- * return true.
- * The header is extracted from the specified HFileBlock while the
- * data-to-be-verified is extracted from 'data'.
+ * Validates that the data in the specified HFileBlock matches the checksum. Generates the
+ * checksums for the data and then validate that it matches those stored in the end of the data.
+ * @param buffer Contains the data in following order: HFileBlock header, data, checksums.
+ * @param pathName Path of the HFile to which the {@code data} belongs. Only used for logging.
+ * @param offset offset of the data being validated. Only used for logging.
+ * @param hdrSize Size of the block header in {@code data}. Only used for logging.
+ * @return True if checksum matches, else false.
*/
- static boolean validateBlockChecksum(String pathName, long offset, HFileBlock block,
- byte[] data, int hdrSize) throws IOException {
-
- // If this is an older version of the block that does not have
- // checksums, then return false indicating that checksum verification
- // did not succeed. Actually, this method should never be called
- // when the minorVersion is 0, thus this is a defensive check for a
- // cannot-happen case. Since this is a cannot-happen case, it is
- // better to return false to indicate a checksum validation failure.
- if (!block.getHFileContext().isUseHBaseChecksum()) {
- return false;
- }
-
- // Get a checksum object based on the type of checksum that is
- // set in the HFileBlock header. A ChecksumType.NULL indicates that
- // the caller is not interested in validating checksums, so we
- // always return true.
- ChecksumType cktype = ChecksumType.codeToType(block.getChecksumType());
+ static boolean validateChecksum(ByteBuffer buffer, String pathName, long offset, int hdrSize)
+ throws IOException {
+ // A ChecksumType.NULL indicates that the caller is not interested in validating checksums,
+ // so we always return true.
+ ChecksumType cktype =
+ ChecksumType.codeToType(buffer.get(HFileBlock.Header.CHECKSUM_TYPE_INDEX));
if (cktype == ChecksumType.NULL) {
return true; // No checksum validations needed for this block.
}
// read in the stored value of the checksum size from the header.
- int bytesPerChecksum = block.getBytesPerChecksum();
+ int bytesPerChecksum = buffer.getInt(HFileBlock.Header.BYTES_PER_CHECKSUM_INDEX);
DataChecksum dataChecksum = DataChecksum.newDataChecksum(
cktype.getDataChecksumType(), bytesPerChecksum);
assert dataChecksum != null;
- int sizeWithHeader = block.getOnDiskDataSizeWithHeader();
+ int onDiskDataSizeWithHeader =
+ buffer.getInt(HFileBlock.Header.ON_DISK_DATA_SIZE_WITH_HEADER_INDEX);
if (LOG.isTraceEnabled()) {
- LOG.info("dataLength=" + data.length
- + ", sizeWithHeader=" + sizeWithHeader
+ LOG.info("dataLength=" + buffer.capacity()
+ + ", sizeWithHeader=" + onDiskDataSizeWithHeader
+ ", checksumType=" + cktype.getName()
+ ", file=" + pathName
+ ", offset=" + offset
@@ -125,8 +114,10 @@ public class ChecksumUtil {
+ ", bytesPerChecksum=" + bytesPerChecksum);
}
try {
- dataChecksum.verifyChunkedSums(ByteBuffer.wrap(data, 0, sizeWithHeader),
- ByteBuffer.wrap(data, sizeWithHeader, data.length - sizeWithHeader), pathName, 0);
+ ByteBuffer data = (ByteBuffer) buffer.duplicate().position(0).limit(onDiskDataSizeWithHeader);
+ ByteBuffer checksums = (ByteBuffer) buffer.duplicate().position(onDiskDataSizeWithHeader)
+ .limit(buffer.capacity());
+ dataChecksum.verifyChunkedSums(data, checksums, pathName, 0);
} catch (ChecksumException e) {
return false;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/48b77dbc/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java
index f3402da..efc9a30 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java
@@ -276,6 +276,26 @@ public class HFileBlock implements Cacheable {
CacheableDeserializerIdManager.registerDeserializer(BLOCK_DESERIALIZER);
}
+ // Todo: encapsulate Header related logic in this inner class.
+ static class Header {
+ // Format of header is:
+ // 8 bytes - block magic
+ // 4 bytes int - onDiskSizeWithoutHeader
+ // 4 bytes int - uncompressedSizeWithoutHeader
+ // 8 bytes long - prevBlockOffset
+ // The following 3 are only present if header contains checksum information
+ // 1 byte - checksum type
+ // 4 byte int - bytes per checksum
+ // 4 byte int - onDiskDataSizeWithHeader
+ static int BLOCK_MAGIC_INDEX = 0;
+ static int ON_DISK_SIZE_WITHOUT_HEADER_INDEX = 8;
+ static int UNCOMPRESSED_SIZE_WITHOUT_HEADER_INDEX = 12;
+ static int PREV_BLOCK_OFFSET_INDEX = 16;
+ static int CHECKSUM_TYPE_INDEX = 24;
+ static int BYTES_PER_CHECKSUM_INDEX = 25;
+ static int ON_DISK_DATA_SIZE_WITH_HEADER_INDEX = 29;
+ }
+
/**
* Copy constructor. Creates a shallow copy of {@code that}'s buffer.
*/
@@ -305,7 +325,7 @@ public class HFileBlock implements Cacheable {
* @param onDiskSizeWithoutHeader see {@link #onDiskSizeWithoutHeader}
* @param uncompressedSizeWithoutHeader see {@link #uncompressedSizeWithoutHeader}
* @param prevBlockOffset see {@link #prevBlockOffset}
- * @param buf block header ({@link HConstants#HFILEBLOCK_HEADER_SIZE} bytes) followed by
+ * @param b block header ({@link HConstants#HFILEBLOCK_HEADER_SIZE} bytes) followed by
* uncompressed data.
* @param fillHeader when true, write the first 4 header fields into passed buffer.
* @param offset the file offset the block was read from
@@ -335,12 +355,13 @@ public class HFileBlock implements Cacheable {
final int nextBlockOnDiskSize, HFileContext fileContext) throws IOException {
buf.rewind();
final BlockType blockType = BlockType.read(buf);
- final int onDiskSizeWithoutHeader = buf.getInt();
- final int uncompressedSizeWithoutHeader = buf.getInt();
- final long prevBlockOffset = buf.getLong();
- byte checksumType = buf.get();
- int bytesPerChecksum = buf.getInt();
- int onDiskDataSizeWithHeader = buf.getInt();
+ final int onDiskSizeWithoutHeader = buf.getInt(Header.ON_DISK_SIZE_WITHOUT_HEADER_INDEX);
+ final int uncompressedSizeWithoutHeader =
+ buf.getInt(Header.UNCOMPRESSED_SIZE_WITHOUT_HEADER_INDEX);
+ final long prevBlockOffset = buf.getLong(Header.PREV_BLOCK_OFFSET_INDEX);
+ byte checksumType = buf.get(Header.CHECKSUM_TYPE_INDEX);
+ int bytesPerChecksum = buf.getInt(Header.BYTES_PER_CHECKSUM_INDEX);
+ int onDiskDataSizeWithHeader = buf.getInt(Header.ON_DISK_DATA_SIZE_WITH_HEADER_INDEX);
// This constructor is called when we deserialize a block from cache and when we read a block in
// from the fs. fileCache is null when deserialized from cache so need to make up one.
HFileContextBuilder fileContextBuilder = fileContext != null?
@@ -384,14 +405,13 @@ public class HFileBlock implements Cacheable {
}
/**
- * Parse total ondisk size including header and checksum. Its second field in header after
- * the magic bytes.
+ * Parse total ondisk size including header and checksum.
* @param headerBuf Header ByteBuffer. Presumed exact size of header.
* @return Size of the block with header included.
*/
private static int getOnDiskSizeWithHeader(final ByteBuffer headerBuf) {
// Set hbase checksum to true always calling headerSize.
- return headerBuf.getInt(BlockType.MAGIC_LENGTH) + headerSize(true);
+ return headerBuf.getInt(Header.ON_DISK_SIZE_WITHOUT_HEADER_INDEX) + headerSize(true);
}
/**
@@ -1459,8 +1479,7 @@ public class HFileBlock implements Cacheable {
* @throws IOException
*/
protected int readAtOffset(FSDataInputStream istream, byte [] dest, int destOffset, int size,
- boolean peekIntoNextBlock, long fileOffset, boolean pread)
- throws IOException {
+ boolean peekIntoNextBlock, long fileOffset, boolean pread) throws IOException {
if (peekIntoNextBlock && destOffset + size + hdrSize > dest.length) {
// We are asked to read the next block's header as well, but there is
// not enough room in the array.
@@ -1647,9 +1666,8 @@ public class HFileBlock implements Cacheable {
* If HBase checksum is switched off, then use HDFS checksum.
* @return the HFileBlock or null if there is a HBase checksum mismatch
*/
- private HFileBlock readBlockDataInternal(FSDataInputStream is, long offset,
- long onDiskSizeWithHeaderL, boolean pread, boolean verifyChecksum)
- throws IOException {
+ protected HFileBlock readBlockDataInternal(FSDataInputStream is, long offset,
+ long onDiskSizeWithHeaderL, boolean pread, boolean verifyChecksum) throws IOException {
if (offset < 0) {
throw new IOException("Invalid offset=" + offset + " trying to read "
+ "block (onDiskSize=" + onDiskSizeWithHeaderL + ")");
@@ -1691,20 +1709,23 @@ public class HFileBlock implements Cacheable {
// Do a few checks before we go instantiate HFileBlock.
assert onDiskSizeWithHeader > this.hdrSize;
verifyOnDiskSizeMatchesHeader(onDiskSizeWithHeader, headerBuf, offset);
+ ByteBuffer onDiskBlockByteBuffer = ByteBuffer.wrap(onDiskBlock, 0, onDiskSizeWithHeader);
+ // Verify checksum of the data before using it for building HFileBlock.
+ if (verifyChecksum &&
+ !validateChecksum(offset, onDiskBlockByteBuffer.asReadOnlyBuffer(), hdrSize)) {
+ return null;
+ }
// The onDiskBlock will become the headerAndDataBuffer for this block.
// If nextBlockOnDiskSizeWithHeader is not zero, the onDiskBlock already
// contains the header of next block, so no need to set next block's header in it.
HFileBlock hFileBlock =
- new HFileBlock(new SingleByteBuff(ByteBuffer.wrap(onDiskBlock, 0, onDiskSizeWithHeader)),
+ new HFileBlock(new SingleByteBuff(onDiskBlockByteBuffer),
this.fileContext.isUseHBaseChecksum(), MemoryType.EXCLUSIVE, offset,
nextBlockOnDiskSize, fileContext);
// Run check on uncompressed sizings.
if (!fileContext.isCompressedOrEncrypted()) {
hFileBlock.sanityCheckUncompressed();
}
- if (verifyChecksum && !validateBlockChecksum(hFileBlock, offset, onDiskBlock, hdrSize)) {
- return null;
- }
if (LOG.isTraceEnabled()) {
LOG.trace("Read " + hFileBlock);
}
@@ -1737,15 +1758,21 @@ public class HFileBlock implements Cacheable {
}
/**
- * Generates the checksum for the header as well as the data and
- * then validates that it matches the value stored in the header.
- * If there is a checksum mismatch, then return false. Otherwise
- * return true.
+ * Generates the checksum for the header as well as the data and then validates it.
+ * If the block doesn't uses checksum, returns false.
+ * @return True if checksum matches, else false.
*/
- protected boolean validateBlockChecksum(HFileBlock block, long offset, byte[] data,
- int hdrSize)
- throws IOException {
- return ChecksumUtil.validateBlockChecksum(pathName, offset, block, data, hdrSize);
+ protected boolean validateChecksum(long offset, ByteBuffer data, int hdrSize)
+ throws IOException {
+ // If this is an older version of the block that does not have checksums, then return false
+ // indicating that checksum verification did not succeed. Actually, this method should never
+ // be called when the minorVersion is 0, thus this is a defensive check for a cannot-happen
+ // case. Since this is a cannot-happen case, it is better to return false to indicate a
+ // checksum validation failure.
+ if (!fileContext.isUseHBaseChecksum()) {
+ return false;
+ }
+ return ChecksumUtil.validateChecksum(data, pathName, offset, hdrSize);
}
@Override
http://git-wip-us.apache.org/repos/asf/hbase/blob/48b77dbc/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestChecksum.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestChecksum.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestChecksum.java
index d91a811..c4950c3 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestChecksum.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestChecksum.java
@@ -108,9 +108,9 @@ public class TestChecksum {
ChecksumType cktype = itr.next();
Path path = new Path(TEST_UTIL.getDataTestDir(), "checksum" + cktype.getName());
FSDataOutputStream os = fs.create(path);
- HFileContext meta = new HFileContextBuilder().
- withChecksumType(cktype).
- build();
+ HFileContext meta = new HFileContextBuilder()
+ .withChecksumType(cktype)
+ .build();
HFileBlock.Writer hbw = new HFileBlock.Writer(null, meta);
DataOutputStream dos = hbw.startWriting(BlockType.DATA);
for (int i = 0; i < 1000; ++i) {
@@ -189,7 +189,7 @@ public class TestChecksum {
.withIncludesTags(useTags)
.withHBaseCheckSum(true)
.build();
- HFileBlock.FSReader hbr = new FSReaderImplTest(is, totalSize, fs, path, meta);
+ HFileBlock.FSReader hbr = new CorruptedFSReaderImpl(is, totalSize, fs, path, meta);
HFileBlock b = hbr.readBlockData(0, -1, pread);
b.sanityCheck();
assertEquals(4936, b.getUncompressedSizeWithoutHeader());
@@ -231,7 +231,7 @@ public class TestChecksum {
HFileSystem newfs = new HFileSystem(TEST_UTIL.getConfiguration(), false);
assertEquals(false, newfs.useHBaseChecksum());
is = new FSDataInputStreamWrapper(newfs, path);
- hbr = new FSReaderImplTest(is, totalSize, newfs, path, meta);
+ hbr = new CorruptedFSReaderImpl(is, totalSize, newfs, path, meta);
b = hbr.readBlockData(0, -1, pread);
is.close();
b.sanityCheck();
@@ -252,7 +252,7 @@ public class TestChecksum {
}
}
- /**
+ /**
* Test different values of bytesPerChecksum
*/
@Test
@@ -340,20 +340,56 @@ public class TestChecksum {
}
/**
- * A class that introduces hbase-checksum failures while
- * reading data from hfiles. This should trigger the hdfs level
- * checksum validations.
+ * This class is to test checksum behavior when data is corrupted. It mimics the following
+ * behavior:
+ * - When fs checksum is disabled, hbase may get corrupted data from hdfs. If verifyChecksum
+ * is true, it means hbase checksum is on and fs checksum is off, so we corrupt the data.
+ * - When fs checksum is enabled, hdfs will get a different copy from another node, and will
+ * always return correct data. So we don't corrupt the data when verifyChecksum for hbase is
+ * off.
*/
- static private class FSReaderImplTest extends HFileBlock.FSReaderImpl {
- public FSReaderImplTest(FSDataInputStreamWrapper istream, long fileSize, FileSystem fs,
+ static private class CorruptedFSReaderImpl extends HFileBlock.FSReaderImpl {
+ /**
+ * If set to true, corrupt reads using readAtOffset(...).
+ */
+ boolean corruptDataStream = false;
+
+ public CorruptedFSReaderImpl(FSDataInputStreamWrapper istream, long fileSize, FileSystem fs,
Path path, HFileContext meta) throws IOException {
super(istream, fileSize, (HFileSystem) fs, path, meta);
}
@Override
- protected boolean validateBlockChecksum(HFileBlock block, long offset,
- byte[] data, int hdrSize) throws IOException {
- return false; // checksum validation failure
+ protected HFileBlock readBlockDataInternal(FSDataInputStream is, long offset,
+ long onDiskSizeWithHeaderL, boolean pread, boolean verifyChecksum) throws IOException {
+ if (verifyChecksum) {
+ corruptDataStream = true;
+ }
+ HFileBlock b = super.readBlockDataInternal(is, offset, onDiskSizeWithHeaderL, pread,
+ verifyChecksum);
+ corruptDataStream = false;
+ return b;
+ }
+
+ @Override
+ protected int readAtOffset(FSDataInputStream istream, byte [] dest, int destOffset, int size,
+ boolean peekIntoNextBlock, long fileOffset, boolean pread) throws IOException {
+ int returnValue = super.readAtOffset(istream, dest, destOffset, size, peekIntoNextBlock,
+ fileOffset, pread);
+ if (!corruptDataStream) {
+ return returnValue;
+ }
+ // Corrupt 3rd character of block magic of next block's header.
+ if (peekIntoNextBlock) {
+ dest[destOffset + size + 3] = 0b00000000;
+ }
+ // We might be reading this block's header too, corrupt it.
+ dest[destOffset + 1] = 0b00000000;
+ // Corrupt non header data
+ if (size > hdrSize) {
+ dest[destOffset + hdrSize + 1] = 0b00000000;
+ }
+ return returnValue;
}
}
}
\ No newline at end of file
[02/50] [abbrv] hbase git commit: HBASE-15607 Remove PB references
from Admin for 2.0 (Ram)
Posted by sy...@apache.org.
HBASE-15607 Remove PB references from Admin for 2.0 (Ram)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/c06a976a
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/c06a976a
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/c06a976a
Branch: refs/heads/hbase-12439
Commit: c06a976a98f04b3c834c2d8ee2e889b7882591d2
Parents: d77972f
Author: Ramkrishna <ra...@intel.com>
Authored: Tue May 3 10:51:46 2016 +0530
Committer: Ramkrishna <ra...@intel.com>
Committed: Tue May 3 10:51:46 2016 +0530
----------------------------------------------------------------------
.../org/apache/hadoop/hbase/client/Admin.java | 61 +++--------
.../apache/hadoop/hbase/client/CompactType.java | 35 +++++++
.../hadoop/hbase/client/CompactionState.java | 29 ++++++
.../apache/hadoop/hbase/client/HBaseAdmin.java | 104 +++++++++++++------
.../hadoop/hbase/client/MasterSwitchType.java | 29 ++++++
.../hbase/client/SnapshotDescription.java | 84 +++++++++++++++
.../hadoop/hbase/client/SnapshotType.java | 29 ++++++
.../hadoop/hbase/protobuf/ProtobufUtil.java | 75 +++++++++++++
.../hadoop/hbase/protobuf/RequestConverter.java | 14 +--
.../hbase/client/TestSnapshotFromAdmin.java | 25 +++--
.../hbase/chaos/actions/CompactMobAction.java | 5 +-
.../hbase/rsgroup/RSGroupAdminEndpoint.java | 6 +-
.../hbase/tmpl/master/MasterStatusTmpl.jamon | 2 +-
.../BaseMasterAndRegionObserver.java | 6 +-
.../hbase/coprocessor/BaseMasterObserver.java | 6 +-
.../hbase/coprocessor/MasterObserver.java | 6 +-
.../hadoop/hbase/master/AssignmentManager.java | 6 +-
.../org/apache/hadoop/hbase/master/HMaster.java | 6 +-
.../hbase/master/MasterCoprocessorHost.java | 6 +-
.../hadoop/hbase/master/MasterRpcServices.java | 12 +--
.../normalizer/SimpleRegionNormalizer.java | 2 +-
.../hbase/security/access/AccessController.java | 6 +-
.../visibility/VisibilityController.java | 6 +-
.../hadoop/hbase/snapshot/CreateSnapshot.java | 8 +-
.../hadoop/hbase/snapshot/SnapshotInfo.java | 38 ++++---
.../org/apache/hadoop/hbase/util/HBaseFsck.java | 3 +-
.../hbase/zookeeper/SplitOrMergeTracker.java | 10 +-
.../resources/hbase-webapps/master/snapshot.jsp | 2 +-
.../resources/hbase-webapps/master/table.jsp | 2 +-
.../hadoop/hbase/client/TestFromClientSide.java | 3 +-
.../hbase/client/TestSnapshotFromClient.java | 33 +++---
.../hbase/client/TestSplitOrMergeStatus.java | 32 +++---
.../hbase/coprocessor/TestMasterObserver.java | 5 +-
.../master/TestDistributedLogSplitting.java | 5 +-
.../hadoop/hbase/master/TestWarmupRegion.java | 5 +-
.../procedure/TestCloneSnapshotProcedure.java | 10 +-
.../procedure/TestRestoreSnapshotProcedure.java | 8 +-
.../hbase/mob/compactions/TestMobCompactor.java | 13 +--
.../hbase/regionserver/TestCompactionState.java | 4 +-
.../hadoop/hbase/regionserver/TestTags.java | 2 +-
.../security/access/TestAccessController.java | 3 +-
.../hbase/snapshot/SnapshotTestingUtils.java | 43 ++++----
.../snapshot/TestFlushSnapshotFromClient.java | 68 +++++++-----
.../TestRestoreFlushSnapshotFromClient.java | 8 +-
.../hadoop/hbase/util/TestHBaseFsckOneRS.java | 16 +--
45 files changed, 611 insertions(+), 270 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/c06a976a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
index 97356a2..51a26bc 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
@@ -41,9 +41,6 @@ import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.classification.InterfaceStability;
import org.apache.hadoop.hbase.client.security.SecurityCapability;
import org.apache.hadoop.hbase.ipc.CoprocessorRpcChannel;
-import org.apache.hadoop.hbase.protobuf.generated.AdminProtos;
-import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos;
-import org.apache.hadoop.hbase.protobuf.generated.MasterProtos;
import org.apache.hadoop.hbase.quotas.QuotaFilter;
import org.apache.hadoop.hbase.quotas.QuotaRetriever;
import org.apache.hadoop.hbase.quotas.QuotaSettings;
@@ -1153,7 +1150,7 @@ public interface Admin extends Abortable, Closeable {
* @return the current compaction state
* @throws IOException if a remote or network exception occurs
*/
- AdminProtos.GetRegionInfoResponse.CompactionState getCompactionState(final TableName tableName)
+ CompactionState getCompactionState(final TableName tableName)
throws IOException;
/**
@@ -1164,7 +1161,7 @@ public interface Admin extends Abortable, Closeable {
* @return the current compaction state
* @throws IOException if a remote or network exception occurs
*/
- AdminProtos.GetRegionInfoResponse.CompactionState getCompactionStateForRegion(
+ CompactionState getCompactionStateForRegion(
final byte[] regionName) throws IOException;
/**
@@ -1244,7 +1241,7 @@ public interface Admin extends Abortable, Closeable {
*/
void snapshot(final String snapshotName,
final TableName tableName,
- HBaseProtos.SnapshotDescription.Type type) throws IOException, SnapshotCreationException,
+ SnapshotType type) throws IOException, SnapshotCreationException,
IllegalArgumentException;
/**
@@ -1265,7 +1262,7 @@ public interface Admin extends Abortable, Closeable {
* @throws SnapshotCreationException if snapshot failed to be taken
* @throws IllegalArgumentException if the snapshot request is formatted incorrectly
*/
- void snapshot(HBaseProtos.SnapshotDescription snapshot)
+ void snapshot(SnapshotDescription snapshot)
throws IOException, SnapshotCreationException, IllegalArgumentException;
/**
@@ -1273,12 +1270,11 @@ public interface Admin extends Abortable, Closeable {
* single snapshot should be taken at a time, or results may be undefined.
*
* @param snapshot snapshot to take
- * @return response from the server indicating the max time to wait for the snapshot
* @throws IOException if the snapshot did not succeed or we lose contact with the master.
* @throws SnapshotCreationException if snapshot creation failed
* @throws IllegalArgumentException if the snapshot request is formatted incorrectly
*/
- MasterProtos.SnapshotResponse takeSnapshotAsync(HBaseProtos.SnapshotDescription snapshot)
+ void takeSnapshotAsync(SnapshotDescription snapshot)
throws IOException, SnapshotCreationException;
/**
@@ -1297,7 +1293,7 @@ public interface Admin extends Abortable, Closeable {
* @throws org.apache.hadoop.hbase.snapshot.UnknownSnapshotException if the requested snapshot is
* unknown
*/
- boolean isSnapshotFinished(final HBaseProtos.SnapshotDescription snapshot)
+ boolean isSnapshotFinished(final SnapshotDescription snapshot)
throws IOException, HBaseSnapshotException, UnknownSnapshotException;
/**
@@ -1470,7 +1466,7 @@ public interface Admin extends Abortable, Closeable {
* @return a list of snapshot descriptors for completed snapshots
* @throws IOException if a network error occurs
*/
- List<HBaseProtos.SnapshotDescription> listSnapshots() throws IOException;
+ List<SnapshotDescription> listSnapshots() throws IOException;
/**
* List all the completed snapshots matching the given regular expression.
@@ -1479,7 +1475,7 @@ public interface Admin extends Abortable, Closeable {
* @return - returns a List of SnapshotDescription
* @throws IOException if a remote or network exception occurs
*/
- List<HBaseProtos.SnapshotDescription> listSnapshots(String regex) throws IOException;
+ List<SnapshotDescription> listSnapshots(String regex) throws IOException;
/**
* List all the completed snapshots matching the given pattern.
@@ -1488,7 +1484,7 @@ public interface Admin extends Abortable, Closeable {
* @return - returns a List of SnapshotDescription
* @throws IOException if a remote or network exception occurs
*/
- List<HBaseProtos.SnapshotDescription> listSnapshots(Pattern pattern) throws IOException;
+ List<SnapshotDescription> listSnapshots(Pattern pattern) throws IOException;
/**
* List all the completed snapshots matching the given table name regular expression and snapshot
@@ -1498,7 +1494,7 @@ public interface Admin extends Abortable, Closeable {
* @return - returns a List of completed SnapshotDescription
* @throws IOException if a remote or network exception occurs
*/
- List<HBaseProtos.SnapshotDescription> listTableSnapshots(String tableNameRegex,
+ List<SnapshotDescription> listTableSnapshots(String tableNameRegex,
String snapshotNameRegex) throws IOException;
/**
@@ -1509,7 +1505,7 @@ public interface Admin extends Abortable, Closeable {
* @return - returns a List of completed SnapshotDescription
* @throws IOException if a remote or network exception occurs
*/
- List<HBaseProtos.SnapshotDescription> listTableSnapshots(Pattern tableNamePattern,
+ List<SnapshotDescription> listTableSnapshots(Pattern tableNamePattern,
Pattern snapshotNamePattern) throws IOException;
/**
@@ -1651,7 +1647,7 @@ public interface Admin extends Abortable, Closeable {
* Compact a table. Asynchronous operation.
*
* @param tableName table to compact
- * @param compactType {@link org.apache.hadoop.hbase.client.Admin.CompactType}
+ * @param compactType {@link org.apache.hadoop.hbase.client.CompactType}
* @throws IOException
* @throws InterruptedException
*/
@@ -1663,7 +1659,7 @@ public interface Admin extends Abortable, Closeable {
*
* @param tableName table to compact
* @param columnFamily column family within a table
- * @param compactType {@link org.apache.hadoop.hbase.client.Admin.CompactType}
+ * @param compactType {@link org.apache.hadoop.hbase.client.CompactType}
* @throws IOException if not a mob column family or if a remote or network exception occurs
* @throws InterruptedException
*/
@@ -1674,7 +1670,7 @@ public interface Admin extends Abortable, Closeable {
* Major compact a table. Asynchronous operation.
*
* @param tableName table to compact
- * @param compactType {@link org.apache.hadoop.hbase.client.Admin.CompactType}
+ * @param compactType {@link org.apache.hadoop.hbase.client.CompactType}
* @throws IOException
* @throws InterruptedException
*/
@@ -1686,7 +1682,7 @@ public interface Admin extends Abortable, Closeable {
*
* @param tableName table to compact
* @param columnFamily column family within a table
- * @param compactType {@link org.apache.hadoop.hbase.client.Admin.CompactType}
+ * @param compactType {@link org.apache.hadoop.hbase.client.CompactType}
* @throws IOException if not a mob column family or if a remote or network exception occurs
* @throws InterruptedException
*/
@@ -1697,11 +1693,11 @@ public interface Admin extends Abortable, Closeable {
* Get the current compaction state of a table. It could be in a compaction, or none.
*
* @param tableName table to examine
- * @param compactType {@link org.apache.hadoop.hbase.client.Admin.CompactType}
+ * @param compactType {@link org.apache.hadoop.hbase.client.CompactType}
* @return the current compaction state
* @throws IOException if a remote or network exception occurs
*/
- AdminProtos.GetRegionInfoResponse.CompactionState getCompactionState(final TableName tableName,
+ CompactionState getCompactionState(final TableName tableName,
CompactType compactType) throws IOException;
/**
@@ -1741,27 +1737,4 @@ public interface Admin extends Abortable, Closeable {
* and rollback the switch state to be original state before you change switch
* */
void releaseSplitOrMergeLockAndRollback() throws IOException;
-
- /**
- * Currently, there are only two compact types:
- * {@code NORMAL} means do store files compaction;
- * {@code MOB} means do mob files compaction.
- * */
- @InterfaceAudience.Public
- @InterfaceStability.Unstable
- public enum CompactType {
-
- NORMAL (0),
- MOB (1);
-
- CompactType(int value) {}
- }
-
- @InterfaceAudience.Public
- @InterfaceStability.Evolving
- public enum MasterSwitchType {
- SPLIT,
- MERGE
- }
-
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/c06a976a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/CompactType.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/CompactType.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/CompactType.java
new file mode 100644
index 0000000..17fec2b
--- /dev/null
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/CompactType.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.client;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+
+/**
+ * Currently, there are only two compact types:
+ * {@code NORMAL} means do store files compaction;
+ * {@code MOB} means do mob files compaction.
+ * */
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+public enum CompactType {
+
+ NORMAL (0),
+ MOB (1);
+
+ CompactType(int value) {}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/c06a976a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/CompactionState.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/CompactionState.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/CompactionState.java
new file mode 100644
index 0000000..b4824ef
--- /dev/null
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/CompactionState.java
@@ -0,0 +1,29 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.client;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+
+/**
+ * POJO representing the compaction state
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public enum CompactionState {
+ NONE, MINOR, MAJOR, MAJOR_AND_MINOR;
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/c06a976a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
index ea1f591..7371f03 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
@@ -73,6 +73,7 @@ import org.apache.hadoop.hbase.ipc.RegionServerCoprocessorRpcChannel;
import org.apache.hadoop.hbase.ipc.RpcControllerFactory;
import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.protobuf.RequestConverter;
+import org.apache.hadoop.hbase.protobuf.generated.AdminProtos;
import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService;
import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.CloseRegionRequest;
import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.CloseRegionResponse;
@@ -80,7 +81,6 @@ import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.CompactRegionReque
import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.FlushRegionRequest;
import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoRequest;
import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoResponse;
-import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoResponse.CompactionState;
import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.RollWALWriterRequest;
import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.RollWALWriterResponse;
import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.StopServerRequest;
@@ -89,7 +89,6 @@ import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameStringPair;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ProcedureDescription;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier.RegionSpecifierType;
-import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.TableSchema;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.AbortProcedureRequest;
@@ -2265,7 +2264,10 @@ public class HBaseAdmin implements Admin {
PayloadCarryingRpcController controller = rpcControllerFactory.newController();
// TODO: this does not do retries, it should. Set priority and timeout in controller
GetRegionInfoResponse response = admin.getRegionInfo(controller, request);
- return response.getCompactionState();
+ if (response.getCompactionState() != null) {
+ return ProtobufUtil.createCompactionState(response.getCompactionState());
+ }
+ return null;
} catch (ServiceException se) {
throw ProtobufUtil.getRemoteException(se);
}
@@ -2275,33 +2277,30 @@ public class HBaseAdmin implements Admin {
public void snapshot(final String snapshotName,
final TableName tableName) throws IOException,
SnapshotCreationException, IllegalArgumentException {
- snapshot(snapshotName, tableName, SnapshotDescription.Type.FLUSH);
+ snapshot(snapshotName, tableName, SnapshotType.FLUSH);
}
@Override
public void snapshot(final byte[] snapshotName, final TableName tableName)
throws IOException, SnapshotCreationException, IllegalArgumentException {
- snapshot(Bytes.toString(snapshotName), tableName, SnapshotDescription.Type.FLUSH);
+ snapshot(Bytes.toString(snapshotName), tableName, SnapshotType.FLUSH);
}
@Override
public void snapshot(final String snapshotName, final TableName tableName,
- SnapshotDescription.Type type)
+ SnapshotType type)
throws IOException, SnapshotCreationException, IllegalArgumentException {
- SnapshotDescription.Builder builder = SnapshotDescription.newBuilder();
- builder.setTable(tableName.getNameAsString());
- builder.setName(snapshotName);
- builder.setType(type);
- snapshot(builder.build());
+ snapshot(new SnapshotDescription(snapshotName, tableName.getNameAsString(), type));
}
@Override
- public void snapshot(SnapshotDescription snapshot) throws IOException, SnapshotCreationException,
- IllegalArgumentException {
+ public void snapshot(SnapshotDescription snapshotDesc)
+ throws IOException, SnapshotCreationException, IllegalArgumentException {
// actually take the snapshot
- SnapshotResponse response = takeSnapshotAsync(snapshot);
- final IsSnapshotDoneRequest request = IsSnapshotDoneRequest.newBuilder().setSnapshot(snapshot)
- .build();
+ HBaseProtos.SnapshotDescription snapshot = createHBaseProtosSnapshotDesc(snapshotDesc);
+ SnapshotResponse response = asyncSnapshot(snapshot);
+ final IsSnapshotDoneRequest request =
+ IsSnapshotDoneRequest.newBuilder().setSnapshot(snapshot).build();
IsSnapshotDoneResponse done = null;
long start = EnvironmentEdgeManager.currentTime();
long max = response.getExpectedTimeout();
@@ -2339,8 +2338,37 @@ public class HBaseAdmin implements Admin {
}
@Override
- public SnapshotResponse takeSnapshotAsync(SnapshotDescription snapshot) throws IOException,
+ public void takeSnapshotAsync(SnapshotDescription snapshotDesc) throws IOException,
SnapshotCreationException {
+ HBaseProtos.SnapshotDescription snapshot = createHBaseProtosSnapshotDesc(snapshotDesc);
+ asyncSnapshot(snapshot);
+ }
+
+ private HBaseProtos.SnapshotDescription
+ createHBaseProtosSnapshotDesc(SnapshotDescription snapshotDesc) {
+ HBaseProtos.SnapshotDescription.Builder builder = HBaseProtos.SnapshotDescription.newBuilder();
+ if (snapshotDesc.getTable() != null) {
+ builder.setTable(snapshotDesc.getTable());
+ }
+ if (snapshotDesc.getName() != null) {
+ builder.setName(snapshotDesc.getName());
+ }
+ if (snapshotDesc.getOwner() != null) {
+ builder.setOwner(snapshotDesc.getOwner());
+ }
+ if (snapshotDesc.getCreationTime() != -1) {
+ builder.setCreationTime(snapshotDesc.getCreationTime());
+ }
+ if (snapshotDesc.getVersion() != -1) {
+ builder.setVersion(snapshotDesc.getVersion());
+ }
+ builder.setType(ProtobufUtil.createProtosSnapShotDescType(snapshotDesc.getType()));
+ HBaseProtos.SnapshotDescription snapshot = builder.build();
+ return snapshot;
+ }
+
+ private SnapshotResponse asyncSnapshot(HBaseProtos.SnapshotDescription snapshot)
+ throws IOException {
ClientSnapshotDescriptionUtils.assertSnapshotRequestIsValid(snapshot);
final SnapshotRequest request = SnapshotRequest.newBuilder().setSnapshot(snapshot)
.build();
@@ -2356,9 +2384,9 @@ public class HBaseAdmin implements Admin {
}
@Override
- public boolean isSnapshotFinished(final SnapshotDescription snapshot)
+ public boolean isSnapshotFinished(final SnapshotDescription snapshotDesc)
throws IOException, HBaseSnapshotException, UnknownSnapshotException {
-
+ final HBaseProtos.SnapshotDescription snapshot = createHBaseProtosSnapshotDesc(snapshotDesc);
return executeCallable(new MasterCallable<IsSnapshotDoneResponse>(getConnection()) {
@Override
public IsSnapshotDoneResponse call(int callTimeout) throws ServiceException {
@@ -2643,7 +2671,7 @@ public class HBaseAdmin implements Admin {
private Future<Void> internalRestoreSnapshotAsync(
final String snapshotName,
final TableName tableName) throws IOException, RestoreSnapshotException {
- final SnapshotDescription snapshot = SnapshotDescription.newBuilder()
+ final HBaseProtos.SnapshotDescription snapshot = HBaseProtos.SnapshotDescription.newBuilder()
.setName(snapshotName).setTable(tableName.getNameAsString()).build();
// actually restore the snapshot
@@ -2671,7 +2699,7 @@ public class HBaseAdmin implements Admin {
private static class RestoreSnapshotFuture extends TableFuture<Void> {
public RestoreSnapshotFuture(
final HBaseAdmin admin,
- final SnapshotDescription snapshot,
+ final HBaseProtos.SnapshotDescription snapshot,
final TableName tableName,
final RestoreSnapshotResponse response) {
super(admin, tableName,
@@ -2702,8 +2730,16 @@ public class HBaseAdmin implements Admin {
public List<SnapshotDescription> call(int callTimeout) throws ServiceException {
PayloadCarryingRpcController controller = rpcControllerFactory.newController();
controller.setCallTimeout(callTimeout);
- return master.getCompletedSnapshots(controller,
- GetCompletedSnapshotsRequest.newBuilder().build()).getSnapshotsList();
+ List<HBaseProtos.SnapshotDescription> snapshotsList = master
+ .getCompletedSnapshots(controller, GetCompletedSnapshotsRequest.newBuilder().build())
+ .getSnapshotsList();
+ List<SnapshotDescription> result = new ArrayList<SnapshotDescription>(snapshotsList.size());
+ for (HBaseProtos.SnapshotDescription snapshot : snapshotsList) {
+ result.add(new SnapshotDescription(snapshot.getName(), snapshot.getTable(),
+ ProtobufUtil.createSnapshotType(snapshot.getType()), snapshot.getOwner(),
+ snapshot.getCreationTime(), snapshot.getVersion()));
+ }
+ return result;
}
});
}
@@ -2765,7 +2801,9 @@ public class HBaseAdmin implements Admin {
controller.setCallTimeout(callTimeout);
master.deleteSnapshot(controller,
DeleteSnapshotRequest.newBuilder().
- setSnapshot(SnapshotDescription.newBuilder().setName(snapshotName).build()).build()
+ setSnapshot(
+ HBaseProtos.SnapshotDescription.newBuilder().setName(snapshotName).build())
+ .build()
);
return null;
}
@@ -2798,7 +2836,7 @@ public class HBaseAdmin implements Admin {
PayloadCarryingRpcController controller = rpcControllerFactory.newController();
controller.setCallTimeout(callTimeout);
this.master.deleteSnapshot(controller, DeleteSnapshotRequest.newBuilder()
- .setSnapshot(snapshot).build());
+ .setSnapshot(createHBaseProtosSnapshotDesc(snapshot)).build());
return null;
}
});
@@ -3002,7 +3040,8 @@ public class HBaseAdmin implements Admin {
@Override
public CompactionState getCompactionState(TableName tableName,
CompactType compactType) throws IOException {
- CompactionState state = CompactionState.NONE;
+ AdminProtos.GetRegionInfoResponse.CompactionState state =
+ AdminProtos.GetRegionInfoResponse.CompactionState.NONE;
checkTableExists(tableName);
PayloadCarryingRpcController controller = rpcControllerFactory.newController();
switch (compactType) {
@@ -3044,16 +3083,16 @@ public class HBaseAdmin implements Admin {
case MAJOR_AND_MINOR:
return CompactionState.MAJOR_AND_MINOR;
case MAJOR:
- if (state == CompactionState.MINOR) {
+ if (state == AdminProtos.GetRegionInfoResponse.CompactionState.MINOR) {
return CompactionState.MAJOR_AND_MINOR;
}
- state = CompactionState.MAJOR;
+ state = AdminProtos.GetRegionInfoResponse.CompactionState.MAJOR;
break;
case MINOR:
- if (state == CompactionState.MAJOR) {
+ if (state == AdminProtos.GetRegionInfoResponse.CompactionState.MAJOR) {
return CompactionState.MAJOR_AND_MINOR;
}
- state = CompactionState.MINOR;
+ state = AdminProtos.GetRegionInfoResponse.CompactionState.MINOR;
break;
case NONE:
default: // nothing, continue
@@ -3084,7 +3123,10 @@ public class HBaseAdmin implements Admin {
}
break;
}
- return state;
+ if(state != null) {
+ return ProtobufUtil.createCompactionState(state);
+ }
+ return null;
}
/**
http://git-wip-us.apache.org/repos/asf/hbase/blob/c06a976a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MasterSwitchType.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MasterSwitchType.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MasterSwitchType.java
new file mode 100644
index 0000000..7e31b25
--- /dev/null
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MasterSwitchType.java
@@ -0,0 +1,29 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.client;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+/**
+ * Represents the master switch type
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public enum MasterSwitchType {
+ SPLIT,
+ MERGE
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/c06a976a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/SnapshotDescription.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/SnapshotDescription.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/SnapshotDescription.java
new file mode 100644
index 0000000..a455937
--- /dev/null
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/SnapshotDescription.java
@@ -0,0 +1,84 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.client;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+
+/**
+ * The POJO equivalent of HBaseProtos.SnapshotDescription
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class SnapshotDescription {
+ private String name;
+ private String table;
+ private SnapshotType snapShotType = SnapshotType.DISABLED;
+ private String owner;
+ private long creationTime = -1L;
+ private int version = -1;
+
+ public SnapshotDescription(String name) {
+ this(name, null);
+ }
+
+ public SnapshotDescription(String name, String table) {
+ this(name, table, SnapshotType.DISABLED, null);
+ }
+
+ public SnapshotDescription(String name, String table, SnapshotType type) {
+ this(name, table, type, null);
+ }
+
+ public SnapshotDescription(String name, String table, SnapshotType type, String owner) {
+ this(name, table, type, owner, -1, -1);
+ }
+
+ public SnapshotDescription(String name, String table, SnapshotType type, String owner,
+ long creationTime, int version) {
+ this.name = name;
+ this.table = table;
+ this.snapShotType = type;
+ this.owner = owner;
+ this.creationTime = creationTime;
+ this.version = version;
+ }
+
+ public String getName() {
+ return this.name;
+ }
+
+ public String getTable() {
+ return this.table;
+ }
+
+ public SnapshotType getType() {
+ return this.snapShotType;
+ }
+
+ public String getOwner() {
+ return this.owner;
+ }
+
+ public long getCreationTime() {
+ return this.creationTime;
+ }
+
+ public int getVersion() {
+ return this.version;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/c06a976a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/SnapshotType.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/SnapshotType.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/SnapshotType.java
new file mode 100644
index 0000000..e3e12bd
--- /dev/null
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/SnapshotType.java
@@ -0,0 +1,29 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.client;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+
+/**
+ * POJO representing the snapshot type
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public enum SnapshotType {
+ DISABLED, FLUSH, SKIPFLUSH;
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/c06a976a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
index 90516ec..9978c77 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
@@ -58,6 +58,7 @@ import org.apache.hadoop.hbase.Tag;
import org.apache.hadoop.hbase.TagUtil;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.client.Append;
+import org.apache.hadoop.hbase.client.CompactionState;
import org.apache.hadoop.hbase.client.Consistency;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Durability;
@@ -67,6 +68,8 @@ import org.apache.hadoop.hbase.client.Mutation;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.SnapshotDescription;
+import org.apache.hadoop.hbase.client.SnapshotType;
import org.apache.hadoop.hbase.client.metrics.ScanMetrics;
import org.apache.hadoop.hbase.client.security.SecurityCapability;
import org.apache.hadoop.hbase.exceptions.DeserializationException;
@@ -3397,4 +3400,76 @@ public final class ProtobufUtil {
}
return htd;
}
+
+ /**
+ * Creates {@link CompactionState} from
+ * {@link org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoResponse.CompactionState}
+ * state
+ * @param state the protobuf CompactionState
+ * @return CompactionState
+ */
+ public static CompactionState createCompactionState(GetRegionInfoResponse.CompactionState state) {
+ return CompactionState.valueOf(state.toString());
+ }
+
+ /**
+ * Creates {@link org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription.Type}
+ * from {@link SnapshotType}
+ * @param type the SnapshotDescription type
+ * @return the protobuf SnapshotDescription type
+ */
+ public static HBaseProtos.SnapshotDescription.Type
+ createProtosSnapShotDescType(SnapshotType type) {
+ return HBaseProtos.SnapshotDescription.Type.valueOf(type.name());
+ }
+
+ /**
+ * Creates {@link org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription.Type}
+ * from the type of SnapshotDescription string
+ * @param snapshotDesc string representing the snapshot description type
+ * @return the protobuf SnapshotDescription type
+ */
+ public static HBaseProtos.SnapshotDescription.Type
+ createProtosSnapShotDescType(String snapshotDesc) {
+ return HBaseProtos.SnapshotDescription.Type.valueOf(snapshotDesc.toUpperCase());
+ }
+
+ /**
+ * Creates {@link SnapshotType} from the type of
+ * {@link org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription}
+ * @param type the snapshot description type
+ * @return the protobuf SnapshotDescription type
+ */
+ public static SnapshotType createSnapshotType(HBaseProtos.SnapshotDescription.Type type) {
+ return SnapshotType.valueOf(type.toString());
+ }
+
+ /**
+ * Convert from {@link SnapshotDescription} to
+ * {@link org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription}
+ * @param snapshotDesc the POJO SnapshotDescription
+ * @return the protobuf SnapshotDescription
+ */
+ public static HBaseProtos.SnapshotDescription
+ createHBaseProtosSnapshotDesc(SnapshotDescription snapshotDesc) {
+ HBaseProtos.SnapshotDescription.Builder builder = HBaseProtos.SnapshotDescription.newBuilder();
+ if (snapshotDesc.getTable() != null) {
+ builder.setTable(snapshotDesc.getTable());
+ }
+ if (snapshotDesc.getName() != null) {
+ builder.setName(snapshotDesc.getName());
+ }
+ if (snapshotDesc.getOwner() != null) {
+ builder.setOwner(snapshotDesc.getOwner());
+ }
+ if (snapshotDesc.getCreationTime() != -1L) {
+ builder.setCreationTime(snapshotDesc.getCreationTime());
+ }
+ if (snapshotDesc.getVersion() != -1) {
+ builder.setVersion(snapshotDesc.getVersion());
+ }
+ builder.setType(ProtobufUtil.createProtosSnapShotDescType(snapshotDesc.getType()));
+ HBaseProtos.SnapshotDescription snapshot = builder.build();
+ return snapshot;
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/c06a976a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/RequestConverter.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/RequestConverter.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/RequestConverter.java
index 7502323..ce01e1e 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/RequestConverter.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/RequestConverter.java
@@ -31,12 +31,12 @@ import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.client.Action;
-import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.Append;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Durability;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Increment;
+import org.apache.hadoop.hbase.client.MasterSwitchType;
import org.apache.hadoop.hbase.client.Mutation;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.RegionCoprocessorServiceExec;
@@ -1701,11 +1701,11 @@ public final class RequestConverter {
/**
* Creates a protocol buffer IsSplitOrMergeEnabledRequest
*
- * @param switchType see {@link org.apache.hadoop.hbase.client.Admin.MasterSwitchType}
+ * @param switchType see {@link org.apache.hadoop.hbase.client.MasterSwitchType}
* @return a IsSplitOrMergeEnabledRequest
*/
public static IsSplitOrMergeEnabledRequest buildIsSplitOrMergeEnabledRequest(
- Admin.MasterSwitchType switchType) {
+ MasterSwitchType switchType) {
IsSplitOrMergeEnabledRequest.Builder builder = IsSplitOrMergeEnabledRequest.newBuilder();
builder.setSwitchType(convert(switchType));
return builder.build();
@@ -1723,23 +1723,23 @@ public final class RequestConverter {
*
* @param enabled switch is enabled or not
* @param synchronous set switch sync?
- * @param switchTypes see {@link org.apache.hadoop.hbase.client.Admin.MasterSwitchType}, it is
+ * @param switchTypes see {@link org.apache.hadoop.hbase.client.MasterSwitchType}, it is
* a list.
* @return a SetSplitOrMergeEnabledRequest
*/
public static SetSplitOrMergeEnabledRequest buildSetSplitOrMergeEnabledRequest(boolean enabled,
- boolean synchronous, boolean skipLock, Admin.MasterSwitchType... switchTypes) {
+ boolean synchronous, boolean skipLock, MasterSwitchType... switchTypes) {
SetSplitOrMergeEnabledRequest.Builder builder = SetSplitOrMergeEnabledRequest.newBuilder();
builder.setEnabled(enabled);
builder.setSynchronous(synchronous);
builder.setSkipLock(skipLock);
- for (Admin.MasterSwitchType switchType : switchTypes) {
+ for (MasterSwitchType switchType : switchTypes) {
builder.addSwitchTypes(convert(switchType));
}
return builder.build();
}
- private static MasterProtos.MasterSwitchType convert(Admin.MasterSwitchType switchType) {
+ private static MasterProtos.MasterSwitchType convert(MasterSwitchType switchType) {
switch (switchType) {
case SPLIT:
return MasterProtos.MasterSwitchType.SPLIT;
http://git-wip-us.apache.org/repos/asf/hbase/blob/c06a976a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromAdmin.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromAdmin.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromAdmin.java
index 4d55c33..b41c859 100644
--- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromAdmin.java
+++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromAdmin.java
@@ -30,7 +30,6 @@ import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.ipc.PayloadCarryingRpcController;
import org.apache.hadoop.hbase.ipc.RpcControllerFactory;
-import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsSnapshotDoneRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsSnapshotDoneResponse;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.SnapshotRequest;
@@ -141,18 +140,17 @@ public class TestSnapshotFromAdmin {
Mockito.when(mockConnection.getRpcRetryingCallerFactory()).thenReturn(callerFactory);
Mockito.when(mockConnection.getRpcControllerFactory()).thenReturn(controllerFactory);
Admin admin = new HBaseAdmin(mockConnection);
- SnapshotDescription.Builder builder = SnapshotDescription.newBuilder();
// check that invalid snapshot names fail
- failSnapshotStart(admin, builder.setName(HConstants.SNAPSHOT_DIR_NAME).build());
- failSnapshotStart(admin, builder.setName("-snapshot").build());
- failSnapshotStart(admin, builder.setName("snapshot fails").build());
- failSnapshotStart(admin, builder.setName("snap$hot").build());
- failSnapshotStart(admin, builder.setName("snap:hot").build());
+ failSnapshotStart(admin, new SnapshotDescription(HConstants.SNAPSHOT_DIR_NAME));
+ failSnapshotStart(admin, new SnapshotDescription("-snapshot"));
+ failSnapshotStart(admin, new SnapshotDescription("snapshot fails"));
+ failSnapshotStart(admin, new SnapshotDescription("snap$hot"));
+ failSnapshotStart(admin, new SnapshotDescription("snap:hot"));
// check the table name also get verified
- failSnapshotStart(admin, builder.setName("snapshot").setTable(".table").build());
- failSnapshotStart(admin, builder.setName("snapshot").setTable("-table").build());
- failSnapshotStart(admin, builder.setName("snapshot").setTable("table fails").build());
- failSnapshotStart(admin, builder.setName("snapshot").setTable("tab%le").build());
+ failSnapshotStart(admin, new SnapshotDescription("snapshot", ".table"));
+ failSnapshotStart(admin, new SnapshotDescription("snapshot", "-table"));
+ failSnapshotStart(admin, new SnapshotDescription("snapshot", "table fails"));
+ failSnapshotStart(admin, new SnapshotDescription("snapshot", "tab%le"));
// mock the master connection
MasterKeepAliveConnection master = Mockito.mock(MasterKeepAliveConnection.class);
@@ -167,10 +165,11 @@ public class TestSnapshotFromAdmin {
Mockito.any(IsSnapshotDoneRequest.class))).thenReturn(doneResponse);
// make sure that we can use valid names
- admin.snapshot(builder.setName("snapshot").setTable("table").build());
+ admin.snapshot(new SnapshotDescription("snapshot", "table"));
}
- private void failSnapshotStart(Admin admin, SnapshotDescription snapshot) throws IOException {
+ private void failSnapshotStart(Admin admin, SnapshotDescription snapshot)
+ throws IOException {
try {
admin.snapshot(snapshot);
fail("Snapshot should not have succeed with name:" + snapshot.getName());
http://git-wip-us.apache.org/repos/asf/hbase/blob/c06a976a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/CompactMobAction.java
----------------------------------------------------------------------
diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/CompactMobAction.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/CompactMobAction.java
index 2fff77a..b75bbbd 100644
--- a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/CompactMobAction.java
+++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/CompactMobAction.java
@@ -22,6 +22,7 @@ import org.apache.commons.lang.math.RandomUtils;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.CompactType;
/**
* Action that queues a table compaction.
@@ -56,9 +57,9 @@ public class CompactMobAction extends Action {
LOG.info("Performing action: Compact mob of table " + tableName + ", major=" + major);
try {
if (major) {
- admin.majorCompact(tableName, Admin.CompactType.MOB);
+ admin.majorCompact(tableName, CompactType.MOB);
} else {
- admin.compact(tableName, Admin.CompactType.MOB);
+ admin.compact(tableName, CompactType.MOB);
}
} catch (Exception ex) {
LOG.warn("Mob Compaction failed, might be caused by other chaos: " + ex.getMessage());
http://git-wip-us.apache.org/repos/asf/hbase/blob/c06a976a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminEndpoint.java
----------------------------------------------------------------------
diff --git a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminEndpoint.java b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminEndpoint.java
index 22bad72..6bc7508 100644
--- a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminEndpoint.java
+++ b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminEndpoint.java
@@ -43,7 +43,7 @@ import org.apache.hadoop.hbase.NamespaceDescriptor;
import org.apache.hadoop.hbase.ProcedureInfo;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.MasterSwitchType;
import org.apache.hadoop.hbase.constraint.ConstraintException;
import org.apache.hadoop.hbase.coprocessor.CoprocessorService;
import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment;
@@ -827,13 +827,13 @@ public class RSGroupAdminEndpoint extends RSGroupAdminService
@Override
public boolean preSetSplitOrMergeEnabled(final ObserverContext<MasterCoprocessorEnvironment> ctx,
- final boolean newValue, final Admin.MasterSwitchType switchType) throws IOException {
+ final boolean newValue, final MasterSwitchType switchType) throws IOException {
return false;
}
@Override
public void postSetSplitOrMergeEnabled(final ObserverContext<MasterCoprocessorEnvironment> ctx,
- final boolean newValue, final Admin.MasterSwitchType switchType) throws IOException {
+ final boolean newValue, final MasterSwitchType switchType) throws IOException {
}
@Override
http://git-wip-us.apache.org/repos/asf/hbase/blob/c06a976a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon
index 951a95e..4b1c192 100644
--- a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon
+++ b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon
@@ -39,13 +39,13 @@ org.apache.hadoop.hbase.ServerLoad;
org.apache.hadoop.hbase.ServerName;
org.apache.hadoop.hbase.TableName;
org.apache.hadoop.hbase.client.Admin;
+org.apache.hadoop.hbase.client.SnapshotDescription;
org.apache.hadoop.hbase.master.AssignmentManager;
org.apache.hadoop.hbase.master.DeadServer;
org.apache.hadoop.hbase.master.HMaster;
org.apache.hadoop.hbase.master.RegionState;
org.apache.hadoop.hbase.master.ServerManager;
org.apache.hadoop.hbase.protobuf.ProtobufUtil;
-org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
org.apache.hadoop.hbase.quotas.QuotaUtil;
org.apache.hadoop.hbase.security.access.AccessControlLists;
org.apache.hadoop.hbase.security.visibility.VisibilityConstants;
http://git-wip-us.apache.org/repos/asf/hbase/blob/c06a976a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseMasterAndRegionObserver.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseMasterAndRegionObserver.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseMasterAndRegionObserver.java
index 65398c2..74d9fe1 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseMasterAndRegionObserver.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseMasterAndRegionObserver.java
@@ -36,7 +36,7 @@ import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.classification.InterfaceStability;
-import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.MasterSwitchType;
import org.apache.hadoop.hbase.master.RegionPlan;
import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
@@ -450,7 +450,7 @@ public class BaseMasterAndRegionObserver extends BaseRegionObserver
@Override
public boolean preSetSplitOrMergeEnabled(ObserverContext<MasterCoprocessorEnvironment> ctx,
boolean newValue,
- Admin.MasterSwitchType switchType)
+ MasterSwitchType switchType)
throws IOException {
return false;
}
@@ -458,7 +458,7 @@ public class BaseMasterAndRegionObserver extends BaseRegionObserver
@Override
public void postSetSplitOrMergeEnabled(ObserverContext<MasterCoprocessorEnvironment> ctx,
boolean newValue,
- Admin.MasterSwitchType switchType)
+ MasterSwitchType switchType)
throws IOException {
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/c06a976a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseMasterObserver.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseMasterObserver.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseMasterObserver.java
index 18c6a0a..3574bbd 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseMasterObserver.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseMasterObserver.java
@@ -36,7 +36,7 @@ import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.classification.InterfaceStability;
-import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.MasterSwitchType;
import org.apache.hadoop.hbase.master.RegionPlan;
import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
@@ -442,13 +442,13 @@ public class BaseMasterObserver implements MasterObserver {
@Override
public boolean preSetSplitOrMergeEnabled(final ObserverContext<MasterCoprocessorEnvironment> ctx,
- final boolean newValue, final Admin.MasterSwitchType switchType) throws IOException {
+ final boolean newValue, final MasterSwitchType switchType) throws IOException {
return false;
}
@Override
public void postSetSplitOrMergeEnabled(final ObserverContext<MasterCoprocessorEnvironment> ctx,
- final boolean newValue, final Admin.MasterSwitchType switchType) throws IOException {
+ final boolean newValue, final MasterSwitchType switchType) throws IOException {
}
@Override
http://git-wip-us.apache.org/repos/asf/hbase/blob/c06a976a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterObserver.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterObserver.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterObserver.java
index 81f97aa..4b43cfe 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterObserver.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterObserver.java
@@ -36,7 +36,7 @@ import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.classification.InterfaceStability;
-import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.MasterSwitchType;
import org.apache.hadoop.hbase.master.RegionPlan;
import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
@@ -807,7 +807,7 @@ public interface MasterObserver extends Coprocessor {
* @param switchType type of switch
*/
boolean preSetSplitOrMergeEnabled(final ObserverContext<MasterCoprocessorEnvironment> ctx,
- final boolean newValue, final Admin.MasterSwitchType switchType) throws IOException;
+ final boolean newValue, final MasterSwitchType switchType) throws IOException;
/**
* Called after setting split / merge switch
@@ -816,7 +816,7 @@ public interface MasterObserver extends Coprocessor {
* @param switchType type of switch
*/
void postSetSplitOrMergeEnabled(final ObserverContext<MasterCoprocessorEnvironment> ctx,
- final boolean newValue, final Admin.MasterSwitchType switchType) throws IOException;
+ final boolean newValue, final MasterSwitchType switchType) throws IOException;
/**
* Called prior to modifying the flag used to enable/disable region balancing.
http://git-wip-us.apache.org/repos/asf/hbase/blob/c06a976a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
index 2b7713b..b81dcf4 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
@@ -62,7 +62,7 @@ import org.apache.hadoop.hbase.RegionStateListener;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.TableNotFoundException;
-import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.MasterSwitchType;
import org.apache.hadoop.hbase.client.RegionReplicaUtil;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.TableState;
@@ -2364,7 +2364,7 @@ public class AssignmentManager {
}
if (!((HMaster)server).getSplitOrMergeTracker().isSplitOrMergeEnabled(
- Admin.MasterSwitchType.SPLIT)) {
+ MasterSwitchType.SPLIT)) {
return "split switch is off!";
}
@@ -2527,7 +2527,7 @@ public class AssignmentManager {
}
if (!((HMaster)server).getSplitOrMergeTracker().isSplitOrMergeEnabled(
- Admin.MasterSwitchType.MERGE)) {
+ MasterSwitchType.MERGE)) {
return "merge switch is off!";
}
// Just return in case of retrying
http://git-wip-us.apache.org/repos/asf/hbase/blob/c06a976a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 3b5af42..2f1cd3c 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -83,7 +83,7 @@ import org.apache.hadoop.hbase.TableNotDisabledException;
import org.apache.hadoop.hbase.TableNotFoundException;
import org.apache.hadoop.hbase.UnknownRegionException;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.MasterSwitchType;
import org.apache.hadoop.hbase.client.RegionReplicaUtil;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.TableState;
@@ -2821,10 +2821,10 @@ public class HMaster extends HRegionServer implements MasterServices {
/**
* Queries the state of the {@link SplitOrMergeTracker}. If it is not initialized,
* false is returned. If switchType is illegal, false will return.
- * @param switchType see {@link org.apache.hadoop.hbase.client.Admin.MasterSwitchType}
+ * @param switchType see {@link org.apache.hadoop.hbase.client.MasterSwitchType}
* @return The state of the switch
*/
- public boolean isSplitOrMergeEnabled(Admin.MasterSwitchType switchType) {
+ public boolean isSplitOrMergeEnabled(MasterSwitchType switchType) {
if (null == splitOrMergeTracker) {
return false;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/c06a976a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java
index 313b1ad..14e8c20 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java
@@ -38,7 +38,7 @@ import org.apache.hadoop.hbase.ProcedureInfo;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.MasterSwitchType;
import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
import org.apache.hadoop.hbase.coprocessor.CoprocessorService;
import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment;
@@ -779,7 +779,7 @@ public class MasterCoprocessorHost
}
public boolean preSetSplitOrMergeEnabled(final boolean newValue,
- final Admin.MasterSwitchType switchType) throws IOException {
+ final MasterSwitchType switchType) throws IOException {
return execOperation(coprocessors.isEmpty() ? null : new CoprocessorOperation() {
@Override
public void call(MasterObserver oserver, ObserverContext<MasterCoprocessorEnvironment> ctx)
@@ -790,7 +790,7 @@ public class MasterCoprocessorHost
}
public void postSetSplitOrMergeEnabled(final boolean newValue,
- final Admin.MasterSwitchType switchType) throws IOException {
+ final MasterSwitchType switchType) throws IOException {
execOperation(coprocessors.isEmpty() ? null : new CoprocessorOperation() {
@Override
public void call(MasterObserver oserver, ObserverContext<MasterCoprocessorEnvironment> ctx)
http://git-wip-us.apache.org/repos/asf/hbase/blob/c06a976a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
index edfb3ce..18ee4fe 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
@@ -41,7 +41,7 @@ import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.UnknownRegionException;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.MasterSwitchType;
import org.apache.hadoop.hbase.client.TableState;
import org.apache.hadoop.hbase.errorhandling.ForeignException;
import org.apache.hadoop.hbase.exceptions.DeserializationException;
@@ -1500,8 +1500,8 @@ public class MasterRpcServices extends RSRpcServices
if (!master.getSplitOrMergeTracker().lock(skipLock)) {
throw new DoNotRetryIOException("can't set splitOrMerge switch due to lock");
}
- for (MasterSwitchType masterSwitchType : request.getSwitchTypesList()) {
- Admin.MasterSwitchType switchType = convert(masterSwitchType);
+ for (MasterProtos.MasterSwitchType masterSwitchType : request.getSwitchTypesList()) {
+ MasterSwitchType switchType = convert(masterSwitchType);
boolean oldValue = master.isSplitOrMergeEnabled(switchType);
response.addPrevValue(oldValue);
boolean bypass = false;
@@ -1619,12 +1619,12 @@ public class MasterRpcServices extends RSRpcServices
return response.build();
}
- private Admin.MasterSwitchType convert(MasterSwitchType switchType) {
+ private MasterSwitchType convert(MasterProtos.MasterSwitchType switchType) {
switch (switchType) {
case SPLIT:
- return Admin.MasterSwitchType.SPLIT;
+ return MasterSwitchType.SPLIT;
case MERGE:
- return Admin.MasterSwitchType.MERGE;
+ return MasterSwitchType.MERGE;
default:
break;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/c06a976a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java
index 13b5fab..583f873 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java
@@ -28,7 +28,7 @@ import org.apache.hadoop.hbase.RegionLoad;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.client.Admin.MasterSwitchType;
+import org.apache.hadoop.hbase.client.MasterSwitchType;
import org.apache.hadoop.hbase.master.MasterRpcServices;
import org.apache.hadoop.hbase.master.MasterServices;
import org.apache.hadoop.hbase.master.normalizer.NormalizationPlan.PlanType;
http://git-wip-us.apache.org/repos/asf/hbase/blob/c06a976a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
index fb19a96..e866f29 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
@@ -60,12 +60,12 @@ import org.apache.hadoop.hbase.Tag;
import org.apache.hadoop.hbase.TagRewriteCell;
import org.apache.hadoop.hbase.TagUtil;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.Append;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Durability;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Increment;
+import org.apache.hadoop.hbase.client.MasterSwitchType;
import org.apache.hadoop.hbase.client.Mutation;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Query;
@@ -1262,14 +1262,14 @@ public class AccessController extends BaseMasterAndRegionObserver
@Override
public boolean preSetSplitOrMergeEnabled(final ObserverContext<MasterCoprocessorEnvironment> ctx,
- final boolean newValue, final Admin.MasterSwitchType switchType) throws IOException {
+ final boolean newValue, final MasterSwitchType switchType) throws IOException {
requirePermission("setSplitOrMergeEnabled", Action.ADMIN);
return false;
}
@Override
public void postSetSplitOrMergeEnabled(final ObserverContext<MasterCoprocessorEnvironment> ctx,
- final boolean newValue, final Admin.MasterSwitchType switchType) throws IOException {
+ final boolean newValue, final MasterSwitchType switchType) throws IOException {
}
@Override
http://git-wip-us.apache.org/repos/asf/hbase/blob/c06a976a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityController.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityController.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityController.java
index 1b18f42..3fa3fa3 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityController.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityController.java
@@ -51,11 +51,11 @@ import org.apache.hadoop.hbase.TagRewriteCell;
import org.apache.hadoop.hbase.TagType;
import org.apache.hadoop.hbase.TagUtil;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.Append;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Increment;
+import org.apache.hadoop.hbase.client.MasterSwitchType;
import org.apache.hadoop.hbase.client.Mutation;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
@@ -310,13 +310,13 @@ public class VisibilityController extends BaseMasterAndRegionObserver implements
@Override
public boolean preSetSplitOrMergeEnabled(final ObserverContext<MasterCoprocessorEnvironment> ctx,
- final boolean newValue, final Admin.MasterSwitchType switchType) throws IOException {
+ final boolean newValue, final MasterSwitchType switchType) throws IOException {
return false;
}
@Override
public void postSetSplitOrMergeEnabled(final ObserverContext<MasterCoprocessorEnvironment> ctx,
- final boolean newValue, final Admin.MasterSwitchType switchType) throws IOException {
+ final boolean newValue, final MasterSwitchType switchType) throws IOException {
}
@Override
http://git-wip-us.apache.org/repos/asf/hbase/blob/c06a976a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/CreateSnapshot.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/CreateSnapshot.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/CreateSnapshot.java
index 06b6017..09284e3 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/CreateSnapshot.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/CreateSnapshot.java
@@ -23,6 +23,8 @@ import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.client.SnapshotDescription;
+import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos;
import org.apache.hadoop.hbase.util.AbstractHBaseTool;
import java.util.Arrays;
@@ -66,10 +68,10 @@ public class CreateSnapshot extends AbstractHBaseTool {
admin = connection.getAdmin();
HBaseProtos.SnapshotDescription.Type type = HBaseProtos.SnapshotDescription.Type.FLUSH;
if (snapshotType != null) {
- type = HBaseProtos.SnapshotDescription.Type.valueOf(snapshotName.toUpperCase());
+ type = ProtobufUtil.createProtosSnapShotDescType(snapshotName);
}
-
- admin.snapshot(snapshotName, TableName.valueOf(tableName), type);
+ admin.snapshot(new SnapshotDescription(snapshotName, tableName,
+ ProtobufUtil.createSnapshotType(type)));
} catch (Exception e) {
return -1;
} finally {
http://git-wip-us.apache.org/repos/asf/hbase/blob/c06a976a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotInfo.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotInfo.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotInfo.java
index 0a359f8..c944fc4 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotInfo.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotInfo.java
@@ -36,6 +36,7 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.classification.InterfaceStability;
+import org.apache.hadoop.hbase.client.SnapshotDescription;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.TableName;
@@ -47,7 +48,8 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.io.HFileLink;
import org.apache.hadoop.hbase.io.WALLink;
-import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos;
import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
import org.apache.hadoop.hbase.util.FSUtils;
@@ -126,14 +128,15 @@ public final class SnapshotInfo extends Configured implements Tool {
private AtomicLong hfilesMobSize = new AtomicLong();
private AtomicLong logSize = new AtomicLong();
- private final SnapshotDescription snapshot;
+ private final HBaseProtos.SnapshotDescription snapshot;
private final TableName snapshotTable;
private final Configuration conf;
private final FileSystem fs;
- SnapshotStats(final Configuration conf, final FileSystem fs, final SnapshotDescription snapshot)
+ SnapshotStats(final Configuration conf, final FileSystem fs,
+ final SnapshotDescription snapshot)
{
- this.snapshot = snapshot;
+ this.snapshot = ProtobufUtil.createHBaseProtosSnapshotDesc(snapshot);
this.snapshotTable = TableName.valueOf(snapshot.getTable());
this.conf = conf;
this.fs = fs;
@@ -141,7 +144,9 @@ public final class SnapshotInfo extends Configured implements Tool {
/** @return the snapshot descriptor */
public SnapshotDescription getSnapshotDescription() {
- return this.snapshot;
+ return new SnapshotDescription(this.snapshot.getName(), this.snapshot.getTable(),
+ ProtobufUtil.createSnapshotType(this.snapshot.getType()), this.snapshot.getOwner(),
+ this.snapshot.getCreationTime(), this.snapshot.getVersion());
}
/** @return true if the snapshot is corrupted */
@@ -371,7 +376,8 @@ public final class SnapshotInfo extends Configured implements Tool {
return false;
}
- SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
+ HBaseProtos.SnapshotDescription snapshotDesc =
+ SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
snapshotManifest = SnapshotManifest.open(getConf(), fs, snapshotDir, snapshotDesc);
return true;
}
@@ -380,7 +386,7 @@ public final class SnapshotInfo extends Configured implements Tool {
* Dump the {@link SnapshotDescription}
*/
private void printInfo() {
- SnapshotDescription snapshotDesc = snapshotManifest.getSnapshotDescription();
+ HBaseProtos.SnapshotDescription snapshotDesc = snapshotManifest.getSnapshotDescription();
SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
System.out.println("Snapshot Info");
System.out.println("----------------------------------------");
@@ -413,9 +419,12 @@ public final class SnapshotInfo extends Configured implements Tool {
}
// Collect information about hfiles and logs in the snapshot
- final SnapshotDescription snapshotDesc = snapshotManifest.getSnapshotDescription();
+ final HBaseProtos.SnapshotDescription snapshotDesc = snapshotManifest.getSnapshotDescription();
final String table = snapshotDesc.getTable();
- final SnapshotStats stats = new SnapshotStats(this.getConf(), this.fs, snapshotDesc);
+ SnapshotDescription desc = new SnapshotDescription(snapshotDesc.getName(),
+ snapshotDesc.getTable(), ProtobufUtil.createSnapshotType(snapshotDesc.getType()),
+ snapshotDesc.getOwner(), snapshotDesc.getCreationTime(), snapshotDesc.getVersion());
+ final SnapshotStats stats = new SnapshotStats(this.getConf(), this.fs, desc);
SnapshotReferenceUtil.concurrentVisitReferencedFiles(getConf(), fs, snapshotManifest,
new SnapshotReferenceUtil.SnapshotVisitor() {
@Override
@@ -492,10 +501,11 @@ public final class SnapshotInfo extends Configured implements Tool {
*/
public static SnapshotStats getSnapshotStats(final Configuration conf,
final SnapshotDescription snapshot) throws IOException {
+ HBaseProtos.SnapshotDescription snapshotDesc = ProtobufUtil.createHBaseProtosSnapshotDesc(snapshot);
Path rootDir = FSUtils.getRootDir(conf);
FileSystem fs = FileSystem.get(rootDir.toUri(), conf);
- Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
- SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshot);
+ Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotDesc, rootDir);
+ SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc);
final SnapshotStats stats = new SnapshotStats(conf, fs, snapshot);
SnapshotReferenceUtil.concurrentVisitReferencedFiles(conf, fs, manifest,
new SnapshotReferenceUtil.SnapshotVisitor() {
@@ -525,7 +535,11 @@ public final class SnapshotInfo extends Configured implements Tool {
List<SnapshotDescription> snapshotLists =
new ArrayList<SnapshotDescription>(snapshots.length);
for (FileStatus snapshotDirStat: snapshots) {
- snapshotLists.add(SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDirStat.getPath()));
+ HBaseProtos.SnapshotDescription snapshotDesc =
+ SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDirStat.getPath());
+ snapshotLists.add(new SnapshotDescription(snapshotDesc.getName(),
+ snapshotDesc.getTable(), ProtobufUtil.createSnapshotType(snapshotDesc.getType()),
+ snapshotDesc.getOwner(), snapshotDesc.getCreationTime(), snapshotDesc.getVersion()));
}
return snapshotLists;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/c06a976a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
index 5af0634..31ae925 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
@@ -104,6 +104,7 @@ import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.HConnection;
+import org.apache.hadoop.hbase.client.MasterSwitchType;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.RegionReplicaUtil;
import org.apache.hadoop.hbase.client.Result;
@@ -691,7 +692,7 @@ public class HBaseFsck extends Configured implements Closeable {
if (shouldDisableSplitAndMerge()) {
admin.releaseSplitOrMergeLockAndRollback();
oldSplitAndMerge = admin.setSplitOrMergeEnabled(false, false, false,
- Admin.MasterSwitchType.SPLIT, Admin.MasterSwitchType.MERGE);
+ MasterSwitchType.SPLIT, MasterSwitchType.MERGE);
}
try {
http://git-wip-us.apache.org/repos/asf/hbase/blob/c06a976a/hbase-server/src/main/java/org/apache/hadoop/hbase/zookeeper/SplitOrMergeTracker.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/zookeeper/SplitOrMergeTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/zookeeper/SplitOrMergeTracker.java
index e548245..b975c43 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/zookeeper/SplitOrMergeTracker.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/zookeeper/SplitOrMergeTracker.java
@@ -24,7 +24,7 @@ import java.util.List;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Abortable;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.MasterSwitchType;
import org.apache.hadoop.hbase.exceptions.DeserializationException;
import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos;
@@ -80,7 +80,7 @@ public class SplitOrMergeTracker {
mergeStateTracker.start();
}
- public boolean isSplitOrMergeEnabled(Admin.MasterSwitchType switchType) {
+ public boolean isSplitOrMergeEnabled(MasterSwitchType switchType) {
switch (switchType) {
case SPLIT:
return splitStateTracker.isSwitchEnabled();
@@ -92,7 +92,7 @@ public class SplitOrMergeTracker {
return false;
}
- public void setSplitOrMergeEnabled(boolean enabled, Admin.MasterSwitchType switchType)
+ public void setSplitOrMergeEnabled(boolean enabled, MasterSwitchType switchType)
throws KeeperException {
switch (switchType) {
case SPLIT:
@@ -164,8 +164,8 @@ public class SplitOrMergeTracker {
}
private void saveOriginalState() throws KeeperException {
- boolean splitEnabled = isSplitOrMergeEnabled(Admin.MasterSwitchType.SPLIT);
- boolean mergeEnabled = isSplitOrMergeEnabled(Admin.MasterSwitchType.MERGE);
+ boolean splitEnabled = isSplitOrMergeEnabled(MasterSwitchType.SPLIT);
+ boolean mergeEnabled = isSplitOrMergeEnabled(MasterSwitchType.MERGE);
String splitOrMergeStates = ZKUtil.joinZNode(watcher.getSwitchLockZNode(),
SplitOrMergeTracker.STATE);
ZooKeeperProtos.SplitAndMergeState.Builder builder
http://git-wip-us.apache.org/repos/asf/hbase/blob/c06a976a/hbase-server/src/main/resources/hbase-webapps/master/snapshot.jsp
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/resources/hbase-webapps/master/snapshot.jsp b/hbase-server/src/main/resources/hbase-webapps/master/snapshot.jsp
index 5900383..6bb1aa3 100644
--- a/hbase-server/src/main/resources/hbase-webapps/master/snapshot.jsp
+++ b/hbase-server/src/main/resources/hbase-webapps/master/snapshot.jsp
@@ -21,9 +21,9 @@
import="java.util.Date"
import="org.apache.hadoop.conf.Configuration"
import="org.apache.hadoop.hbase.client.Admin"
+ import="org.apache.hadoop.hbase.client.SnapshotDescription"
import="org.apache.hadoop.hbase.master.HMaster"
import="org.apache.hadoop.hbase.snapshot.SnapshotInfo"
- import="org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription"
import="org.apache.hadoop.util.StringUtils"
import="org.apache.hadoop.hbase.TableName"
import="org.apache.hadoop.hbase.HBaseConfiguration" %>
http://git-wip-us.apache.org/repos/asf/hbase/blob/c06a976a/hbase-server/src/main/resources/hbase-webapps/master/table.jsp
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/resources/hbase-webapps/master/table.jsp b/hbase-server/src/main/resources/hbase-webapps/master/table.jsp
index 4a151e7..693a663 100644
--- a/hbase-server/src/main/resources/hbase-webapps/master/table.jsp
+++ b/hbase-server/src/main/resources/hbase-webapps/master/table.jsp
@@ -28,6 +28,7 @@
import="org.apache.hadoop.conf.Configuration"
import="org.apache.hadoop.hbase.client.HTable"
import="org.apache.hadoop.hbase.client.Admin"
+ import="org.apache.hadoop.hbase.client.CompactionState"
import="org.apache.hadoop.hbase.client.RegionLocator"
import="org.apache.hadoop.hbase.HRegionInfo"
import="org.apache.hadoop.hbase.HRegionLocation"
@@ -39,7 +40,6 @@
import="org.apache.hadoop.hbase.zookeeper.MetaTableLocator"
import="org.apache.hadoop.hbase.util.Bytes"
import="org.apache.hadoop.hbase.util.FSUtils"
- import="org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoResponse.CompactionState"
import="org.apache.hadoop.hbase.TableName"
import="org.apache.hadoop.hbase.HColumnDescriptor"
import="org.apache.hadoop.hbase.client.RegionReplicaUtil"
http://git-wip-us.apache.org/repos/asf/hbase/blob/c06a976a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
index d24118f..b35a58f 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
@@ -85,7 +85,6 @@ import org.apache.hadoop.hbase.io.hfile.CacheConfig;
import org.apache.hadoop.hbase.ipc.CoprocessorRpcChannel;
import org.apache.hadoop.hbase.master.HMaster;
import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
-import org.apache.hadoop.hbase.protobuf.generated.AdminProtos;
import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto;
import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto.MutationType;
import org.apache.hadoop.hbase.protobuf.generated.MultiRowMutationProtos.MultiRowMutationService;
@@ -278,7 +277,7 @@ public class TestFromClientSide {
@Override
public boolean evaluate() throws IOException {
return TEST_UTIL.getHBaseAdmin().getCompactionState(TABLENAME) ==
- AdminProtos.GetRegionInfoResponse.CompactionState.NONE;
+ CompactionState.NONE;
}
});
[30/50] [abbrv] hbase git commit: HBASE-15745 Refactor RPC classes to
better accept async changes
Posted by sy...@apache.org.
HBASE-15745 Refactor RPC classes to better accept async changes
Signed-off-by: stack <st...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/56358a0f
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/56358a0f
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/56358a0f
Branch: refs/heads/hbase-12439
Commit: 56358a0fd3e93b7ba8dc2a9cd8d92d961a1c24a9
Parents: 86ca09e
Author: Jurriaan Mous <ju...@jurmo.us>
Authored: Sun May 1 15:01:01 2016 +0200
Committer: stack <st...@apache.org>
Committed: Fri May 6 13:43:45 2016 -0700
----------------------------------------------------------------------
.../client/AbstractRegionServerCallable.java | 158 ++++
.../client/FastFailInterceptorContext.java | 5 +-
.../client/NoOpRetryingInterceptorContext.java | 4 +-
.../hbase/client/RegionServerCallable.java | 107 +--
.../hadoop/hbase/client/RetryingCallable.java | 34 +-
.../hbase/client/RetryingCallableBase.java | 60 ++
.../RetryingCallerInterceptorContext.java | 4 +-
.../hadoop/hbase/ipc/AsyncRpcChannel.java | 700 +----------------
.../hadoop/hbase/ipc/AsyncRpcChannelImpl.java | 743 +++++++++++++++++++
.../apache/hadoop/hbase/ipc/AsyncRpcClient.java | 8 +-
.../hbase/ipc/AsyncServerResponseHandler.java | 4 +-
.../hadoop/hbase/ipc/CoprocessorRpcChannel.java | 64 +-
.../hbase/ipc/MasterCoprocessorRpcChannel.java | 2 +-
.../hbase/ipc/RegionCoprocessorRpcChannel.java | 12 +-
.../ipc/RegionServerCoprocessorRpcChannel.java | 2 +-
.../hbase/ipc/SyncCoprocessorRpcChannel.java | 79 ++
16 files changed, 1106 insertions(+), 880 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/56358a0f/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AbstractRegionServerCallable.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AbstractRegionServerCallable.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AbstractRegionServerCallable.java
new file mode 100644
index 0000000..ee9a781
--- /dev/null
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AbstractRegionServerCallable.java
@@ -0,0 +1,158 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.client;
+
+import java.io.IOException;
+import java.net.ConnectException;
+import java.net.SocketTimeoutException;
+
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HRegionLocation;
+import org.apache.hadoop.hbase.NotServingRegionException;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.exceptions.RegionMovedException;
+import org.apache.hadoop.hbase.util.Bytes;
+
+/**
+ * Implementations call a RegionServer.
+ * Passed to a {@link RpcRetryingCaller} so we retry on fail.
+ * TODO: this class is actually tied to one region, because most of the paths make use of
+ * the regioninfo part of location when building requests. The only reason it works for
+ * multi-region requests (e.g. batch) is that they happen to not use the region parts.
+ * This could be done cleaner (e.g. having a generic parameter and 2 derived classes,
+ * RegionCallable and actual RegionServerCallable with ServerName.
+ * @param <T> the class that the ServerCallable handles
+ */
+@InterfaceAudience.Private
+abstract class AbstractRegionServerCallable<T> implements RetryingCallableBase {
+ protected final Connection connection;
+ protected final TableName tableName;
+ protected final byte[] row;
+ protected HRegionLocation location;
+
+ protected final static int MIN_WAIT_DEAD_SERVER = 10000;
+
+ /**
+ * @param connection Connection to use.
+ * @param tableName Table name to which <code>row</code> belongs.
+ * @param row The row we want in <code>tableName</code>.
+ */
+ public AbstractRegionServerCallable(Connection connection, TableName tableName, byte[] row) {
+ this.connection = connection;
+ this.tableName = tableName;
+ this.row = row;
+ }
+
+ /**
+ * @return {@link ClusterConnection} instance used by this Callable.
+ */
+ ClusterConnection getConnection() {
+ return (ClusterConnection) this.connection;
+ }
+
+ protected HRegionLocation getLocation() {
+ return this.location;
+ }
+
+ protected void setLocation(final HRegionLocation location) {
+ this.location = location;
+ }
+
+ public TableName getTableName() {
+ return this.tableName;
+ }
+
+ public byte [] getRow() {
+ return this.row;
+ }
+
+ @Override
+ public void throwable(Throwable t, boolean retrying) {
+ if (t instanceof SocketTimeoutException ||
+ t instanceof ConnectException ||
+ t instanceof RetriesExhaustedException ||
+ (location != null && getConnection().isDeadServer(location.getServerName()))) {
+ // if thrown these exceptions, we clear all the cache entries that
+ // map to that slow/dead server; otherwise, let cache miss and ask
+ // hbase:meta again to find the new location
+ if (this.location != null) {
+ getConnection().clearCaches(location.getServerName());
+ }
+ } else if (t instanceof RegionMovedException) {
+ getConnection().updateCachedLocations(tableName, row, t, location);
+ } else if (t instanceof NotServingRegionException && !retrying) {
+ // Purge cache entries for this specific region from hbase:meta cache
+ // since we don't call connect(true) when number of retries is 1.
+ getConnection().deleteCachedRegionLocation(location);
+ }
+ }
+
+ @Override
+ public String getExceptionMessageAdditionalDetail() {
+ return "row '" + Bytes.toString(row) + "' on table '" + tableName + "' at " + location;
+ }
+
+ @Override
+ public long sleep(long pause, int tries) {
+ // Tries hasn't been bumped up yet so we use "tries + 1" to get right pause time
+ long sleep = ConnectionUtils.getPauseTime(pause, tries + 1);
+ if (sleep < MIN_WAIT_DEAD_SERVER
+ && (location == null || getConnection().isDeadServer(location.getServerName()))) {
+ sleep = ConnectionUtils.addJitter(MIN_WAIT_DEAD_SERVER, 0.10f);
+ }
+ return sleep;
+ }
+
+ /**
+ * @return the HRegionInfo for the current region
+ */
+ public HRegionInfo getHRegionInfo() {
+ if (this.location == null) {
+ return null;
+ }
+ return this.location.getRegionInfo();
+ }
+
+ /**
+ * Prepare for connection to the server hosting region with row from tablename. Does lookup
+ * to find region location and hosting server.
+ * @param reload Set this to true if connection should re-find the region
+ * @throws IOException e
+ */
+ @Override
+ public void prepare(final boolean reload) throws IOException {
+ try (RegionLocator regionLocator = connection.getRegionLocator(tableName)) {
+ this.location = regionLocator.getRegionLocation(row, reload);
+ }
+ if (this.location == null) {
+ throw new IOException("Failed to find location, tableName=" + tableName +
+ ", row=" + Bytes.toString(row) + ", reload=" + reload);
+ }
+ setClientByServiceName(this.location.getServerName());
+ }
+
+ /**
+ * Set the Rpc client for Client services
+ * @param serviceName to get client for
+ * @throws IOException When client could not be created
+ */
+ abstract void setClientByServiceName(ServerName serviceName) throws IOException;
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/56358a0f/hbase-client/src/main/java/org/apache/hadoop/hbase/client/FastFailInterceptorContext.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/FastFailInterceptorContext.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/FastFailInterceptorContext.java
index 3cbdfb3..c9d2324 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/FastFailInterceptorContext.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/FastFailInterceptorContext.java
@@ -118,12 +118,11 @@ class FastFailInterceptorContext extends
tries = 0;
}
- public FastFailInterceptorContext prepare(RetryingCallable<?> callable) {
+ public FastFailInterceptorContext prepare(RetryingCallableBase callable) {
return prepare(callable, 0);
}
- public FastFailInterceptorContext prepare(RetryingCallable<?> callable,
- int tries) {
+ public FastFailInterceptorContext prepare(RetryingCallableBase callable, int tries) {
if (callable instanceof RegionServerCallable) {
RegionServerCallable<?> retryingCallable = (RegionServerCallable<?>) callable;
server = retryingCallable.getLocation().getServerName();
http://git-wip-us.apache.org/repos/asf/hbase/blob/56358a0f/hbase-client/src/main/java/org/apache/hadoop/hbase/client/NoOpRetryingInterceptorContext.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/NoOpRetryingInterceptorContext.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/NoOpRetryingInterceptorContext.java
index 1ccf43c..f8542bd 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/NoOpRetryingInterceptorContext.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/NoOpRetryingInterceptorContext.java
@@ -29,14 +29,14 @@ class NoOpRetryingInterceptorContext extends RetryingCallerInterceptorContext {
@Override
public RetryingCallerInterceptorContext prepare(
- RetryingCallable<?> callable) {
+ RetryingCallableBase callable) {
// Do Nothing
return this;
}
@Override
public RetryingCallerInterceptorContext prepare(
- RetryingCallable<?> callable, int tries) {
+ RetryingCallableBase callable, int tries) {
// Do Nothing
return this;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/56358a0f/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionServerCallable.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionServerCallable.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionServerCallable.java
index 6e0752b..d878bae 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionServerCallable.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionServerCallable.java
@@ -21,15 +21,10 @@ package org.apache.hadoop.hbase.client;
import java.io.IOException;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.HRegionLocation;
+import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.TableNotEnabledException;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ClientService;
-import org.apache.hadoop.hbase.util.Bytes;
/**
* Implementations call a RegionServer and implement {@link #call(int)}.
@@ -42,16 +37,10 @@ import org.apache.hadoop.hbase.util.Bytes;
* @param <T> the class that the ServerCallable handles
*/
@InterfaceAudience.Private
-public abstract class RegionServerCallable<T> implements RetryingCallable<T> {
- // Public because used outside of this package over in ipc.
- private static final Log LOG = LogFactory.getLog(RegionServerCallable.class);
- protected final Connection connection;
- protected final TableName tableName;
- protected final byte[] row;
- protected HRegionLocation location;
- private ClientService.BlockingInterface stub;
+public abstract class RegionServerCallable<T> extends AbstractRegionServerCallable<T> implements
+ RetryingCallable<T> {
- protected final static int MIN_WAIT_DEAD_SERVER = 10000;
+ private ClientService.BlockingInterface stub;
/**
* @param connection Connection to use.
@@ -59,97 +48,25 @@ public abstract class RegionServerCallable<T> implements RetryingCallable<T> {
* @param row The row we want in <code>tableName</code>.
*/
public RegionServerCallable(Connection connection, TableName tableName, byte [] row) {
- this.connection = connection;
- this.tableName = tableName;
- this.row = row;
+ super(connection, tableName, row);
}
- /**
- * Prepare for connection to the server hosting region with row from tablename. Does lookup
- * to find region location and hosting server.
- * @param reload Set to true to re-check the table state
- * @throws IOException e
- */
- @Override
- public void prepare(final boolean reload) throws IOException {
- // check table state if this is a retry
- if (reload &&
- !tableName.equals(TableName.META_TABLE_NAME) &&
- getConnection().isTableDisabled(tableName)) {
- throw new TableNotEnabledException(tableName.getNameAsString() + " is disabled.");
- }
- try (RegionLocator regionLocator = connection.getRegionLocator(tableName)) {
- this.location = regionLocator.getRegionLocation(row);
- }
- if (this.location == null) {
- throw new IOException("Failed to find location, tableName=" + tableName +
- ", row=" + Bytes.toString(row) + ", reload=" + reload);
- }
- setStub(getConnection().getClient(this.location.getServerName()));
+ void setClientByServiceName(ServerName service) throws IOException {
+ this.setStub(getConnection().getClient(service));
}
/**
- * @return {@link HConnection} instance used by this Callable.
+ * @return Client Rpc protobuf communication stub
*/
- HConnection getConnection() {
- return (HConnection) this.connection;
- }
-
protected ClientService.BlockingInterface getStub() {
return this.stub;
}
- void setStub(final ClientService.BlockingInterface stub) {
- this.stub = stub;
- }
-
- protected HRegionLocation getLocation() {
- return this.location;
- }
-
- protected void setLocation(final HRegionLocation location) {
- this.location = location;
- }
-
- public TableName getTableName() {
- return this.tableName;
- }
-
- public byte [] getRow() {
- return this.row;
- }
-
- @Override
- public void throwable(Throwable t, boolean retrying) {
- if (location != null) {
- getConnection().updateCachedLocations(tableName, location.getRegionInfo().getRegionName(),
- row, t, location.getServerName());
- }
- }
-
- @Override
- public String getExceptionMessageAdditionalDetail() {
- return "row '" + Bytes.toString(row) + "' on table '" + tableName + "' at " + location;
- }
-
- @Override
- public long sleep(long pause, int tries) {
- // Tries hasn't been bumped up yet so we use "tries + 1" to get right pause time
- long sleep = ConnectionUtils.getPauseTime(pause, tries + 1);
- if (sleep < MIN_WAIT_DEAD_SERVER
- && (location == null || getConnection().isDeadServer(location.getServerName()))) {
- sleep = ConnectionUtils.addJitter(MIN_WAIT_DEAD_SERVER, 0.10f);
- }
- return sleep;
- }
-
/**
- * @return the HRegionInfo for the current region
+ * Set the client protobuf communication stub
+ * @param stub to set
*/
- public HRegionInfo getHRegionInfo() {
- if (this.location == null) {
- return null;
- }
- return this.location.getRegionInfo();
+ void setStub(final ClientService.BlockingInterface stub) {
+ this.stub = stub;
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/56358a0f/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RetryingCallable.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RetryingCallable.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RetryingCallable.java
index 7a32175..2377a0d 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RetryingCallable.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RetryingCallable.java
@@ -19,8 +19,6 @@
package org.apache.hadoop.hbase.client;
-import java.io.IOException;
-
import org.apache.hadoop.hbase.classification.InterfaceAudience;
/**
@@ -29,23 +27,7 @@ import org.apache.hadoop.hbase.classification.InterfaceAudience;
* @param <T> result class from executing <tt>this</tt>
*/
@InterfaceAudience.Private
-public interface RetryingCallable<T> {
- /**
- * Prepare by setting up any connections to servers, etc., ahead of {@link #call(int)} invocation.
- * @param reload Set this to true if need to requery locations
- * @throws IOException e
- */
- void prepare(final boolean reload) throws IOException;
-
- /**
- * Called when {@link #call(int)} throws an exception and we are going to retry; take action to
- * make it so we succeed on next call (clear caches, do relookup of locations, etc.).
- * @param t
- * @param retrying True if we are in retrying mode (we are not in retrying mode when max
- * retries == 1; we ARE in retrying mode if retries > 1 even when we are the last attempt)
- */
- void throwable(final Throwable t, boolean retrying);
-
+public interface RetryingCallable<T> extends RetryingCallableBase {
/**
* Computes a result, or throws an exception if unable to do so.
*
@@ -54,18 +36,4 @@ public interface RetryingCallable<T> {
* @throws Exception if unable to compute a result
*/
T call(int callTimeout) throws Exception;
-
- /**
- * @return Some details from the implementation that we would like to add to a terminating
- * exception; i.e. a fatal exception is being thrown ending retries and we might like to add
- * more implementation-specific detail on to the exception being thrown.
- */
- String getExceptionMessageAdditionalDetail();
-
- /**
- * @param pause
- * @param tries
- * @return Suggestion on how much to sleep between retries
- */
- long sleep(final long pause, final int tries);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/56358a0f/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RetryingCallableBase.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RetryingCallableBase.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RetryingCallableBase.java
new file mode 100644
index 0000000..483f6c2
--- /dev/null
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RetryingCallableBase.java
@@ -0,0 +1,60 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.client;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+
+/**
+ * All generic methods for a Callable that can be retried. It is extended with Sync and
+ * Async versions.
+ */
+@InterfaceAudience.Private
+public interface RetryingCallableBase {
+ /**
+ * Prepare by setting up any connections to servers, etc., ahead of call invocation.
+ * @param reload Set this to true if need to requery locations
+ * @throws IOException e
+ */
+ void prepare(final boolean reload) throws IOException;
+
+ /**
+ * Called when call throws an exception and we are going to retry; take action to
+ * make it so we succeed on next call (clear caches, do relookup of locations, etc.).
+ * @param t throwable which was thrown
+ * @param retrying True if we are in retrying mode (we are not in retrying mode when max
+ * retries == 1; we ARE in retrying mode if retries > 1 even when we are the
+ * last attempt)
+ */
+ void throwable(final Throwable t, boolean retrying);
+
+ /**
+ * @return Some details from the implementation that we would like to add to a terminating
+ * exception; i.e. a fatal exception is being thrown ending retries and we might like to
+ * add more implementation-specific detail on to the exception being thrown.
+ */
+ String getExceptionMessageAdditionalDetail();
+
+ /**
+ * @param pause time to pause
+ * @param tries amount of tries until till sleep
+ * @return Suggestion on how much to sleep between retries
+ */
+ long sleep(final long pause, final int tries);
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/56358a0f/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RetryingCallerInterceptorContext.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RetryingCallerInterceptorContext.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RetryingCallerInterceptorContext.java
index a9f414f..b0ba9f5 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RetryingCallerInterceptorContext.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RetryingCallerInterceptorContext.java
@@ -50,7 +50,7 @@ abstract class RetryingCallerInterceptorContext {
* used for use in the current retrying call
*/
public abstract RetryingCallerInterceptorContext prepare(
- RetryingCallable<?> callable);
+ RetryingCallableBase callable);
/**
* Telescopic extension that takes which of the many retries we are currently
@@ -65,5 +65,5 @@ abstract class RetryingCallerInterceptorContext {
* retrying call
*/
public abstract RetryingCallerInterceptorContext prepare(
- RetryingCallable<?> callable, int tries);
+ RetryingCallableBase callable, int tries);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/56358a0f/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncRpcChannel.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncRpcChannel.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncRpcChannel.java
index 53eb824..60dc5e4 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncRpcChannel.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncRpcChannel.java
@@ -17,275 +17,22 @@
*/
package org.apache.hadoop.hbase.ipc;
-import java.io.IOException;
-import java.net.ConnectException;
-import java.net.InetSocketAddress;
-import java.nio.ByteBuffer;
-import java.security.PrivilegedExceptionAction;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-import java.util.concurrent.TimeUnit;
-
-import javax.security.sasl.SaslException;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.client.MetricsConnection;
-import org.apache.hadoop.hbase.exceptions.ConnectionClosingException;
-import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
-import org.apache.hadoop.hbase.protobuf.generated.AuthenticationProtos;
-import org.apache.hadoop.hbase.protobuf.generated.AuthenticationProtos.TokenIdentifier.Kind;
-import org.apache.hadoop.hbase.protobuf.generated.RPCProtos;
-import org.apache.hadoop.hbase.protobuf.generated.TracingProtos;
-import org.apache.hadoop.hbase.security.AuthMethod;
-import org.apache.hadoop.hbase.security.SaslClientHandler;
-import org.apache.hadoop.hbase.security.SaslUtil;
-import org.apache.hadoop.hbase.security.SecurityInfo;
-import org.apache.hadoop.hbase.security.User;
-import org.apache.hadoop.hbase.security.token.AuthenticationTokenSelector;
-import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.ipc.RemoteException;
-import org.apache.hadoop.security.SecurityUtil;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.security.token.TokenIdentifier;
-import org.apache.hadoop.security.token.TokenSelector;
-import org.apache.htrace.Span;
-import org.apache.htrace.Trace;
-
import com.google.protobuf.Descriptors;
import com.google.protobuf.Message;
-import com.google.protobuf.RpcCallback;
-import io.netty.bootstrap.Bootstrap;
-import io.netty.buffer.ByteBuf;
-import io.netty.buffer.ByteBufOutputStream;
-import io.netty.channel.Channel;
-import io.netty.channel.ChannelFuture;
-import io.netty.channel.ChannelFutureListener;
-import io.netty.handler.codec.LengthFieldBasedFrameDecoder;
-import io.netty.util.Timeout;
-import io.netty.util.TimerTask;
-import io.netty.util.concurrent.GenericFutureListener;
+import io.netty.util.concurrent.EventExecutor;
import io.netty.util.concurrent.Promise;
+import java.net.InetSocketAddress;
+
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.client.MetricsConnection;
+
/**
- * Netty RPC channel
+ * Interface for Async Rpc Channels
*/
@InterfaceAudience.Private
-public class AsyncRpcChannel {
- private static final Log LOG = LogFactory.getLog(AsyncRpcChannel.class.getName());
-
- private static final int MAX_SASL_RETRIES = 5;
-
- protected final static Map<Kind, TokenSelector<? extends TokenIdentifier>> TOKEN_HANDDLERS
- = new HashMap<>();
-
- static {
- TOKEN_HANDDLERS.put(AuthenticationProtos.TokenIdentifier.Kind.HBASE_AUTH_TOKEN,
- new AuthenticationTokenSelector());
- }
-
- final AsyncRpcClient client;
-
- // Contains the channel to work with.
- // Only exists when connected
- private Channel channel;
-
- String name;
- final User ticket;
- final String serviceName;
- final InetSocketAddress address;
-
- private int failureCounter = 0;
-
- boolean useSasl;
- AuthMethod authMethod;
- private int reloginMaxBackoff;
- private Token<? extends TokenIdentifier> token;
- private String serverPrincipal;
-
- // NOTE: closed and connected flags below are only changed when a lock on pendingCalls
- private final Map<Integer, AsyncCall> pendingCalls = new HashMap<Integer, AsyncCall>();
- private boolean connected = false;
- private boolean closed = false;
-
- private Timeout cleanupTimer;
-
- private final TimerTask timeoutTask = new TimerTask() {
- @Override
- public void run(Timeout timeout) throws Exception {
- cleanupCalls();
- }
- };
-
- /**
- * Constructor for netty RPC channel
- * @param bootstrap to construct channel on
- * @param client to connect with
- * @param ticket of user which uses connection
- * @param serviceName name of service to connect to
- * @param address to connect to
- */
- public AsyncRpcChannel(Bootstrap bootstrap, final AsyncRpcClient client, User ticket,
- String serviceName, InetSocketAddress address) {
- this.client = client;
-
- this.ticket = ticket;
- this.serviceName = serviceName;
- this.address = address;
-
- this.channel = connect(bootstrap).channel();
-
- name = ("IPC Client (" + channel.hashCode() + ") to " + address.toString()
- + ((ticket == null) ? " from unknown user" : (" from " + ticket.getName())));
- }
-
- /**
- * Connect to channel
- * @param bootstrap to connect to
- * @return future of connection
- */
- private ChannelFuture connect(final Bootstrap bootstrap) {
- return bootstrap.remoteAddress(address).connect()
- .addListener(new GenericFutureListener<ChannelFuture>() {
- @Override
- public void operationComplete(final ChannelFuture f) throws Exception {
- if (!f.isSuccess()) {
- retryOrClose(bootstrap, failureCounter++, client.failureSleep, f.cause());
- return;
- }
- channel = f.channel();
-
- setupAuthorization();
-
- ByteBuf b = channel.alloc().directBuffer(6);
- createPreamble(b, authMethod);
- channel.writeAndFlush(b).addListener(ChannelFutureListener.CLOSE_ON_FAILURE);
- if (useSasl) {
- UserGroupInformation ticket = AsyncRpcChannel.this.ticket.getUGI();
- if (authMethod == AuthMethod.KERBEROS) {
- if (ticket != null && ticket.getRealUser() != null) {
- ticket = ticket.getRealUser();
- }
- }
- SaslClientHandler saslHandler;
- if (ticket == null) {
- throw new FatalConnectionException("ticket/user is null");
- }
- final UserGroupInformation realTicket = ticket;
- saslHandler = ticket.doAs(new PrivilegedExceptionAction<SaslClientHandler>() {
- @Override
- public SaslClientHandler run() throws IOException {
- return getSaslHandler(realTicket, bootstrap);
- }
- });
- if (saslHandler != null) {
- // Sasl connect is successful. Let's set up Sasl channel handler
- channel.pipeline().addFirst(saslHandler);
- } else {
- // fall back to simple auth because server told us so.
- authMethod = AuthMethod.SIMPLE;
- useSasl = false;
- }
- } else {
- startHBaseConnection(f.channel());
- }
- }
- });
- }
-
- /**
- * Start HBase connection
- * @param ch channel to start connection on
- */
- private void startHBaseConnection(Channel ch) {
- ch.pipeline().addLast("frameDecoder",
- new LengthFieldBasedFrameDecoder(Integer.MAX_VALUE, 0, 4, 0, 4));
- ch.pipeline().addLast(new AsyncServerResponseHandler(this));
- try {
- writeChannelHeader(ch).addListener(new GenericFutureListener<ChannelFuture>() {
- @Override
- public void operationComplete(ChannelFuture future) throws Exception {
- if (!future.isSuccess()) {
- close(future.cause());
- return;
- }
- List<AsyncCall> callsToWrite;
- synchronized (pendingCalls) {
- connected = true;
- callsToWrite = new ArrayList<AsyncCall>(pendingCalls.values());
- }
- for (AsyncCall call : callsToWrite) {
- writeRequest(call);
- }
- }
- });
- } catch (IOException e) {
- close(e);
- }
- }
-
- /**
- * Get SASL handler
- * @param bootstrap to reconnect to
- * @return new SASL handler
- * @throws java.io.IOException if handler failed to create
- */
- private SaslClientHandler getSaslHandler(final UserGroupInformation realTicket,
- final Bootstrap bootstrap) throws IOException {
- return new SaslClientHandler(realTicket, authMethod, token, serverPrincipal,
- client.fallbackAllowed,
- client.conf.get("hbase.rpc.protection",
- SaslUtil.QualityOfProtection.AUTHENTICATION.name().toLowerCase()),
- new SaslClientHandler.SaslExceptionHandler() {
- @Override
- public void handle(int retryCount, Random random, Throwable cause) {
- try {
- // Handle Sasl failure. Try to potentially get new credentials
- handleSaslConnectionFailure(retryCount, cause, realTicket);
-
- retryOrClose(bootstrap, failureCounter++, random.nextInt(reloginMaxBackoff) + 1,
- cause);
- } catch (IOException | InterruptedException e) {
- close(e);
- }
- }
- }, new SaslClientHandler.SaslSuccessfulConnectHandler() {
- @Override
- public void onSuccess(Channel channel) {
- startHBaseConnection(channel);
- }
- });
- }
-
- /**
- * Retry to connect or close
- * @param bootstrap to connect with
- * @param failureCount failure count
- * @param e exception of fail
- */
- private void retryOrClose(final Bootstrap bootstrap, int failureCount, long timeout,
- Throwable e) {
- if (failureCount < client.maxRetries) {
- client.newTimeout(new TimerTask() {
- @Override
- public void run(Timeout timeout) throws Exception {
- connect(bootstrap);
- }
- }, timeout, TimeUnit.MILLISECONDS);
- } else {
- client.failedServers.addToFailedServers(address);
- close(e);
- }
- }
+public interface AsyncRpcChannel {
/**
* Calls method on channel
@@ -294,439 +41,32 @@ public class AsyncRpcChannel {
* @param request to send
* @param responsePrototype to construct response with
*/
- public Promise<Message> callMethod(final Descriptors.MethodDescriptor method,
+ Promise<Message> callMethod(final Descriptors.MethodDescriptor method,
final PayloadCarryingRpcController controller, final Message request,
- final Message responsePrototype, MetricsConnection.CallStats callStats) {
- final AsyncCall call = new AsyncCall(channel.eventLoop(), client.callIdCnt.getAndIncrement(),
- method, request, controller, responsePrototype, callStats);
- controller.notifyOnCancel(new RpcCallback<Object>() {
- @Override
- public void run(Object parameter) {
- // TODO: do not need to call AsyncCall.setFailed?
- synchronized (pendingCalls) {
- pendingCalls.remove(call.id);
- }
- }
- });
- // TODO: this should be handled by PayloadCarryingRpcController.
- if (controller.isCanceled()) {
- // To finish if the call was cancelled before we set the notification (race condition)
- call.cancel(true);
- return call;
- }
-
- synchronized (pendingCalls) {
- if (closed) {
- Promise<Message> promise = channel.eventLoop().newPromise();
- promise.setFailure(new ConnectException());
- return promise;
- }
- pendingCalls.put(call.id, call);
- // Add timeout for cleanup if none is present
- if (cleanupTimer == null && call.getRpcTimeout() > 0) {
- cleanupTimer = client.newTimeout(timeoutTask, call.getRpcTimeout(), TimeUnit.MILLISECONDS);
- }
- if (!connected) {
- return call;
- }
- }
- writeRequest(call);
- return call;
- }
-
- AsyncCall removePendingCall(int id) {
- synchronized (pendingCalls) {
- return pendingCalls.remove(id);
- }
- }
+ final Message responsePrototype, MetricsConnection.CallStats callStats);
/**
- * Write the channel header
- * @param channel to write to
- * @return future of write
- * @throws java.io.IOException on failure to write
+ * Get the EventLoop on which this channel operated
+ * @return EventLoop
*/
- private ChannelFuture writeChannelHeader(Channel channel) throws IOException {
- RPCProtos.ConnectionHeader.Builder headerBuilder = RPCProtos.ConnectionHeader.newBuilder()
- .setServiceName(serviceName);
-
- RPCProtos.UserInformation userInfoPB = buildUserInfo(ticket.getUGI(), authMethod);
- if (userInfoPB != null) {
- headerBuilder.setUserInfo(userInfoPB);
- }
-
- if (client.codec != null) {
- headerBuilder.setCellBlockCodecClass(client.codec.getClass().getCanonicalName());
- }
- if (client.compressor != null) {
- headerBuilder.setCellBlockCompressorClass(client.compressor.getClass().getCanonicalName());
- }
-
- headerBuilder.setVersionInfo(ProtobufUtil.getVersionInfo());
- RPCProtos.ConnectionHeader header = headerBuilder.build();
-
- int totalSize = IPCUtil.getTotalSizeWhenWrittenDelimited(header);
-
- ByteBuf b = channel.alloc().directBuffer(totalSize);
-
- b.writeInt(header.getSerializedSize());
- b.writeBytes(header.toByteArray());
-
- return channel.writeAndFlush(b);
- }
-
- /**
- * Write request to channel
- * @param call to write
- */
- private void writeRequest(final AsyncCall call) {
- try {
- final RPCProtos.RequestHeader.Builder requestHeaderBuilder = RPCProtos.RequestHeader
- .newBuilder();
- requestHeaderBuilder.setCallId(call.id).setMethodName(call.method.getName())
- .setRequestParam(call.param != null);
-
- if (Trace.isTracing()) {
- Span s = Trace.currentSpan();
- requestHeaderBuilder.setTraceInfo(TracingProtos.RPCTInfo.newBuilder()
- .setParentId(s.getSpanId()).setTraceId(s.getTraceId()));
- }
-
- ByteBuffer cellBlock = client.buildCellBlock(call.controller.cellScanner());
- if (cellBlock != null) {
- final RPCProtos.CellBlockMeta.Builder cellBlockBuilder = RPCProtos.CellBlockMeta
- .newBuilder();
- cellBlockBuilder.setLength(cellBlock.limit());
- requestHeaderBuilder.setCellBlockMeta(cellBlockBuilder.build());
- }
- // Only pass priority if there one. Let zero be same as no priority.
- if (call.controller.getPriority() != PayloadCarryingRpcController.PRIORITY_UNSET) {
- requestHeaderBuilder.setPriority(call.controller.getPriority());
- }
-
- RPCProtos.RequestHeader rh = requestHeaderBuilder.build();
-
- int totalSize = IPCUtil.getTotalSizeWhenWrittenDelimited(rh, call.param);
- if (cellBlock != null) {
- totalSize += cellBlock.remaining();
- }
-
- ByteBuf b = channel.alloc().directBuffer(4 + totalSize);
- try (ByteBufOutputStream out = new ByteBufOutputStream(b)) {
- call.callStats.setRequestSizeBytes(IPCUtil.write(out, rh, call.param, cellBlock));
- }
-
- channel.writeAndFlush(b).addListener(new CallWriteListener(this, call.id));
- } catch (IOException e) {
- close(e);
- }
- }
-
- /**
- * Set up server authorization
- * @throws java.io.IOException if auth setup failed
- */
- private void setupAuthorization() throws IOException {
- SecurityInfo securityInfo = SecurityInfo.getInfo(serviceName);
- this.useSasl = client.userProvider.isHBaseSecurityEnabled();
-
- this.token = null;
- if (useSasl && securityInfo != null) {
- AuthenticationProtos.TokenIdentifier.Kind tokenKind = securityInfo.getTokenKind();
- if (tokenKind != null) {
- TokenSelector<? extends TokenIdentifier> tokenSelector = TOKEN_HANDDLERS.get(tokenKind);
- if (tokenSelector != null) {
- token = tokenSelector.selectToken(new Text(client.clusterId),
- ticket.getUGI().getTokens());
- } else if (LOG.isDebugEnabled()) {
- LOG.debug("No token selector found for type " + tokenKind);
- }
- }
- String serverKey = securityInfo.getServerPrincipal();
- if (serverKey == null) {
- throw new IOException("Can't obtain server Kerberos config key from SecurityInfo");
- }
- this.serverPrincipal = SecurityUtil.getServerPrincipal(client.conf.get(serverKey),
- address.getAddress().getCanonicalHostName().toLowerCase());
- if (LOG.isDebugEnabled()) {
- LOG.debug("RPC Server Kerberos principal name for service=" + serviceName + " is "
- + serverPrincipal);
- }
- }
-
- if (!useSasl) {
- authMethod = AuthMethod.SIMPLE;
- } else if (token != null) {
- authMethod = AuthMethod.DIGEST;
- } else {
- authMethod = AuthMethod.KERBEROS;
- }
-
- if (LOG.isDebugEnabled()) {
- LOG.debug(
- "Use " + authMethod + " authentication for service " + serviceName + ", sasl=" + useSasl);
- }
- reloginMaxBackoff = client.conf.getInt("hbase.security.relogin.maxbackoff", 5000);
- }
-
- /**
- * Build the user information
- * @param ugi User Group Information
- * @param authMethod Authorization method
- * @return UserInformation protobuf
- */
- private RPCProtos.UserInformation buildUserInfo(UserGroupInformation ugi, AuthMethod authMethod) {
- if (ugi == null || authMethod == AuthMethod.DIGEST) {
- // Don't send user for token auth
- return null;
- }
- RPCProtos.UserInformation.Builder userInfoPB = RPCProtos.UserInformation.newBuilder();
- if (authMethod == AuthMethod.KERBEROS) {
- // Send effective user for Kerberos auth
- userInfoPB.setEffectiveUser(ugi.getUserName());
- } else if (authMethod == AuthMethod.SIMPLE) {
- // Send both effective user and real user for simple auth
- userInfoPB.setEffectiveUser(ugi.getUserName());
- if (ugi.getRealUser() != null) {
- userInfoPB.setRealUser(ugi.getRealUser().getUserName());
- }
- }
- return userInfoPB.build();
- }
-
- /**
- * Create connection preamble
- * @param byteBuf to write to
- * @param authMethod to write
- */
- private void createPreamble(ByteBuf byteBuf, AuthMethod authMethod) {
- byteBuf.writeBytes(HConstants.RPC_HEADER);
- byteBuf.writeByte(HConstants.RPC_CURRENT_VERSION);
- byteBuf.writeByte(authMethod.code);
- }
-
- private void close0(Throwable e) {
- List<AsyncCall> toCleanup;
- synchronized (pendingCalls) {
- if (closed) {
- return;
- }
- closed = true;
- toCleanup = new ArrayList<AsyncCall>(pendingCalls.values());
- pendingCalls.clear();
- }
- IOException closeException = null;
- if (e != null) {
- if (e instanceof IOException) {
- closeException = (IOException) e;
- } else {
- closeException = new IOException(e);
- }
- }
- // log the info
- if (LOG.isDebugEnabled() && closeException != null) {
- LOG.debug(name + ": closing ipc connection to " + address, closeException);
- }
- if (cleanupTimer != null) {
- cleanupTimer.cancel();
- cleanupTimer = null;
- }
- for (AsyncCall call : toCleanup) {
- call.setFailed(closeException != null ? closeException
- : new ConnectionClosingException(
- "Call id=" + call.id + " on server " + address + " aborted: connection is closing"));
- }
- channel.disconnect().addListener(ChannelFutureListener.CLOSE);
- if (LOG.isDebugEnabled()) {
- LOG.debug(name + ": closed");
- }
- }
+ EventExecutor getEventExecutor();
/**
* Close connection
- * @param e exception on close
+ * @param cause of closure.
*/
- public void close(final Throwable e) {
- client.removeConnection(this);
-
- // Move closing from the requesting thread to the channel thread
- if (channel.eventLoop().inEventLoop()) {
- close0(e);
- } else {
- channel.eventLoop().execute(new Runnable() {
- @Override
- public void run() {
- close0(e);
- }
- });
- }
- }
-
- /**
- * Clean up calls.
- */
- private void cleanupCalls() {
- List<AsyncCall> toCleanup = new ArrayList<AsyncCall>();
- long currentTime = EnvironmentEdgeManager.currentTime();
- long nextCleanupTaskDelay = -1L;
- synchronized (pendingCalls) {
- for (Iterator<AsyncCall> iter = pendingCalls.values().iterator(); iter.hasNext();) {
- AsyncCall call = iter.next();
- long timeout = call.getRpcTimeout();
- if (timeout > 0) {
- if (currentTime - call.getStartTime() >= timeout) {
- iter.remove();
- toCleanup.add(call);
- } else {
- if (nextCleanupTaskDelay < 0 || timeout < nextCleanupTaskDelay) {
- nextCleanupTaskDelay = timeout;
- }
- }
- }
- }
- if (nextCleanupTaskDelay > 0) {
- cleanupTimer = client.newTimeout(timeoutTask, nextCleanupTaskDelay, TimeUnit.MILLISECONDS);
- } else {
- cleanupTimer = null;
- }
- }
- for (AsyncCall call : toCleanup) {
- call.setFailed(new CallTimeoutException("Call id=" + call.id + ", waitTime="
- + (currentTime - call.getStartTime()) + ", rpcTimeout=" + call.getRpcTimeout()));
- }
- }
+ void close(Throwable cause);
/**
* Check if the connection is alive
+ *
* @return true if alive
*/
- public boolean isAlive() {
- return channel.isOpen();
- }
-
- /**
- * Check if user should authenticate over Kerberos
- * @return true if should be authenticated over Kerberos
- * @throws java.io.IOException on failure of check
- */
- private synchronized boolean shouldAuthenticateOverKrb() throws IOException {
- UserGroupInformation loginUser = UserGroupInformation.getLoginUser();
- UserGroupInformation currentUser = UserGroupInformation.getCurrentUser();
- UserGroupInformation realUser = currentUser.getRealUser();
- return authMethod == AuthMethod.KERBEROS && loginUser != null &&
- // Make sure user logged in using Kerberos either keytab or TGT
- loginUser.hasKerberosCredentials() &&
- // relogin only in case it is the login user (e.g. JT)
- // or superuser (like oozie).
- (loginUser.equals(currentUser) || loginUser.equals(realUser));
- }
+ boolean isAlive();
/**
- * If multiple clients with the same principal try to connect to the same server at the same time,
- * the server assumes a replay attack is in progress. This is a feature of kerberos. In order to
- * work around this, what is done is that the client backs off randomly and tries to initiate the
- * connection again. The other problem is to do with ticket expiry. To handle that, a relogin is
- * attempted.
- * <p>
- * The retry logic is governed by the {@link #shouldAuthenticateOverKrb} method. In case when the
- * user doesn't have valid credentials, we don't need to retry (from cache or ticket). In such
- * cases, it is prudent to throw a runtime exception when we receive a SaslException from the
- * underlying authentication implementation, so there is no retry from other high level (for eg,
- * HCM or HBaseAdmin).
- * </p>
- * @param currRetries retry count
- * @param ex exception describing fail
- * @param user which is trying to connect
- * @throws java.io.IOException if IO fail
- * @throws InterruptedException if thread is interrupted
+ * Get the address on which this channel operates
+ * @return InetSocketAddress
*/
- private void handleSaslConnectionFailure(final int currRetries, final Throwable ex,
- final UserGroupInformation user) throws IOException, InterruptedException {
- user.doAs(new PrivilegedExceptionAction<Void>() {
- @Override
- public Void run() throws IOException, InterruptedException {
- if (shouldAuthenticateOverKrb()) {
- if (currRetries < MAX_SASL_RETRIES) {
- LOG.debug("Exception encountered while connecting to the server : " + ex);
- // try re-login
- if (UserGroupInformation.isLoginKeytabBased()) {
- UserGroupInformation.getLoginUser().reloginFromKeytab();
- } else {
- UserGroupInformation.getLoginUser().reloginFromTicketCache();
- }
-
- // Should reconnect
- return null;
- } else {
- String msg = "Couldn't setup connection for "
- + UserGroupInformation.getLoginUser().getUserName() + " to " + serverPrincipal;
- LOG.warn(msg);
- throw (IOException) new IOException(msg).initCause(ex);
- }
- } else {
- LOG.warn("Exception encountered while connecting to " + "the server : " + ex);
- }
- if (ex instanceof RemoteException) {
- throw (RemoteException) ex;
- }
- if (ex instanceof SaslException) {
- String msg = "SASL authentication failed."
- + " The most likely cause is missing or invalid credentials." + " Consider 'kinit'.";
- LOG.fatal(msg, ex);
- throw new RuntimeException(msg, ex);
- }
- throw new IOException(ex);
- }
- });
- }
-
- public int getConnectionHashCode() {
- return ConnectionId.hashCode(ticket, serviceName, address);
- }
-
- @Override
- public int hashCode() {
- return getConnectionHashCode();
- }
-
- @Override
- public boolean equals(Object obj) {
- if (obj instanceof AsyncRpcChannel) {
- AsyncRpcChannel channel = (AsyncRpcChannel) obj;
- return channel.hashCode() == obj.hashCode();
- }
- return false;
- }
-
- @Override
- public String toString() {
- return this.address.toString() + "/" + this.serviceName + "/" + this.ticket;
- }
-
- /**
- * Listens to call writes and fails if write failed
- */
- private static final class CallWriteListener implements ChannelFutureListener {
- private final AsyncRpcChannel rpcChannel;
- private final int id;
-
- public CallWriteListener(AsyncRpcChannel asyncRpcChannel, int id) {
- this.rpcChannel = asyncRpcChannel;
- this.id = id;
- }
-
- @Override
- public void operationComplete(ChannelFuture future) throws Exception {
- if (!future.isSuccess()) {
- AsyncCall call = rpcChannel.removePendingCall(id);
- if (call != null) {
- if (future.cause() instanceof IOException) {
- call.setFailed((IOException) future.cause());
- } else {
- call.setFailed(new IOException(future.cause()));
- }
- }
- }
- }
- }
+ InetSocketAddress getAddress();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/56358a0f/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncRpcChannelImpl.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncRpcChannelImpl.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncRpcChannelImpl.java
new file mode 100644
index 0000000..7cc9e78
--- /dev/null
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncRpcChannelImpl.java
@@ -0,0 +1,743 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.ipc;
+
+import com.google.protobuf.Descriptors;
+import com.google.protobuf.Message;
+import com.google.protobuf.RpcCallback;
+
+import io.netty.bootstrap.Bootstrap;
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.ByteBufOutputStream;
+import io.netty.channel.Channel;
+import io.netty.channel.ChannelFuture;
+import io.netty.channel.ChannelFutureListener;
+import io.netty.channel.EventLoop;
+import io.netty.handler.codec.LengthFieldBasedFrameDecoder;
+import io.netty.util.Timeout;
+import io.netty.util.TimerTask;
+import io.netty.util.concurrent.GenericFutureListener;
+import io.netty.util.concurrent.Promise;
+
+import java.io.IOException;
+import java.net.ConnectException;
+import java.net.InetSocketAddress;
+import java.nio.ByteBuffer;
+import java.security.PrivilegedExceptionAction;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+
+import javax.security.sasl.SaslException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.client.MetricsConnection;
+import org.apache.hadoop.hbase.exceptions.ConnectionClosingException;
+import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
+import org.apache.hadoop.hbase.protobuf.generated.AuthenticationProtos;
+import org.apache.hadoop.hbase.protobuf.generated.AuthenticationProtos.TokenIdentifier.Kind;
+import org.apache.hadoop.hbase.protobuf.generated.RPCProtos;
+import org.apache.hadoop.hbase.protobuf.generated.TracingProtos;
+import org.apache.hadoop.hbase.security.AuthMethod;
+import org.apache.hadoop.hbase.security.SaslClientHandler;
+import org.apache.hadoop.hbase.security.SaslUtil;
+import org.apache.hadoop.hbase.security.SecurityInfo;
+import org.apache.hadoop.hbase.security.User;
+import org.apache.hadoop.hbase.security.token.AuthenticationTokenSelector;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.ipc.RemoteException;
+import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
+import org.apache.hadoop.security.token.TokenSelector;
+import org.apache.htrace.Span;
+import org.apache.htrace.Trace;
+
+/**
+ * Netty RPC channel
+ */
+@InterfaceAudience.Private
+public class AsyncRpcChannelImpl implements AsyncRpcChannel {
+ private static final Log LOG = LogFactory.getLog(AsyncRpcChannelImpl.class.getName());
+
+ private static final int MAX_SASL_RETRIES = 5;
+
+ protected final static Map<Kind, TokenSelector<? extends TokenIdentifier>> TOKEN_HANDDLERS
+ = new HashMap<>();
+
+ static {
+ TOKEN_HANDDLERS.put(AuthenticationProtos.TokenIdentifier.Kind.HBASE_AUTH_TOKEN,
+ new AuthenticationTokenSelector());
+ }
+
+ final AsyncRpcClient client;
+
+ // Contains the channel to work with.
+ // Only exists when connected
+ private Channel channel;
+
+ String name;
+ final User ticket;
+ final String serviceName;
+ final InetSocketAddress address;
+
+ private int failureCounter = 0;
+
+ boolean useSasl;
+ AuthMethod authMethod;
+ private int reloginMaxBackoff;
+ private Token<? extends TokenIdentifier> token;
+ private String serverPrincipal;
+
+ // NOTE: closed and connected flags below are only changed when a lock on pendingCalls
+ private final Map<Integer, AsyncCall> pendingCalls = new HashMap<Integer, AsyncCall>();
+ private boolean connected = false;
+ private boolean closed = false;
+
+ private Timeout cleanupTimer;
+
+ private final TimerTask timeoutTask = new TimerTask() {
+ @Override
+ public void run(Timeout timeout) throws Exception {
+ cleanupCalls();
+ }
+ };
+
+ /**
+ * Constructor for netty RPC channel
+ * @param bootstrap to construct channel on
+ * @param client to connect with
+ * @param ticket of user which uses connection
+ * @param serviceName name of service to connect to
+ * @param address to connect to
+ */
+ public AsyncRpcChannelImpl(Bootstrap bootstrap, final AsyncRpcClient client, User ticket,
+ String serviceName, InetSocketAddress address) {
+ this.client = client;
+
+ this.ticket = ticket;
+ this.serviceName = serviceName;
+ this.address = address;
+
+ this.channel = connect(bootstrap).channel();
+
+ name = ("IPC Client (" + channel.hashCode() + ") to " + address.toString()
+ + ((ticket == null) ? " from unknown user" : (" from " + ticket.getName())));
+ }
+
+ /**
+ * Connect to channel
+ * @param bootstrap to connect to
+ * @return future of connection
+ */
+ private ChannelFuture connect(final Bootstrap bootstrap) {
+ return bootstrap.remoteAddress(address).connect()
+ .addListener(new GenericFutureListener<ChannelFuture>() {
+ @Override
+ public void operationComplete(final ChannelFuture f) throws Exception {
+ if (!f.isSuccess()) {
+ retryOrClose(bootstrap, failureCounter++, client.failureSleep, f.cause());
+ return;
+ }
+ channel = f.channel();
+
+ setupAuthorization();
+
+ ByteBuf b = channel.alloc().directBuffer(6);
+ createPreamble(b, authMethod);
+ channel.writeAndFlush(b).addListener(ChannelFutureListener.CLOSE_ON_FAILURE);
+ if (useSasl) {
+ UserGroupInformation ticket = AsyncRpcChannelImpl.this.ticket.getUGI();
+ if (authMethod == AuthMethod.KERBEROS) {
+ if (ticket != null && ticket.getRealUser() != null) {
+ ticket = ticket.getRealUser();
+ }
+ }
+ SaslClientHandler saslHandler;
+ if (ticket == null) {
+ throw new FatalConnectionException("ticket/user is null");
+ }
+ final UserGroupInformation realTicket = ticket;
+ saslHandler = ticket.doAs(new PrivilegedExceptionAction<SaslClientHandler>() {
+ @Override
+ public SaslClientHandler run() throws IOException {
+ return getSaslHandler(realTicket, bootstrap);
+ }
+ });
+ if (saslHandler != null) {
+ // Sasl connect is successful. Let's set up Sasl channel handler
+ channel.pipeline().addFirst(saslHandler);
+ } else {
+ // fall back to simple auth because server told us so.
+ authMethod = AuthMethod.SIMPLE;
+ useSasl = false;
+ }
+ } else {
+ startHBaseConnection(f.channel());
+ }
+ }
+ });
+ }
+
+ /**
+ * Start HBase connection
+ * @param ch channel to start connection on
+ */
+ private void startHBaseConnection(Channel ch) {
+ ch.pipeline().addLast("frameDecoder",
+ new LengthFieldBasedFrameDecoder(Integer.MAX_VALUE, 0, 4, 0, 4));
+ ch.pipeline().addLast(new AsyncServerResponseHandler(this));
+ try {
+ writeChannelHeader(ch).addListener(new GenericFutureListener<ChannelFuture>() {
+ @Override
+ public void operationComplete(ChannelFuture future) throws Exception {
+ if (!future.isSuccess()) {
+ close(future.cause());
+ return;
+ }
+ List<AsyncCall> callsToWrite;
+ synchronized (pendingCalls) {
+ connected = true;
+ callsToWrite = new ArrayList<AsyncCall>(pendingCalls.values());
+ }
+ for (AsyncCall call : callsToWrite) {
+ writeRequest(call);
+ }
+ }
+ });
+ } catch (IOException e) {
+ close(e);
+ }
+ }
+
+ /**
+ * Get SASL handler
+ * @param bootstrap to reconnect to
+ * @return new SASL handler
+ * @throws java.io.IOException if handler failed to create
+ */
+ private SaslClientHandler getSaslHandler(final UserGroupInformation realTicket,
+ final Bootstrap bootstrap) throws IOException {
+ return new SaslClientHandler(realTicket, authMethod, token, serverPrincipal,
+ client.fallbackAllowed,
+ client.conf.get("hbase.rpc.protection",
+ SaslUtil.QualityOfProtection.AUTHENTICATION.name().toLowerCase()),
+ new SaslClientHandler.SaslExceptionHandler() {
+ @Override
+ public void handle(int retryCount, Random random, Throwable cause) {
+ try {
+ // Handle Sasl failure. Try to potentially get new credentials
+ handleSaslConnectionFailure(retryCount, cause, realTicket);
+
+ retryOrClose(bootstrap, failureCounter++, random.nextInt(reloginMaxBackoff) + 1,
+ cause);
+ } catch (IOException | InterruptedException e) {
+ close(e);
+ }
+ }
+ }, new SaslClientHandler.SaslSuccessfulConnectHandler() {
+ @Override
+ public void onSuccess(Channel channel) {
+ startHBaseConnection(channel);
+ }
+ });
+ }
+
+ /**
+ * Retry to connect or close
+ * @param bootstrap to connect with
+ * @param failureCount failure count
+ * @param e exception of fail
+ */
+ private void retryOrClose(final Bootstrap bootstrap, int failureCount, long timeout,
+ Throwable e) {
+ if (failureCount < client.maxRetries) {
+ client.newTimeout(new TimerTask() {
+ @Override
+ public void run(Timeout timeout) throws Exception {
+ connect(bootstrap);
+ }
+ }, timeout, TimeUnit.MILLISECONDS);
+ } else {
+ client.failedServers.addToFailedServers(address);
+ close(e);
+ }
+ }
+
+ /**
+ * Calls method on channel
+ * @param method to call
+ * @param controller to run call with
+ * @param request to send
+ * @param responsePrototype to construct response with
+ */
+ public Promise<Message> callMethod(final Descriptors.MethodDescriptor method,
+ final PayloadCarryingRpcController controller, final Message request,
+ final Message responsePrototype, MetricsConnection.CallStats callStats) {
+ final AsyncCall call = new AsyncCall(channel.eventLoop(), client.callIdCnt.getAndIncrement(),
+ method, request, controller, responsePrototype, callStats);
+ controller.notifyOnCancel(new RpcCallback<Object>() {
+ @Override
+ public void run(Object parameter) {
+ // TODO: do not need to call AsyncCall.setFailed?
+ synchronized (pendingCalls) {
+ pendingCalls.remove(call.id);
+ }
+ }
+ });
+ // TODO: this should be handled by PayloadCarryingRpcController.
+ if (controller.isCanceled()) {
+ // To finish if the call was cancelled before we set the notification (race condition)
+ call.cancel(true);
+ return call;
+ }
+
+ synchronized (pendingCalls) {
+ if (closed) {
+ Promise<Message> promise = channel.eventLoop().newPromise();
+ promise.setFailure(new ConnectException());
+ return promise;
+ }
+ pendingCalls.put(call.id, call);
+ // Add timeout for cleanup if none is present
+ if (cleanupTimer == null && call.getRpcTimeout() > 0) {
+ cleanupTimer = client.newTimeout(timeoutTask, call.getRpcTimeout(), TimeUnit.MILLISECONDS);
+ }
+ if (!connected) {
+ return call;
+ }
+ }
+ writeRequest(call);
+ return call;
+ }
+
+ @Override
+ public EventLoop getEventExecutor() {
+ return this.channel.eventLoop();
+ }
+
+ AsyncCall removePendingCall(int id) {
+ synchronized (pendingCalls) {
+ return pendingCalls.remove(id);
+ }
+ }
+
+ /**
+ * Write the channel header
+ * @param channel to write to
+ * @return future of write
+ * @throws java.io.IOException on failure to write
+ */
+ private ChannelFuture writeChannelHeader(Channel channel) throws IOException {
+ RPCProtos.ConnectionHeader.Builder headerBuilder = RPCProtos.ConnectionHeader.newBuilder()
+ .setServiceName(serviceName);
+
+ RPCProtos.UserInformation userInfoPB = buildUserInfo(ticket.getUGI(), authMethod);
+ if (userInfoPB != null) {
+ headerBuilder.setUserInfo(userInfoPB);
+ }
+
+ if (client.codec != null) {
+ headerBuilder.setCellBlockCodecClass(client.codec.getClass().getCanonicalName());
+ }
+ if (client.compressor != null) {
+ headerBuilder.setCellBlockCompressorClass(client.compressor.getClass().getCanonicalName());
+ }
+
+ headerBuilder.setVersionInfo(ProtobufUtil.getVersionInfo());
+ RPCProtos.ConnectionHeader header = headerBuilder.build();
+
+ int totalSize = IPCUtil.getTotalSizeWhenWrittenDelimited(header);
+
+ ByteBuf b = channel.alloc().directBuffer(totalSize);
+
+ b.writeInt(header.getSerializedSize());
+ b.writeBytes(header.toByteArray());
+
+ return channel.writeAndFlush(b);
+ }
+
+ /**
+ * Write request to channel
+ * @param call to write
+ */
+ private void writeRequest(final AsyncCall call) {
+ try {
+ final RPCProtos.RequestHeader.Builder requestHeaderBuilder = RPCProtos.RequestHeader
+ .newBuilder();
+ requestHeaderBuilder.setCallId(call.id).setMethodName(call.method.getName())
+ .setRequestParam(call.param != null);
+
+ if (Trace.isTracing()) {
+ Span s = Trace.currentSpan();
+ requestHeaderBuilder.setTraceInfo(TracingProtos.RPCTInfo.newBuilder()
+ .setParentId(s.getSpanId()).setTraceId(s.getTraceId()));
+ }
+
+ ByteBuffer cellBlock = client.buildCellBlock(call.controller.cellScanner());
+ if (cellBlock != null) {
+ final RPCProtos.CellBlockMeta.Builder cellBlockBuilder = RPCProtos.CellBlockMeta
+ .newBuilder();
+ cellBlockBuilder.setLength(cellBlock.limit());
+ requestHeaderBuilder.setCellBlockMeta(cellBlockBuilder.build());
+ }
+ // Only pass priority if there one. Let zero be same as no priority.
+ if (call.controller.getPriority() != PayloadCarryingRpcController.PRIORITY_UNSET) {
+ requestHeaderBuilder.setPriority(call.controller.getPriority());
+ }
+
+ RPCProtos.RequestHeader rh = requestHeaderBuilder.build();
+
+ int totalSize = IPCUtil.getTotalSizeWhenWrittenDelimited(rh, call.param);
+ if (cellBlock != null) {
+ totalSize += cellBlock.remaining();
+ }
+
+ ByteBuf b = channel.alloc().directBuffer(4 + totalSize);
+ try (ByteBufOutputStream out = new ByteBufOutputStream(b)) {
+ call.callStats.setRequestSizeBytes(IPCUtil.write(out, rh, call.param, cellBlock));
+ }
+
+ channel.writeAndFlush(b).addListener(new CallWriteListener(this, call.id));
+ } catch (IOException e) {
+ close(e);
+ }
+ }
+
+ /**
+ * Set up server authorization
+ * @throws java.io.IOException if auth setup failed
+ */
+ private void setupAuthorization() throws IOException {
+ SecurityInfo securityInfo = SecurityInfo.getInfo(serviceName);
+ this.useSasl = client.userProvider.isHBaseSecurityEnabled();
+
+ this.token = null;
+ if (useSasl && securityInfo != null) {
+ AuthenticationProtos.TokenIdentifier.Kind tokenKind = securityInfo.getTokenKind();
+ if (tokenKind != null) {
+ TokenSelector<? extends TokenIdentifier> tokenSelector = TOKEN_HANDDLERS.get(tokenKind);
+ if (tokenSelector != null) {
+ token = tokenSelector.selectToken(new Text(client.clusterId),
+ ticket.getUGI().getTokens());
+ } else if (LOG.isDebugEnabled()) {
+ LOG.debug("No token selector found for type " + tokenKind);
+ }
+ }
+ String serverKey = securityInfo.getServerPrincipal();
+ if (serverKey == null) {
+ throw new IOException("Can't obtain server Kerberos config key from SecurityInfo");
+ }
+ this.serverPrincipal = SecurityUtil.getServerPrincipal(client.conf.get(serverKey),
+ address.getAddress().getCanonicalHostName().toLowerCase());
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("RPC Server Kerberos principal name for service=" + serviceName + " is "
+ + serverPrincipal);
+ }
+ }
+
+ if (!useSasl) {
+ authMethod = AuthMethod.SIMPLE;
+ } else if (token != null) {
+ authMethod = AuthMethod.DIGEST;
+ } else {
+ authMethod = AuthMethod.KERBEROS;
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(
+ "Use " + authMethod + " authentication for service " + serviceName + ", sasl=" + useSasl);
+ }
+ reloginMaxBackoff = client.conf.getInt("hbase.security.relogin.maxbackoff", 5000);
+ }
+
+ /**
+ * Build the user information
+ * @param ugi User Group Information
+ * @param authMethod Authorization method
+ * @return UserInformation protobuf
+ */
+ private RPCProtos.UserInformation buildUserInfo(UserGroupInformation ugi, AuthMethod authMethod) {
+ if (ugi == null || authMethod == AuthMethod.DIGEST) {
+ // Don't send user for token auth
+ return null;
+ }
+ RPCProtos.UserInformation.Builder userInfoPB = RPCProtos.UserInformation.newBuilder();
+ if (authMethod == AuthMethod.KERBEROS) {
+ // Send effective user for Kerberos auth
+ userInfoPB.setEffectiveUser(ugi.getUserName());
+ } else if (authMethod == AuthMethod.SIMPLE) {
+ // Send both effective user and real user for simple auth
+ userInfoPB.setEffectiveUser(ugi.getUserName());
+ if (ugi.getRealUser() != null) {
+ userInfoPB.setRealUser(ugi.getRealUser().getUserName());
+ }
+ }
+ return userInfoPB.build();
+ }
+
+ /**
+ * Create connection preamble
+ * @param byteBuf to write to
+ * @param authMethod to write
+ */
+ private void createPreamble(ByteBuf byteBuf, AuthMethod authMethod) {
+ byteBuf.writeBytes(HConstants.RPC_HEADER);
+ byteBuf.writeByte(HConstants.RPC_CURRENT_VERSION);
+ byteBuf.writeByte(authMethod.code);
+ }
+
+ private void close0(Throwable e) {
+ List<AsyncCall> toCleanup;
+ synchronized (pendingCalls) {
+ if (closed) {
+ return;
+ }
+ closed = true;
+ toCleanup = new ArrayList<AsyncCall>(pendingCalls.values());
+ pendingCalls.clear();
+ }
+ IOException closeException = null;
+ if (e != null) {
+ if (e instanceof IOException) {
+ closeException = (IOException) e;
+ } else {
+ closeException = new IOException(e);
+ }
+ }
+ // log the info
+ if (LOG.isDebugEnabled() && closeException != null) {
+ LOG.debug(name + ": closing ipc connection to " + address, closeException);
+ }
+ if (cleanupTimer != null) {
+ cleanupTimer.cancel();
+ cleanupTimer = null;
+ }
+ for (AsyncCall call : toCleanup) {
+ call.setFailed(closeException != null ? closeException
+ : new ConnectionClosingException(
+ "Call id=" + call.id + " on server " + address + " aborted: connection is closing"));
+ }
+ channel.disconnect().addListener(ChannelFutureListener.CLOSE);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(name + ": closed");
+ }
+ }
+
+ /**
+ * Close connection
+ * @param e exception on close
+ */
+ public void close(final Throwable e) {
+ client.removeConnection(this);
+
+ // Move closing from the requesting thread to the channel thread
+ if (channel.eventLoop().inEventLoop()) {
+ close0(e);
+ } else {
+ channel.eventLoop().execute(new Runnable() {
+ @Override
+ public void run() {
+ close0(e);
+ }
+ });
+ }
+ }
+
+ /**
+ * Clean up calls.
+ */
+ private void cleanupCalls() {
+ List<AsyncCall> toCleanup = new ArrayList<AsyncCall>();
+ long currentTime = EnvironmentEdgeManager.currentTime();
+ long nextCleanupTaskDelay = -1L;
+ synchronized (pendingCalls) {
+ for (Iterator<AsyncCall> iter = pendingCalls.values().iterator(); iter.hasNext();) {
+ AsyncCall call = iter.next();
+ long timeout = call.getRpcTimeout();
+ if (timeout > 0) {
+ if (currentTime - call.getStartTime() >= timeout) {
+ iter.remove();
+ toCleanup.add(call);
+ } else {
+ if (nextCleanupTaskDelay < 0 || timeout < nextCleanupTaskDelay) {
+ nextCleanupTaskDelay = timeout;
+ }
+ }
+ }
+ }
+ if (nextCleanupTaskDelay > 0) {
+ cleanupTimer = client.newTimeout(timeoutTask, nextCleanupTaskDelay, TimeUnit.MILLISECONDS);
+ } else {
+ cleanupTimer = null;
+ }
+ }
+ for (AsyncCall call : toCleanup) {
+ call.setFailed(new CallTimeoutException("Call id=" + call.id + ", waitTime="
+ + (currentTime - call.getStartTime()) + ", rpcTimeout=" + call.getRpcTimeout()));
+ }
+ }
+
+ /**
+ * Check if the connection is alive
+ * @return true if alive
+ */
+ public boolean isAlive() {
+ return channel.isOpen();
+ }
+
+ @Override
+ public InetSocketAddress getAddress() {
+ return this.address;
+ }
+
+ /**
+ * Check if user should authenticate over Kerberos
+ * @return true if should be authenticated over Kerberos
+ * @throws java.io.IOException on failure of check
+ */
+ private synchronized boolean shouldAuthenticateOverKrb() throws IOException {
+ UserGroupInformation loginUser = UserGroupInformation.getLoginUser();
+ UserGroupInformation currentUser = UserGroupInformation.getCurrentUser();
+ UserGroupInformation realUser = currentUser.getRealUser();
+ return authMethod == AuthMethod.KERBEROS && loginUser != null &&
+ // Make sure user logged in using Kerberos either keytab or TGT
+ loginUser.hasKerberosCredentials() &&
+ // relogin only in case it is the login user (e.g. JT)
+ // or superuser (like oozie).
+ (loginUser.equals(currentUser) || loginUser.equals(realUser));
+ }
+
+ /**
+ * If multiple clients with the same principal try to connect to the same server at the same time,
+ * the server assumes a replay attack is in progress. This is a feature of kerberos. In order to
+ * work around this, what is done is that the client backs off randomly and tries to initiate the
+ * connection again. The other problem is to do with ticket expiry. To handle that, a relogin is
+ * attempted.
+ * <p>
+ * The retry logic is governed by the {@link #shouldAuthenticateOverKrb} method. In case when the
+ * user doesn't have valid credentials, we don't need to retry (from cache or ticket). In such
+ * cases, it is prudent to throw a runtime exception when we receive a SaslException from the
+ * underlying authentication implementation, so there is no retry from other high level (for eg,
+ * HCM or HBaseAdmin).
+ * </p>
+ * @param currRetries retry count
+ * @param ex exception describing fail
+ * @param user which is trying to connect
+ * @throws java.io.IOException if IO fail
+ * @throws InterruptedException if thread is interrupted
+ */
+ private void handleSaslConnectionFailure(final int currRetries, final Throwable ex,
+ final UserGroupInformation user) throws IOException, InterruptedException {
+ user.doAs(new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws IOException, InterruptedException {
+ if (shouldAuthenticateOverKrb()) {
+ if (currRetries < MAX_SASL_RETRIES) {
+ LOG.debug("Exception encountered while connecting to the server : " + ex);
+ // try re-login
+ if (UserGroupInformation.isLoginKeytabBased()) {
+ UserGroupInformation.getLoginUser().reloginFromKeytab();
+ } else {
+ UserGroupInformation.getLoginUser().reloginFromTicketCache();
+ }
+
+ // Should reconnect
+ return null;
+ } else {
+ String msg = "Couldn't setup connection for "
+ + UserGroupInformation.getLoginUser().getUserName() + " to " + serverPrincipal;
+ LOG.warn(msg);
+ throw (IOException) new IOException(msg).initCause(ex);
+ }
+ } else {
+ LOG.warn("Exception encountered while connecting to " + "the server : " + ex);
+ }
+ if (ex instanceof RemoteException) {
+ throw (RemoteException) ex;
+ }
+ if (ex instanceof SaslException) {
+ String msg = "SASL authentication failed."
+ + " The most likely cause is missing or invalid credentials." + " Consider 'kinit'.";
+ LOG.fatal(msg, ex);
+ throw new RuntimeException(msg, ex);
+ }
+ throw new IOException(ex);
+ }
+ });
+ }
+
+ public int getConnectionHashCode() {
+ return ConnectionId.hashCode(ticket, serviceName, address);
+ }
+
+ @Override
+ public int hashCode() {
+ return getConnectionHashCode();
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj instanceof AsyncRpcChannelImpl) {
+ AsyncRpcChannelImpl channel = (AsyncRpcChannelImpl) obj;
+ return channel.hashCode() == obj.hashCode();
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return this.address.toString() + "/" + this.serviceName + "/" + this.ticket;
+ }
+
+ /**
+ * Listens to call writes and fails if write failed
+ */
+ private static final class CallWriteListener implements ChannelFutureListener {
+ private final AsyncRpcChannelImpl rpcChannel;
+ private final int id;
+
+ public CallWriteListener(AsyncRpcChannelImpl asyncRpcChannelImpl, int id) {
+ this.rpcChannel = asyncRpcChannelImpl;
+ this.id = id;
+ }
+
+ @Override
+ public void operationComplete(ChannelFuture future) throws Exception {
+ if (!future.isSuccess()) {
+ AsyncCall call = rpcChannel.removePendingCall(id);
+ if (call != null) {
+ if (future.cause() instanceof IOException) {
+ call.setFailed((IOException) future.cause());
+ } else {
+ call.setFailed(new IOException(future.cause()));
+ }
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/56358a0f/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncRpcClient.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncRpcClient.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncRpcClient.java
index c2bd457..8d9a5b3 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncRpcClient.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncRpcClient.java
@@ -388,11 +388,11 @@ public class AsyncRpcClient extends AbstractRpcClient {
}
rpcChannel = connections.get(hashCode);
if (rpcChannel != null && !rpcChannel.isAlive()) {
- LOG.debug("Removing dead channel from server="+rpcChannel.address.toString());
+ LOG.debug("Removing dead channel from server="+rpcChannel.getAddress().toString());
connections.remove(hashCode);
}
if (rpcChannel == null) {
- rpcChannel = new AsyncRpcChannel(this.bootstrap, this, ticket, serviceName, location);
+ rpcChannel = new AsyncRpcChannelImpl(this.bootstrap, this, ticket, serviceName, location);
connections.put(hashCode, rpcChannel);
}
}
@@ -415,8 +415,8 @@ public class AsyncRpcClient extends AbstractRpcClient {
synchronized (connections) {
for (AsyncRpcChannel rpcChannel : connections.values()) {
if (rpcChannel.isAlive() &&
- rpcChannel.address.getPort() == sn.getPort() &&
- rpcChannel.address.getHostName().contentEquals(sn.getHostname())) {
+ rpcChannel.getAddress().getPort() == sn.getPort() &&
+ rpcChannel.getAddress().getHostName().contentEquals(sn.getHostname())) {
LOG.info("The server on " + sn.toString() +
" is dead - stopping the connection " + rpcChannel.toString());
rpcChannel.close(null);
http://git-wip-us.apache.org/repos/asf/hbase/blob/56358a0f/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncServerResponseHandler.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncServerResponseHandler.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncServerResponseHandler.java
index e0c7586..a0928b1 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncServerResponseHandler.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncServerResponseHandler.java
@@ -37,13 +37,13 @@ import io.netty.channel.SimpleChannelInboundHandler;
*/
@InterfaceAudience.Private
public class AsyncServerResponseHandler extends SimpleChannelInboundHandler<ByteBuf> {
- private final AsyncRpcChannel channel;
+ private final AsyncRpcChannelImpl channel;
/**
* Constructor
* @param channel on which this response handler operates
*/
- public AsyncServerResponseHandler(AsyncRpcChannel channel) {
+ public AsyncServerResponseHandler(AsyncRpcChannelImpl channel) {
this.channel = channel;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/56358a0f/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/CoprocessorRpcChannel.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/CoprocessorRpcChannel.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/CoprocessorRpcChannel.java
index b1d54a4..45c3700 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/CoprocessorRpcChannel.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/CoprocessorRpcChannel.java
@@ -1,4 +1,4 @@
-/*
+/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -7,7 +7,7 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -15,68 +15,20 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.hadoop.hbase.ipc;
-import java.io.IOException;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.classification.InterfaceStability;
-import org.apache.hadoop.hbase.protobuf.ResponseConverter;
-
import com.google.protobuf.BlockingRpcChannel;
-import com.google.protobuf.Descriptors;
-import com.google.protobuf.Message;
-import com.google.protobuf.RpcCallback;
import com.google.protobuf.RpcChannel;
-import com.google.protobuf.RpcController;
-import com.google.protobuf.ServiceException;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
/**
- * Base class which provides clients with an RPC connection to
+ * Base interface which provides clients with an RPC connection to
* call coprocessor endpoint {@link com.google.protobuf.Service}s.
* Note that clients should not use this class directly, except through
- * {@link org.apache.hadoop.hbase.client.HTableInterface#coprocessorService(byte[])}.
+ * {@link org.apache.hadoop.hbase.client.Table#coprocessorService(byte[])}.
*/
@InterfaceAudience.Public
@InterfaceStability.Evolving
-public abstract class CoprocessorRpcChannel implements RpcChannel, BlockingRpcChannel {
- private static final Log LOG = LogFactory.getLog(CoprocessorRpcChannel.class);
-
- @Override
- @InterfaceAudience.Private
- public void callMethod(Descriptors.MethodDescriptor method,
- RpcController controller,
- Message request, Message responsePrototype,
- RpcCallback<Message> callback) {
- Message response = null;
- try {
- response = callExecService(controller, method, request, responsePrototype);
- } catch (IOException ioe) {
- LOG.warn("Call failed on IOException", ioe);
- ResponseConverter.setControllerException(controller, ioe);
- }
- if (callback != null) {
- callback.run(response);
- }
- }
-
- @Override
- @InterfaceAudience.Private
- public Message callBlockingMethod(Descriptors.MethodDescriptor method,
- RpcController controller,
- Message request, Message responsePrototype)
- throws ServiceException {
- try {
- return callExecService(controller, method, request, responsePrototype);
- } catch (IOException ioe) {
- throw new ServiceException("Error calling method "+method.getFullName(), ioe);
- }
- }
-
- protected abstract Message callExecService(RpcController controller,
- Descriptors.MethodDescriptor method, Message request, Message responsePrototype)
- throws IOException;
-}
+public interface CoprocessorRpcChannel extends RpcChannel, BlockingRpcChannel {
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/56358a0f/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/MasterCoprocessorRpcChannel.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/MasterCoprocessorRpcChannel.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/MasterCoprocessorRpcChannel.java
index ac6a022..68798ed 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/MasterCoprocessorRpcChannel.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/MasterCoprocessorRpcChannel.java
@@ -42,7 +42,7 @@ import com.google.protobuf.RpcController;
* @see org.apache.hadoop.hbase.client.HBaseAdmin#coprocessorService()
*/
@InterfaceAudience.Private
-public class MasterCoprocessorRpcChannel extends CoprocessorRpcChannel{
+public class MasterCoprocessorRpcChannel extends SyncCoprocessorRpcChannel {
private static final Log LOG = LogFactory.getLog(MasterCoprocessorRpcChannel.class);
private final ClusterConnection connection;
http://git-wip-us.apache.org/repos/asf/hbase/blob/56358a0f/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RegionCoprocessorRpcChannel.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RegionCoprocessorRpcChannel.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RegionCoprocessorRpcChannel.java
index 0f06af1..4d3a453 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RegionCoprocessorRpcChannel.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RegionCoprocessorRpcChannel.java
@@ -45,7 +45,7 @@ import com.google.protobuf.RpcController;
* @see org.apache.hadoop.hbase.client.Table#coprocessorService(byte[])
*/
@InterfaceAudience.Private
-public class RegionCoprocessorRpcChannel extends CoprocessorRpcChannel{
+public class RegionCoprocessorRpcChannel extends SyncCoprocessorRpcChannel {
private static final Log LOG = LogFactory.getLog(RegionCoprocessorRpcChannel.class);
private final ClusterConnection connection;
@@ -57,6 +57,12 @@ public class RegionCoprocessorRpcChannel extends CoprocessorRpcChannel{
private RpcRetryingCallerFactory rpcCallerFactory;
private RpcControllerFactory rpcControllerFactory;
+ /**
+ * Constructor
+ * @param conn connection to use
+ * @param table to connect to
+ * @param row to locate region with
+ */
public RegionCoprocessorRpcChannel(ClusterConnection conn, TableName table, byte[] row) {
this.connection = conn;
this.table = table;
@@ -114,6 +120,10 @@ public class RegionCoprocessorRpcChannel extends CoprocessorRpcChannel{
return response;
}
+ /**
+ * Get last region this RpcChannel communicated with
+ * @return region name as byte array
+ */
public byte[] getLastRegion() {
return lastRegion;
}
[50/50] [abbrv] hbase git commit: HBASE-15798 Add Async RpcChannels
to all RpcClients
Posted by sy...@apache.org.
HBASE-15798 Add Async RpcChannels to all RpcClients
Signed-off-by: stack <st...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/a11091c4
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/a11091c4
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/a11091c4
Branch: refs/heads/hbase-12439
Commit: a11091c49cdd67c0ad37e8a55e87f430e24da4f1
Parents: 3b74b6f
Author: Jurriaan Mous <ju...@jurmo.us>
Authored: Sun May 8 10:20:15 2016 +0200
Committer: stack <st...@apache.org>
Committed: Tue May 10 07:44:51 2016 -0700
----------------------------------------------------------------------
.../hadoop/hbase/ipc/AbstractRpcClient.java | 34 +++-
.../org/apache/hadoop/hbase/ipc/AsyncCall.java | 26 ++-
.../hadoop/hbase/ipc/AsyncRpcChannel.java | 9 +-
.../hadoop/hbase/ipc/AsyncRpcChannelImpl.java | 8 +-
.../apache/hadoop/hbase/ipc/AsyncRpcClient.java | 66 +++---
.../hadoop/hbase/ipc/MessageConverter.java | 2 +-
.../org/apache/hadoop/hbase/ipc/RpcClient.java | 33 ++-
.../apache/hadoop/hbase/ipc/RpcClientImpl.java | 201 +++++++++++++++++-
.../hadoop/hbase/ipc/AbstractTestIPC.java | 203 +++++++++++++++----
.../apache/hadoop/hbase/ipc/TestAsyncIPC.java | 121 +----------
.../org/apache/hadoop/hbase/ipc/TestIPC.java | 7 +-
11 files changed, 483 insertions(+), 227 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/a11091c4/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AbstractRpcClient.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AbstractRpcClient.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AbstractRpcClient.java
index c091d1d..71c8875 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AbstractRpcClient.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AbstractRpcClient.java
@@ -271,6 +271,27 @@ public abstract class AbstractRpcClient implements RpcClient {
}
/**
+ * Configure a payload carrying controller
+ * @param controller to configure
+ * @param channelOperationTimeout timeout for operation
+ * @return configured payload controller
+ */
+ static PayloadCarryingRpcController configurePayloadCarryingRpcController(
+ RpcController controller, int channelOperationTimeout) {
+ PayloadCarryingRpcController pcrc;
+ if (controller != null && controller instanceof PayloadCarryingRpcController) {
+ pcrc = (PayloadCarryingRpcController) controller;
+ if (!pcrc.hasCallTimeout()) {
+ pcrc.setCallTimeout(channelOperationTimeout);
+ }
+ } else {
+ pcrc = new PayloadCarryingRpcController();
+ pcrc.setCallTimeout(channelOperationTimeout);
+ }
+ return pcrc;
+ }
+
+ /**
* Takes an Exception and the address we were trying to connect to and return an IOException with
* the input exception as the cause. The new exception provides the stack trace of the place where
* the exception is thrown and some extra diagnostics information. If the exception is
@@ -321,16 +342,9 @@ public abstract class AbstractRpcClient implements RpcClient {
@Override
public Message callBlockingMethod(Descriptors.MethodDescriptor md, RpcController controller,
Message param, Message returnType) throws ServiceException {
- PayloadCarryingRpcController pcrc;
- if (controller != null && controller instanceof PayloadCarryingRpcController) {
- pcrc = (PayloadCarryingRpcController) controller;
- if (!pcrc.hasCallTimeout()) {
- pcrc.setCallTimeout(channelOperationTimeout);
- }
- } else {
- pcrc = new PayloadCarryingRpcController();
- pcrc.setCallTimeout(channelOperationTimeout);
- }
+ PayloadCarryingRpcController pcrc = configurePayloadCarryingRpcController(
+ controller,
+ channelOperationTimeout);
return this.rpcClient.callBlockingMethod(md, pcrc, param, returnType, this.ticket, this.isa);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/a11091c4/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncCall.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncCall.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncCall.java
index 3acf280..89e6ca4 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncCall.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncCall.java
@@ -51,14 +51,15 @@ public class AsyncCall<M extends Message, T> extends Promise<T> {
final Message responseDefaultType;
private final MessageConverter<M,T> messageConverter;
- final long startTime;
- final long rpcTimeout;
private final IOExceptionConverter exceptionConverter;
+ final long rpcTimeout;
+
// For only the request
private final CellScanner cellScanner;
private final int priority;
+ final MetricsConnection clientMetrics;
final MetricsConnection.CallStats callStats;
/**
@@ -71,13 +72,15 @@ public class AsyncCall<M extends Message, T> extends Promise<T> {
* @param cellScanner cellScanner containing cells to send as request
* @param responseDefaultType the default response type
* @param messageConverter converts the messages to what is the expected output
+ * @param exceptionConverter converts exceptions to expected format. Can be null
* @param rpcTimeout timeout for this call in ms
* @param priority for this request
+ * @param metrics MetricsConnection to which the metrics are stored for this request
*/
public AsyncCall(AsyncRpcChannelImpl channel, int connectId, Descriptors.MethodDescriptor
md, Message param, CellScanner cellScanner, M responseDefaultType, MessageConverter<M, T>
messageConverter, IOExceptionConverter exceptionConverter, long rpcTimeout, int priority,
- MetricsConnection.CallStats callStats) {
+ MetricsConnection metrics) {
super(channel.getEventExecutor());
this.channel = channel;
@@ -90,13 +93,15 @@ public class AsyncCall<M extends Message, T> extends Promise<T> {
this.messageConverter = messageConverter;
this.exceptionConverter = exceptionConverter;
- this.startTime = EnvironmentEdgeManager.currentTime();
this.rpcTimeout = rpcTimeout;
this.priority = priority;
this.cellScanner = cellScanner;
- this.callStats = callStats;
+ this.callStats = MetricsConnection.newCallStats();
+ callStats.setStartTime(EnvironmentEdgeManager.currentTime());
+
+ this.clientMetrics = metrics;
}
/**
@@ -105,7 +110,7 @@ public class AsyncCall<M extends Message, T> extends Promise<T> {
* @return start time for the call
*/
public long getStartTime() {
- return this.startTime;
+ return this.callStats.getStartTime();
}
@Override
@@ -122,9 +127,14 @@ public class AsyncCall<M extends Message, T> extends Promise<T> {
* @param cellBlockScanner to set
*/
public void setSuccess(M value, CellScanner cellBlockScanner) {
+ callStats.setCallTimeMs(EnvironmentEdgeManager.currentTime() - callStats.getStartTime());
+
if (LOG.isTraceEnabled()) {
- long callTime = EnvironmentEdgeManager.currentTime() - startTime;
- LOG.trace("Call: " + method.getName() + ", callTime: " + callTime + "ms");
+ LOG.trace("Call: " + method.getName() + ", callTime: " + callStats.getCallTimeMs() + "ms");
+ }
+
+ if (clientMetrics != null) {
+ clientMetrics.updateRpc(method, param, callStats);
}
try {
http://git-wip-us.apache.org/repos/asf/hbase/blob/a11091c4/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncRpcChannel.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncRpcChannel.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncRpcChannel.java
index bd4be5a..8cc730f 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncRpcChannel.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncRpcChannel.java
@@ -27,7 +27,6 @@ import java.net.InetSocketAddress;
import org.apache.hadoop.hbase.CellScanner;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.client.Future;
-import org.apache.hadoop.hbase.client.MetricsConnection;
/**
* Interface for Async Rpc Channels
@@ -45,15 +44,13 @@ public interface AsyncRpcChannel {
* @param exceptionConverter for converting exceptions
* @param rpcTimeout timeout for request
* @param priority for request
- * @param callStats collects stats of the call
* @return Promise for the response Message
*/
-
<R extends Message, O> Future<O> callMethod(
final Descriptors.MethodDescriptor method,
- final Message request,final CellScanner cellScanner,
- R responsePrototype, MessageConverter<R, O> messageConverter, IOExceptionConverter
- exceptionConverter, long rpcTimeout, int priority, MetricsConnection.CallStats callStats);
+ final Message request, final CellScanner cellScanner,
+ R responsePrototype, MessageConverter<R, O> messageConverter,
+ IOExceptionConverter exceptionConverter, long rpcTimeout, int priority);
/**
http://git-wip-us.apache.org/repos/asf/hbase/blob/a11091c4/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncRpcChannelImpl.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncRpcChannelImpl.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncRpcChannelImpl.java
index 5af2354..cd61b61 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncRpcChannelImpl.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncRpcChannelImpl.java
@@ -30,7 +30,6 @@ import io.netty.handler.codec.LengthFieldBasedFrameDecoder;
import io.netty.util.Timeout;
import io.netty.util.TimerTask;
import io.netty.util.concurrent.GenericFutureListener;
-
import java.io.IOException;
import java.net.ConnectException;
import java.net.InetSocketAddress;
@@ -52,7 +51,6 @@ import org.apache.hadoop.hbase.CellScanner;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.client.Future;
-import org.apache.hadoop.hbase.client.MetricsConnection;
import org.apache.hadoop.hbase.exceptions.ConnectionClosingException;
import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.protobuf.generated.AuthenticationProtos;
@@ -297,14 +295,16 @@ public class AsyncRpcChannelImpl implements AsyncRpcChannel {
* @param priority for request
* @return Promise for the response Message
*/
+ @Override
public <R extends Message, O> Future<O> callMethod(
final Descriptors.MethodDescriptor method,
final Message request,final CellScanner cellScanner,
R responsePrototype, MessageConverter<R, O> messageConverter, IOExceptionConverter
- exceptionConverter, long rpcTimeout, int priority, MetricsConnection.CallStats callStats) {
+ exceptionConverter, long rpcTimeout, int priority) {
final AsyncCall<R, O> call = new AsyncCall<>(this, client.callIdCnt.getAndIncrement(),
method, request, cellScanner, responsePrototype, messageConverter, exceptionConverter,
- rpcTimeout, priority, callStats);
+ rpcTimeout, priority, client.metrics);
+
synchronized (pendingCalls) {
if (closed) {
call.setFailure(new ConnectException());
http://git-wip-us.apache.org/repos/asf/hbase/blob/a11091c4/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncRpcClient.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncRpcClient.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncRpcClient.java
index 2fdc1ec..c1ed748 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncRpcClient.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncRpcClient.java
@@ -27,6 +27,7 @@ import io.netty.bootstrap.Bootstrap;
import io.netty.channel.Channel;
import io.netty.channel.ChannelInitializer;
import io.netty.channel.ChannelOption;
+import io.netty.channel.EventLoop;
import io.netty.channel.EventLoopGroup;
import io.netty.channel.epoll.EpollEventLoopGroup;
import io.netty.channel.epoll.EpollSocketChannel;
@@ -59,7 +60,6 @@ import org.apache.hadoop.hbase.client.Future;
import org.apache.hadoop.hbase.client.MetricsConnection;
import org.apache.hadoop.hbase.client.ResponseFutureListener;
import org.apache.hadoop.hbase.security.User;
-import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.JVM;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.hbase.util.PoolMap;
@@ -103,7 +103,7 @@ public class AsyncRpcClient extends AbstractRpcClient {
@VisibleForTesting
static Pair<EventLoopGroup, Class<? extends Channel>> GLOBAL_EVENT_LOOP_GROUP;
- private synchronized static Pair<EventLoopGroup, Class<? extends Channel>>
+ synchronized static Pair<EventLoopGroup, Class<? extends Channel>>
getGlobalEventLoopGroup(Configuration conf) {
if (GLOBAL_EVENT_LOOP_GROUP == null) {
GLOBAL_EVENT_LOOP_GROUP = createEventLoopGroup(conf);
@@ -241,8 +241,8 @@ public class AsyncRpcClient extends AbstractRpcClient {
final AsyncRpcChannel connection = createRpcChannel(md.getService().getName(), addr, ticket);
final Future<Message> promise = connection.callMethod(md, param, pcrc.cellScanner(), returnType,
- getMessageConverterWithRpcController(pcrc), null, pcrc.getCallTimeout(), pcrc.getPriority(),
- callStats);
+ getMessageConverterWithRpcController(pcrc), null, pcrc.getCallTimeout(),
+ pcrc.getPriority());
pcrc.notifyOnCancel(new RpcCallback<Object>() {
@Override
@@ -289,19 +289,11 @@ public class AsyncRpcClient extends AbstractRpcClient {
final AsyncRpcChannel connection;
try {
connection = createRpcChannel(md.getService().getName(), addr, ticket);
- final MetricsConnection.CallStats cs = MetricsConnection.newCallStats();
ResponseFutureListener<Message> listener =
new ResponseFutureListener<Message>() {
@Override
public void operationComplete(Future<Message> future) throws Exception {
- cs.setCallTimeMs(EnvironmentEdgeManager.currentTime() - cs.getStartTime());
- if (metrics != null) {
- metrics.updateRpc(md, param, cs);
- }
- if (LOG.isTraceEnabled()) {
- LOG.trace("Call: " + md.getName() + ", callTime: " + cs.getCallTimeMs() + "ms");
- }
if (!future.isSuccess()) {
Throwable cause = future.cause();
if (cause instanceof IOException) {
@@ -325,10 +317,9 @@ public class AsyncRpcClient extends AbstractRpcClient {
}
}
};
- cs.setStartTime(EnvironmentEdgeManager.currentTime());
connection.callMethod(md, param, pcrc.cellScanner(), returnType,
getMessageConverterWithRpcController(pcrc), null,
- pcrc.getCallTimeout(), pcrc.getPriority(), cs)
+ pcrc.getCallTimeout(), pcrc.getPriority())
.addListener(listener);
} catch (StoppedRpcClientException|FailedServerException e) {
pcrc.setFailed(e);
@@ -360,6 +351,11 @@ public class AsyncRpcClient extends AbstractRpcClient {
}
}
+ @Override
+ public EventLoop getEventExecutor() {
+ return this.bootstrap.group().next();
+ }
+
/**
* Create a cell scanner
*
@@ -382,10 +378,17 @@ public class AsyncRpcClient extends AbstractRpcClient {
return ipcUtil.buildCellBlock(this.codec, this.compressor, cells);
}
+ @Override
+ public AsyncRpcChannel createRpcChannel(String serviceName, ServerName sn, User user)
+ throws StoppedRpcClientException, FailedServerException {
+ return this.createRpcChannel(serviceName,
+ new InetSocketAddress(sn.getHostname(), sn.getPort()), user);
+ }
+
/**
* Creates an RPC client
*
- * @param serviceName name of servicce
+ * @param serviceName name of service
* @param location to connect to
* @param ticket for current user
* @return new RpcChannel
@@ -452,6 +455,7 @@ public class AsyncRpcClient extends AbstractRpcClient {
/**
* Remove connection from pool
+ * @param connection to remove
*/
public void removeConnection(AsyncRpcChannel connection) {
int connectionHashCode = connection.hashCode();
@@ -469,17 +473,8 @@ public class AsyncRpcClient extends AbstractRpcClient {
}
}
- /**
- * Creates a "channel" that can be used by a protobuf service. Useful setting up
- * protobuf stubs.
- *
- * @param sn server name describing location of server
- * @param user which is to use the connection
- * @param rpcTimeout default rpc operation timeout
- *
- * @return A rpc channel that goes via this rpc client instance.
- */
- public RpcChannel createRpcChannel(final ServerName sn, final User user, int rpcTimeout) {
+ @Override
+ public RpcChannel createProtobufRpcChannel(final ServerName sn, final User user, int rpcTimeout) {
return new RpcChannelImplementation(this, sn, user, rpcTimeout);
}
@@ -507,21 +502,20 @@ public class AsyncRpcClient extends AbstractRpcClient {
@Override
public void callMethod(Descriptors.MethodDescriptor md, RpcController controller,
Message param, Message returnType, RpcCallback<Message> done) {
- PayloadCarryingRpcController pcrc;
- if (controller != null) {
- pcrc = (PayloadCarryingRpcController) controller;
- if (!pcrc.hasCallTimeout()) {
- pcrc.setCallTimeout(channelOperationTimeout);
- }
- } else {
- pcrc = new PayloadCarryingRpcController();
- pcrc.setCallTimeout(channelOperationTimeout);
- }
+ PayloadCarryingRpcController pcrc =
+ configurePayloadCarryingRpcController(controller, channelOperationTimeout);
this.rpcClient.callMethod(md, pcrc, param, returnType, this.ticket, this.isa, done);
}
}
+ /**
+ * Get a new timeout on this RPC client
+ * @param task to run at timeout
+ * @param delay for the timeout
+ * @param unit time unit for the timeout
+ * @return Timeout
+ */
Timeout newTimeout(TimerTask task, long delay, TimeUnit unit) {
return WHEEL_TIMER.newTimeout(task, delay, unit);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/a11091c4/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/MessageConverter.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/MessageConverter.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/MessageConverter.java
index 527ac95..a85225a 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/MessageConverter.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/MessageConverter.java
@@ -41,7 +41,7 @@ public interface MessageConverter<M,O> {
MessageConverter<Message,Message> NO_CONVERTER = new MessageConverter<Message, Message>() {
@Override
public Message convert(Message msg, CellScanner cellScanner) throws IOException {
- return null;
+ return msg;
}
};
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/a11091c4/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcClient.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcClient.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcClient.java
index f77f1ec..9d05c21 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcClient.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcClient.java
@@ -18,7 +18,8 @@
package org.apache.hadoop.hbase.ipc;
import com.google.protobuf.BlockingRpcChannel;
-
+import com.google.protobuf.RpcChannel;
+import io.netty.util.concurrent.EventExecutor;
import java.io.Closeable;
import java.io.IOException;
@@ -69,6 +70,30 @@ import org.apache.hadoop.hbase.security.User;
throws IOException;
/**
+ * Create or fetch AsyncRpcChannel
+ * @param serviceName to connect to
+ * @param sn ServerName of the channel to create
+ * @param user for the service
+ * @return An async RPC channel fitting given parameters
+ * @throws FailedServerException if server failed
+ * @throws StoppedRpcClientException if the RPC client has stopped
+ */
+ AsyncRpcChannel createRpcChannel(String serviceName, ServerName sn, User user)
+ throws StoppedRpcClientException, FailedServerException;
+
+ /**
+ * Creates a "channel" that can be used by a protobuf service. Useful setting up
+ * protobuf stubs.
+ *
+ * @param sn server name describing location of server
+ * @param user which is to use the connection
+ * @param rpcTimeout default rpc operation timeout
+ *
+ * @return A rpc channel that goes via this rpc client instance.
+ */
+ RpcChannel createProtobufRpcChannel(final ServerName sn, final User user, int rpcTimeout);
+
+ /**
* Interrupt the connections to the given server. This should be called if the server
* is known as actually dead. This will not prevent current operation to be retried, and,
* depending on their own behavior, they may retry on the same server. This can be a feature,
@@ -91,4 +116,10 @@ import org.apache.hadoop.hbase.security.User;
* supports cell blocks.
*/
boolean hasCellBlockSupport();
+
+ /**
+ * Get an event loop to operate on
+ * @return EventLoop
+ */
+ EventExecutor getEventExecutor();
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/a11091c4/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcClientImpl.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcClientImpl.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcClientImpl.java
index 7b2500c..a5d2482 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcClientImpl.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcClientImpl.java
@@ -22,6 +22,9 @@ import com.google.protobuf.Descriptors.MethodDescriptor;
import com.google.protobuf.Message;
import com.google.protobuf.Message.Builder;
import com.google.protobuf.RpcCallback;
+import com.google.protobuf.RpcChannel;
+import com.google.protobuf.RpcController;
+import io.netty.util.concurrent.EventExecutor;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
@@ -51,7 +54,6 @@ import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ConcurrentSkipListMap;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
-
import javax.net.SocketFactory;
import javax.security.sasl.SaslException;
@@ -63,6 +65,7 @@ import org.apache.hadoop.hbase.DoNotRetryIOException;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.client.Future;
import org.apache.hadoop.hbase.client.MetricsConnection;
import org.apache.hadoop.hbase.codec.Codec;
import org.apache.hadoop.hbase.exceptions.ConnectionClosingException;
@@ -118,9 +121,7 @@ public class RpcClientImpl extends AbstractRpcClient {
protected final SocketFactory socketFactory; // how to create sockets
protected final static Map<AuthenticationProtos.TokenIdentifier.Kind,
- TokenSelector<? extends TokenIdentifier>> tokenHandlers =
- new HashMap<AuthenticationProtos.TokenIdentifier.Kind,
- TokenSelector<? extends TokenIdentifier>>();
+ TokenSelector<? extends TokenIdentifier>> tokenHandlers = new HashMap<>();
static {
tokenHandlers.put(AuthenticationProtos.TokenIdentifier.Kind.HBASE_AUTH_TOKEN,
new AuthenticationTokenSelector());
@@ -1217,7 +1218,13 @@ public class RpcClientImpl extends AbstractRpcClient {
}
}
- /** Make a call, passing <code>param</code>, to the IPC server running at
+ @Override
+ public EventExecutor getEventExecutor() {
+ return AsyncRpcClient.getGlobalEventLoopGroup(this.conf).getFirst().next();
+ }
+
+ /**
+ * Make a call, passing <code>param</code>, to the IPC server running at
* <code>address</code> which is servicing the <code>protocol</code> protocol,
* with the <code>ticket</code> credentials, returning the value.
* Throws exceptions if there are network problems or if the remote code
@@ -1226,7 +1233,7 @@ public class RpcClientImpl extends AbstractRpcClient {
* {@link UserProvider#getCurrent()} makes a new instance of User each time so will be a
* new Connection each time.
* @return A pair with the Message response and the Cell data (if any).
- * @throws InterruptedException if the call is interupted
+ * @throws InterruptedException if the call is interrupted
* @throws IOException if something fails on the connection
*/
@Override
@@ -1237,10 +1244,35 @@ public class RpcClientImpl extends AbstractRpcClient {
if (pcrc == null) {
pcrc = new PayloadCarryingRpcController();
}
+
+ Call call = this.call(md, param, returnType, pcrc, ticket, addr, callStats);
+
+ return new Pair<>(call.response, call.cells);
+ }
+
+
+ /**
+ * Make a call, passing <code>param</code>, to the IPC server running at
+ * <code>address</code> which is servicing the <code>protocol</code> protocol,
+ * with the <code>ticket</code> credentials, returning the value.
+ * Throws exceptions if there are network problems or if the remote code
+ * threw an exception.
+ * @param ticket Be careful which ticket you pass. A new user will mean a new Connection.
+ * {@link UserProvider#getCurrent()} makes a new instance of User each time so will be a
+ * new Connection each time.
+ * @return A Call
+ * @throws InterruptedException if the call is interrupted
+ * @throws IOException if something fails on the connection
+ */
+ private <R extends Message> Call call(MethodDescriptor method, Message request,
+ R responsePrototype, PayloadCarryingRpcController pcrc, User ticket,
+ InetSocketAddress addr, MetricsConnection.CallStats callStats)
+ throws IOException, InterruptedException {
+
CellScanner cells = pcrc.cellScanner();
- final Call call = new Call(this.callIdCnt.getAndIncrement(), md, param, cells, returnType,
- pcrc.getCallTimeout(), MetricsConnection.newCallStats());
+ final Call call = new Call(callIdCnt.getAndIncrement(), method, request, cells,
+ responsePrototype, pcrc.getCallTimeout(), callStats);
final Connection connection = getConnection(ticket, call, addr);
@@ -1256,7 +1288,7 @@ public class RpcClientImpl extends AbstractRpcClient {
if (pcrc.isCanceled()) {
// To finish if the call was cancelled before we set the notification (race condition)
call.callComplete();
- return new Pair<>(call.response, call.cells);
+ return call;
}
} else {
cts = null;
@@ -1299,9 +1331,19 @@ public class RpcClientImpl extends AbstractRpcClient {
throw wrapException(addr, call.error);
}
- return new Pair<>(call.response, call.cells);
+ return call;
}
+ @Override
+ public org.apache.hadoop.hbase.ipc.AsyncRpcChannel createRpcChannel(String serviceName,
+ ServerName sn, User user) throws StoppedRpcClientException, FailedServerException {
+ return new AsyncRpcChannel(sn, user);
+ }
+
+ @Override
+ public RpcChannel createProtobufRpcChannel(ServerName sn, User user, int rpcTimeout) {
+ return new RpcChannelImplementation(sn, user, rpcTimeout);
+ }
/**
* Interrupt the connections to the given ip:port server. This should be called if the server
@@ -1349,4 +1391,143 @@ public class RpcClientImpl extends AbstractRpcClient {
return connection;
}
+
+ /**
+ * Simulated async call
+ */
+ private class RpcChannelImplementation implements RpcChannel {
+ private final InetSocketAddress isa;
+ private final User ticket;
+ private final int channelOperationTimeout;
+ private final EventExecutor executor;
+
+ /**
+ * @param channelOperationTimeout - the default timeout when no timeout is given
+ */
+ protected RpcChannelImplementation(
+ final ServerName sn, final User ticket, int channelOperationTimeout) {
+ this.isa = new InetSocketAddress(sn.getHostname(), sn.getPort());
+ this.ticket = ticket;
+ this.channelOperationTimeout = channelOperationTimeout;
+
+ this.executor = RpcClientImpl.this.getEventExecutor();
+ }
+
+ @Override
+ public void callMethod(final MethodDescriptor method, RpcController controller,
+ final Message request, final Message responsePrototype, final RpcCallback<Message> done) {
+ final PayloadCarryingRpcController pcrc = configurePayloadCarryingRpcController(
+ controller,
+ channelOperationTimeout);
+
+ executor.execute(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ final MetricsConnection.CallStats cs = MetricsConnection.newCallStats();
+ cs.setStartTime(EnvironmentEdgeManager.currentTime());
+ Call call = call(method, request, responsePrototype, pcrc, ticket, isa, cs);
+ cs.setCallTimeMs(EnvironmentEdgeManager.currentTime() - cs.getStartTime());
+ if (metrics != null) {
+ metrics.updateRpc(method, request, cs);
+ }
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("Call: " + method.getName() + ", callTime: " + cs.getCallTimeMs() + "ms");
+ }
+
+ done.run(call.response);
+ } catch (IOException e) {
+ pcrc.setFailed(e);
+ } catch (InterruptedException e) {
+ pcrc.startCancel();
+ }
+ }
+ });
+ }
+ }
+
+ /**
+ * Wraps the call in an async channel.
+ */
+ private class AsyncRpcChannel implements org.apache.hadoop.hbase.ipc.AsyncRpcChannel {
+ private final EventExecutor executor;
+ private final InetSocketAddress isa;
+
+ private final User ticket;
+
+ /**
+ * Constructor
+ * @param sn servername to connect to
+ * @param user to connect with
+ */
+ public AsyncRpcChannel(ServerName sn, User user) {
+ this.isa = new InetSocketAddress(sn.getHostname(), sn.getPort());
+ this.executor = RpcClientImpl.this.getEventExecutor();
+ this.ticket = user;
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public <R extends Message, O> Future<O> callMethod(final MethodDescriptor method,
+ final Message request, CellScanner cellScanner, final R responsePrototype,
+ final MessageConverter<R, O> messageConverter,
+ final IOExceptionConverter exceptionConverter, long rpcTimeout, int priority) {
+ final PayloadCarryingRpcController pcrc = new PayloadCarryingRpcController(cellScanner);
+ pcrc.setPriority(priority);
+ pcrc.setCallTimeout((int) rpcTimeout);
+
+ final Promise<O> promise = new Promise<>(executor);
+
+ executor.execute(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ final MetricsConnection.CallStats cs = MetricsConnection.newCallStats();
+ cs.setStartTime(EnvironmentEdgeManager.currentTime());
+ Call call = call(method, request, responsePrototype, pcrc, ticket, isa, cs);
+ cs.setCallTimeMs(EnvironmentEdgeManager.currentTime() - cs.getStartTime());
+ if (metrics != null) {
+ metrics.updateRpc(method, request, cs);
+ }
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("Call: " + method.getName() + ", callTime: " + cs.getCallTimeMs() + "ms");
+ }
+
+ promise.setSuccess(
+ messageConverter.convert((R) call.response, call.cells)
+ );
+ } catch (InterruptedException e) {
+ promise.cancel(true);
+ } catch (IOException e) {
+ if(exceptionConverter != null) {
+ e = exceptionConverter.convert(e);
+ }
+ promise.setFailure(e);
+ }
+ }
+ });
+
+ return promise;
+ }
+
+ @Override
+ public EventExecutor getEventExecutor() {
+ return this.executor;
+ }
+
+ @Override
+ public void close(Throwable cause) {
+ this.executor.shutdownGracefully();
+ }
+
+ @Override
+ public boolean isAlive() {
+ return !this.executor.isShuttingDown() && !this.executor.isShutdown();
+ }
+
+ @Override
+ public InetSocketAddress getAddress() {
+ return isa;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/a11091c4/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/AbstractTestIPC.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/AbstractTestIPC.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/AbstractTestIPC.java
index 69c8fe2..ceb945b 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/AbstractTestIPC.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/AbstractTestIPC.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hbase.ipc;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.mockito.Matchers.anyObject;
@@ -25,6 +26,15 @@ import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.verify;
import static org.mockito.internal.verification.VerificationModeFactory.times;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+import com.google.protobuf.BlockingRpcChannel;
+import com.google.protobuf.BlockingService;
+import com.google.protobuf.Descriptors.MethodDescriptor;
+import com.google.protobuf.Message;
+import com.google.protobuf.RpcChannel;
+import com.google.protobuf.RpcController;
+import com.google.protobuf.ServiceException;
import java.io.IOException;
import java.net.ConnectException;
import java.net.InetAddress;
@@ -32,7 +42,9 @@ import java.net.InetSocketAddress;
import java.net.SocketTimeoutException;
import java.util.ArrayList;
import java.util.List;
-
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -40,9 +52,12 @@ import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellScanner;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.Waiter;
+import org.apache.hadoop.hbase.client.Future;
import org.apache.hadoop.hbase.client.MetricsConnection;
import org.apache.hadoop.hbase.exceptions.ConnectionClosingException;
import org.apache.hadoop.hbase.ipc.protobuf.generated.TestProtos.EchoRequestProto;
@@ -60,15 +75,6 @@ import org.apache.hadoop.util.StringUtils;
import org.junit.Assert;
import org.junit.Test;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Lists;
-import com.google.protobuf.BlockingRpcChannel;
-import com.google.protobuf.BlockingService;
-import com.google.protobuf.Descriptors.MethodDescriptor;
-import com.google.protobuf.Message;
-import com.google.protobuf.RpcController;
-import com.google.protobuf.ServiceException;
-
/**
* Some basic ipc tests.
*/
@@ -76,8 +82,10 @@ public abstract class AbstractTestIPC {
private static final Log LOG = LogFactory.getLog(AbstractTestIPC.class);
- private static byte[] CELL_BYTES = Bytes.toBytes("xyz");
- private static KeyValue CELL = new KeyValue(CELL_BYTES, CELL_BYTES, CELL_BYTES, CELL_BYTES);
+ private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+
+ private static final byte[] CELL_BYTES = Bytes.toBytes("xyz");
+ private static final KeyValue CELL = new KeyValue(CELL_BYTES, CELL_BYTES, CELL_BYTES, CELL_BYTES);
static byte[] BIG_CELL_BYTES = new byte[10 * 1024];
static KeyValue BIG_CELL = new KeyValue(CELL_BYTES, CELL_BYTES, CELL_BYTES, BIG_CELL_BYTES);
static final Configuration CONF = HBaseConfiguration.create();
@@ -114,7 +122,7 @@ public abstract class AbstractTestIPC {
CellScanner cellScanner = pcrc.cellScanner();
List<Cell> list = null;
if (cellScanner != null) {
- list = new ArrayList<Cell>();
+ list = new ArrayList<>();
try {
while (cellScanner.advance()) {
list.add(cellScanner.current());
@@ -168,9 +176,8 @@ public abstract class AbstractTestIPC {
@Test
public void testNoCodec() throws InterruptedException, IOException {
Configuration conf = HBaseConfiguration.create();
- AbstractRpcClient client = createRpcClientNoCodec(conf);
TestRpcServer rpcServer = new TestRpcServer();
- try {
+ try (AbstractRpcClient client = createRpcClientNoCodec(conf)) {
rpcServer.start();
MethodDescriptor md = SERVICE.getDescriptorForType().findMethodByName("echo");
final String message = "hello";
@@ -186,7 +193,6 @@ public abstract class AbstractTestIPC {
// Silly assertion that the message is in the returned pb.
assertTrue(r.getFirst().toString().contains(message));
} finally {
- client.close();
rpcServer.stop();
}
}
@@ -207,14 +213,13 @@ public abstract class AbstractTestIPC {
NoSuchMethodException, ServiceException {
Configuration conf = new Configuration(HBaseConfiguration.create());
conf.set("hbase.client.rpc.compressor", GzipCodec.class.getCanonicalName());
- List<Cell> cells = new ArrayList<Cell>();
+ List<Cell> cells = new ArrayList<>();
int count = 3;
for (int i = 0; i < count; i++) {
cells.add(CELL);
}
- AbstractRpcClient client = createRpcClient(conf);
TestRpcServer rpcServer = new TestRpcServer();
- try {
+ try (AbstractRpcClient client = createRpcClient(conf)) {
rpcServer.start();
MethodDescriptor md = SERVICE.getDescriptorForType().findMethodByName("echo");
EchoRequestProto param = EchoRequestProto.newBuilder().setMessage("hello").build();
@@ -234,7 +239,6 @@ public abstract class AbstractTestIPC {
}
assertEquals(count, index);
} finally {
- client.close();
rpcServer.stop();
}
}
@@ -246,8 +250,7 @@ public abstract class AbstractTestIPC {
public void testRTEDuringConnectionSetup() throws Exception {
Configuration conf = HBaseConfiguration.create();
TestRpcServer rpcServer = new TestRpcServer();
- AbstractRpcClient client = createRpcClientRTEDuringConnectionSetup(conf);
- try {
+ try (AbstractRpcClient client = createRpcClientRTEDuringConnectionSetup(conf)) {
rpcServer.start();
MethodDescriptor md = SERVICE.getDescriptorForType().findMethodByName("echo");
EchoRequestProto param = EchoRequestProto.newBuilder().setMessage("hello").build();
@@ -262,7 +265,6 @@ public abstract class AbstractTestIPC {
LOG.info("Caught expected exception: " + e.toString());
assertTrue(StringUtils.stringifyException(e).contains("Injected fault"));
} finally {
- client.close();
rpcServer.stop();
}
}
@@ -332,7 +334,7 @@ public abstract class AbstractTestIPC {
*/
static class TestRpcServer1 extends RpcServer {
- private static BlockingInterface SERVICE1 =
+ private static final BlockingInterface SERVICE1 =
new TestRpcServiceProtos.TestProtobufRpcProto.BlockingInterface() {
@Override
public EmptyResponseProto ping(RpcController unused, EmptyRequestProto request)
@@ -378,26 +380,22 @@ public abstract class AbstractTestIPC {
final RpcScheduler scheduler = new FifoRpcScheduler(CONF, 1);
final TestRpcServer1 rpcServer = new TestRpcServer1(scheduler);
final InetSocketAddress localAddr = new InetSocketAddress("localhost", 0);
- final AbstractRpcClient client =
- new RpcClientImpl(CONF, HConstants.CLUSTER_ID_DEFAULT, localAddr, null);
- try {
+ try (AbstractRpcClient client = new RpcClientImpl(CONF, HConstants.CLUSTER_ID_DEFAULT,
+ localAddr, null)) {
rpcServer.start();
final InetSocketAddress isa = rpcServer.getListenerAddress();
if (isa == null) {
throw new IOException("Listener channel is closed");
}
- final BlockingRpcChannel channel =
- client.createBlockingRpcChannel(
- ServerName.valueOf(isa.getHostName(), isa.getPort(), System.currentTimeMillis()),
- User.getCurrent(), 0);
- TestRpcServiceProtos.TestProtobufRpcProto.BlockingInterface stub =
- TestRpcServiceProtos.TestProtobufRpcProto.newBlockingStub(channel);
+ final BlockingRpcChannel channel = client.createBlockingRpcChannel(
+ ServerName.valueOf(isa.getHostName(), isa.getPort(), System.currentTimeMillis()),
+ User.getCurrent(), 0);
+ BlockingInterface stub = TestRpcServiceProtos.TestProtobufRpcProto.newBlockingStub(channel);
final EchoRequestProto echoRequest =
EchoRequestProto.newBuilder().setMessage("GetRemoteAddress").build();
final EchoResponseProto echoResponse = stub.echo(null, echoRequest);
Assert.assertEquals(localAddr.getAddress().getHostAddress(), echoResponse.getMessage());
} finally {
- client.close();
rpcServer.stop();
}
}
@@ -416,4 +414,141 @@ public abstract class AbstractTestIPC {
.wrapException(address, new CallTimeoutException("Test AbstractRpcClient#wrapException"))
.getCause() instanceof CallTimeoutException);
}
+
+ @Test
+ public void testAsyncProtobufConnectionSetup() throws Exception {
+ TestRpcServer rpcServer = new TestRpcServer();
+ try (RpcClient client = createRpcClient(CONF)) {
+ rpcServer.start();
+ InetSocketAddress address = rpcServer.getListenerAddress();
+ if (address == null) {
+ throw new IOException("Listener channel is closed");
+ }
+ MethodDescriptor md = SERVICE.getDescriptorForType().findMethodByName("echo");
+ EchoRequestProto param = EchoRequestProto.newBuilder().setMessage("hello").build();
+
+ RpcChannel channel = client.createProtobufRpcChannel(
+ ServerName.valueOf(address.getHostName(), address.getPort(), System.currentTimeMillis()),
+ User.getCurrent(), 0);
+
+ final AtomicBoolean done = new AtomicBoolean(false);
+
+ channel
+ .callMethod(md, new PayloadCarryingRpcController(), param, md.getOutputType().toProto(),
+ new com.google.protobuf.RpcCallback<Message>() {
+ @Override
+ public void run(Message parameter) {
+ done.set(true);
+ }
+ });
+
+ TEST_UTIL.waitFor(1000, new Waiter.Predicate<Exception>() {
+ @Override
+ public boolean evaluate() throws Exception {
+ return done.get();
+ }
+ });
+ } finally {
+ rpcServer.stop();
+ }
+ }
+
+ @Test
+ public void testRTEDuringAsyncProtobufConnectionSetup() throws Exception {
+ TestRpcServer rpcServer = new TestRpcServer();
+ try (RpcClient client = createRpcClientRTEDuringConnectionSetup(CONF)) {
+ rpcServer.start();
+ InetSocketAddress address = rpcServer.getListenerAddress();
+ if (address == null) {
+ throw new IOException("Listener channel is closed");
+ }
+ MethodDescriptor md = SERVICE.getDescriptorForType().findMethodByName("echo");
+ EchoRequestProto param = EchoRequestProto.newBuilder().setMessage("hello").build();
+
+ RpcChannel channel = client.createProtobufRpcChannel(
+ ServerName.valueOf(address.getHostName(), address.getPort(), System.currentTimeMillis()),
+ User.getCurrent(), 0);
+
+ final AtomicBoolean done = new AtomicBoolean(false);
+
+ PayloadCarryingRpcController controller = new PayloadCarryingRpcController();
+ controller.notifyOnFail(new com.google.protobuf.RpcCallback<IOException>() {
+ @Override
+ public void run(IOException e) {
+ done.set(true);
+ LOG.info("Caught expected exception: " + e.toString());
+ assertTrue(StringUtils.stringifyException(e).contains("Injected fault"));
+ }
+ });
+
+ channel.callMethod(md, controller, param, md.getOutputType().toProto(),
+ new com.google.protobuf.RpcCallback<Message>() {
+ @Override
+ public void run(Message parameter) {
+ done.set(true);
+ fail("Expected an exception to have been thrown!");
+ }
+ });
+
+ TEST_UTIL.waitFor(1000, new Waiter.Predicate<Exception>() {
+ @Override
+ public boolean evaluate() throws Exception {
+ return done.get();
+ }
+ });
+ } finally {
+ rpcServer.stop();
+ }
+ }
+
+ @Test
+ public void testAsyncConnectionSetup() throws Exception {
+ TestRpcServer rpcServer = new TestRpcServer();
+ try (RpcClient client = createRpcClient(CONF)) {
+ rpcServer.start();
+ Message msg = setupAsyncConnection(rpcServer, client);
+
+ assertNotNull(msg);
+ } finally {
+ rpcServer.stop();
+ }
+ }
+
+ @Test
+ public void testRTEDuringAsyncConnectionSetup() throws Exception {
+ TestRpcServer rpcServer = new TestRpcServer();
+ try (RpcClient client = createRpcClientRTEDuringConnectionSetup(CONF)) {
+ rpcServer.start();
+ setupAsyncConnection(rpcServer, client);
+
+ fail("Expected an exception to have been thrown!");
+ } catch (ExecutionException e) {
+ assertTrue(StringUtils.stringifyException(e).contains("Injected fault"));
+ } finally {
+ rpcServer.stop();
+ }
+ }
+
+ private Message setupAsyncConnection(TestRpcServer rpcServer, RpcClient client)
+ throws IOException, InterruptedException, ExecutionException,
+ java.util.concurrent.TimeoutException {
+ InetSocketAddress address = rpcServer.getListenerAddress();
+ if (address == null) {
+ throw new IOException("Listener channel is closed");
+ }
+ MethodDescriptor md = SERVICE.getDescriptorForType().findMethodByName("echo");
+ EchoRequestProto param = EchoRequestProto.newBuilder().setMessage("hello").build();
+
+ ServerName serverName =
+ ServerName.valueOf(address.getHostName(), address.getPort(), System.currentTimeMillis());
+
+ AsyncRpcChannel channel =
+ client.createRpcChannel(md.getService().getName(), serverName, User.getCurrent());
+
+ final Future<Message> f = channel
+ .callMethod(md, param, null, md.getOutputType().toProto(), MessageConverter.NO_CONVERTER,
+ null, 1000, HConstants.NORMAL_QOS);
+
+ return f.get(1, TimeUnit.SECONDS);
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/a11091c4/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestAsyncIPC.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestAsyncIPC.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestAsyncIPC.java
index b9d390a..7efe198 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestAsyncIPC.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestAsyncIPC.java
@@ -17,8 +17,8 @@
*/
package org.apache.hadoop.hbase.ipc;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
+import com.google.protobuf.ByteString;
+import com.google.protobuf.Descriptors.MethodDescriptor;
import io.netty.channel.ChannelHandlerContext;
import io.netty.channel.ChannelInitializer;
import io.netty.channel.ChannelOutboundHandlerAdapter;
@@ -32,7 +32,6 @@ import java.net.InetSocketAddress;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
-import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -40,12 +39,9 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.CellScannable;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.ServerName;
-import org.apache.hadoop.hbase.Waiter;
import org.apache.hadoop.hbase.client.MetricsConnection;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.RowMutations;
@@ -60,30 +56,20 @@ import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier.Re
import org.apache.hadoop.hbase.security.User;
import org.apache.hadoop.hbase.testclassification.RPCTests;
import org.apache.hadoop.hbase.testclassification.SmallTests;
-import org.apache.hadoop.util.StringUtils;
-import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameters;
-import com.google.protobuf.ByteString;
-import com.google.protobuf.Descriptors.MethodDescriptor;
-import com.google.protobuf.Message;
-import com.google.protobuf.RpcCallback;
-import com.google.protobuf.RpcChannel;
-
@RunWith(Parameterized.class)
@Category({ RPCTests.class, SmallTests.class })
public class TestAsyncIPC extends AbstractTestIPC {
private static final Log LOG = LogFactory.getLog(TestAsyncIPC.class);
- private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
-
@Parameters
public static Collection<Object[]> parameters() {
- List<Object[]> paramList = new ArrayList<Object[]>();
+ List<Object[]> paramList = new ArrayList<>();
paramList.add(new Object[] { false, false });
paramList.add(new Object[] { false, true });
paramList.add(new Object[] { true, false });
@@ -150,95 +136,6 @@ public class TestAsyncIPC extends AbstractTestIPC {
});
}
- @Test
- public void testAsyncConnectionSetup() throws Exception {
- TestRpcServer rpcServer = new TestRpcServer();
- AsyncRpcClient client = createRpcClient(CONF);
- try {
- rpcServer.start();
- InetSocketAddress address = rpcServer.getListenerAddress();
- if (address == null) {
- throw new IOException("Listener channel is closed");
- }
- MethodDescriptor md = SERVICE.getDescriptorForType().findMethodByName("echo");
- EchoRequestProto param = EchoRequestProto.newBuilder().setMessage("hello").build();
-
- RpcChannel channel =
- client.createRpcChannel(ServerName.valueOf(address.getHostName(), address.getPort(),
- System.currentTimeMillis()), User.getCurrent(), 0);
-
- final AtomicBoolean done = new AtomicBoolean(false);
-
- channel.callMethod(md, new PayloadCarryingRpcController(), param, md.getOutputType()
- .toProto(), new RpcCallback<Message>() {
- @Override
- public void run(Message parameter) {
- done.set(true);
- }
- });
-
- TEST_UTIL.waitFor(1000, new Waiter.Predicate<Exception>() {
- @Override
- public boolean evaluate() throws Exception {
- return done.get();
- }
- });
- } finally {
- client.close();
- rpcServer.stop();
- }
- }
-
- @Test
- public void testRTEDuringAsyncConnectionSetup() throws Exception {
- TestRpcServer rpcServer = new TestRpcServer();
- AsyncRpcClient client = createRpcClientRTEDuringConnectionSetup(CONF);
- try {
- rpcServer.start();
- InetSocketAddress address = rpcServer.getListenerAddress();
- if (address == null) {
- throw new IOException("Listener channel is closed");
- }
- MethodDescriptor md = SERVICE.getDescriptorForType().findMethodByName("echo");
- EchoRequestProto param = EchoRequestProto.newBuilder().setMessage("hello").build();
-
- RpcChannel channel =
- client.createRpcChannel(ServerName.valueOf(address.getHostName(), address.getPort(),
- System.currentTimeMillis()), User.getCurrent(), 0);
-
- final AtomicBoolean done = new AtomicBoolean(false);
-
- PayloadCarryingRpcController controller = new PayloadCarryingRpcController();
- controller.notifyOnFail(new RpcCallback<IOException>() {
- @Override
- public void run(IOException e) {
- done.set(true);
- LOG.info("Caught expected exception: " + e.toString());
- assertTrue(StringUtils.stringifyException(e).contains("Injected fault"));
- }
- });
-
- channel.callMethod(md, controller, param, md.getOutputType().toProto(),
- new RpcCallback<Message>() {
- @Override
- public void run(Message parameter) {
- done.set(true);
- fail("Expected an exception to have been thrown!");
- }
- });
-
- TEST_UTIL.waitFor(1000, new Waiter.Predicate<Exception>() {
- @Override
- public boolean evaluate() throws Exception {
- return done.get();
- }
- });
- } finally {
- client.close();
- rpcServer.stop();
- }
- }
-
public static void main(String[] args) throws IOException, SecurityException,
NoSuchMethodException, InterruptedException {
if (args.length != 2) {
@@ -253,7 +150,6 @@ public class TestAsyncIPC extends AbstractTestIPC {
TestRpcServer rpcServer = new TestRpcServer();
MethodDescriptor md = SERVICE.getDescriptorForType().findMethodByName("echo");
EchoRequestProto param = EchoRequestProto.newBuilder().setMessage("hello").build();
- AsyncRpcClient client = new AsyncRpcClient(conf);
KeyValue kv = BIG_CELL;
Put p = new Put(CellUtil.cloneRow(kv));
for (int i = 0; i < cellcount; i++) {
@@ -261,7 +157,7 @@ public class TestAsyncIPC extends AbstractTestIPC {
}
RowMutations rm = new RowMutations(CellUtil.cloneRow(kv));
rm.add(p);
- try {
+ try (AsyncRpcClient client = new AsyncRpcClient(conf)) {
rpcServer.start();
InetSocketAddress address = rpcServer.getListenerAddress();
if (address == null) {
@@ -270,17 +166,17 @@ public class TestAsyncIPC extends AbstractTestIPC {
long startTime = System.currentTimeMillis();
User user = User.getCurrent();
for (int i = 0; i < cycles; i++) {
- List<CellScannable> cells = new ArrayList<CellScannable>();
+ List<CellScannable> cells = new ArrayList<>();
// Message param = RequestConverter.buildMultiRequest(HConstants.EMPTY_BYTE_ARRAY, rm);
ClientProtos.RegionAction.Builder builder =
RequestConverter.buildNoDataRegionAction(HConstants.EMPTY_BYTE_ARRAY, rm, cells,
- RegionAction.newBuilder(), ClientProtos.Action.newBuilder(),
- MutationProto.newBuilder());
+ RegionAction.newBuilder(), ClientProtos.Action.newBuilder(),
+ MutationProto.newBuilder());
builder.setRegion(RegionSpecifier
.newBuilder()
.setType(RegionSpecifierType.REGION_NAME)
.setValue(
- ByteString.copyFrom(HRegionInfo.FIRST_META_REGIONINFO.getEncodedNameAsBytes())));
+ ByteString.copyFrom(HRegionInfo.FIRST_META_REGIONINFO.getEncodedNameAsBytes())));
if (i % 100000 == 0) {
LOG.info("" + i);
// Uncomment this for a thread dump every so often.
@@ -300,7 +196,6 @@ public class TestAsyncIPC extends AbstractTestIPC {
LOG.info("Cycled " + cycles + " time(s) with " + cellcount + " cell(s) in "
+ (System.currentTimeMillis() - startTime) + "ms");
} finally {
- client.close();
rpcServer.stop();
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/a11091c4/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestIPC.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestIPC.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestIPC.java
index 3fc1259..56de07d 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestIPC.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestIPC.java
@@ -22,6 +22,8 @@ import static org.mockito.Matchers.anyInt;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.spy;
+import com.google.protobuf.ByteString;
+import com.google.protobuf.Descriptors.MethodDescriptor;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.net.Socket;
@@ -59,9 +61,6 @@ import org.mockito.Mockito;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
-import com.google.protobuf.ByteString;
-import com.google.protobuf.Descriptors.MethodDescriptor;
-
@Category({ RPCTests.class, SmallTests.class })
public class TestIPC extends AbstractTestIPC {
@@ -129,7 +128,7 @@ public class TestIPC extends AbstractTestIPC {
throw new IOException("Listener channel is closed");
}
for (int i = 0; i < cycles; i++) {
- List<CellScannable> cells = new ArrayList<CellScannable>();
+ List<CellScannable> cells = new ArrayList<>();
// Message param = RequestConverter.buildMultiRequest(HConstants.EMPTY_BYTE_ARRAY, rm);
ClientProtos.RegionAction.Builder builder =
RequestConverter.buildNoDataRegionAction(HConstants.EMPTY_BYTE_ARRAY, rm, cells,
[36/50] [abbrv] hbase git commit: HBASE-15796 TestMetaCache fails
after HBASE-15745
Posted by sy...@apache.org.
HBASE-15796 TestMetaCache fails after HBASE-15745
Signed-off-by: stack <st...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/9ee0cbb9
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/9ee0cbb9
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/9ee0cbb9
Branch: refs/heads/hbase-12439
Commit: 9ee0cbb995c1d7de905f4138a199f115762725e8
Parents: fa033b6
Author: Jurriaan Mous <ju...@jurmo.us>
Authored: Sat May 7 22:02:26 2016 +0200
Committer: stack <st...@apache.org>
Committed: Sat May 7 15:29:07 2016 -0700
----------------------------------------------------------------------
.../client/AbstractRegionServerCallable.java | 44 +++++++++-----------
1 file changed, 19 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/9ee0cbb9/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AbstractRegionServerCallable.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AbstractRegionServerCallable.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AbstractRegionServerCallable.java
index ee9a781..4a0ea28 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AbstractRegionServerCallable.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AbstractRegionServerCallable.java
@@ -15,20 +15,17 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.hadoop.hbase.client;
import java.io.IOException;
-import java.net.ConnectException;
-import java.net.SocketTimeoutException;
-
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HRegionLocation;
-import org.apache.hadoop.hbase.NotServingRegionException;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.TableNotEnabledException;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.exceptions.RegionMovedException;
import org.apache.hadoop.hbase.util.Bytes;
/**
@@ -42,10 +39,14 @@ import org.apache.hadoop.hbase.util.Bytes;
* @param <T> the class that the ServerCallable handles
*/
@InterfaceAudience.Private
-abstract class AbstractRegionServerCallable<T> implements RetryingCallableBase {
+abstract class AbstractRegionServerCallable<T> implements RetryingCallable<T> {
+ // Public because used outside of this package over in ipc.
+ private static final Log LOG = LogFactory.getLog(AbstractRegionServerCallable.class);
+
protected final Connection connection;
protected final TableName tableName;
protected final byte[] row;
+
protected HRegionLocation location;
protected final static int MIN_WAIT_DEAD_SERVER = 10000;
@@ -86,22 +87,9 @@ abstract class AbstractRegionServerCallable<T> implements RetryingCallableBase {
@Override
public void throwable(Throwable t, boolean retrying) {
- if (t instanceof SocketTimeoutException ||
- t instanceof ConnectException ||
- t instanceof RetriesExhaustedException ||
- (location != null && getConnection().isDeadServer(location.getServerName()))) {
- // if thrown these exceptions, we clear all the cache entries that
- // map to that slow/dead server; otherwise, let cache miss and ask
- // hbase:meta again to find the new location
- if (this.location != null) {
- getConnection().clearCaches(location.getServerName());
- }
- } else if (t instanceof RegionMovedException) {
- getConnection().updateCachedLocations(tableName, row, t, location);
- } else if (t instanceof NotServingRegionException && !retrying) {
- // Purge cache entries for this specific region from hbase:meta cache
- // since we don't call connect(true) when number of retries is 1.
- getConnection().deleteCachedRegionLocation(location);
+ if (location != null) {
+ getConnection().updateCachedLocations(tableName, location.getRegionInfo().getRegionName(),
+ row, t, location.getServerName());
}
}
@@ -134,13 +122,19 @@ abstract class AbstractRegionServerCallable<T> implements RetryingCallableBase {
/**
* Prepare for connection to the server hosting region with row from tablename. Does lookup
* to find region location and hosting server.
- * @param reload Set this to true if connection should re-find the region
+ * @param reload Set to true to re-check the table state
* @throws IOException e
*/
@Override
public void prepare(final boolean reload) throws IOException {
+ // check table state if this is a retry
+ if (reload &&
+ !tableName.equals(TableName.META_TABLE_NAME) &&
+ getConnection().isTableDisabled(tableName)) {
+ throw new TableNotEnabledException(tableName.getNameAsString() + " is disabled.");
+ }
try (RegionLocator regionLocator = connection.getRegionLocator(tableName)) {
- this.location = regionLocator.getRegionLocation(row, reload);
+ this.location = regionLocator.getRegionLocation(row);
}
if (this.location == null) {
throw new IOException("Failed to find location, tableName=" + tableName +
[46/50] [abbrv] hbase git commit: HBASE-15797 TestIPCUtil fails after
HBASE-15795
Posted by sy...@apache.org.
HBASE-15797 TestIPCUtil fails after HBASE-15795
Signed-off-by: stack <st...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/9d17e3d2
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/9d17e3d2
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/9d17e3d2
Branch: refs/heads/hbase-12439
Commit: 9d17e3d22a03dbff99b67837184bfe731049d56f
Parents: 1174057
Author: Jurriaan Mous <ju...@jurmo.us>
Authored: Sun May 8 08:43:28 2016 +0200
Committer: stack <st...@apache.org>
Committed: Mon May 9 17:04:58 2016 -0700
----------------------------------------------------------------------
.../main/java/org/apache/hadoop/hbase/ipc/IPCUtil.java | 10 ++++++----
1 file changed, 6 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/9d17e3d2/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/IPCUtil.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/IPCUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/IPCUtil.java
index a87bc8d..74466b5 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/IPCUtil.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/IPCUtil.java
@@ -139,16 +139,16 @@ public class IPCUtil {
baos = new ByteBufferOutputStream(bufferSize);
}
Compressor poolCompressor = null;
- try (OutputStream os = baos) {
- OutputStream os2Compress = os;
+ OutputStream os = baos;
+ try {
if (compressor != null) {
if (compressor instanceof Configurable) {
((Configurable) compressor).setConf(this.conf);
}
poolCompressor = CodecPool.getCompressor(compressor);
- os2Compress = compressor.createOutputStream(os, poolCompressor);
+ os = compressor.createOutputStream(os, poolCompressor);
}
- Codec.Encoder encoder = codec.getEncoder(os2Compress);
+ Codec.Encoder encoder = codec.getEncoder(os);
int count = 0;
while (cellScanner.advance()) {
encoder.write(cellScanner.current());
@@ -163,6 +163,8 @@ public class IPCUtil {
} catch (BufferOverflowException e) {
throw new DoNotRetryIOException(e);
} finally {
+ os.close();
+
if (poolCompressor != null) {
CodecPool.returnCompressor(poolCompressor);
}
[44/50] [abbrv] hbase git commit: HBASE-15612: Minor improvements to
CellCounter and RowCounter documentation
Posted by sy...@apache.org.
HBASE-15612: Minor improvements to CellCounter and RowCounter documentation
Signed-off-by: stack <st...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/a6e29676
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/a6e29676
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/a6e29676
Branch: refs/heads/hbase-12439
Commit: a6e29676db1d2aa8ff40624573bfd93bf356d2b4
Parents: 2eced6f
Author: Esteban Gutierrez <es...@apache.org>
Authored: Thu Apr 7 10:04:07 2016 -0700
Committer: stack <st...@apache.org>
Committed: Mon May 9 14:12:43 2016 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hbase/mapreduce/CellCounter.java | 5 +++--
src/main/asciidoc/_chapters/ops_mgt.adoc | 10 ++++++++--
2 files changed, 11 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/a6e29676/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/CellCounter.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/CellCounter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/CellCounter.java
index 73f9b93..e2af762 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/CellCounter.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/CellCounter.java
@@ -64,10 +64,11 @@ import com.google.common.base.Preconditions;
* 6. Total number of versions of each qualifier.
* </pre>
*
- * The cellcounter takes two optional parameters one to use a user
+ * The cellcounter can take optional parameters to use a user
* supplied row/family/qualifier string to use in the report and
* second a regex based or prefix based row filter to restrict the
- * count operation to a limited subset of rows from the table.
+ * count operation to a limited subset of rows from the table or a
+ * start time and/or end time to limit the count to a time range.
*/
@InterfaceAudience.Public
@InterfaceStability.Stable
http://git-wip-us.apache.org/repos/asf/hbase/blob/a6e29676/src/main/asciidoc/_chapters/ops_mgt.adoc
----------------------------------------------------------------------
diff --git a/src/main/asciidoc/_chapters/ops_mgt.adoc b/src/main/asciidoc/_chapters/ops_mgt.adoc
index 4c9c7c5..583a872 100644
--- a/src/main/asciidoc/_chapters/ops_mgt.adoc
+++ b/src/main/asciidoc/_chapters/ops_mgt.adoc
@@ -304,12 +304,15 @@ The following utilities are available:
`RowCounter`::
Count rows in an HBase table.
+`CellCounter`::
+ Count cells in an HBase table.
+
`replication.VerifyReplication`::
Compare the data from tables in two different clusters.
WARNING: It doesn't work for incrementColumnValues'd cells since the timestamp is changed.
Note that this command is in a different package than the others.
-Each command except `RowCounter` accepts a single `--help` argument to print usage instructions.
+Each command except `RowCounter` and `CellCounter` accept a single `--help` argument to print usage instructions.
[[hbck]]
=== HBase `hbck`
@@ -619,7 +622,8 @@ To NOT run WALPlayer as a mapreduce job on your cluster, force it to run all in
link:http://hbase.apache.org/apidocs/org/apache/hadoop/hbase/mapreduce/RowCounter.html[RowCounter] is a mapreduce job to count all the rows of a table.
This is a good utility to use as a sanity check to ensure that HBase can read all the blocks of a table if there are any concerns of metadata inconsistency.
-It will run the mapreduce all in a single process but it will run faster if you have a MapReduce cluster in place for it to exploit.
+It will run the mapreduce all in a single process but it will run faster if you have a MapReduce cluster in place for it to exploit. It is also possible to limit
+the time range of data to be scanned by using the `--starttime=[starttime]` and `--endtime=[endtime]` flags.
----
$ bin/hbase org.apache.hadoop.hbase.mapreduce.RowCounter <tablename> [<column1> <column2>...]
@@ -642,6 +646,8 @@ The statistics gathered by RowCounter are more fine-grained and include:
The program allows you to limit the scope of the run.
Provide a row regex or prefix to limit the rows to analyze.
+Specify a time range to scan the table by using the `--starttime=[starttime]` and `--endtime=[endtime]` flags.
+
Use `hbase.mapreduce.scan.column.family` to specify scanning a single column family.
----
[48/50] [abbrv] hbase git commit: HBASE-15415 Improve Master WebUI
snapshot information (huaxiang sun)
Posted by sy...@apache.org.
HBASE-15415 Improve Master WebUI snapshot information (huaxiang sun)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/8604f9ee
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/8604f9ee
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/8604f9ee
Branch: refs/heads/hbase-12439
Commit: 8604f9eebb08eef10ea99043ea5758d08d8dbf6b
Parents: 224b03b
Author: Matteo Bertozzi <ma...@cloudera.com>
Authored: Mon May 9 20:18:13 2016 -0700
Committer: Matteo Bertozzi <ma...@cloudera.com>
Committed: Mon May 9 20:20:49 2016 -0700
----------------------------------------------------------------------
.../hbase/tmpl/master/MasterStatusTmpl.jamon | 2 +-
.../hadoop/hbase/snapshot/SnapshotInfo.java | 205 +++++++++++++++++--
.../hbase/snapshot/SnapshotReferenceUtil.java | 70 ++++---
.../hbase-webapps/master/snapshotsStats.jsp | 149 ++++++++++++++
4 files changed, 384 insertions(+), 42 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/8604f9ee/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon
index 38bcf22..e39ad8a 100644
--- a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon
+++ b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon
@@ -477,7 +477,7 @@ AssignmentManager assignmentManager = master.getAssignmentManager();
<td><% new Date(snapshotDesc.getCreationTime()) %></td>
</tr>
</%for>
- <p><% snapshots.size() %> snapshot(s) in set.</p>
+ <p><% snapshots.size() %> snapshot(s) in set. [<a href="/snapshotsStats.jsp">Snapshot Storefile stats</a>]</p>
</table>
</%if>
</%def>
http://git-wip-us.apache.org/repos/asf/hbase/blob/8604f9ee/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotInfo.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotInfo.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotInfo.java
index c944fc4..3c5033e 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotInfo.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotInfo.java
@@ -23,8 +23,12 @@ import java.io.FileNotFoundException;
import java.net.URI;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.Date;
import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutorService;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
@@ -126,6 +130,7 @@ public final class SnapshotInfo extends Configured implements Tool {
private AtomicLong hfilesArchiveSize = new AtomicLong();
private AtomicLong hfilesSize = new AtomicLong();
private AtomicLong hfilesMobSize = new AtomicLong();
+ private AtomicLong nonSharedHfilesArchiveSize = new AtomicLong();
private AtomicLong logSize = new AtomicLong();
private final HBaseProtos.SnapshotDescription snapshot;
@@ -142,6 +147,15 @@ public final class SnapshotInfo extends Configured implements Tool {
this.fs = fs;
}
+ SnapshotStats(final Configuration conf, final FileSystem fs,
+ final HBaseProtos.SnapshotDescription snapshot) {
+ this.snapshot = snapshot;
+ this.snapshotTable = TableName.valueOf(snapshot.getTable());
+ this.conf = conf;
+ this.fs = fs;
+ }
+
+
/** @return the snapshot descriptor */
public SnapshotDescription getSnapshotDescription() {
return new SnapshotDescription(this.snapshot.getName(), this.snapshot.getTable(),
@@ -207,6 +221,17 @@ public final class SnapshotInfo extends Configured implements Tool {
/** @return the total size of the store files in the mob store*/
public long getMobStoreFilesSize() { return hfilesMobSize.get(); }
+ /** @return the total size of the store files in the archive which is not shared
+ * with other snapshots and tables
+ *
+ * This is only calculated when
+ * {@link #getSnapshotStats(Configuration, HBaseProtos.SnapshotDescription, Map)}
+ * is called with a non-null Map
+ */
+ public long getNonSharedArchivedStoreFilesSize() {
+ return nonSharedHfilesArchiveSize.get();
+ }
+
/** @return the percentage of the shared store files */
public float getSharedStoreFilePercentage() {
return ((float) hfilesSize.get() / (getStoreFilesSize())) * 100;
@@ -222,15 +247,46 @@ public final class SnapshotInfo extends Configured implements Tool {
return logSize.get();
}
+ /** Check if for a give file in archive, if there are other snapshots/tables still
+ * reference it.
+ * @param filePath file path in archive
+ * @param snapshotFilesMap a map for store files in snapshots about how many snapshots refer
+ * to it.
+ * @return true or false
+ */
+ private boolean isArchivedFileStillReferenced(final Path filePath,
+ final Map<Path, Integer> snapshotFilesMap) {
+
+ Integer c = snapshotFilesMap.get(filePath);
+
+ // Check if there are other snapshots or table from clone_snapshot() (via back-reference)
+ // still reference to it.
+ if ((c != null) && (c == 1)) {
+ Path parentDir = filePath.getParent();
+ Path backRefDir = HFileLink.getBackReferencesDir(parentDir, filePath.getName());
+ try {
+ if (FSUtils.listStatus(fs, backRefDir) == null) {
+ return false;
+ }
+ } catch (IOException e) {
+ // For the purpose of this function, IOException is ignored and treated as
+ // the file is still being referenced.
+ }
+ }
+ return true;
+ }
+
/**
* Add the specified store file to the stats
* @param region region encoded Name
* @param family family name
* @param storeFile store file name
+ * @param filesMap store files map for all snapshots, it may be null
* @return the store file information
*/
FileInfo addStoreFile(final HRegionInfo region, final String family,
- final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
+ final SnapshotRegionManifest.StoreFile storeFile,
+ final Map<Path, Integer> filesMap) throws IOException {
HFileLink link = HFileLink.build(conf, snapshotTable, region.getEncodedName(),
family, storeFile.getName());
boolean isCorrupted = false;
@@ -241,6 +297,13 @@ public final class SnapshotInfo extends Configured implements Tool {
size = fs.getFileStatus(link.getArchivePath()).getLen();
hfilesArchiveSize.addAndGet(size);
hfilesArchiveCount.incrementAndGet();
+
+ // If store file is not shared with other snapshots and tables,
+ // increase nonSharedHfilesArchiveSize
+ if ((filesMap != null) &&
+ !isArchivedFileStillReferenced(link.getArchivePath(), filesMap)) {
+ nonSharedHfilesArchiveSize.addAndGet(size);
+ }
} else if (inArchive = fs.exists(link.getMobPath())) {
size = fs.getFileStatus(link.getMobPath()).getLen();
hfilesMobSize.addAndGet(size);
@@ -426,13 +489,14 @@ public final class SnapshotInfo extends Configured implements Tool {
snapshotDesc.getOwner(), snapshotDesc.getCreationTime(), snapshotDesc.getVersion());
final SnapshotStats stats = new SnapshotStats(this.getConf(), this.fs, desc);
SnapshotReferenceUtil.concurrentVisitReferencedFiles(getConf(), fs, snapshotManifest,
+ "SnapshotInfo",
new SnapshotReferenceUtil.SnapshotVisitor() {
@Override
public void storeFile(final HRegionInfo regionInfo, final String family,
final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
if (storeFile.hasReference()) return;
- SnapshotStats.FileInfo info = stats.addStoreFile(regionInfo, family, storeFile);
+ SnapshotStats.FileInfo info = stats.addStoreFile(regionInfo, family, storeFile, null);
if (showFiles) {
String state = info.getStateToString();
System.out.printf("%8s %s/%s/%s/%s %s%n",
@@ -501,22 +565,36 @@ public final class SnapshotInfo extends Configured implements Tool {
*/
public static SnapshotStats getSnapshotStats(final Configuration conf,
final SnapshotDescription snapshot) throws IOException {
- HBaseProtos.SnapshotDescription snapshotDesc = ProtobufUtil.createHBaseProtosSnapshotDesc(snapshot);
+ HBaseProtos.SnapshotDescription snapshotDesc = ProtobufUtil.createHBaseProtosSnapshotDesc(
+ snapshot);
+
+ return getSnapshotStats(conf, snapshotDesc, null);
+ }
+
+ /**
+ * Returns the snapshot stats
+ * @param conf the {@link Configuration} to use
+ * @param snapshotDesc HBaseProtos.SnapshotDescription to get stats from
+ * @param filesMap {@link Map} store files map for all snapshots, it may be null
+ * @return the snapshot stats
+ */
+ public static SnapshotStats getSnapshotStats(final Configuration conf,
+ final HBaseProtos.SnapshotDescription snapshotDesc,
+ final Map<Path, Integer> filesMap) throws IOException {
Path rootDir = FSUtils.getRootDir(conf);
FileSystem fs = FileSystem.get(rootDir.toUri(), conf);
Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotDesc, rootDir);
SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc);
- final SnapshotStats stats = new SnapshotStats(conf, fs, snapshot);
+ final SnapshotStats stats = new SnapshotStats(conf, fs, snapshotDesc);
SnapshotReferenceUtil.concurrentVisitReferencedFiles(conf, fs, manifest,
- new SnapshotReferenceUtil.SnapshotVisitor() {
- @Override
- public void storeFile(final HRegionInfo regionInfo, final String family,
- final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
- if (!storeFile.hasReference()) {
- stats.addStoreFile(regionInfo, family, storeFile);
- }
- }
- });
+ "SnapshotsStatsAggregation", new SnapshotReferenceUtil.SnapshotVisitor() {
+ @Override
+ public void storeFile(final HRegionInfo regionInfo, final String family,
+ final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
+ if (!storeFile.hasReference()) {
+ stats.addStoreFile(regionInfo, family, storeFile, filesMap);
+ }
+ }});
return stats;
}
@@ -531,7 +609,7 @@ public final class SnapshotInfo extends Configured implements Tool {
FileSystem fs = FileSystem.get(rootDir.toUri(), conf);
Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
FileStatus[] snapshots = fs.listStatus(snapshotDir,
- new SnapshotDescriptionUtils.CompletedSnaphotDirectoriesFilter(fs));
+ new SnapshotDescriptionUtils.CompletedSnaphotDirectoriesFilter(fs));
List<SnapshotDescription> snapshotLists =
new ArrayList<SnapshotDescription>(snapshots.length);
for (FileStatus snapshotDirStat: snapshots) {
@@ -545,6 +623,105 @@ public final class SnapshotInfo extends Configured implements Tool {
}
/**
+ * Gets the store files map for snapshot
+ * @param conf the {@link Configuration} to use
+ * @param snapshot {@link SnapshotDescription} to get stats from
+ * @param exec the {@link ExecutorService} to use
+ * @param filesMap {@link Map} the map to put the mapping entries
+ * @param uniqueHFilesArchiveSize {@link AtomicLong} the accumulated store file size in archive
+ * @param uniqueHFilesSize {@link AtomicLong} the accumulated store file size shared
+ * @param uniqueHFilesMobSize {@link AtomicLong} the accumulated mob store file size shared
+ * @return the snapshot stats
+ */
+ private static void getSnapshotFilesMap(final Configuration conf,
+ final SnapshotDescription snapshot, final ExecutorService exec,
+ final ConcurrentHashMap<Path, Integer> filesMap,
+ final AtomicLong uniqueHFilesArchiveSize, final AtomicLong uniqueHFilesSize,
+ final AtomicLong uniqueHFilesMobSize) throws IOException {
+ HBaseProtos.SnapshotDescription snapshotDesc = ProtobufUtil.createHBaseProtosSnapshotDesc(
+ snapshot);
+ Path rootDir = FSUtils.getRootDir(conf);
+ final FileSystem fs = FileSystem.get(rootDir.toUri(), conf);
+
+ Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotDesc, rootDir);
+ SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc);
+ SnapshotReferenceUtil.concurrentVisitReferencedFiles(conf, fs, manifest, exec,
+ new SnapshotReferenceUtil.SnapshotVisitor() {
+ @Override public void storeFile(final HRegionInfo regionInfo, final String family,
+ final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
+ if (!storeFile.hasReference()) {
+ HFileLink link = HFileLink
+ .build(conf, TableName.valueOf(snapshot.getTable()), regionInfo.getEncodedName(),
+ family, storeFile.getName());
+ long size;
+ Integer count;
+ Path p;
+ AtomicLong al;
+ int c = 0;
+
+ if (fs.exists(link.getArchivePath())) {
+ p = link.getArchivePath();
+ al = uniqueHFilesArchiveSize;
+ size = fs.getFileStatus(p).getLen();
+ } else if (fs.exists(link.getMobPath())) {
+ p = link.getMobPath();
+ al = uniqueHFilesMobSize;
+ size = fs.getFileStatus(p).getLen();
+ } else {
+ p = link.getOriginPath();
+ al = uniqueHFilesSize;
+ size = link.getFileStatus(fs).getLen();
+ }
+
+ // If it has been counted, do not double count
+ count = filesMap.get(p);
+ if (count != null) {
+ c = count.intValue();
+ } else {
+ al.addAndGet(size);
+ }
+
+ filesMap.put(p, ++c);
+ }
+ }
+ });
+ }
+
+ /**
+ * Returns the map of store files based on path for all snapshots
+ * @param conf the {@link Configuration} to use
+ * @param uniqueHFilesArchiveSize pass out the size for store files in archive
+ * @param uniqueHFilesSize pass out the size for store files shared
+ * @param uniqueHFilesMobSize pass out the size for mob store files shared
+ * @return the map of store files
+ */
+ public static Map<Path, Integer> getSnapshotsFilesMap(final Configuration conf,
+ AtomicLong uniqueHFilesArchiveSize, AtomicLong uniqueHFilesSize,
+ AtomicLong uniqueHFilesMobSize) throws IOException {
+ List<SnapshotDescription> snapshotList = getSnapshotList(conf);
+
+
+ if (snapshotList.size() == 0) {
+ return Collections.emptyMap();
+ }
+
+ ConcurrentHashMap<Path, Integer> fileMap = new ConcurrentHashMap<>();
+
+ ExecutorService exec = SnapshotManifest.createExecutor(conf, "SnapshotsFilesMapping");
+
+ try {
+ for (final SnapshotDescription snapshot : snapshotList) {
+ getSnapshotFilesMap(conf, snapshot, exec, fileMap, uniqueHFilesArchiveSize,
+ uniqueHFilesSize, uniqueHFilesMobSize);
+ }
+ } finally {
+ exec.shutdown();
+ }
+
+ return fileMap;
+ }
+
+ /**
* The guts of the {@link #main} method.
* Call this method to avoid the {@link #main(String[])} System.exit.
* @param args
http://git-wip-us.apache.org/repos/asf/hbase/blob/8604f9ee/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotReferenceUtil.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotReferenceUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotReferenceUtil.java
index ba8087b..c655dd8 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotReferenceUtil.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotReferenceUtil.java
@@ -169,7 +169,7 @@ public final class SnapshotReferenceUtil {
final SnapshotManifest manifest) throws IOException {
final SnapshotDescription snapshotDesc = manifest.getSnapshotDescription();
final Path snapshotDir = manifest.getSnapshotDir();
- concurrentVisitReferencedFiles(conf, fs, manifest, new StoreFileVisitor() {
+ concurrentVisitReferencedFiles(conf, fs, manifest, "VerifySnapshot", new StoreFileVisitor() {
@Override
public void storeFile(final HRegionInfo regionInfo, final String family,
final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
@@ -179,7 +179,28 @@ public final class SnapshotReferenceUtil {
}
public static void concurrentVisitReferencedFiles(final Configuration conf, final FileSystem fs,
- final SnapshotManifest manifest, final StoreFileVisitor visitor) throws IOException {
+ final SnapshotManifest manifest, final String desc, final StoreFileVisitor visitor)
+ throws IOException {
+
+ final Path snapshotDir = manifest.getSnapshotDir();
+ List<SnapshotRegionManifest> regionManifests = manifest.getRegionManifests();
+ if (regionManifests == null || regionManifests.size() == 0) {
+ LOG.debug("No manifest files present: " + snapshotDir);
+ return;
+ }
+
+ ExecutorService exec = SnapshotManifest.createExecutor(conf, desc);
+
+ try {
+ concurrentVisitReferencedFiles(conf, fs, manifest, exec, visitor);
+ } finally {
+ exec.shutdown();
+ }
+ }
+
+ public static void concurrentVisitReferencedFiles(final Configuration conf, final FileSystem fs,
+ final SnapshotManifest manifest, final ExecutorService exec, final StoreFileVisitor visitor)
+ throws IOException {
final SnapshotDescription snapshotDesc = manifest.getSnapshotDescription();
final Path snapshotDir = manifest.getSnapshotDir();
@@ -189,37 +210,32 @@ public final class SnapshotReferenceUtil {
return;
}
- ExecutorService exec = SnapshotManifest.createExecutor(conf, "VerifySnapshot");
final ExecutorCompletionService<Void> completionService =
new ExecutorCompletionService<Void>(exec);
+
+ for (final SnapshotRegionManifest regionManifest : regionManifests) {
+ completionService.submit(new Callable<Void>() {
+ @Override public Void call() throws IOException {
+ visitRegionStoreFiles(regionManifest, visitor);
+ return null;
+ }
+ });
+ }
try {
- for (final SnapshotRegionManifest regionManifest: regionManifests) {
- completionService.submit(new Callable<Void>() {
- @Override
- public Void call() throws IOException {
- visitRegionStoreFiles(regionManifest, visitor);
- return null;
- }
- });
+ for (int i = 0; i < regionManifests.size(); ++i) {
+ completionService.take().get();
}
- try {
- for (int i = 0; i < regionManifests.size(); ++i) {
- completionService.take().get();
- }
- } catch (InterruptedException e) {
- throw new InterruptedIOException(e.getMessage());
- } catch (ExecutionException e) {
- if (e.getCause() instanceof CorruptedSnapshotException) {
- throw new CorruptedSnapshotException(e.getCause().getMessage(),
+ } catch (InterruptedException e) {
+ throw new InterruptedIOException(e.getMessage());
+ } catch (ExecutionException e) {
+ if (e.getCause() instanceof CorruptedSnapshotException) {
+ throw new CorruptedSnapshotException(e.getCause().getMessage(),
ProtobufUtil.createSnapshotDesc(snapshotDesc));
- } else {
- IOException ex = new IOException();
- ex.initCause(e.getCause());
- throw ex;
- }
+ } else {
+ IOException ex = new IOException();
+ ex.initCause(e.getCause());
+ throw ex;
}
- } finally {
- exec.shutdown();
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/8604f9ee/hbase-server/src/main/resources/hbase-webapps/master/snapshotsStats.jsp
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/resources/hbase-webapps/master/snapshotsStats.jsp b/hbase-server/src/main/resources/hbase-webapps/master/snapshotsStats.jsp
new file mode 100644
index 0000000..54c7437
--- /dev/null
+++ b/hbase-server/src/main/resources/hbase-webapps/master/snapshotsStats.jsp
@@ -0,0 +1,149 @@
+<%--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+--%>
+<%@ page contentType="text/html;charset=UTF-8"
+ import="java.util.concurrent.atomic.AtomicLong"
+ import="java.util.Date"
+ import="java.util.List"
+ import="java.util.Map"
+ import="org.apache.hadoop.conf.Configuration"
+ import="org.apache.hadoop.fs.Path"
+ import="org.apache.hadoop.hbase.HBaseConfiguration"
+ import="org.apache.hadoop.hbase.master.HMaster"
+ import="org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription"
+ import="org.apache.hadoop.hbase.snapshot.SnapshotInfo"
+ import="org.apache.hadoop.hbase.TableName"
+ import="org.apache.hadoop.util.StringUtils" %>
+<%
+ HMaster master = (HMaster)getServletContext().getAttribute(HMaster.MASTER);
+ Configuration conf = master.getConfiguration();
+ AtomicLong totalSharedSize = new AtomicLong();
+ AtomicLong totalArchivedSize = new AtomicLong();
+ AtomicLong totalMobSize = new AtomicLong();
+ long totalSize = 0;
+ long totalUnsharedArchivedSize = 0;
+
+ Map<Path, Integer> filesMap = null;
+
+ List<SnapshotDescription> snapshots = master.isInitialized() ?
+ master.getSnapshotManager().getCompletedSnapshots() : null;
+
+ if (snapshots != null && snapshots.size() > 0) {
+ filesMap = SnapshotInfo.getSnapshotsFilesMap(master.getConfiguration(),
+ totalArchivedSize, totalSharedSize, totalMobSize);
+ totalSize = totalSharedSize.get() + totalArchivedSize.get() + totalMobSize.get();
+ }
+%>
+<!--[if IE]>
+<!DOCTYPE html>
+<![endif]-->
+<?xml version="1.0" encoding="UTF-8" ?>
+<html xmlns="http://www.w3.org/1999/xhtml">
+<head>
+ <meta charset="utf-8">
+ <title>HBase Master Snapshots: <%= master.getServerName() %></title>
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+ <meta name="description" content="">
+ <meta name="author" content="">
+
+ <link href="/static/css/bootstrap.min.css" rel="stylesheet">
+ <link href="/static/css/bootstrap-theme.min.css" rel="stylesheet">
+ <link href="/static/css/hbase.css" rel="stylesheet">
+ </head>
+<body>
+<div class="navbar navbar-fixed-top navbar-default">
+ <div class="container-fluid">
+ <div class="navbar-header">
+ <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-collapse">
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+ <a class="navbar-brand" href="/master-status"><img src="/static/hbase_logo_small.png" alt="HBase Logo"/></a>
+ </div>
+ <div class="collapse navbar-collapse">
+ <ul class="nav navbar-nav">
+ <li><a href="/master-status">Home</a></li>
+ <li><a href="/tablesDetailed.jsp">Table Details</a></li>
+ <li><a href="/procedures.jsp">Procedures</a></li>
+ <li><a href="/logs/">Local Logs</a></li>
+ <li><a href="/logLevel">Log Level</a></li>
+ <li><a href="/dump">Debug Dump</a></li>
+ <li><a href="/jmx">Metrics Dump</a></li>
+ <% if (HBaseConfiguration.isShowConfInServlet()) { %>
+ <li><a href="/conf">HBase Configuration</a></li>
+ <% } %>
+ </ul>
+ </div><!--/.nav-collapse -->
+ </div>
+</div>
+<div class="container-fluid content">
+ <div class="row">
+ <div class="page-header">
+ <h1>Snapshot Storefile Stats</h1>
+ </div>
+ </div>
+ <table class="table table-striped" width="90%" >
+ <tr>
+ <th>Snapshot Name</th>
+ <th>Table</th>
+ <th>Creation Time</th>
+ <th>Shared Storefile Size</th>
+ <th>Mob Storefile Size</th>
+ <th>Archived Storefile Size</th>
+ </tr>
+ <%for (SnapshotDescription snapshotDesc : snapshots) { %>
+ <tr>
+ <td><a href="/snapshot.jsp?name=<%= snapshotDesc.getName() %>">
+ <%= snapshotDesc.getName() %></a></td>
+ <%
+ TableName snapshotTable = TableName.valueOf(snapshotDesc.getTable());
+ SnapshotInfo.SnapshotStats stats = SnapshotInfo.getSnapshotStats(master.getConfiguration(),
+ snapshotDesc, filesMap);
+ totalUnsharedArchivedSize += stats.getNonSharedArchivedStoreFilesSize();
+ %>
+ <td><a href="/table.jsp?name=<%= snapshotTable.getNameAsString() %>">
+ <%= snapshotTable.getNameAsString() %></a></td>
+ <td><%= new Date(snapshotDesc.getCreationTime()) %></td>
+ <td><%= StringUtils.humanReadableInt(stats.getSharedStoreFilesSize()) %></td>
+ <td><%= StringUtils.humanReadableInt(stats.getMobStoreFilesSize()) %></td>
+ <td><%= StringUtils.humanReadableInt(stats.getArchivedStoreFileSize()) %>
+ (<%= StringUtils.humanReadableInt(stats.getNonSharedArchivedStoreFilesSize()) %>)</td>
+ </tr>
+ <% } %>
+ <p><%= snapshots.size() %> snapshot(s) in set.</p>
+ <p>Total Storefile Size: <%= StringUtils.humanReadableInt(totalSize) %></p>
+ <p>Total Shared Storefile Size: <%= StringUtils.humanReadableInt(totalSharedSize.get()) %>,
+ Total Mob Storefile Size: <%= StringUtils.humanReadableInt(totalMobSize.get()) %>,
+ Total Archived Storefile Size: <%= StringUtils.humanReadableInt(totalArchivedSize.get()) %>
+ (<%= StringUtils.humanReadableInt(totalUnsharedArchivedSize) %>)</p>
+ <p>Shared Storefile Size is the Storefile size shared between snapshots and active tables.
+ Mob Storefile Size is the Mob Storefile size shared between snapshots and active tables.
+ Archived Storefile Size is the Storefile size in Archive.
+ The format of Archived Storefile Size is NNN(MMM). NNN is the total Storefile
+ size in Archive, MMM is the total Storefile size in Archive that is specific
+ to the snapshot (not shared with other snapshots and tables)</p>
+ </table>
+</div>
+
+<script src="/static/js/jquery.min.js" type="text/javascript"></script>
+<script src="/static/js/bootstrap.min.js" type="text/javascript"></script>
+
+</body>
+</html>
[17/50] [abbrv] hbase git commit: HBASE-15575 Rename table DDL
*Handler methods in MasterObserver to more meaningful names (Stephen Yuan
Jiang)
Posted by sy...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase/blob/b7ce55f0/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TruncateTableProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TruncateTableProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TruncateTableProcedure.java
index 40f8157..9abed52 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TruncateTableProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TruncateTableProcedure.java
@@ -264,7 +264,7 @@ public class TruncateTableProcedure
user.doAs(new PrivilegedExceptionAction<Void>() {
@Override
public Void run() throws Exception {
- cpHost.preTruncateTableHandler(tableName);
+ cpHost.preTruncateTableAction(tableName);
return null;
}
});
@@ -280,7 +280,7 @@ public class TruncateTableProcedure
user.doAs(new PrivilegedExceptionAction<Void>() {
@Override
public Void run() throws Exception {
- cpHost.postTruncateTableHandler(tableName);
+ cpHost.postCompletedTruncateTableAction(tableName);
return null;
}
});
http://git-wip-us.apache.org/repos/asf/hbase/blob/b7ce55f0/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
index e866f29..f21d8e2 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
@@ -1005,8 +1005,10 @@ public class AccessController extends BaseMasterAndRegionObserver
}
@Override
- public void postCreateTableHandler(final ObserverContext<MasterCoprocessorEnvironment> c,
- HTableDescriptor desc, HRegionInfo[] regions) throws IOException {
+ public void postCompletedCreateTableAction(
+ final ObserverContext<MasterCoprocessorEnvironment> c,
+ final HTableDescriptor desc,
+ final HRegionInfo[] regions) throws IOException {
// When AC is used, it should be configured as the 1st CP.
// In Master, the table operations like create, are handled by a Thread pool but the max size
// for this pool is 1. So if multiple CPs create tables on startup, these creations will happen
http://git-wip-us.apache.org/repos/asf/hbase/blob/b7ce55f0/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java
index 5257cee..73917cd 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java
@@ -152,24 +152,24 @@ public class TestMasterObserver {
private boolean postRestoreSnapshotCalled;
private boolean preDeleteSnapshotCalled;
private boolean postDeleteSnapshotCalled;
- private boolean preCreateTableHandlerCalled;
- private boolean postCreateTableHandlerCalled;
- private boolean preDeleteTableHandlerCalled;
- private boolean postDeleteTableHandlerCalled;
- private boolean preTruncateTableHandlerCalled;
- private boolean postTruncateTableHandlerCalled;
- private boolean preAddColumnHandlerCalled;
- private boolean postAddColumnHandlerCalled;
- private boolean preModifyColumnHandlerCalled;
- private boolean postModifyColumnHandlerCalled;
- private boolean preDeleteColumnHandlerCalled;
- private boolean postDeleteColumnHandlerCalled;
- private boolean preEnableTableHandlerCalled;
- private boolean postEnableTableHandlerCalled;
- private boolean preDisableTableHandlerCalled;
- private boolean postDisableTableHandlerCalled;
- private boolean preModifyTableHandlerCalled;
- private boolean postModifyTableHandlerCalled;
+ private boolean preCreateTableActionCalled;
+ private boolean postCompletedCreateTableActionCalled;
+ private boolean preDeleteTableActionCalled;
+ private boolean postCompletedDeleteTableActionCalled;
+ private boolean preTruncateTableActionCalled;
+ private boolean postCompletedTruncateTableActionCalled;
+ private boolean preAddColumnFamilyActionCalled;
+ private boolean postCompletedAddColumnFamilyActionCalled;
+ private boolean preModifyColumnFamilyActionCalled;
+ private boolean postCompletedModifyColumnFamilyActionCalled;
+ private boolean preDeleteColumnFamilyActionCalled;
+ private boolean postCompletedDeleteColumnFamilyActionCalled;
+ private boolean preEnableTableActionCalled;
+ private boolean postCompletedEnableTableActionCalled;
+ private boolean preDisableTableActionCalled;
+ private boolean postCompletedDisableTableActionCalled;
+ private boolean preModifyTableActionCalled;
+ private boolean postCompletedModifyTableActionCalled;
private boolean preGetTableDescriptorsCalled;
private boolean postGetTableDescriptorsCalled;
private boolean postGetTableNamesCalled;
@@ -236,24 +236,24 @@ public class TestMasterObserver {
postRestoreSnapshotCalled = false;
preDeleteSnapshotCalled = false;
postDeleteSnapshotCalled = false;
- preCreateTableHandlerCalled = false;
- postCreateTableHandlerCalled = false;
- preDeleteTableHandlerCalled = false;
- postDeleteTableHandlerCalled = false;
- preTruncateTableHandlerCalled = false;
- postTruncateTableHandlerCalled = false;
- preModifyTableHandlerCalled = false;
- postModifyTableHandlerCalled = false;
- preAddColumnHandlerCalled = false;
- postAddColumnHandlerCalled = false;
- preModifyColumnHandlerCalled = false;
- postModifyColumnHandlerCalled = false;
- preDeleteColumnHandlerCalled = false;
- postDeleteColumnHandlerCalled = false;
- preEnableTableHandlerCalled = false;
- postEnableTableHandlerCalled = false;
- preDisableTableHandlerCalled = false;
- postDisableTableHandlerCalled = false;
+ preCreateTableActionCalled = false;
+ postCompletedCreateTableActionCalled = false;
+ preDeleteTableActionCalled = false;
+ postCompletedDeleteTableActionCalled = false;
+ preTruncateTableActionCalled = false;
+ postCompletedTruncateTableActionCalled = false;
+ preModifyTableActionCalled = false;
+ postCompletedModifyTableActionCalled = false;
+ preAddColumnFamilyActionCalled = false;
+ postCompletedAddColumnFamilyActionCalled = false;
+ preModifyColumnFamilyActionCalled = false;
+ postCompletedModifyColumnFamilyActionCalled = false;
+ preDeleteColumnFamilyActionCalled = false;
+ postCompletedDeleteColumnFamilyActionCalled = false;
+ preEnableTableActionCalled = false;
+ postCompletedEnableTableActionCalled = false;
+ preDisableTableActionCalled = false;
+ postCompletedDisableTableActionCalled = false;
preGetTableDescriptorsCalled = false;
postGetTableDescriptorsCalled = false;
postGetTableNamesCalled = false;
@@ -962,109 +962,169 @@ public class TestMasterObserver {
return preDeleteSnapshotCalled && postDeleteSnapshotCalled;
}
+ @Deprecated
@Override
public void preCreateTableHandler(
ObserverContext<MasterCoprocessorEnvironment> env,
HTableDescriptor desc, HRegionInfo[] regions) throws IOException {
+ }
+
+ @Override
+ public void preCreateTableAction(
+ final ObserverContext<MasterCoprocessorEnvironment> env,
+ final HTableDescriptor desc,
+ final HRegionInfo[] regions) throws IOException {
if (bypass) {
env.bypass();
}
- preCreateTableHandlerCalled = true;
+ preCreateTableActionCalled = true;
}
+ @Deprecated
@Override
public void postCreateTableHandler(
ObserverContext<MasterCoprocessorEnvironment> ctx,
HTableDescriptor desc, HRegionInfo[] regions) throws IOException {
- postCreateTableHandlerCalled = true;
+ }
+
+ @Override
+ public void postCompletedCreateTableAction(
+ final ObserverContext<MasterCoprocessorEnvironment> ctx,
+ final HTableDescriptor desc,
+ final HRegionInfo[] regions) throws IOException {
+ postCompletedCreateTableActionCalled = true;
tableCreationLatch.countDown();
}
- public boolean wasPreCreateTableHandlerCalled(){
- return preCreateTableHandlerCalled;
+ public boolean wasPreCreateTableActionCalled(){
+ return preCreateTableActionCalled;
}
- public boolean wasCreateTableHandlerCalled() {
- return preCreateTableHandlerCalled && postCreateTableHandlerCalled;
+ public boolean wasCreateTableActionCalled() {
+ return preCreateTableActionCalled && postCompletedCreateTableActionCalled;
}
- public boolean wasCreateTableHandlerCalledOnly() {
- return preCreateTableHandlerCalled && !postCreateTableHandlerCalled;
+ public boolean wasCreateTableActionCalledOnly() {
+ return preCreateTableActionCalled && !postCompletedCreateTableActionCalled;
}
+ @Deprecated
@Override
public void preDeleteTableHandler(
ObserverContext<MasterCoprocessorEnvironment> env, TableName tableName)
throws IOException {
+ }
+
+ @Deprecated
+ @Override
+ public void postDeleteTableHandler(
+ ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName)
+ throws IOException {
+ }
+
+ @Override
+ public void preDeleteTableAction(
+ final ObserverContext<MasterCoprocessorEnvironment> env, final TableName tableName)
+ throws IOException {
if (bypass) {
env.bypass();
}
- preDeleteTableHandlerCalled = true;
+ preDeleteTableActionCalled = true;
}
@Override
- public void postDeleteTableHandler(
- ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName)
+ public void postCompletedDeleteTableAction(
+ final ObserverContext<MasterCoprocessorEnvironment> ctx, final TableName tableName)
throws IOException {
- postDeleteTableHandlerCalled = true;
+ postCompletedDeleteTableActionCalled = true;
tableDeletionLatch.countDown();
}
- public boolean wasDeleteTableHandlerCalled() {
- return preDeleteTableHandlerCalled && postDeleteTableHandlerCalled;
+ public boolean wasDeleteTableActionCalled() {
+ return preDeleteTableActionCalled && postCompletedDeleteTableActionCalled;
}
- public boolean wasDeleteTableHandlerCalledOnly() {
- return preDeleteTableHandlerCalled && !postDeleteTableHandlerCalled;
+ public boolean wasDeleteTableActionCalledOnly() {
+ return preDeleteTableActionCalled && !postCompletedDeleteTableActionCalled;
}
+ @Deprecated
@Override
public void preTruncateTableHandler(
ObserverContext<MasterCoprocessorEnvironment> env, TableName tableName)
throws IOException {
+ }
+
+ @Deprecated
+ @Override
+ public void postTruncateTableHandler(
+ ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName)
+ throws IOException {
+ }
+
+ @Override
+ public void preTruncateTableAction(
+ final ObserverContext<MasterCoprocessorEnvironment> env, final TableName tableName)
+ throws IOException {
if (bypass) {
env.bypass();
}
- preTruncateTableHandlerCalled = true;
+ preTruncateTableActionCalled = true;
}
@Override
- public void postTruncateTableHandler(
- ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName)
+ public void postCompletedTruncateTableAction(
+ final ObserverContext<MasterCoprocessorEnvironment> ctx, final TableName tableName)
throws IOException {
- postTruncateTableHandlerCalled = true;
+ postCompletedTruncateTableActionCalled = true;
}
- public boolean wasTruncateTableHandlerCalled() {
- return preTruncateTableHandlerCalled && postTruncateTableHandlerCalled;
+ public boolean wasTruncateTableActionCalled() {
+ return preTruncateTableActionCalled && postCompletedTruncateTableActionCalled;
}
- public boolean wasTruncateTableHandlerCalledOnly() {
- return preTruncateTableHandlerCalled && !postTruncateTableHandlerCalled;
+ public boolean wasTruncateTableActionCalledOnly() {
+ return preTruncateTableActionCalled && !postCompletedTruncateTableActionCalled;
}
+ @Deprecated
@Override
public void preModifyTableHandler(
ObserverContext<MasterCoprocessorEnvironment> env, TableName tableName,
HTableDescriptor htd) throws IOException {
- if (bypass) {
- env.bypass();
- }
- preModifyTableHandlerCalled = true;
}
+ @Deprecated
@Override
public void postModifyTableHandler(
ObserverContext<MasterCoprocessorEnvironment> env, TableName tableName,
HTableDescriptor htd) throws IOException {
- postModifyTableHandlerCalled = true;
}
- public boolean wasModifyTableHandlerCalled() {
- return preModifyColumnHandlerCalled && postModifyColumnHandlerCalled;
+ @Override
+ public void preModifyTableAction(
+ final ObserverContext<MasterCoprocessorEnvironment> env,
+ final TableName tableName,
+ final HTableDescriptor htd) throws IOException {
+ if (bypass) {
+ env.bypass();
+ }
+ preModifyTableActionCalled = true;
+ }
+
+ @Override
+ public void postCompletedModifyTableAction(
+ final ObserverContext<MasterCoprocessorEnvironment> env,
+ final TableName tableName,
+ final HTableDescriptor htd) throws IOException {
+ postCompletedModifyTableActionCalled = true;
+ }
+
+ public boolean wasModifyTableActionCalled() {
+ return preModifyColumnFamilyActionCalled && postCompletedModifyColumnFamilyActionCalled;
}
- public boolean wasModifyTableHandlerCalledOnly() {
- return preModifyColumnHandlerCalled && !postModifyColumnHandlerCalled;
+ public boolean wasModifyTableActionCalledOnly() {
+ return preModifyColumnFamilyActionCalled && !postCompletedModifyColumnFamilyActionCalled;
}
@Deprecated
@@ -1075,13 +1135,14 @@ public class TestMasterObserver {
}
@Override
- public void preAddColumnFamilyHandler(
- ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName,
- HColumnDescriptor columnFamily) throws IOException {
+ public void preAddColumnFamilyAction(
+ final ObserverContext<MasterCoprocessorEnvironment> ctx,
+ final TableName tableName,
+ final HColumnDescriptor columnFamily) throws IOException {
if (bypass) {
ctx.bypass();
}
- preAddColumnHandlerCalled = true;
+ preAddColumnFamilyActionCalled = true;
}
@Deprecated
@@ -1092,18 +1153,19 @@ public class TestMasterObserver {
}
@Override
- public void postAddColumnFamilyHandler(
- ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName,
- HColumnDescriptor columnFamily) throws IOException {
- postAddColumnHandlerCalled = true;
+ public void postCompletedAddColumnFamilyAction(
+ final ObserverContext<MasterCoprocessorEnvironment> ctx,
+ final TableName tableName,
+ final HColumnDescriptor columnFamily) throws IOException {
+ postCompletedAddColumnFamilyActionCalled = true;
}
- public boolean wasAddColumnHandlerCalled() {
- return preAddColumnHandlerCalled && postAddColumnHandlerCalled;
+ public boolean wasAddColumnFamilyActionCalled() {
+ return preAddColumnFamilyActionCalled && postCompletedAddColumnFamilyActionCalled;
}
- public boolean preAddColumnHandlerCalledOnly() {
- return preAddColumnHandlerCalled && !postAddColumnHandlerCalled;
+ public boolean preAddColumnFamilyActionCalledOnly() {
+ return preAddColumnFamilyActionCalled && !postCompletedAddColumnFamilyActionCalled;
}
@Deprecated
@@ -1114,13 +1176,14 @@ public class TestMasterObserver {
}
@Override
- public void preModifyColumnFamilyHandler(
- ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName,
- HColumnDescriptor columnFamily) throws IOException {
+ public void preModifyColumnFamilyAction(
+ final ObserverContext<MasterCoprocessorEnvironment> ctx,
+ final TableName tableName,
+ final HColumnDescriptor columnFamily) throws IOException {
if (bypass) {
ctx.bypass();
}
- preModifyColumnHandlerCalled = true;
+ preModifyColumnFamilyActionCalled = true;
}
@Deprecated
@@ -1131,18 +1194,18 @@ public class TestMasterObserver {
}
@Override
- public void postModifyColumnFamilyHandler(
+ public void postCompletedModifyColumnFamilyAction(
ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName,
HColumnDescriptor columnFamily) throws IOException {
- postModifyColumnHandlerCalled = true;
+ postCompletedModifyColumnFamilyActionCalled = true;
}
- public boolean wasModifyColumnHandlerCalled() {
- return preModifyColumnHandlerCalled && postModifyColumnHandlerCalled;
+ public boolean wasModifyColumnFamilyActionCalled() {
+ return preModifyColumnFamilyActionCalled && postCompletedModifyColumnFamilyActionCalled;
}
- public boolean preModifyColumnHandlerCalledOnly() {
- return preModifyColumnHandlerCalled && !postModifyColumnHandlerCalled;
+ public boolean preModifyColumnFamilyActionCalledOnly() {
+ return preModifyColumnFamilyActionCalled && !postCompletedModifyColumnFamilyActionCalled;
}
@Deprecated
@@ -1153,13 +1216,14 @@ public class TestMasterObserver {
}
@Override
- public void preDeleteColumnFamilyHandler(
- ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName,
- byte[] columnFamily) throws IOException {
+ public void preDeleteColumnFamilyAction(
+ final ObserverContext<MasterCoprocessorEnvironment> ctx,
+ final TableName tableName,
+ final byte[] columnFamily) throws IOException {
if (bypass) {
ctx.bypass();
}
- preDeleteColumnHandlerCalled = true;
+ preDeleteColumnFamilyActionCalled = true;
}
@Deprecated
@@ -1170,68 +1234,95 @@ public class TestMasterObserver {
}
@Override
- public void postDeleteColumnFamilyHandler(
- ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName,
- byte[] columnFamily) throws IOException {
- postDeleteColumnHandlerCalled = true;
+ public void postCompletedDeleteColumnFamilyAction(
+ final ObserverContext<MasterCoprocessorEnvironment> ctx,
+ final TableName tableName,
+ final byte[] columnFamily) throws IOException {
+ postCompletedDeleteColumnFamilyActionCalled = true;
}
- public boolean wasDeleteColumnHandlerCalled() {
- return preDeleteColumnHandlerCalled && postDeleteColumnHandlerCalled;
+ public boolean wasDeleteColumnFamilyActionCalled() {
+ return preDeleteColumnFamilyActionCalled && postCompletedDeleteColumnFamilyActionCalled;
}
- public boolean preDeleteColumnHandlerCalledOnly() {
- return preDeleteColumnHandlerCalled && !postDeleteColumnHandlerCalled;
+ public boolean preDeleteColumnFamilyActionCalledOnly() {
+ return preDeleteColumnFamilyActionCalled && !postCompletedDeleteColumnFamilyActionCalled;
}
+ @Deprecated
@Override
public void preEnableTableHandler(
ObserverContext<MasterCoprocessorEnvironment> env, TableName tableName)
throws IOException {
- if (bypass) {
- env.bypass();
- }
- preEnableTableHandlerCalled = true;
}
+ @Deprecated
@Override
public void postEnableTableHandler(
ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName)
throws IOException {
- postEnableTableHandlerCalled = true;
}
- public boolean wasEnableTableHandlerCalled() {
- return preEnableTableHandlerCalled && postEnableTableHandlerCalled;
+ @Override
+ public void preEnableTableAction(
+ final ObserverContext<MasterCoprocessorEnvironment> ctx, final TableName tableName)
+ throws IOException {
+ if (bypass) {
+ ctx.bypass();
+ }
+ preEnableTableActionCalled = true;
+ }
+
+ @Override
+ public void postCompletedEnableTableAction(
+ final ObserverContext<MasterCoprocessorEnvironment> ctx, final TableName tableName)
+ throws IOException {
+ postCompletedEnableTableActionCalled = true;
}
- public boolean preEnableTableHandlerCalledOnly() {
- return preEnableTableHandlerCalled && !postEnableTableHandlerCalled;
+ public boolean wasEnableTableActionCalled() {
+ return preEnableTableActionCalled && postCompletedEnableTableActionCalled;
+ }
+
+ public boolean preEnableTableActionCalledOnly() {
+ return preEnableTableActionCalled && !postCompletedEnableTableActionCalled;
}
@Override
public void preDisableTableHandler(
ObserverContext<MasterCoprocessorEnvironment> env, TableName tableName)
throws IOException {
- if (bypass) {
- env.bypass();
- }
- preDisableTableHandlerCalled = true;
}
@Override
public void postDisableTableHandler(
ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName)
throws IOException {
- postDisableTableHandlerCalled = true;
}
- public boolean wasDisableTableHandlerCalled() {
- return preDisableTableHandlerCalled && postDisableTableHandlerCalled;
+ @Override
+ public void preDisableTableAction(
+ final ObserverContext<MasterCoprocessorEnvironment> ctx, final TableName tableName)
+ throws IOException {
+ if (bypass) {
+ ctx.bypass();
+ }
+ preDisableTableActionCalled = true;
+ }
+
+ @Override
+ public void postCompletedDisableTableAction(
+ final ObserverContext<MasterCoprocessorEnvironment> ctx, final TableName tableName)
+ throws IOException {
+ postCompletedDisableTableActionCalled = true;
+ }
+
+ public boolean wasDisableTableActionCalled() {
+ return preDisableTableActionCalled && postCompletedDisableTableActionCalled;
}
- public boolean preDisableTableHandlerCalledOnly() {
- return preDisableTableHandlerCalled && !postDisableTableHandlerCalled;
+ public boolean preDisableTableActionCalledOnly() {
+ return preDisableTableActionCalled && !postCompletedDisableTableActionCalled;
}
@Override
@@ -1445,9 +1536,9 @@ public class TestMasterObserver {
assertTrue("Test table should be created", cp.wasCreateTableCalled());
tableCreationLatch.await();
assertTrue("Table pre create handler called.", cp
- .wasPreCreateTableHandlerCalled());
+ .wasPreCreateTableActionCalled());
assertTrue("Table create handler should be called.",
- cp.wasCreateTableHandlerCalled());
+ cp.wasCreateTableActionCalled());
RegionLocator regionLocator = connection.getRegionLocator(htd.getTableName());
List<HRegionLocation> regions = regionLocator.getAllRegionLocations();
@@ -1464,7 +1555,7 @@ public class TestMasterObserver {
assertTrue("Coprocessor should have been called on table disable",
cp.wasDisableTableCalled());
assertTrue("Disable table handler should be called.",
- cp.wasDisableTableHandlerCalled());
+ cp.wasDisableTableActionCalled());
// enable
assertFalse(cp.wasEnableTableCalled());
@@ -1474,7 +1565,7 @@ public class TestMasterObserver {
assertTrue("Coprocessor should have been called on table enable",
cp.wasEnableTableCalled());
assertTrue("Enable table handler should be called.",
- cp.wasEnableTableHandlerCalled());
+ cp.wasEnableTableActionCalled());
admin.disableTable(tableName);
assertTrue(admin.isTableDisabled(tableName));
@@ -1511,7 +1602,7 @@ public class TestMasterObserver {
assertTrue("Coprocessor should have been called on table delete",
cp.wasDeleteTableCalled());
assertTrue("Delete table handler should be called.",
- cp.wasDeleteTableHandlerCalled());
+ cp.wasDeleteTableActionCalled());
// turn off bypass, run the tests again
cp.enableBypass(false);
@@ -1521,19 +1612,19 @@ public class TestMasterObserver {
assertTrue("Test table should be created", cp.wasCreateTableCalled());
tableCreationLatch.await();
assertTrue("Table pre create handler called.", cp
- .wasPreCreateTableHandlerCalled());
+ .wasPreCreateTableActionCalled());
assertTrue("Table create handler should be called.",
- cp.wasCreateTableHandlerCalled());
+ cp.wasCreateTableActionCalled());
// disable
assertFalse(cp.wasDisableTableCalled());
- assertFalse(cp.wasDisableTableHandlerCalled());
+ assertFalse(cp.wasDisableTableActionCalled());
admin.disableTable(tableName);
assertTrue(admin.isTableDisabled(tableName));
assertTrue("Coprocessor should have been called on table disable",
cp.wasDisableTableCalled());
assertTrue("Disable table handler should be called.",
- cp.wasDisableTableHandlerCalled());
+ cp.wasDisableTableActionCalled());
// modify table
htd.setMaxFileSize(512 * 1024 * 1024);
@@ -1545,7 +1636,7 @@ public class TestMasterObserver {
assertTrue("New column family should have been added to test table",
cp.wasAddColumnCalled());
assertTrue("Add column handler should be called.",
- cp.wasAddColumnHandlerCalled());
+ cp.wasAddColumnFamilyActionCalled());
// modify a column family
HColumnDescriptor hcd = new HColumnDescriptor(TEST_FAMILY2);
@@ -1554,17 +1645,17 @@ public class TestMasterObserver {
assertTrue("Second column family should be modified",
cp.wasModifyColumnCalled());
assertTrue("Modify table handler should be called.",
- cp.wasModifyColumnHandlerCalled());
+ cp.wasModifyColumnFamilyActionCalled());
// enable
assertFalse(cp.wasEnableTableCalled());
- assertFalse(cp.wasEnableTableHandlerCalled());
+ assertFalse(cp.wasEnableTableActionCalled());
admin.enableTable(tableName);
assertTrue(admin.isTableEnabled(tableName));
assertTrue("Coprocessor should have been called on table enable",
cp.wasEnableTableCalled());
assertTrue("Enable table handler should be called.",
- cp.wasEnableTableHandlerCalled());
+ cp.wasEnableTableActionCalled());
// disable again
admin.disableTable(tableName);
@@ -1573,7 +1664,7 @@ public class TestMasterObserver {
// delete column
assertFalse("No column family deleted yet", cp.wasDeleteColumnCalled());
assertFalse("Delete table column handler should not be called.",
- cp.wasDeleteColumnHandlerCalled());
+ cp.wasDeleteColumnFamilyActionCalled());
admin.deleteColumnFamily(tableName, TEST_FAMILY2);
HTableDescriptor tableDesc = admin.getTableDescriptor(tableName);
assertNull("'"+Bytes.toString(TEST_FAMILY2)+"' should have been removed",
@@ -1581,19 +1672,19 @@ public class TestMasterObserver {
assertTrue("Coprocessor should have been called on column delete",
cp.wasDeleteColumnCalled());
assertTrue("Delete table column handler should be called.",
- cp.wasDeleteColumnHandlerCalled());
+ cp.wasDeleteColumnFamilyActionCalled());
// delete table
assertFalse("No table deleted yet", cp.wasDeleteTableCalled());
assertFalse("Delete table handler should not be called.",
- cp.wasDeleteTableHandlerCalled());
+ cp.wasDeleteTableActionCalled());
deleteTable(admin, tableName);
assertFalse("Test table should have been deleted",
admin.tableExists(tableName));
assertTrue("Coprocessor should have been called on table delete",
cp.wasDeleteTableCalled());
assertTrue("Delete table handler should be called.",
- cp.wasDeleteTableHandlerCalled());
+ cp.wasDeleteTableActionCalled());
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b7ce55f0/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestTableLockManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestTableLockManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestTableLockManager.java
index 14e4844..573fdcb 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestTableLockManager.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestTableLockManager.java
@@ -104,18 +104,19 @@ public class TestTableLockManager {
public static class TestLockTimeoutExceptionMasterObserver extends BaseMasterObserver {
@Override
- public void preDeleteColumnFamilyHandler(ObserverContext<MasterCoprocessorEnvironment> ctx,
+ public void preDeleteColumnFamilyAction(ObserverContext<MasterCoprocessorEnvironment> ctx,
TableName tableName, byte[] columnFamily) throws IOException {
deleteColumn.countDown();
}
@Override
- public void postDeleteColumnFamilyHandler(ObserverContext<MasterCoprocessorEnvironment> ctx,
+ public void postCompletedDeleteColumnFamilyAction(
+ ObserverContext<MasterCoprocessorEnvironment> ctx,
TableName tableName, byte[] columnFamily) throws IOException {
Threads.sleep(10000);
}
@Override
- public void preAddColumnFamilyHandler(ObserverContext<MasterCoprocessorEnvironment> ctx,
+ public void preAddColumnFamilyAction(ObserverContext<MasterCoprocessorEnvironment> ctx,
TableName tableName, HColumnDescriptor columnFamily) throws IOException {
fail("Add column should have timeouted out for acquiring the table lock");
}
@@ -169,14 +170,15 @@ public class TestTableLockManager {
public static class TestAlterAndDisableMasterObserver extends BaseMasterObserver {
@Override
- public void preAddColumnFamilyHandler(ObserverContext<MasterCoprocessorEnvironment> ctx,
+ public void preAddColumnFamilyAction(ObserverContext<MasterCoprocessorEnvironment> ctx,
TableName tableName, HColumnDescriptor columnFamily) throws IOException {
LOG.debug("addColumn called");
addColumn.countDown();
}
@Override
- public void postAddColumnFamilyHandler(ObserverContext<MasterCoprocessorEnvironment> ctx,
+ public void postCompletedAddColumnFamilyAction(
+ ObserverContext<MasterCoprocessorEnvironment> ctx,
TableName tableName, HColumnDescriptor columnFamily) throws IOException {
Threads.sleep(6000);
try {
http://git-wip-us.apache.org/repos/asf/hbase/blob/b7ce55f0/hbase-server/src/test/java/org/apache/hadoop/hbase/master/handler/TestEnableTableHandler.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/handler/TestEnableTableHandler.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/handler/TestEnableTableHandler.java
index a260d50..a01ccc4 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/handler/TestEnableTableHandler.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/handler/TestEnableTableHandler.java
@@ -192,17 +192,21 @@ public class TestEnableTableHandler {
volatile CountDownLatch tableDeletionLatch = null;
@Override
- public void postCreateTableHandler(final ObserverContext<MasterCoprocessorEnvironment> ctx,
- HTableDescriptor desc, HRegionInfo[] regions) throws IOException {
- // the AccessController test, some times calls only and directly the postCreateTableHandler()
+ public void postCompletedCreateTableAction(
+ final ObserverContext<MasterCoprocessorEnvironment> ctx,
+ final HTableDescriptor desc,
+ final HRegionInfo[] regions) throws IOException {
+ // the AccessController test, some times calls only and directly the
+ // postCompletedCreateTableAction()
if (tableCreationLatch != null) {
tableCreationLatch.countDown();
}
}
@Override
- public void postDeleteTableHandler(final ObserverContext<MasterCoprocessorEnvironment> ctx,
- TableName tableName)
+ public void postCompletedDeleteTableAction(
+ final ObserverContext<MasterCoprocessorEnvironment> ctx,
+ final TableName tableName)
throws IOException {
// the AccessController test, some times calls only and directly the postDeleteTableHandler()
if (tableDeletionLatch != null) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/b7ce55f0/hbase-server/src/test/java/org/apache/hadoop/hbase/namespace/TestNamespaceAuditor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/namespace/TestNamespaceAuditor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/namespace/TestNamespaceAuditor.java
index 40732f6..d8b1b2e 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/namespace/TestNamespaceAuditor.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/namespace/TestNamespaceAuditor.java
@@ -513,7 +513,7 @@ public class TestNamespaceAuditor {
byte[] columnFamily = Bytes.toBytes("info");
HTableDescriptor tableDescOne = new HTableDescriptor(tableOne);
tableDescOne.addFamily(new HColumnDescriptor(columnFamily));
- MasterSyncObserver.throwExceptionInPreCreateTableHandler = true;
+ MasterSyncObserver.throwExceptionInPreCreateTableAction = true;
try {
try {
ADMIN.createTable(tableDescOne);
@@ -527,7 +527,7 @@ public class TestNamespaceAuditor {
assertEquals("First table creation failed in namespace so number of tables in namespace "
+ "should be 0.", 0, nstate.getTables().size());
- MasterSyncObserver.throwExceptionInPreCreateTableHandler = false;
+ MasterSyncObserver.throwExceptionInPreCreateTableAction = false;
try {
ADMIN.createTable(tableDescOne);
} catch (Exception e) {
@@ -539,7 +539,7 @@ public class TestNamespaceAuditor {
assertEquals("First table was created successfully so table size in namespace should "
+ "be one now.", 1, nstate.getTables().size());
} finally {
- MasterSyncObserver.throwExceptionInPreCreateTableHandler = false;
+ MasterSyncObserver.throwExceptionInPreCreateTableAction = false;
if (ADMIN.tableExists(tableOne)) {
ADMIN.disableTable(tableOne);
deleteTable(tableOne);
@@ -660,7 +660,7 @@ public class TestNamespaceAuditor {
public static class MasterSyncObserver extends BaseMasterObserver {
volatile CountDownLatch tableDeletionLatch;
- static boolean throwExceptionInPreCreateTableHandler;
+ static boolean throwExceptionInPreCreateTableAction;
@Override
public void preDeleteTable(ObserverContext<MasterCoprocessorEnvironment> ctx,
@@ -669,16 +669,16 @@ public class TestNamespaceAuditor {
}
@Override
- public void postDeleteTableHandler(
- final ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName)
- throws IOException {
+ public void postCompletedDeleteTableAction(
+ final ObserverContext<MasterCoprocessorEnvironment> ctx,
+ final TableName tableName) throws IOException {
tableDeletionLatch.countDown();
}
@Override
- public void preCreateTableHandler(ObserverContext<MasterCoprocessorEnvironment> ctx,
+ public void preCreateTableAction(ObserverContext<MasterCoprocessorEnvironment> ctx,
HTableDescriptor desc, HRegionInfo[] regions) throws IOException {
- if (throwExceptionInPreCreateTableHandler) {
+ if (throwExceptionInPreCreateTableAction) {
throw new IOException("Throw exception as it is demanded.");
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b7ce55f0/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/SecureTestUtil.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/SecureTestUtil.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/SecureTestUtil.java
index 22a9748..f0e7ac9 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/SecureTestUtil.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/SecureTestUtil.java
@@ -628,20 +628,22 @@ public class SecureTestUtil {
volatile CountDownLatch tableDeletionLatch = null;
@Override
- public void postCreateTableHandler(
+ public void postCompletedCreateTableAction(
final ObserverContext<MasterCoprocessorEnvironment> ctx,
HTableDescriptor desc, HRegionInfo[] regions) throws IOException {
- // the AccessController test, some times calls only and directly the postCreateTableHandler()
+ // the AccessController test, some times calls only and directly the
+ // postCompletedCreateTableAction()
if (tableCreationLatch != null) {
tableCreationLatch.countDown();
}
}
@Override
- public void postDeleteTableHandler(
- final ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName)
- throws IOException {
- // the AccessController test, some times calls only and directly the postDeleteTableHandler()
+ public void postCompletedDeleteTableAction(
+ final ObserverContext<MasterCoprocessorEnvironment> ctx,
+ final TableName tableName) throws IOException {
+ // the AccessController test, some times calls only and directly the
+ // postCompletedDeleteTableAction()
if (tableDeletionLatch != null) {
tableDeletionLatch.countDown();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b7ce55f0/hbase-server/src/test/java/org/apache/hadoop/hbase/util/BaseTestHBaseFsck.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/BaseTestHBaseFsck.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/BaseTestHBaseFsck.java
index 35560f5..1dfb526 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/BaseTestHBaseFsck.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/BaseTestHBaseFsck.java
@@ -608,19 +608,23 @@ public class BaseTestHBaseFsck {
volatile CountDownLatch tableDeletionLatch = null;
@Override
- public void postCreateTableHandler(final ObserverContext<MasterCoprocessorEnvironment> ctx,
- HTableDescriptor desc, HRegionInfo[] regions) throws IOException {
- // the AccessController test, some times calls only and directly the postCreateTableHandler()
+ public void postCompletedCreateTableAction(
+ final ObserverContext<MasterCoprocessorEnvironment> ctx,
+ final HTableDescriptor desc,
+ final HRegionInfo[] regions) throws IOException {
+ // the AccessController test, some times calls only and directly the
+ // postCompletedCreateTableAction()
if (tableCreationLatch != null) {
tableCreationLatch.countDown();
}
}
@Override
- public void postDeleteTableHandler(final ObserverContext<MasterCoprocessorEnvironment> ctx,
- TableName tableName)
- throws IOException {
- // the AccessController test, some times calls only and directly the postDeleteTableHandler()
+ public void postCompletedDeleteTableAction(
+ final ObserverContext<MasterCoprocessorEnvironment> ctx,
+ final TableName tableName) throws IOException {
+ // the AccessController test, some times calls only and directly the
+ // postCompletedDeleteTableAction()
if (tableDeletionLatch != null) {
tableDeletionLatch.countDown();
}
[14/50] [abbrv] hbase git commit: HBASE-15755
SnapshotDescriptionUtils and SnapshotTestingUtils do not have any Interface
audience marked
Posted by sy...@apache.org.
HBASE-15755 SnapshotDescriptionUtils and SnapshotTestingUtils do not have any Interface audience marked
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/c0ce47d1
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/c0ce47d1
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/c0ce47d1
Branch: refs/heads/hbase-12439
Commit: c0ce47d1cb1c6741acf464ec02fe2c28b2625b6a
Parents: d23d600
Author: Matteo Bertozzi <ma...@cloudera.com>
Authored: Thu May 5 07:24:29 2016 -0700
Committer: Matteo Bertozzi <ma...@cloudera.com>
Committed: Thu May 5 07:24:29 2016 -0700
----------------------------------------------------------------------
.../hadoop/hbase/snapshot/SnapshotDescriptionUtils.java | 4 +++-
.../apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java | 8 +++++++-
2 files changed, 10 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/c0ce47d1/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDescriptionUtils.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDescriptionUtils.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDescriptionUtils.java
index 79e7312..b6b608c 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDescriptionUtils.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDescriptionUtils.java
@@ -29,6 +29,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
import org.apache.hadoop.hbase.security.User;
import org.apache.hadoop.hbase.snapshot.SnapshotManifestV2;
@@ -71,7 +72,8 @@ import org.apache.hadoop.hbase.util.FSUtils;
* {@link #completeSnapshot}, and writing the
* {@link SnapshotDescription} to the working snapshot directory.
*/
-public class SnapshotDescriptionUtils {
+@InterfaceAudience.Private
+public final class SnapshotDescriptionUtils {
/**
* Filter that only accepts completed snapshot directories
http://git-wip-us.apache.org/repos/asf/hbase/blob/c0ce47d1/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java
index 6615a8f..b816200 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java
@@ -47,6 +47,7 @@ import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.TableNotEnabledException;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.SnapshotType;
import org.apache.hadoop.hbase.client.BufferedMutator;
@@ -80,11 +81,16 @@ import com.google.protobuf.ServiceException;
/**
* Utilities class for snapshots
*/
-public class SnapshotTestingUtils {
+@InterfaceAudience.Private
+public final class SnapshotTestingUtils {
private static final Log LOG = LogFactory.getLog(SnapshotTestingUtils.class);
private static byte[] KEYS = Bytes.toBytes("0123456789");
+ private SnapshotTestingUtils() {
+ // private constructor for utility class
+ }
+
/**
* Assert that we don't have any snapshots lists
*
[07/50] [abbrv] hbase git commit: HBASE-15754 Add testcase for AES
encryption
Posted by sy...@apache.org.
HBASE-15754 Add testcase for AES encryption
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/6abe1879
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/6abe1879
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/6abe1879
Branch: refs/heads/hbase-12439
Commit: 6abe1879dd5776c887bf2ca704c12b3adc75713f
Parents: 66213c9
Author: zhangduo <zh...@apache.org>
Authored: Tue May 3 17:59:52 2016 +0800
Committer: zhangduo <zh...@apache.org>
Committed: Wed May 4 09:40:57 2016 +0800
----------------------------------------------------------------------
.../FanOutOneBlockAsyncDFSOutputSaslHelper.java | 17 +++++++-----
.../TestSaslFanOutOneBlockAsyncDFSOutput.java | 28 ++++++++++++++------
2 files changed, 31 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/6abe1879/hbase-server/src/main/java/org/apache/hadoop/hbase/io/asyncfs/FanOutOneBlockAsyncDFSOutputSaslHelper.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/asyncfs/FanOutOneBlockAsyncDFSOutputSaslHelper.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/asyncfs/FanOutOneBlockAsyncDFSOutputSaslHelper.java
index 870262e..22c4e04 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/asyncfs/FanOutOneBlockAsyncDFSOutputSaslHelper.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/asyncfs/FanOutOneBlockAsyncDFSOutputSaslHelper.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.hbase.io.asyncfs;
import static io.netty.handler.timeout.IdleState.READER_IDLE;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Charsets;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Lists;
@@ -107,9 +108,13 @@ public final class FanOutOneBlockAsyncDFSOutputSaslHelper {
private static final String MECHANISM = "DIGEST-MD5";
private static final int SASL_TRANSFER_MAGIC_NUMBER = 0xDEADBEEF;
private static final String NAME_DELIMITER = " ";
- private static final String DFS_ENCRYPT_DATA_TRANSFER_CIPHER_SUITES_KEY =
+
+ @VisibleForTesting
+ static final String DFS_ENCRYPT_DATA_TRANSFER_CIPHER_SUITES_KEY =
"dfs.encrypt.data.transfer.cipher.suites";
- private static final String AES_CTR_NOPADDING = "AES/CTR/NoPadding";
+
+ @VisibleForTesting
+ static final String AES_CTR_NOPADDING = "AES/CTR/NoPadding";
private interface SaslAdaptor {
@@ -184,11 +189,11 @@ public final class FanOutOneBlockAsyncDFSOutputSaslHelper {
CREATE_DECRYPTOR = cryptoCodecClass.getMethod("createDecryptor");
Class<?> encryptorClass = Class.forName("org.apache.hadoop.crypto.Encryptor");
- INIT_ENCRYPTOR = encryptorClass.getMethod("init");
+ INIT_ENCRYPTOR = encryptorClass.getMethod("init", byte[].class, byte[].class);
ENCRYPT = encryptorClass.getMethod("encrypt", ByteBuffer.class, ByteBuffer.class);
Class<?> decryptorClass = Class.forName("org.apache.hadoop.crypto.Decryptor");
- INIT_DECRYPTOR = decryptorClass.getMethod("init");
+ INIT_DECRYPTOR = decryptorClass.getMethod("init", byte[].class, byte[].class);
DECRYPT = decryptorClass.getMethod("decrypt", ByteBuffer.class, ByteBuffer.class);
} catch (NoSuchMethodException | ClassNotFoundException e) {
throw new Error(e);
@@ -879,7 +884,7 @@ public final class FanOutOneBlockAsyncDFSOutputSaslHelper {
}
ByteBuffer inBuffer = inBuf.nioBuffer();
ByteBuf outBuf = ctx.alloc().directBuffer(inBuf.readableBytes());
- ByteBuffer outBuffer = outBuf.nioBuffer();
+ ByteBuffer outBuffer = outBuf.nioBuffer(0, inBuf.readableBytes());
codec.decrypt(inBuffer, outBuffer);
outBuf.writerIndex(inBuf.readableBytes());
if (release) {
@@ -920,7 +925,7 @@ public final class FanOutOneBlockAsyncDFSOutputSaslHelper {
release = true;
}
ByteBuffer inBuffer = inBuf.nioBuffer();
- ByteBuffer outBuffer = out.nioBuffer();
+ ByteBuffer outBuffer = out.nioBuffer(0, inBuf.readableBytes());
codec.encrypt(inBuffer, outBuffer);
out.writerIndex(inBuf.readableBytes());
if (release) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/6abe1879/hbase-server/src/test/java/org/apache/hadoop/hbase/io/asyncfs/TestSaslFanOutOneBlockAsyncDFSOutput.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/asyncfs/TestSaslFanOutOneBlockAsyncDFSOutput.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/asyncfs/TestSaslFanOutOneBlockAsyncDFSOutput.java
index 7501849..545a39e 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/asyncfs/TestSaslFanOutOneBlockAsyncDFSOutput.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/asyncfs/TestSaslFanOutOneBlockAsyncDFSOutput.java
@@ -17,6 +17,8 @@
*/
package org.apache.hadoop.hbase.io.asyncfs;
+import static org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.AES_CTR_NOPADDING;
+import static org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.DFS_ENCRYPT_DATA_TRANSFER_CIPHER_SUITES_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_SOCKET_TIMEOUT_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATA_ENCRYPTION_ALGORITHM_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ENCRYPT_DATA_TRANSFER_KEY;
@@ -47,8 +49,6 @@ import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.http.HttpConfig;
import org.apache.hadoop.minikdc.MiniKdc;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.log4j.Level;
-import org.apache.log4j.Logger;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
@@ -95,12 +95,17 @@ public class TestSaslFanOutOneBlockAsyncDFSOutput {
@Parameter(1)
public String encryptionAlgorithm;
- @Parameters(name = "{index}: protection={0}, encryption={1}")
+ @Parameter(2)
+ public String cipherSuite;
+
+ @Parameters(name = "{index}: protection={0}, encryption={1}, cipherSuite={2}")
public static Iterable<Object[]> data() {
List<Object[]> params = new ArrayList<>();
for (String protection : Arrays.asList("authentication", "integrity", "privacy")) {
for (String encryptionAlgorithm : Arrays.asList("", "3des", "rc4")) {
- params.add(new Object[] { protection, encryptionAlgorithm });
+ for (String cipherSuite : Arrays.asList("", AES_CTR_NOPADDING)) {
+ params.add(new Object[] { protection, encryptionAlgorithm, cipherSuite });
+ }
}
}
return params;
@@ -129,8 +134,6 @@ public class TestSaslFanOutOneBlockAsyncDFSOutput {
@BeforeClass
public static void setUpBeforeClass() throws Exception {
- Logger.getLogger("org.apache.hadoop.hdfs.StateChange").setLevel(Level.DEBUG);
- Logger.getLogger("BlockStateChange").setLevel(Level.DEBUG);
EVENT_LOOP_GROUP = new NioEventLoopGroup();
TEST_UTIL.getConfiguration().setInt(DFS_CLIENT_SOCKET_TIMEOUT_KEY, READ_TIMEOUT_MS);
Properties conf = MiniKdc.createConf();
@@ -161,13 +164,22 @@ public class TestSaslFanOutOneBlockAsyncDFSOutput {
@Before
public void setUp() throws Exception {
TEST_UTIL.getConfiguration().set("dfs.data.transfer.protection", protection);
- if (StringUtils.isBlank(encryptionAlgorithm)) {
+ if (StringUtils.isBlank(encryptionAlgorithm) && StringUtils.isBlank(cipherSuite)) {
TEST_UTIL.getConfiguration().setBoolean(DFS_ENCRYPT_DATA_TRANSFER_KEY, false);
- TEST_UTIL.getConfiguration().unset(DFS_DATA_ENCRYPTION_ALGORITHM_KEY);
} else {
TEST_UTIL.getConfiguration().setBoolean(DFS_ENCRYPT_DATA_TRANSFER_KEY, true);
+ }
+ if (StringUtils.isBlank(encryptionAlgorithm)) {
+ TEST_UTIL.getConfiguration().unset(DFS_DATA_ENCRYPTION_ALGORITHM_KEY);
+ } else {
TEST_UTIL.getConfiguration().set(DFS_DATA_ENCRYPTION_ALGORITHM_KEY, encryptionAlgorithm);
}
+ if (StringUtils.isBlank(cipherSuite)) {
+ TEST_UTIL.getConfiguration().unset(DFS_ENCRYPT_DATA_TRANSFER_CIPHER_SUITES_KEY);
+ } else {
+ TEST_UTIL.getConfiguration().set(DFS_ENCRYPT_DATA_TRANSFER_CIPHER_SUITES_KEY, cipherSuite);
+ }
+
TEST_UTIL.startMiniDFSCluster(3);
FS = TEST_UTIL.getDFSCluster().getFileSystem();
}
[41/50] [abbrv] hbase git commit: HBASE-15791 Improve javadoc around
ScheduledChore
Posted by sy...@apache.org.
HBASE-15791 Improve javadoc around ScheduledChore
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/541d1da5
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/541d1da5
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/541d1da5
Branch: refs/heads/hbase-12439
Commit: 541d1da5fee9e52f3d6dc86f754dbdec9dab1919
Parents: 05378cb
Author: Jonathan M Hsieh <jm...@apache.org>
Authored: Sat May 7 15:33:20 2016 -0700
Committer: Jonathan M Hsieh <jm...@apache.org>
Committed: Mon May 9 08:41:28 2016 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hbase/ScheduledChore.java | 37 +++++++++++++-------
1 file changed, 25 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/541d1da5/hbase-common/src/main/java/org/apache/hadoop/hbase/ScheduledChore.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/ScheduledChore.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/ScheduledChore.java
index 2c54b9f..5c5bcd8 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/ScheduledChore.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/ScheduledChore.java
@@ -55,9 +55,9 @@ public abstract class ScheduledChore implements Runnable {
/**
* Scheduling parameters. Used by ChoreService when scheduling the chore to run periodically
*/
- private final int period;
+ private final int period; // in TimeUnit units
private final TimeUnit timeUnit;
- private final long initialDelay;
+ private final long initialDelay; // in TimeUnit units
/**
* Interface to the ChoreService that this ScheduledChore is scheduled with. null if the chore is
@@ -68,8 +68,8 @@ public abstract class ScheduledChore implements Runnable {
/**
* Variables that encapsulate the meaningful state information
*/
- private long timeOfLastRun = -1;
- private long timeOfThisRun = -1;
+ private long timeOfLastRun = -1; // system time millis
+ private long timeOfThisRun = -1; // system time millis
private boolean initialChoreComplete = false;
/**
@@ -127,7 +127,7 @@ public abstract class ScheduledChore implements Runnable {
/**
* @param name Name assigned to Chore. Useful for identification amongst chores of the same type
* @param stopper When {@link Stoppable#isStopped()} is true, this chore will cancel and cleanup
- * @param period Period with which this Chore repeats execution when scheduled.
+ * @param period Period in millis with which this Chore repeats execution when scheduled.
*/
public ScheduledChore(final String name, Stoppable stopper, final int period) {
this(name, stopper, period, DEFAULT_INITIAL_DELAY);
@@ -136,7 +136,7 @@ public abstract class ScheduledChore implements Runnable {
/**
* @param name Name assigned to Chore. Useful for identification amongst chores of the same type
* @param stopper When {@link Stoppable#isStopped()} is true, this chore will cancel and cleanup
- * @param period Period with which this Chore repeats execution when scheduled.
+ * @param period Period in millis with which this Chore repeats execution when scheduled.
* @param initialDelay Delay before this Chore begins to execute once it has been scheduled. A
* value of 0 means the chore will begin to execute immediately. Negative delays are
* invalid and will be corrected to a value of 0.
@@ -149,10 +149,10 @@ public abstract class ScheduledChore implements Runnable {
/**
* @param name Name assigned to Chore. Useful for identification amongst chores of the same type
* @param stopper When {@link Stoppable#isStopped()} is true, this chore will cancel and cleanup
- * @param period Period with which this Chore repeats execution when scheduled.
- * @param initialDelay Delay before this Chore begins to execute once it has been scheduled. A
- * value of 0 means the chore will begin to execute immediately. Negative delays are
- * invalid and will be corrected to a value of 0.
+ * @param period Period in Timeunit unit with which this Chore repeats execution when scheduled.
+ * @param initialDelay Delay in Timeunit unit before this Chore begins to execute once it has been
+ * scheduled. A value of 0 means the chore will begin to execute immediately. Negative
+ * delays are invalid and will be corrected to a value of 0.
* @param unit The unit that is used to measure period and initialDelay
*/
public ScheduledChore(final String name, Stoppable stopper, final int period,
@@ -213,8 +213,8 @@ public abstract class ScheduledChore implements Runnable {
}
/**
- * @return How long has it been since this chore last run. Useful for checking if the chore has
- * missed its scheduled start time by too large of a margin
+ * @return How long in millis has it been since this chore last run. Useful for checking if the
+ * chore has missed its scheduled start time by too large of a margin
*/
synchronized long getTimeBetweenRuns() {
return timeOfThisRun - timeOfLastRun;
@@ -228,11 +228,18 @@ public abstract class ScheduledChore implements Runnable {
&& getTimeBetweenRuns() > getMaximumAllowedTimeBetweenRuns();
}
+ /**
+ * @return max allowed time in millis between runs.
+ */
private double getMaximumAllowedTimeBetweenRuns() {
// Threshold used to determine if the Chore's current run started too late
return 1.5 * timeUnit.toMillis(period);
}
+ /**
+ * @param time in system millis
+ * @return true if time is earlier or equal to current milli time
+ */
private synchronized boolean isValidTime(final long time) {
return time > 0 && time <= System.currentTimeMillis();
}
@@ -276,10 +283,16 @@ public abstract class ScheduledChore implements Runnable {
return stopper;
}
+ /**
+ * @return period to execute chore in getTimeUnit() units
+ */
public int getPeriod() {
return period;
}
+ /**
+ * @return initial delay before executing chore in getTimeUnit() units
+ */
public long getInitialDelay() {
return initialDelay;
}
[34/50] [abbrv] hbase git commit: HBASE-15794 Fix Findbugs instanceof
always true issue in MultiServerCallable
Posted by sy...@apache.org.
HBASE-15794 Fix Findbugs instanceof always true issue in
MultiServerCallable
Signed-off-by: stack <st...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/ac31ceb8
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/ac31ceb8
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/ac31ceb8
Branch: refs/heads/hbase-12439
Commit: ac31ceb835b303287b367780c17723cb49d54fba
Parents: e71ae60
Author: Jurriaan Mous <ju...@jurmo.us>
Authored: Sat May 7 14:40:51 2016 +0200
Committer: stack <st...@apache.org>
Committed: Sat May 7 10:10:23 2016 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hbase/client/MultiServerCallable.java | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/ac31ceb8/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MultiServerCallable.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MultiServerCallable.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MultiServerCallable.java
index f78f348..e764ceb 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MultiServerCallable.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MultiServerCallable.java
@@ -137,9 +137,8 @@ class MultiServerCallable<R> extends PayloadCarryingServerCallable<MultiResponse
private boolean isCellBlock() {
// This is not exact -- the configuration could have changed on us after connection was set up
// but it will do for now.
- HConnection connection = getConnection();
- if (!(connection instanceof ClusterConnection)) return true; // Default is to do cellblocks.
- return ((ClusterConnection) connection).hasCellBlockSupport();
+ ClusterConnection conn = getConnection();
+ return conn.hasCellBlockSupport();
}
@Override
[22/50] [abbrv] hbase git commit: HBASE-15759
RegionObserver.preStoreScannerOpen() doesn't have acces to current readpoint
Posted by sy...@apache.org.
HBASE-15759 RegionObserver.preStoreScannerOpen() doesn't have acces to current readpoint
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/532b914f
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/532b914f
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/532b914f
Branch: refs/heads/hbase-12439
Commit: 532b914f08af31d34a9b4ede7e8f5a567a9b211e
Parents: c716a75
Author: tedyu <yu...@gmail.com>
Authored: Thu May 5 15:20:37 2016 -0700
Committer: tedyu <yu...@gmail.com>
Committed: Thu May 5 15:20:37 2016 -0700
----------------------------------------------------------------------
.../hbase/coprocessor/BaseRegionObserver.java | 7 +++++
.../hbase/coprocessor/RegionObserver.java | 32 ++++++++++++++++++++
.../hadoop/hbase/regionserver/HStore.java | 2 +-
.../regionserver/RegionCoprocessorHost.java | 4 +--
.../TestRegionObserverScannerOpenHook.java | 5 ++-
.../hbase/util/TestCoprocessorScanPolicy.java | 5 ++-
6 files changed, 46 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/532b914f/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseRegionObserver.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseRegionObserver.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseRegionObserver.java
index 67093f8..eaf9f43 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseRegionObserver.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseRegionObserver.java
@@ -401,6 +401,13 @@ public class BaseRegionObserver implements RegionObserver {
}
@Override
+ public KeyValueScanner preStoreScannerOpen(final ObserverContext<RegionCoprocessorEnvironment> c,
+ final Store store, final Scan scan, final NavigableSet<byte[]> targetCols,
+ final KeyValueScanner s, final long readPt) throws IOException {
+ return preStoreScannerOpen(c, store, scan, targetCols, s);
+ }
+
+ @Override
public RegionScanner postScannerOpen(final ObserverContext<RegionCoprocessorEnvironment> e,
final Scan scan, final RegionScanner s) throws IOException {
return s;
http://git-wip-us.apache.org/repos/asf/hbase/blob/532b914f/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java
index cbad200..1d985df 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java
@@ -1036,12 +1036,44 @@ public interface RegionObserver extends Coprocessor {
* @param s the base scanner, if not {@code null}, from previous RegionObserver in the chain
* @return a KeyValueScanner instance to use or {@code null} to use the default implementation
* @throws IOException if an error occurred on the coprocessor
+ * @deprecated use {@link #preStoreScannerOpen(ObserverContext, Store, Scan, NavigableSet,
+ * KeyValueScanner, long)} instead
*/
+ @Deprecated
KeyValueScanner preStoreScannerOpen(final ObserverContext<RegionCoprocessorEnvironment> c,
final Store store, final Scan scan, final NavigableSet<byte[]> targetCols,
final KeyValueScanner s) throws IOException;
/**
+ * Called before a store opens a new scanner.
+ * This hook is called when a "user" scanner is opened.
+ * <p>
+ * See {@link #preFlushScannerOpen(ObserverContext, Store, KeyValueScanner, InternalScanner)}
+ * and {@link #preCompactScannerOpen(ObserverContext,
+ * Store, List, ScanType, long, InternalScanner)}
+ * to override scanners created for flushes or compactions, resp.
+ * <p>
+ * Call CoprocessorEnvironment#complete to skip any subsequent chained
+ * coprocessors.
+ * Calling {@link org.apache.hadoop.hbase.coprocessor.ObserverContext#bypass()} has no
+ * effect in this hook.
+ * <p>
+ * Note: Do not retain references to any Cells returned by scanner, beyond the life of this
+ * invocation. If need a Cell reference for later use, copy the cell and use that.
+ * @param c the environment provided by the region server
+ * @param store the store being scanned
+ * @param scan the Scan specification
+ * @param targetCols columns to be used in the scanner
+ * @param s the base scanner, if not {@code null}, from previous RegionObserver in the chain
+ * @param readPt the read point
+ * @return a KeyValueScanner instance to use or {@code null} to use the default implementation
+ * @throws IOException if an error occurred on the coprocessor
+ */
+ KeyValueScanner preStoreScannerOpen(final ObserverContext<RegionCoprocessorEnvironment> c,
+ final Store store, final Scan scan, final NavigableSet<byte[]> targetCols,
+ final KeyValueScanner s, final long readPt) throws IOException;
+
+ /**
* Called after the client opens a new scanner.
* <p>
* Call CoprocessorEnvironment#complete to skip any subsequent chained
http://git-wip-us.apache.org/repos/asf/hbase/blob/532b914f/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
index c02627f..8dfa0e0 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
@@ -1868,7 +1868,7 @@ public class HStore implements Store {
try {
KeyValueScanner scanner = null;
if (this.getCoprocessorHost() != null) {
- scanner = this.getCoprocessorHost().preStoreScannerOpen(this, scan, targetCols);
+ scanner = this.getCoprocessorHost().preStoreScannerOpen(this, scan, targetCols, readPt);
}
scanner = createScanner(scan, targetCols, readPt, scanner);
return scanner;
http://git-wip-us.apache.org/repos/asf/hbase/blob/532b914f/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java
index 6614b3e..25e0200 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java
@@ -1282,13 +1282,13 @@ public class RegionCoprocessorHost
* Store, Scan, NavigableSet, KeyValueScanner)}
*/
public KeyValueScanner preStoreScannerOpen(final Store store, final Scan scan,
- final NavigableSet<byte[]> targetCols) throws IOException {
+ final NavigableSet<byte[]> targetCols, final long readPt) throws IOException {
return execOperationWithResult(null,
coprocessors.isEmpty() ? null : new RegionOperationWithResult<KeyValueScanner>() {
@Override
public void call(RegionObserver oserver, ObserverContext<RegionCoprocessorEnvironment> ctx)
throws IOException {
- setResult(oserver.preStoreScannerOpen(ctx, store, scan, targetCols, getResult()));
+ setResult(oserver.preStoreScannerOpen(ctx, store, scan, targetCols, getResult(), readPt));
}
});
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/532b914f/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverScannerOpenHook.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverScannerOpenHook.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverScannerOpenHook.java
index a8983ef..02bb0d3 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverScannerOpenHook.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverScannerOpenHook.java
@@ -105,11 +105,10 @@ public class TestRegionObserverScannerOpenHook {
public static class NoDataFromScan extends BaseRegionObserver {
@Override
public KeyValueScanner preStoreScannerOpen(ObserverContext<RegionCoprocessorEnvironment> c,
- Store store, Scan scan, NavigableSet<byte[]> targetCols, KeyValueScanner s)
+ Store store, Scan scan, NavigableSet<byte[]> targetCols, KeyValueScanner s, long readPt)
throws IOException {
scan.setFilter(new NoDataFilter());
- return new StoreScanner(store, store.getScanInfo(), scan, targetCols,
- ((HStore)store).getHRegion().getReadPoint(IsolationLevel.READ_COMMITTED));
+ return new StoreScanner(store, store.getScanInfo(), scan, targetCols, readPt);
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/532b914f/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCoprocessorScanPolicy.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCoprocessorScanPolicy.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCoprocessorScanPolicy.java
index b4eb798..58aa56a 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCoprocessorScanPolicy.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCoprocessorScanPolicy.java
@@ -285,7 +285,7 @@ public class TestCoprocessorScanPolicy {
@Override
public KeyValueScanner preStoreScannerOpen(
final ObserverContext<RegionCoprocessorEnvironment> c, Store store, final Scan scan,
- final NavigableSet<byte[]> targetCols, KeyValueScanner s) throws IOException {
+ final NavigableSet<byte[]> targetCols, KeyValueScanner s, long readPt) throws IOException {
TableName tn = store.getTableName();
if (!tn.isSystemTable()) {
Long newTtl = ttls.get(store.getTableName());
@@ -297,8 +297,7 @@ public class TestCoprocessorScanPolicy {
newVersions == null ? family.getMaxVersions() : newVersions,
newTtl == null ? oldSI.getTtl() : newTtl, family.getKeepDeletedCells(),
oldSI.getTimeToPurgeDeletes(), oldSI.getComparator());
- return new StoreScanner(store, scanInfo, scan, targetCols,
- ((HStore) store).getHRegion().getReadPoint(IsolationLevel.READ_COMMITTED));
+ return new StoreScanner(store, scanInfo, scan, targetCols, readPt);
} else {
return s;
}
[06/50] [abbrv] hbase git commit: HBASE-15337 Document Date Tiered
Compaction in the book
Posted by sy...@apache.org.
HBASE-15337 Document Date Tiered Compaction in the book
Signed-off-by: Enis Soztutar <en...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/66213c9f
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/66213c9f
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/66213c9f
Branch: refs/heads/hbase-12439
Commit: 66213c9f28643cfc2ec53d3f683c65a7598b8ace
Parents: f76ffb7
Author: Clara <cl...@flurry.com>
Authored: Thu Apr 28 23:31:25 2016 -0700
Committer: Enis Soztutar <en...@apache.org>
Committed: Tue May 3 15:05:00 2016 -0700
----------------------------------------------------------------------
src/main/asciidoc/_chapters/architecture.adoc | 101 +++++++++++++++++++++
1 file changed, 101 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/66213c9f/src/main/asciidoc/_chapters/architecture.adoc
----------------------------------------------------------------------
diff --git a/src/main/asciidoc/_chapters/architecture.adoc b/src/main/asciidoc/_chapters/architecture.adoc
index 7cc20e5..faa1230 100644
--- a/src/main/asciidoc/_chapters/architecture.adoc
+++ b/src/main/asciidoc/_chapters/architecture.adoc
@@ -2060,6 +2060,107 @@ Why?
NOTE: This information is now included in the configuration parameter table in <<compaction.parameters>>.
+[[ops.date.tiered]]
+===== Date Tiered Compaction
+
+Date tiered compaction is a date-aware store file compaction strategy that is beneficial for time-range scans for time-series data.
+
+[[ops.date.tiered.when]]
+===== When To Use Date Tiered Compactions
+
+Consider using Date Tiered Compaction for reads for limited time ranges, especially scans of recent data
+
+Don't use it for
+
+* random gets without a limited time range
+* frequent deletes and updates
+* Frequent out of order data writes creating long tails, especially writes with future timestamps
+* frequent bulk loads with heavily overlapping time ranges
+
+.Performance Improvements
+Performance testing has shown that the performance of time-range scans improve greatly for limited time ranges, especially scans of recent data.
+
+[[ops.date.tiered.enable]]
+====== Enabling Date Tiered Compaction
+
+You can enable Date Tiered compaction for a table or a column family, by setting its `hbase.hstore.engine.class` to `org.apache.hadoop.hbase.regionserver.DateTieredStoreEngine`.
+
+You also need to set `hbase.hstore.blockingStoreFiles` to a high number, such as 60, if using all default settings, rather than the default value of 12). Use 1.5~2 x projected file count if changing the parameters, Projected file count = windows per tier x tier count + incoming window min + files older than max age
+
+You also need to set `hbase.hstore.compaction.max` to the same value as `hbase.hstore.blockingStoreFiles` to unblock major compaction.
+
+.Procedure: Enable Date Tiered Compaction
+. Run one of following commands in the HBase shell.
+ Replace the table name `orders_table` with the name of your table.
++
+[source,sql]
+----
+alter 'orders_table', CONFIGURATION => {'hbase.hstore.engine.class' => 'org.apache.hadoop.hbase.regionserver.DateTieredStoreEngine', 'hbase.hstore.blockingStoreFiles' => '60', 'hbase.hstore.compaction.min'=>'2', 'hbase.hstore.compaction.max'=>'60'}
+alter 'orders_table', {NAME => 'blobs_cf', CONFIGURATION => {'hbase.hstore.engine.class' => 'org.apache.hadoop.hbase.regionserver.DateTieredStoreEngine', 'hbase.hstore.blockingStoreFiles' => '60', 'hbase.hstore.compaction.min'=>'2', 'hbase.hstore.compaction.max'=>'60'}}
+create 'orders_table', 'blobs_cf', CONFIGURATION => {'hbase.hstore.engine.class' => 'org.apache.hadoop.hbase.regionserver.DateTieredStoreEngine', 'hbase.hstore.blockingStoreFiles' => '60', 'hbase.hstore.compaction.min'=>'2', 'hbase.hstore.compaction.max'=>'60'}
+----
+
+. Configure other options if needed.
+ See <<ops.date.tiered.config>> for more information.
+
+.Procedure: Disable Date Tiered Compaction
+. Set the `hbase.hstore.engine.class` option to either nil or `org.apache.hadoop.hbase.regionserver.DefaultStoreEngine`.
+ Either option has the same effect.
+ Make sure you set the other options you changed to the original settings too.
++
+[source,sql]
+----
+alter 'orders_table', CONFIGURATION => {'hbase.hstore.engine.class' => 'org.apache.hadoop.hbase.regionserver.DefaultStoreEngine'\uff0c 'hbase.hstore.blockingStoreFiles' => '12', 'hbase.hstore.compaction.min'=>'6', 'hbase.hstore.compaction.max'=>'12'}}
+----
+
+When you change the store engine either way, a major compaction will likely be performed on most regions.
+This is not necessary on new tables.
+
+[[ops.date.tiered.config]]
+====== Configuring Date Tiered Compaction
+
+Each of the settings for date tiered compaction should be configured at the table or column family, after disabling the table.
+If you use HBase shell, the general command pattern is as follows:
+
+[source,sql]
+----
+alter 'orders_table', CONFIGURATION => {'key' => 'value', ..., 'key' => 'value'}}
+----
+
+[[ops.date.tiered.config.parameters]]
+.Tier Parameters
+
+You can configure your date tiers by changing the settings for the following parameters:
+
+.Date Tier Parameters
+[cols="1,1a", frame="all", options="header"]
+|===
+| Setting
+| Notes
+
+|`hbase.hstore.compaction.date.tiered.max.storefile.age.millis`
+|Files with max-timestamp smaller than this will no longer be compacted.Default at Long.MAX_VALUE.
+
+| `hbase.hstore.compaction.date.tiered.base.window.millis`
+| Base window size in milliseconds. Default at 6 hours.
+
+| `hbase.hstore.compaction.date.tiered.windows.per.tier`
+| Number of windows per tier. Default at 4.
+
+| `hbase.hstore.compaction.date.tiered.incoming.window.min`
+| Minimal number of files to compact in the incoming window. Set it to expected number of files in the window to avoid wasteful compaction. Default at 6.
+
+| `hbase.hstore.compaction.date.tiered.window.policy.class`
+| The policy to select store files within the same time window. It doesn\u2019t apply to the incoming window. Default at exploring compaction. This is to avoid wasteful compaction.
+|===
+
+[[ops.date.tiered.config.compaction.throttler]]
+.Compaction Throttler
+
+With tiered compaction all servers in the cluster will promote windows to higher tier at the same time, so using a compaction throttle is recommended:
+Set `hbase.regionserver.throughput.controller` to `org.apache.hadoop.hbase.regionserver.compactions.PressureAwareCompactionThroughputController`.
+
+NOTE: For more information about date tiered compaction, please refer to the design specification at https://docs.google.com/document/d/1_AmlNb2N8Us1xICsTeGDLKIqL6T-oHoRLZ323MG_uy8
[[ops.stripe]]
===== Experimental: Stripe Compactions
[09/50] [abbrv] hbase git commit: HBASE-15763 Isolate Wal related
stuff from MasterFileSystem
Posted by sy...@apache.org.
HBASE-15763 Isolate Wal related stuff from MasterFileSystem
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/a8a2c516
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/a8a2c516
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/a8a2c516
Branch: refs/heads/hbase-12439
Commit: a8a2c516a0752ff782f8338502c8b2005049eda3
Parents: 0e37063
Author: Matteo Bertozzi <ma...@cloudera.com>
Authored: Wed May 4 07:59:44 2016 -0700
Committer: Matteo Bertozzi <ma...@cloudera.com>
Committed: Wed May 4 07:59:44 2016 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hbase/master/HMaster.java | 26 +-
.../hadoop/hbase/master/MasterFileSystem.java | 321 +----------------
.../hadoop/hbase/master/MasterServices.java | 5 +
.../hadoop/hbase/master/MasterWalManager.java | 351 +++++++++++++++++++
.../hadoop/hbase/master/ServerManager.java | 4 +-
.../hadoop/hbase/master/SplitLogManager.java | 8 +-
.../master/procedure/ServerCrashProcedure.java | 22 +-
.../hbase/master/MockNoopMasterServices.java | 5 +
.../hadoop/hbase/master/TestCatalogJanitor.java | 11 +-
.../master/TestDistributedLogSplitting.java | 17 +-
.../hbase/master/TestMasterFileSystem.java | 56 +--
.../hbase/master/TestMasterWalManager.java | 104 ++++++
12 files changed, 533 insertions(+), 397 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/a8a2c516/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 2f1cd3c..d7f7c18 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -270,6 +270,7 @@ public class HMaster extends HRegionServer implements MasterServices {
final MetricsMaster metricsMaster;
// file system manager for the master FS operations
private MasterFileSystem fileSystemManager;
+ private MasterWalManager walManager;
// server manager to deal with region server info
volatile ServerManager serverManager;
@@ -656,7 +657,8 @@ public class HMaster extends HRegionServer implements MasterServices {
this.masterActiveTime = System.currentTimeMillis();
// TODO: Do this using Dependency Injection, using PicoContainer, Guice or Spring.
- this.fileSystemManager = new MasterFileSystem(this, this);
+ this.fileSystemManager = new MasterFileSystem(this);
+ this.walManager = new MasterWalManager(this);
// enable table descriptors cache
this.tableDescriptors.setCacheOn();
@@ -715,7 +717,7 @@ public class HMaster extends HRegionServer implements MasterServices {
// we recover hbase:meta region servers inside master initialization and
// handle other failed servers in SSH in order to start up master node ASAP
Set<ServerName> previouslyFailedServers =
- this.fileSystemManager.getFailedServersFromLogFolders();
+ this.walManager.getFailedServersFromLogFolders();
// log splitting for hbase:meta server
ServerName oldMetaServerLocation = metaTableLocator.getMetaRegionLocation(this.getZooKeeper());
@@ -946,11 +948,11 @@ public class HMaster extends HRegionServer implements MasterServices {
// TODO: should we prevent from using state manager before meta was initialized?
// tableStateManager.start();
- if ((RecoveryMode.LOG_REPLAY == this.getMasterFileSystem().getLogRecoveryMode())
+ if ((RecoveryMode.LOG_REPLAY == this.getMasterWalManager().getLogRecoveryMode())
&& (!previouslyFailedMetaRSs.isEmpty())) {
// replay WAL edits mode need new hbase:meta RS is assigned firstly
status.setStatus("replaying log for Meta Region");
- this.fileSystemManager.splitMetaLog(previouslyFailedMetaRSs);
+ this.walManager.splitMetaLog(previouslyFailedMetaRSs);
}
this.assignmentManager.setEnabledTable(TableName.META_TABLE_NAME);
@@ -985,14 +987,14 @@ public class HMaster extends HRegionServer implements MasterServices {
}
private void splitMetaLogBeforeAssignment(ServerName currentMetaServer) throws IOException {
- if (RecoveryMode.LOG_REPLAY == this.getMasterFileSystem().getLogRecoveryMode()) {
+ if (RecoveryMode.LOG_REPLAY == this.getMasterWalManager().getLogRecoveryMode()) {
// In log replay mode, we mark hbase:meta region as recovering in ZK
Set<HRegionInfo> regions = new HashSet<HRegionInfo>();
regions.add(HRegionInfo.FIRST_META_REGIONINFO);
- this.fileSystemManager.prepareLogReplay(currentMetaServer, regions);
+ this.walManager.prepareLogReplay(currentMetaServer, regions);
} else {
// In recovered.edits mode: create recovered edits file for hbase:meta server
- this.fileSystemManager.splitMetaLog(currentMetaServer);
+ this.walManager.splitMetaLog(currentMetaServer);
}
}
@@ -1046,6 +1048,11 @@ public class HMaster extends HRegionServer implements MasterServices {
}
@Override
+ public MasterWalManager getMasterWalManager() {
+ return this.walManager;
+ }
+
+ @Override
public TableStateManager getTableStateManager() {
return tableStateManager;
}
@@ -1082,8 +1089,8 @@ public class HMaster extends HRegionServer implements MasterServices {
int cleanerInterval = conf.getInt("hbase.master.cleaner.interval", 60 * 1000);
this.logCleaner =
new LogCleaner(cleanerInterval,
- this, conf, getMasterFileSystem().getFileSystem(),
- getMasterFileSystem().getOldLogDir());
+ this, conf, getMasterWalManager().getFileSystem(),
+ getMasterWalManager().getOldLogDir());
getChoreService().scheduleChore(logCleaner);
//start the hfile archive cleaner thread
@@ -1132,6 +1139,7 @@ public class HMaster extends HRegionServer implements MasterServices {
if (this.activeMasterManager != null) this.activeMasterManager.stop();
if (this.serverManager != null) this.serverManager.stop();
if (this.assignmentManager != null) this.assignmentManager.stop();
+ if (this.walManager != null) this.walManager.stop();
if (this.fileSystemManager != null) this.fileSystemManager.stop();
if (this.mpmHost != null) this.mpmHost.stop("server shutting down.");
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/a8a2c516/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
index 43ae2f8..ad6e09d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
@@ -19,99 +19,56 @@
package org.apache.hadoop.hbase.master;
import java.io.IOException;
-import java.io.InterruptedIOException;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReentrantLock;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.PathFilter;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hbase.ClusterId;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.Server;
-import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.backup.HFileArchiver;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.exceptions.DeserializationException;
import org.apache.hadoop.hbase.fs.HFileSystem;
import org.apache.hadoop.hbase.mob.MobConstants;
import org.apache.hadoop.hbase.mob.MobUtils;
-import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.SplitLogTask.RecoveryMode;
import org.apache.hadoop.hbase.regionserver.HRegion;
-import org.apache.hadoop.hbase.wal.DefaultWALProvider;
-import org.apache.hadoop.hbase.wal.WALSplitter;
import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.FSTableDescriptors;
import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.ipc.RemoteException;
-import com.google.common.annotations.VisibleForTesting;
-
/**
* This class abstracts a bunch of operations the HMaster needs to interact with
- * the underlying file system, including splitting log files, checking file
+ * the underlying file system like creating the initial layout, checking file
* system status, etc.
*/
@InterfaceAudience.Private
public class MasterFileSystem {
- private static final Log LOG = LogFactory.getLog(MasterFileSystem.class.getName());
+ private static final Log LOG = LogFactory.getLog(MasterFileSystem.class);
+
// HBase configuration
- Configuration conf;
- // master status
- Server master;
- // metrics for master
- private final MetricsMasterFileSystem metricsMasterFilesystem = new MetricsMasterFileSystem();
+ private final Configuration conf;
// Persisted unique cluster ID
private ClusterId clusterId;
// Keep around for convenience.
private final FileSystem fs;
- // Is the fileystem ok?
- private volatile boolean fsOk = true;
- // The Path to the old logs dir
- private final Path oldLogDir;
// root hbase directory on the FS
private final Path rootdir;
// hbase temp directory used for table construction and deletion
private final Path tempdir;
- // create the split log lock
- final Lock splitLogLock = new ReentrantLock();
- final boolean distributedLogReplay;
- final SplitLogManager splitLogManager;
- private final MasterServices services;
-
- final static PathFilter META_FILTER = new PathFilter() {
- @Override
- public boolean accept(Path p) {
- return DefaultWALProvider.isMetaFile(p);
- }
- };
- final static PathFilter NON_META_FILTER = new PathFilter() {
- @Override
- public boolean accept(Path p) {
- return !DefaultWALProvider.isMetaFile(p);
- }
- };
+ private final MasterServices services;
- public MasterFileSystem(Server master, MasterServices services)
- throws IOException {
- this.conf = master.getConfiguration();
- this.master = master;
+ public MasterFileSystem(MasterServices services) throws IOException {
+ this.conf = services.getConfiguration();
this.services = services;
// Set filesystem to be that of this.rootdir else we get complaints about
// mismatched filesystems if hbase.rootdir is hdfs and fs.defaultFS is
@@ -126,18 +83,8 @@ public class MasterFileSystem {
// make sure the fs has the same conf
fs.setConf(conf);
// setup the filesystem variable
- // set up the archived logs path
- this.oldLogDir = createInitialFileSystemLayout();
+ createInitialFileSystemLayout();
HFileSystem.addLocationsOrderInterceptor(conf);
- this.splitLogManager =
- new SplitLogManager(master, master.getConfiguration(), master, services,
- master.getServerName());
- this.distributedLogReplay = this.splitLogManager.isLogReplaying();
- }
-
- @VisibleForTesting
- SplitLogManager getSplitLogManager() {
- return this.splitLogManager;
}
/**
@@ -146,55 +93,23 @@ public class MasterFileSystem {
* <li>Check if the meta region exists and is readable, if not create it.
* Create hbase.version and the hbase:meta directory if not one.
* </li>
- * <li>Create a log archive directory for RS to put archived logs</li>
* </ol>
* Idempotent.
*/
- private Path createInitialFileSystemLayout() throws IOException {
+ private void createInitialFileSystemLayout() throws IOException {
// check if the root directory exists
checkRootDir(this.rootdir, conf, this.fs);
// check if temp directory exists and clean it
checkTempDir(this.tempdir, conf, this.fs);
-
- Path oldLogDir = new Path(this.rootdir, HConstants.HREGION_OLDLOGDIR_NAME);
-
- // Make sure the region servers can archive their old logs
- if(!this.fs.exists(oldLogDir)) {
- this.fs.mkdirs(oldLogDir);
- }
-
- return oldLogDir;
}
public FileSystem getFileSystem() {
return this.fs;
}
- /**
- * Get the directory where old logs go
- * @return the dir
- */
- public Path getOldLogDir() {
- return this.oldLogDir;
- }
-
- /**
- * Checks to see if the file system is still accessible.
- * If not, sets closed
- * @return false if file system is not available
- */
- public boolean checkFileSystem() {
- if (this.fsOk) {
- try {
- FSUtils.checkFileSystemAvailable(this.fs);
- FSUtils.checkDfsSafeMode(this.conf);
- } catch (IOException e) {
- master.abort("Shutting down HBase cluster: file system not available", e);
- this.fsOk = false;
- }
- }
- return this.fsOk;
+ public Configuration getConfiguration() {
+ return this.conf;
}
/**
@@ -219,197 +134,6 @@ public class MasterFileSystem {
}
/**
- * Inspect the log directory to find dead servers which need recovery work
- * @return A set of ServerNames which aren't running but still have WAL files left in file system
- */
- Set<ServerName> getFailedServersFromLogFolders() {
- boolean retrySplitting = !conf.getBoolean("hbase.hlog.split.skip.errors",
- WALSplitter.SPLIT_SKIP_ERRORS_DEFAULT);
-
- Set<ServerName> serverNames = new HashSet<ServerName>();
- Path logsDirPath = new Path(this.rootdir, HConstants.HREGION_LOGDIR_NAME);
-
- do {
- if (master.isStopped()) {
- LOG.warn("Master stopped while trying to get failed servers.");
- break;
- }
- try {
- if (!this.fs.exists(logsDirPath)) return serverNames;
- FileStatus[] logFolders = FSUtils.listStatus(this.fs, logsDirPath, null);
- // Get online servers after getting log folders to avoid log folder deletion of newly
- // checked in region servers . see HBASE-5916
- Set<ServerName> onlineServers = ((HMaster) master).getServerManager().getOnlineServers()
- .keySet();
-
- if (logFolders == null || logFolders.length == 0) {
- LOG.debug("No log files to split, proceeding...");
- return serverNames;
- }
- for (FileStatus status : logFolders) {
- FileStatus[] curLogFiles = FSUtils.listStatus(this.fs, status.getPath(), null);
- if (curLogFiles == null || curLogFiles.length == 0) {
- // Empty log folder. No recovery needed
- continue;
- }
- final ServerName serverName = DefaultWALProvider.getServerNameFromWALDirectoryName(
- status.getPath());
- if (null == serverName) {
- LOG.warn("Log folder " + status.getPath() + " doesn't look like its name includes a " +
- "region server name; leaving in place. If you see later errors about missing " +
- "write ahead logs they may be saved in this location.");
- } else if (!onlineServers.contains(serverName)) {
- LOG.info("Log folder " + status.getPath() + " doesn't belong "
- + "to a known region server, splitting");
- serverNames.add(serverName);
- } else {
- LOG.info("Log folder " + status.getPath() + " belongs to an existing region server");
- }
- }
- retrySplitting = false;
- } catch (IOException ioe) {
- LOG.warn("Failed getting failed servers to be recovered.", ioe);
- if (!checkFileSystem()) {
- LOG.warn("Bad Filesystem, exiting");
- Runtime.getRuntime().halt(1);
- }
- try {
- if (retrySplitting) {
- Thread.sleep(conf.getInt("hbase.hlog.split.failure.retry.interval", 30 * 1000));
- }
- } catch (InterruptedException e) {
- LOG.warn("Interrupted, aborting since cannot return w/o splitting");
- Thread.currentThread().interrupt();
- retrySplitting = false;
- Runtime.getRuntime().halt(1);
- }
- }
- } while (retrySplitting);
-
- return serverNames;
- }
-
- public void splitLog(final ServerName serverName) throws IOException {
- Set<ServerName> serverNames = new HashSet<ServerName>();
- serverNames.add(serverName);
- splitLog(serverNames);
- }
-
- /**
- * Specialized method to handle the splitting for meta WAL
- * @param serverName
- * @throws IOException
- */
- public void splitMetaLog(final ServerName serverName) throws IOException {
- Set<ServerName> serverNames = new HashSet<ServerName>();
- serverNames.add(serverName);
- splitMetaLog(serverNames);
- }
-
- /**
- * Specialized method to handle the splitting for meta WAL
- * @param serverNames
- * @throws IOException
- */
- public void splitMetaLog(final Set<ServerName> serverNames) throws IOException {
- splitLog(serverNames, META_FILTER);
- }
-
- @edu.umd.cs.findbugs.annotations.SuppressWarnings(value="UL_UNRELEASED_LOCK", justification=
- "We only release this lock when we set it. Updates to code that uses it should verify use " +
- "of the guard boolean.")
- private List<Path> getLogDirs(final Set<ServerName> serverNames) throws IOException {
- List<Path> logDirs = new ArrayList<Path>();
- boolean needReleaseLock = false;
- if (!this.services.isInitialized()) {
- // during master initialization, we could have multiple places splitting a same wal
- this.splitLogLock.lock();
- needReleaseLock = true;
- }
- try {
- for (ServerName serverName : serverNames) {
- Path logDir = new Path(this.rootdir,
- DefaultWALProvider.getWALDirectoryName(serverName.toString()));
- Path splitDir = logDir.suffix(DefaultWALProvider.SPLITTING_EXT);
- // Rename the directory so a rogue RS doesn't create more WALs
- if (fs.exists(logDir)) {
- if (!this.fs.rename(logDir, splitDir)) {
- throw new IOException("Failed fs.rename for log split: " + logDir);
- }
- logDir = splitDir;
- LOG.debug("Renamed region directory: " + splitDir);
- } else if (!fs.exists(splitDir)) {
- LOG.info("Log dir for server " + serverName + " does not exist");
- continue;
- }
- logDirs.add(splitDir);
- }
- } finally {
- if (needReleaseLock) {
- this.splitLogLock.unlock();
- }
- }
- return logDirs;
- }
-
- /**
- * Mark regions in recovering state when distributedLogReplay are set true
- * @param serverName Failed region server whose wals to be replayed
- * @param regions Set of regions to be recovered
- * @throws IOException
- */
- public void prepareLogReplay(ServerName serverName, Set<HRegionInfo> regions) throws IOException {
- if (!this.distributedLogReplay) {
- return;
- }
- // mark regions in recovering state
- if (regions == null || regions.isEmpty()) {
- return;
- }
- this.splitLogManager.markRegionsRecovering(serverName, regions);
- }
-
- public void splitLog(final Set<ServerName> serverNames) throws IOException {
- splitLog(serverNames, NON_META_FILTER);
- }
-
- /**
- * Wrapper function on {@link SplitLogManager#removeStaleRecoveringRegions(Set)}
- * @param failedServers
- * @throws IOException
- */
- void removeStaleRecoveringRegionsFromZK(final Set<ServerName> failedServers)
- throws IOException, InterruptedIOException {
- this.splitLogManager.removeStaleRecoveringRegions(failedServers);
- }
-
- /**
- * This method is the base split method that splits WAL files matching a filter. Callers should
- * pass the appropriate filter for meta and non-meta WALs.
- * @param serverNames logs belonging to these servers will be split; this will rename the log
- * directory out from under a soft-failed server
- * @param filter
- * @throws IOException
- */
- public void splitLog(final Set<ServerName> serverNames, PathFilter filter) throws IOException {
- long splitTime = 0, splitLogSize = 0;
- List<Path> logDirs = getLogDirs(serverNames);
-
- splitLogManager.handleDeadWorkers(serverNames);
- splitTime = EnvironmentEdgeManager.currentTime();
- splitLogSize = splitLogManager.splitLogDistributed(serverNames, logDirs, filter);
- splitTime = EnvironmentEdgeManager.currentTime() - splitTime;
-
- if (this.metricsMasterFilesystem != null) {
- if (filter == META_FILTER) {
- this.metricsMasterFilesystem.addMetaWALSplit(splitTime, splitLogSize);
- } else {
- this.metricsMasterFilesystem.addSplit(splitTime, splitLogSize);
- }
- }
- }
-
- /**
* Get the rootdir. Make sure its wholesome and exists before returning.
* @param rd
* @param c
@@ -418,9 +142,8 @@ public class MasterFileSystem {
* needed populating the directory with necessary bootup files).
* @throws IOException
*/
- private Path checkRootDir(final Path rd, final Configuration c,
- final FileSystem fs)
- throws IOException {
+ private Path checkRootDir(final Path rd, final Configuration c, final FileSystem fs)
+ throws IOException {
// If FS is in safe mode wait till out of it.
FSUtils.waitOnSafeMode(c, c.getInt(HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000));
@@ -595,22 +318,6 @@ public class MasterFileSystem {
}
public void stop() {
- if (splitLogManager != null) {
- this.splitLogManager.stop();
- }
- }
-
- /**
- * The function is used in SSH to set recovery mode based on configuration after all outstanding
- * log split tasks drained.
- * @throws IOException
- */
- public void setLogRecoveryMode() throws IOException {
- this.splitLogManager.setRecoveryMode(false);
- }
-
- public RecoveryMode getLogRecoveryMode() {
- return this.splitLogManager.getRecoveryMode();
}
public void logFileSystemState(Log log) throws IOException {
http://git-wip-us.apache.org/repos/asf/hbase/blob/a8a2c516/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java
index d095183..21f14e8 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java
@@ -73,6 +73,11 @@ public interface MasterServices extends Server {
MasterFileSystem getMasterFileSystem();
/**
+ * @return Master's WALs {@link MasterWalManager} utility class.
+ */
+ MasterWalManager getMasterWalManager();
+
+ /**
* @return Master's {@link ServerManager} instance.
*/
ServerManager getServerManager();
http://git-wip-us.apache.org/repos/asf/hbase/blob/a8a2c516/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterWalManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterWalManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterWalManager.java
new file mode 100644
index 0000000..d447e2d
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterWalManager.java
@@ -0,0 +1,351 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master;
+
+import com.google.common.annotations.VisibleForTesting;
+
+import java.io.IOException;
+import java.io.InterruptedIOException;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.SplitLogTask.RecoveryMode;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.wal.DefaultWALProvider;
+import org.apache.hadoop.hbase.wal.WALSplitter;
+
+/**
+ * This class abstracts a bunch of operations the HMaster needs
+ * when splitting log files e.g. finding log files, dirs etc.
+ */
+@InterfaceAudience.Private
+public class MasterWalManager {
+ private static final Log LOG = LogFactory.getLog(MasterWalManager.class);
+
+ final static PathFilter META_FILTER = new PathFilter() {
+ @Override
+ public boolean accept(Path p) {
+ return DefaultWALProvider.isMetaFile(p);
+ }
+ };
+
+ final static PathFilter NON_META_FILTER = new PathFilter() {
+ @Override
+ public boolean accept(Path p) {
+ return !DefaultWALProvider.isMetaFile(p);
+ }
+ };
+
+ // metrics for master
+ // TODO: Rename it, since those metrics are split-manager related
+ private final MetricsMasterFileSystem metricsMasterFilesystem = new MetricsMasterFileSystem();
+
+ // Keep around for convenience.
+ private final MasterServices services;
+ private final Configuration conf;
+ private final FileSystem fs;
+
+ // The Path to the old logs dir
+ private final Path oldLogDir;
+ private final Path rootDir;
+
+ // create the split log lock
+ private final Lock splitLogLock = new ReentrantLock();
+ private final SplitLogManager splitLogManager;
+ private final boolean distributedLogReplay;
+
+ // Is the fileystem ok?
+ private volatile boolean fsOk = true;
+
+ public MasterWalManager(MasterServices services) throws IOException {
+ this(services.getConfiguration(), services.getMasterFileSystem().getFileSystem(),
+ services.getMasterFileSystem().getRootDir(), services);
+ }
+
+ public MasterWalManager(Configuration conf, FileSystem fs, Path rootDir, MasterServices services)
+ throws IOException {
+ this.fs = fs;
+ this.conf = conf;
+ this.rootDir = rootDir;
+ this.services = services;
+ this.splitLogManager = new SplitLogManager(services, conf,
+ services, services, services.getServerName());
+ this.distributedLogReplay = this.splitLogManager.isLogReplaying();
+
+ this.oldLogDir = new Path(rootDir, HConstants.HREGION_OLDLOGDIR_NAME);
+
+ // Make sure the region servers can archive their old logs
+ if (!this.fs.exists(oldLogDir)) {
+ this.fs.mkdirs(oldLogDir);
+ }
+ }
+
+ public void stop() {
+ if (splitLogManager != null) {
+ splitLogManager.stop();
+ }
+ }
+
+ @VisibleForTesting
+ SplitLogManager getSplitLogManager() {
+ return this.splitLogManager;
+ }
+
+ /**
+ * Get the directory where old logs go
+ * @return the dir
+ */
+ Path getOldLogDir() {
+ return this.oldLogDir;
+ }
+
+ public FileSystem getFileSystem() {
+ return this.fs;
+ }
+
+ /**
+ * Checks to see if the file system is still accessible.
+ * If not, sets closed
+ * @return false if file system is not available
+ */
+ private boolean checkFileSystem() {
+ if (this.fsOk) {
+ try {
+ FSUtils.checkFileSystemAvailable(this.fs);
+ FSUtils.checkDfsSafeMode(this.conf);
+ } catch (IOException e) {
+ services.abort("Shutting down HBase cluster: file system not available", e);
+ this.fsOk = false;
+ }
+ }
+ return this.fsOk;
+ }
+
+ /**
+ * Inspect the log directory to find dead servers which need recovery work
+ * @return A set of ServerNames which aren't running but still have WAL files left in file system
+ */
+ Set<ServerName> getFailedServersFromLogFolders() {
+ boolean retrySplitting = !conf.getBoolean("hbase.hlog.split.skip.errors",
+ WALSplitter.SPLIT_SKIP_ERRORS_DEFAULT);
+
+ Set<ServerName> serverNames = new HashSet<ServerName>();
+ Path logsDirPath = new Path(this.rootDir, HConstants.HREGION_LOGDIR_NAME);
+
+ do {
+ if (services.isStopped()) {
+ LOG.warn("Master stopped while trying to get failed servers.");
+ break;
+ }
+ try {
+ if (!this.fs.exists(logsDirPath)) return serverNames;
+ FileStatus[] logFolders = FSUtils.listStatus(this.fs, logsDirPath, null);
+ // Get online servers after getting log folders to avoid log folder deletion of newly
+ // checked in region servers . see HBASE-5916
+ Set<ServerName> onlineServers = services.getServerManager().getOnlineServers().keySet();
+
+ if (logFolders == null || logFolders.length == 0) {
+ LOG.debug("No log files to split, proceeding...");
+ return serverNames;
+ }
+ for (FileStatus status : logFolders) {
+ FileStatus[] curLogFiles = FSUtils.listStatus(this.fs, status.getPath(), null);
+ if (curLogFiles == null || curLogFiles.length == 0) {
+ // Empty log folder. No recovery needed
+ continue;
+ }
+ final ServerName serverName = DefaultWALProvider.getServerNameFromWALDirectoryName(
+ status.getPath());
+ if (null == serverName) {
+ LOG.warn("Log folder " + status.getPath() + " doesn't look like its name includes a " +
+ "region server name; leaving in place. If you see later errors about missing " +
+ "write ahead logs they may be saved in this location.");
+ } else if (!onlineServers.contains(serverName)) {
+ LOG.info("Log folder " + status.getPath() + " doesn't belong "
+ + "to a known region server, splitting");
+ serverNames.add(serverName);
+ } else {
+ LOG.info("Log folder " + status.getPath() + " belongs to an existing region server");
+ }
+ }
+ retrySplitting = false;
+ } catch (IOException ioe) {
+ LOG.warn("Failed getting failed servers to be recovered.", ioe);
+ if (!checkFileSystem()) {
+ LOG.warn("Bad Filesystem, exiting");
+ Runtime.getRuntime().halt(1);
+ }
+ try {
+ if (retrySplitting) {
+ Thread.sleep(conf.getInt("hbase.hlog.split.failure.retry.interval", 30 * 1000));
+ }
+ } catch (InterruptedException e) {
+ LOG.warn("Interrupted, aborting since cannot return w/o splitting");
+ Thread.currentThread().interrupt();
+ retrySplitting = false;
+ Runtime.getRuntime().halt(1);
+ }
+ }
+ } while (retrySplitting);
+
+ return serverNames;
+ }
+
+ public void splitLog(final ServerName serverName) throws IOException {
+ Set<ServerName> serverNames = new HashSet<ServerName>();
+ serverNames.add(serverName);
+ splitLog(serverNames);
+ }
+
+ /**
+ * Specialized method to handle the splitting for meta WAL
+ * @param serverName logs belonging to this server will be split
+ */
+ public void splitMetaLog(final ServerName serverName) throws IOException {
+ Set<ServerName> serverNames = new HashSet<ServerName>();
+ serverNames.add(serverName);
+ splitMetaLog(serverNames);
+ }
+
+ /**
+ * Specialized method to handle the splitting for meta WAL
+ * @param serverNames logs belonging to these servers will be split
+ */
+ public void splitMetaLog(final Set<ServerName> serverNames) throws IOException {
+ splitLog(serverNames, META_FILTER);
+ }
+
+ @edu.umd.cs.findbugs.annotations.SuppressWarnings(value="UL_UNRELEASED_LOCK", justification=
+ "We only release this lock when we set it. Updates to code that uses it should verify use " +
+ "of the guard boolean.")
+ private List<Path> getLogDirs(final Set<ServerName> serverNames) throws IOException {
+ List<Path> logDirs = new ArrayList<Path>();
+ boolean needReleaseLock = false;
+ if (!this.services.isInitialized()) {
+ // during master initialization, we could have multiple places splitting a same wal
+ this.splitLogLock.lock();
+ needReleaseLock = true;
+ }
+ try {
+ for (ServerName serverName : serverNames) {
+ Path logDir = new Path(this.rootDir,
+ DefaultWALProvider.getWALDirectoryName(serverName.toString()));
+ Path splitDir = logDir.suffix(DefaultWALProvider.SPLITTING_EXT);
+ // Rename the directory so a rogue RS doesn't create more WALs
+ if (fs.exists(logDir)) {
+ if (!this.fs.rename(logDir, splitDir)) {
+ throw new IOException("Failed fs.rename for log split: " + logDir);
+ }
+ logDir = splitDir;
+ LOG.debug("Renamed region directory: " + splitDir);
+ } else if (!fs.exists(splitDir)) {
+ LOG.info("Log dir for server " + serverName + " does not exist");
+ continue;
+ }
+ logDirs.add(splitDir);
+ }
+ } finally {
+ if (needReleaseLock) {
+ this.splitLogLock.unlock();
+ }
+ }
+ return logDirs;
+ }
+
+ /**
+ * Mark regions in recovering state when distributedLogReplay are set true
+ * @param serverName Failed region server whose wals to be replayed
+ * @param regions Set of regions to be recovered
+ */
+ public void prepareLogReplay(ServerName serverName, Set<HRegionInfo> regions) throws IOException {
+ if (!this.distributedLogReplay) {
+ return;
+ }
+ // mark regions in recovering state
+ if (regions == null || regions.isEmpty()) {
+ return;
+ }
+ this.splitLogManager.markRegionsRecovering(serverName, regions);
+ }
+
+ public void splitLog(final Set<ServerName> serverNames) throws IOException {
+ splitLog(serverNames, NON_META_FILTER);
+ }
+
+ /**
+ * Wrapper function on {@link SplitLogManager#removeStaleRecoveringRegions(Set)}
+ * @param failedServers A set of known failed servers
+ */
+ void removeStaleRecoveringRegionsFromZK(final Set<ServerName> failedServers)
+ throws IOException, InterruptedIOException {
+ this.splitLogManager.removeStaleRecoveringRegions(failedServers);
+ }
+
+ /**
+ * This method is the base split method that splits WAL files matching a filter. Callers should
+ * pass the appropriate filter for meta and non-meta WALs.
+ * @param serverNames logs belonging to these servers will be split; this will rename the log
+ * directory out from under a soft-failed server
+ */
+ public void splitLog(final Set<ServerName> serverNames, PathFilter filter) throws IOException {
+ long splitTime = 0, splitLogSize = 0;
+ List<Path> logDirs = getLogDirs(serverNames);
+
+ splitLogManager.handleDeadWorkers(serverNames);
+ splitTime = EnvironmentEdgeManager.currentTime();
+ splitLogSize = splitLogManager.splitLogDistributed(serverNames, logDirs, filter);
+ splitTime = EnvironmentEdgeManager.currentTime() - splitTime;
+
+ if (this.metricsMasterFilesystem != null) {
+ if (filter == META_FILTER) {
+ this.metricsMasterFilesystem.addMetaWALSplit(splitTime, splitLogSize);
+ } else {
+ this.metricsMasterFilesystem.addSplit(splitTime, splitLogSize);
+ }
+ }
+ }
+
+ /**
+ * The function is used in SSH to set recovery mode based on configuration after all outstanding
+ * log split tasks drained.
+ */
+ public void setLogRecoveryMode() throws IOException {
+ this.splitLogManager.setRecoveryMode(false);
+ }
+
+ public RecoveryMode getLogRecoveryMode() {
+ return this.splitLogManager.getRecoveryMode();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/a8a2c516/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
index d69c7aa..37659f8 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
@@ -753,7 +753,7 @@ public class ServerManager {
}
OpenRegionRequest request = RequestConverter.buildOpenRegionRequest(server,
region, favoredNodes,
- (RecoveryMode.LOG_REPLAY == this.services.getMasterFileSystem().getLogRecoveryMode()));
+ (RecoveryMode.LOG_REPLAY == this.services.getMasterWalManager().getLogRecoveryMode()));
try {
OpenRegionResponse response = admin.openRegion(null, request);
return ResponseConverter.getRegionOpeningState(response);
@@ -781,7 +781,7 @@ public class ServerManager {
}
OpenRegionRequest request = RequestConverter.buildOpenRegionRequest(server, regionOpenInfos,
- (RecoveryMode.LOG_REPLAY == this.services.getMasterFileSystem().getLogRecoveryMode()));
+ (RecoveryMode.LOG_REPLAY == this.services.getMasterWalManager().getLogRecoveryMode()));
try {
OpenRegionResponse response = admin.openRegion(null, request);
return ResponseConverter.getRegionOpeningStateList(response);
http://git-wip-us.apache.org/repos/asf/hbase/blob/a8a2c516/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
index 4c2e745..5d488dd 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
@@ -75,8 +75,8 @@ import com.google.common.annotations.VisibleForTesting;
* <p>SplitLogManager monitors the tasks that it creates using the
* timeoutMonitor thread. If a task's progress is slow then
* {@link SplitLogManagerCoordination#checkTasks} will take away the
- * task from the owner {@link org.apache.hadoop.hbase.regionserver.SplitLogWorker}
- * and the task will be up for grabs again. When the task is done then it is
+ * task from the owner {@link org.apache.hadoop.hbase.regionserver.SplitLogWorker}
+ * and the task will be up for grabs again. When the task is done then it is
* deleted by SplitLogManager.
*
* <p>Clients call {@link #splitLogDistributed(Path)} to split a region server's
@@ -273,7 +273,7 @@ public class SplitLogManager {
}
waitForSplittingCompletion(batch, status);
// remove recovering regions
- if (filter == MasterFileSystem.META_FILTER /* reference comparison */) {
+ if (filter == MasterWalManager.META_FILTER /* reference comparison */) {
// we split meta regions and user regions separately therefore logfiles are either all for
// meta or user regions but won't for both( we could have mixed situations in tests)
isMetaRecovery = true;
@@ -411,7 +411,7 @@ public class SplitLogManager {
for (ServerName tmpServerName : serverNames) {
recoveredServerNameSet.add(tmpServerName.getServerName());
}
-
+
this.recoveringRegionLock.lock();
try {
((BaseCoordinatedStateManager) server.getCoordinatedStateManager())
http://git-wip-us.apache.org/repos/asf/hbase/blob/a8a2c516/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ServerCrashProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ServerCrashProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ServerCrashProcedure.java
index 7de694c..2fbb559 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ServerCrashProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ServerCrashProcedure.java
@@ -37,8 +37,8 @@ import org.apache.hadoop.hbase.client.ClusterConnection;
import org.apache.hadoop.hbase.client.RegionReplicaUtil;
import org.apache.hadoop.hbase.client.TableState;
import org.apache.hadoop.hbase.master.AssignmentManager;
-import org.apache.hadoop.hbase.master.MasterFileSystem;
import org.apache.hadoop.hbase.master.MasterServices;
+import org.apache.hadoop.hbase.master.MasterWalManager;
import org.apache.hadoop.hbase.master.RegionState;
import org.apache.hadoop.hbase.master.RegionStates;
import org.apache.hadoop.hbase.procedure2.ProcedureYieldException;
@@ -320,10 +320,10 @@ implements ServerProcedureInterface {
* @throws IOException
*/
private void start(final MasterProcedureEnv env) throws IOException {
- MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem();
+ MasterWalManager mwm = env.getMasterServices().getMasterWalManager();
// Set recovery mode late. This is what the old ServerShutdownHandler used do.
- mfs.setLogRecoveryMode();
- this.distributedLogReplay = mfs.getLogRecoveryMode() == RecoveryMode.LOG_REPLAY;
+ mwm.setLogRecoveryMode();
+ this.distributedLogReplay = mwm.getLogRecoveryMode() == RecoveryMode.LOG_REPLAY;
}
/**
@@ -335,7 +335,7 @@ implements ServerProcedureInterface {
private boolean processMeta(final MasterProcedureEnv env)
throws IOException {
if (LOG.isDebugEnabled()) LOG.debug("Processing hbase:meta that was on " + this.serverName);
- MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem();
+ MasterWalManager mwm = env.getMasterServices().getMasterWalManager();
AssignmentManager am = env.getMasterServices().getAssignmentManager();
HRegionInfo metaHRI = HRegionInfo.FIRST_META_REGIONINFO;
if (this.shouldSplitWal) {
@@ -343,7 +343,7 @@ implements ServerProcedureInterface {
prepareLogReplay(env, META_REGION_SET);
} else {
// TODO: Matteo. We BLOCK here but most important thing to be doing at this moment.
- mfs.splitMetaLog(serverName);
+ mwm.splitMetaLog(serverName);
am.getRegionStates().logSplit(metaHRI);
}
}
@@ -360,7 +360,7 @@ implements ServerProcedureInterface {
processed = false;
} else {
// TODO: Matteo. We BLOCK here but most important thing to be doing at this moment.
- mfs.splitMetaLog(serverName);
+ mwm.splitMetaLog(serverName);
}
}
}
@@ -394,9 +394,9 @@ implements ServerProcedureInterface {
LOG.debug("Mark " + size(this.regionsOnCrashedServer) + " regions-in-recovery from " +
this.serverName);
}
- MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem();
+ MasterWalManager mwm = env.getMasterServices().getMasterWalManager();
AssignmentManager am = env.getMasterServices().getAssignmentManager();
- mfs.prepareLogReplay(this.serverName, regions);
+ mwm.prepareLogReplay(this.serverName, regions);
am.getRegionStates().logSplit(this.serverName);
}
@@ -405,10 +405,10 @@ implements ServerProcedureInterface {
LOG.debug("Splitting logs from " + serverName + "; region count=" +
size(this.regionsOnCrashedServer));
}
- MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem();
+ MasterWalManager mwm = env.getMasterServices().getMasterWalManager();
AssignmentManager am = env.getMasterServices().getAssignmentManager();
// TODO: For Matteo. Below BLOCKs!!!! Redo so can relinquish executor while it is running.
- mfs.splitLog(this.serverName);
+ mwm.splitLog(this.serverName);
am.getRegionStates().logSplit(this.serverName);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/a8a2c516/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java
index 0882716..60b62e4 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java
@@ -98,6 +98,11 @@ public class MockNoopMasterServices implements MasterServices, Server {
}
@Override
+ public MasterWalManager getMasterWalManager() {
+ return null;
+ }
+
+ @Override
public MasterCoprocessorHost getMasterCoprocessorHost() {
return null;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/a8a2c516/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
index e850ed0..cc8e2d8 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
@@ -221,9 +221,11 @@ public class TestCatalogJanitor {
class MockMasterServices extends MockNoopMasterServices {
private final MasterFileSystem mfs;
private final AssignmentManager asm;
+ private final Server server;
MockMasterServices(final Server server) throws IOException {
- this.mfs = new MasterFileSystem(server, this);
+ this.server = server;
+ this.mfs = new MasterFileSystem(this);
this.asm = Mockito.mock(AssignmentManager.class);
}
@@ -239,7 +241,12 @@ public class TestCatalogJanitor {
@Override
public Configuration getConfiguration() {
- return mfs.conf;
+ return server.getConfiguration();
+ }
+
+ @Override
+ public ServerName getServerName() {
+ return server.getServerName();
}
@Override
http://git-wip-us.apache.org/repos/asf/hbase/blob/a8a2c516/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java
index 4f2385f..170a882 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java
@@ -1,6 +1,5 @@
/**
*
-
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -213,7 +212,7 @@ public class TestDistributedLogSplitting {
startCluster(NUM_RS);
final int NUM_LOG_LINES = 1000;
- final SplitLogManager slm = master.getMasterFileSystem().splitLogManager;
+ final SplitLogManager slm = master.getMasterWalManager().getSplitLogManager();
// turn off load balancing to prevent regions from moving around otherwise
// they will consume recovered.edits
master.balanceSwitch(false);
@@ -680,7 +679,7 @@ public class TestDistributedLogSplitting {
final ZooKeeperWatcher zkw = master.getZooKeeper();
Table ht = installTable(zkw, "table", "family", 40);
try {
- final SplitLogManager slm = master.getMasterFileSystem().splitLogManager;
+ final SplitLogManager slm = master.getMasterWalManager().getSplitLogManager();
Set<HRegionInfo> regionSet = new HashSet<HRegionInfo>();
HRegionInfo region = null;
@@ -929,7 +928,7 @@ public class TestDistributedLogSplitting {
final ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "table-creation", null);
Table ht = installTable(zkw, "table", "family", NUM_REGIONS_TO_CREATE);
try {
- final SplitLogManager slm = master.getMasterFileSystem().splitLogManager;
+ final SplitLogManager slm = master.getMasterWalManager().getSplitLogManager();
Set<HRegionInfo> regionSet = new HashSet<HRegionInfo>();
HRegionInfo region = null;
@@ -1004,7 +1003,7 @@ public class TestDistributedLogSplitting {
LOG.info("testWorkerAbort");
startCluster(3);
final int NUM_LOG_LINES = 10000;
- final SplitLogManager slm = master.getMasterFileSystem().splitLogManager;
+ final SplitLogManager slm = master.getMasterWalManager().getSplitLogManager();
FileSystem fs = master.getMasterFileSystem().getFileSystem();
final List<RegionServerThread> rsts = cluster.getLiveRegionServerThreads();
@@ -1123,7 +1122,7 @@ public class TestDistributedLogSplitting {
public void testDelayedDeleteOnFailure() throws Exception {
LOG.info("testDelayedDeleteOnFailure");
startCluster(1);
- final SplitLogManager slm = master.getMasterFileSystem().splitLogManager;
+ final SplitLogManager slm = master.getMasterWalManager().getSplitLogManager();
final FileSystem fs = master.getMasterFileSystem().getFileSystem();
final Path logDir = new Path(FSUtils.getRootDir(conf), "x");
fs.mkdirs(logDir);
@@ -1204,10 +1203,10 @@ public class TestDistributedLogSplitting {
LOG.info("#regions = " + regions.size());
Set<HRegionInfo> tmpRegions = new HashSet<HRegionInfo>();
tmpRegions.add(HRegionInfo.FIRST_META_REGIONINFO);
- master.getMasterFileSystem().prepareLogReplay(hrs.getServerName(), tmpRegions);
+ master.getMasterWalManager().prepareLogReplay(hrs.getServerName(), tmpRegions);
Set<HRegionInfo> userRegionSet = new HashSet<HRegionInfo>();
userRegionSet.addAll(regions);
- master.getMasterFileSystem().prepareLogReplay(hrs.getServerName(), userRegionSet);
+ master.getMasterWalManager().prepareLogReplay(hrs.getServerName(), userRegionSet);
boolean isMetaRegionInRecovery = false;
List<String> recoveringRegions =
zkw.getRecoverableZooKeeper().getChildren(zkw.recoveringRegionsZNode, false);
@@ -1219,7 +1218,7 @@ public class TestDistributedLogSplitting {
}
assertTrue(isMetaRegionInRecovery);
- master.getMasterFileSystem().splitMetaLog(hrs.getServerName());
+ master.getMasterWalManager().splitMetaLog(hrs.getServerName());
isMetaRegionInRecovery = false;
recoveringRegions =
http://git-wip-us.apache.org/repos/asf/hbase/blob/a8a2c516/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterFileSystem.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterFileSystem.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterFileSystem.java
index 0534643..bf13e7f 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterFileSystem.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterFileSystem.java
@@ -21,25 +21,15 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
-import java.util.HashSet;
-import java.util.Set;
-
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.ServerName;
-import org.apache.hadoop.hbase.SplitLogTask;
import org.apache.hadoop.hbase.testclassification.MasterTests;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.util.FSUtils;
-import org.apache.hadoop.hbase.zookeeper.ZKSplitLog;
-import org.apache.hadoop.hbase.zookeeper.ZKUtil;
-import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
-import org.apache.zookeeper.CreateMode;
-import org.apache.zookeeper.ZooDefs.Ids;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -50,8 +40,8 @@ import org.junit.experimental.categories.Category;
*/
@Category({MasterTests.class, MediumTests.class})
public class TestMasterFileSystem {
-
private static final Log LOG = LogFactory.getLog(TestMasterFileSystem.class);
+
private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
@BeforeClass
@@ -68,52 +58,12 @@ public class TestMasterFileSystem {
public void testFsUriSetProperly() throws Exception {
HMaster master = UTIL.getMiniHBaseCluster().getMaster();
MasterFileSystem fs = master.getMasterFileSystem();
- Path masterRoot = FSUtils.getRootDir(fs.conf);
+ Path masterRoot = FSUtils.getRootDir(fs.getConfiguration());
Path rootDir = FSUtils.getRootDir(fs.getFileSystem().getConf());
// make sure the fs and the found root dir have the same scheme
LOG.debug("from fs uri:" + FileSystem.getDefaultUri(fs.getFileSystem().getConf()));
- LOG.debug("from configuration uri:" + FileSystem.getDefaultUri(fs.conf));
+ LOG.debug("from configuration uri:" + FileSystem.getDefaultUri(fs.getConfiguration()));
// make sure the set uri matches by forcing it.
assertEquals(masterRoot, rootDir);
}
-
- @Test
- public void testRemoveStaleRecoveringRegionsDuringMasterInitialization() throws Exception {
- // this test is for when distributed log replay is enabled
- if (!UTIL.getConfiguration().getBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, false)) return;
-
- LOG.info("Starting testRemoveStaleRecoveringRegionsDuringMasterInitialization");
- HMaster master = UTIL.getMiniHBaseCluster().getMaster();
- MasterFileSystem fs = master.getMasterFileSystem();
-
- String failedRegion = "failedRegoin1";
- String staleRegion = "staleRegion";
- ServerName inRecoveryServerName = ServerName.valueOf("mgr,1,1");
- ServerName previouselyFaildServerName = ServerName.valueOf("previous,1,1");
- String walPath = "/hbase/data/.logs/" + inRecoveryServerName.getServerName()
- + "-splitting/test";
- // Create a ZKW to use in the test
- ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL);
- zkw.getRecoverableZooKeeper().create(ZKSplitLog.getEncodedNodeName(zkw, walPath),
- new SplitLogTask.Owned(inRecoveryServerName, fs.getLogRecoveryMode()).toByteArray(),
- Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
- String staleRegionPath = ZKUtil.joinZNode(zkw.recoveringRegionsZNode, staleRegion);
- ZKUtil.createWithParents(zkw, staleRegionPath);
- String inRecoveringRegionPath = ZKUtil.joinZNode(zkw.recoveringRegionsZNode, failedRegion);
- inRecoveringRegionPath = ZKUtil.joinZNode(inRecoveringRegionPath,
- inRecoveryServerName.getServerName());
- ZKUtil.createWithParents(zkw, inRecoveringRegionPath);
- Set<ServerName> servers = new HashSet<ServerName>();
- servers.add(previouselyFaildServerName);
- fs.removeStaleRecoveringRegionsFromZK(servers);
-
- // verification
- assertFalse(ZKUtil.checkExists(zkw, staleRegionPath) != -1);
- assertTrue(ZKUtil.checkExists(zkw, inRecoveringRegionPath) != -1);
-
- ZKUtil.deleteChildrenRecursively(zkw, zkw.recoveringRegionsZNode);
- ZKUtil.deleteChildrenRecursively(zkw, zkw.splitLogZNode);
- zkw.close();
- }
-
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/a8a2c516/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterWalManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterWalManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterWalManager.java
new file mode 100644
index 0000000..feb97d9
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterWalManager.java
@@ -0,0 +1,104 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.SplitLogTask;
+import org.apache.hadoop.hbase.testclassification.MasterTests;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.zookeeper.ZKSplitLog;
+import org.apache.hadoop.hbase.zookeeper.ZKUtil;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.ZooDefs.Ids;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+/**
+ * Test the master wal manager in a local cluster
+ */
+@Category({MasterTests.class, MediumTests.class})
+public class TestMasterWalManager {
+ private static final Log LOG = LogFactory.getLog(TestMasterWalManager.class);
+ private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
+
+ @BeforeClass
+ public static void setupTest() throws Exception {
+ UTIL.startMiniCluster();
+ }
+
+ @AfterClass
+ public static void teardownTest() throws Exception {
+ UTIL.shutdownMiniCluster();
+ }
+
+ @Test
+ public void testRemoveStaleRecoveringRegionsDuringMasterInitialization() throws Exception {
+ // this test is for when distributed log replay is enabled
+ if (!UTIL.getConfiguration().getBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, false)) return;
+
+ LOG.info("Starting testRemoveStaleRecoveringRegionsDuringMasterInitialization");
+ HMaster master = UTIL.getMiniHBaseCluster().getMaster();
+ MasterWalManager mwm = master.getMasterWalManager();
+
+ String failedRegion = "failedRegoin1";
+ String staleRegion = "staleRegion";
+ ServerName inRecoveryServerName = ServerName.valueOf("mgr,1,1");
+ ServerName previouselyFaildServerName = ServerName.valueOf("previous,1,1");
+ String walPath = "/hbase/data/.logs/" + inRecoveryServerName.getServerName()
+ + "-splitting/test";
+ // Create a ZKW to use in the test
+ ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL);
+ zkw.getRecoverableZooKeeper().create(ZKSplitLog.getEncodedNodeName(zkw, walPath),
+ new SplitLogTask.Owned(inRecoveryServerName, mwm.getLogRecoveryMode()).toByteArray(),
+ Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
+ String staleRegionPath = ZKUtil.joinZNode(zkw.recoveringRegionsZNode, staleRegion);
+ ZKUtil.createWithParents(zkw, staleRegionPath);
+ String inRecoveringRegionPath = ZKUtil.joinZNode(zkw.recoveringRegionsZNode, failedRegion);
+ inRecoveringRegionPath = ZKUtil.joinZNode(inRecoveringRegionPath,
+ inRecoveryServerName.getServerName());
+ ZKUtil.createWithParents(zkw, inRecoveringRegionPath);
+ Set<ServerName> servers = new HashSet<ServerName>();
+ servers.add(previouselyFaildServerName);
+ mwm.removeStaleRecoveringRegionsFromZK(servers);
+
+ // verification
+ assertFalse(ZKUtil.checkExists(zkw, staleRegionPath) != -1);
+ assertTrue(ZKUtil.checkExists(zkw, inRecoveringRegionPath) != -1);
+
+ ZKUtil.deleteChildrenRecursively(zkw, zkw.recoveringRegionsZNode);
+ ZKUtil.deleteChildrenRecursively(zkw, zkw.splitLogZNode);
+ zkw.close();
+ }
+}
[31/50] [abbrv] hbase git commit: HBASE-15776 Replace
master.am.getTableStateManager() with the direct
master.getTableStateManager()
Posted by sy...@apache.org.
HBASE-15776 Replace master.am.getTableStateManager() with the direct master.getTableStateManager()
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/513ca348
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/513ca348
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/513ca348
Branch: refs/heads/hbase-12439
Commit: 513ca3483f1d32450ffa0c034e7a7f97b63ff582
Parents: 56358a0
Author: Matteo Bertozzi <ma...@cloudera.com>
Authored: Fri May 6 19:30:45 2016 -0700
Committer: Matteo Bertozzi <ma...@cloudera.com>
Committed: Fri May 6 19:30:45 2016 -0700
----------------------------------------------------------------------
.../main/java/org/apache/hadoop/hbase/master/HMaster.java | 10 +++++-----
.../hbase/master/procedure/AddColumnFamilyProcedure.java | 2 +-
.../hbase/master/procedure/CreateTableProcedure.java | 2 +-
.../master/procedure/DeleteColumnFamilyProcedure.java | 2 +-
.../hbase/master/procedure/DeleteTableProcedure.java | 4 ++--
.../hbase/master/procedure/DisableTableProcedure.java | 7 +++----
.../hbase/master/procedure/EnableTableProcedure.java | 6 +++---
.../hbase/master/procedure/MasterDDLOperationHelper.java | 2 +-
.../master/procedure/ModifyColumnFamilyProcedure.java | 2 +-
.../hbase/master/procedure/ModifyTableProcedure.java | 4 ++--
.../hadoop/hbase/master/snapshot/SnapshotManager.java | 6 ++----
.../java/org/apache/hadoop/hbase/client/TestAdmin1.java | 6 +++---
.../java/org/apache/hadoop/hbase/master/TestMaster.java | 3 +--
.../master/TestMasterRestartAfterDisablingTable.java | 8 ++++----
.../master/procedure/MasterProcedureTestingUtility.java | 4 ++--
15 files changed, 32 insertions(+), 36 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/513ca348/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index c2cab33..3829b35 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -944,10 +944,11 @@ public class HMaster extends HRegionServer implements MasterServices {
assigned++;
}
- if (replicaId == HRegionInfo.DEFAULT_REPLICA_ID)
+ if (replicaId == HRegionInfo.DEFAULT_REPLICA_ID) {
+ // TODO: should we prevent from using state manager before meta was initialized?
+ // tableStateManager.start();
getTableStateManager().setTableState(TableName.META_TABLE_NAME, TableState.State.ENABLED);
- // TODO: should we prevent from using state manager before meta was initialized?
- // tableStateManager.start();
+ }
if ((RecoveryMode.LOG_REPLAY == this.getMasterWalManager().getLogRecoveryMode())
&& (!previouslyFailedMetaRSs.isEmpty())) {
@@ -2096,8 +2097,7 @@ public class HMaster extends HRegionServer implements MasterServices {
if (!MetaTableAccessor.tableExists(getConnection(), tableName)) {
throw new TableNotFoundException(tableName);
}
- if (!getAssignmentManager().getTableStateManager().
- isTableState(tableName, TableState.State.DISABLED)) {
+ if (!getTableStateManager().isTableState(tableName, TableState.State.DISABLED)) {
throw new TableNotDisabledException(tableName);
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/513ca348/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/AddColumnFamilyProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/AddColumnFamilyProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/AddColumnFamilyProcedure.java
index 4a6c67d..ce099ed 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/AddColumnFamilyProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/AddColumnFamilyProcedure.java
@@ -339,7 +339,7 @@ public class AddColumnFamilyProcedure
*/
private void reOpenAllRegionsIfTableIsOnline(final MasterProcedureEnv env) throws IOException {
// This operation only run when the table is enabled.
- if (!env.getMasterServices().getAssignmentManager().getTableStateManager()
+ if (!env.getMasterServices().getTableStateManager()
.isTableState(getTableName(), TableState.State.ENABLED)) {
return;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/513ca348/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java
index 110bccd..03d433b 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java
@@ -450,7 +450,7 @@ public class CreateTableProcedure
ModifyRegionUtils.assignRegions(assignmentManager, regions);
// Enable table
- assignmentManager.getTableStateManager()
+ env.getMasterServices().getTableStateManager()
.setTableState(tableName, TableState.State.ENABLED);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/513ca348/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteColumnFamilyProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteColumnFamilyProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteColumnFamilyProcedure.java
index 1e510b3..097aaf1 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteColumnFamilyProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteColumnFamilyProcedure.java
@@ -364,7 +364,7 @@ public class DeleteColumnFamilyProcedure
*/
private void reOpenAllRegionsIfTableIsOnline(final MasterProcedureEnv env) throws IOException {
// This operation only run when the table is enabled.
- if (!env.getMasterServices().getAssignmentManager().getTableStateManager()
+ if (!env.getMasterServices().getTableStateManager()
.isTableState(getTableName(), TableState.State.ENABLED)) {
return;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/513ca348/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteTableProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteTableProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteTableProcedure.java
index 4c92fb5..39ec0a6 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteTableProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteTableProcedure.java
@@ -402,7 +402,7 @@ public class DeleteTableProcedure
protected static void deleteAssignmentState(final MasterProcedureEnv env,
final TableName tableName) throws IOException {
- AssignmentManager am = env.getMasterServices().getAssignmentManager();
+ final AssignmentManager am = env.getMasterServices().getAssignmentManager();
// Clean up regions of the table in RegionStates.
LOG.debug("Removing '" + tableName + "' from region states.");
@@ -410,7 +410,7 @@ public class DeleteTableProcedure
// If entry for this table states, remove it.
LOG.debug("Marking '" + tableName + "' as deleted.");
- am.getTableStateManager().setDeletedTable(tableName);
+ env.getMasterServices().getTableStateManager().setDeletedTable(tableName);
}
protected static void deleteTableDescriptorCache(final MasterProcedureEnv env,
http://git-wip-us.apache.org/repos/asf/hbase/blob/513ca348/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DisableTableProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DisableTableProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DisableTableProcedure.java
index 6a65690..5518b8b 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DisableTableProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DisableTableProcedure.java
@@ -289,8 +289,7 @@ public class DisableTableProcedure
// the state to DISABLING from ENABLED. The implementation was done before table lock
// was implemented. With table lock, there is no need to set the state here (it will
// set the state later on). A quick state check should be enough for us to move forward.
- TableStateManager tsm =
- env.getMasterServices().getAssignmentManager().getTableStateManager();
+ TableStateManager tsm = env.getMasterServices().getTableStateManager();
TableState.State state = tsm.getTableState(tableName);
if(!state.equals(TableState.State.ENABLED)){
LOG.info("Table " + tableName + " isn't enabled;is "+state.name()+"; skipping disable");
@@ -327,7 +326,7 @@ public class DisableTableProcedure
final MasterProcedureEnv env,
final TableName tableName) throws IOException {
// Set table disabling flag up in zk.
- env.getMasterServices().getAssignmentManager().getTableStateManager().setTableState(
+ env.getMasterServices().getTableStateManager().setTableState(
tableName,
TableState.State.DISABLING);
}
@@ -418,7 +417,7 @@ public class DisableTableProcedure
final MasterProcedureEnv env,
final TableName tableName) throws IOException {
// Flip the table to disabled
- env.getMasterServices().getAssignmentManager().getTableStateManager().setTableState(
+ env.getMasterServices().getTableStateManager().setTableState(
tableName,
TableState.State.DISABLED);
LOG.info("Disabled table, " + tableName + ", is completed.");
http://git-wip-us.apache.org/repos/asf/hbase/blob/513ca348/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/EnableTableProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/EnableTableProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/EnableTableProcedure.java
index 862231e..141dcd4 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/EnableTableProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/EnableTableProcedure.java
@@ -310,7 +310,7 @@ public class EnableTableProcedure
// the state to ENABLING from DISABLED. The implementation was done before table lock
// was implemented. With table lock, there is no need to set the state here (it will
// set the state later on). A quick state check should be enough for us to move forward.
- TableStateManager tsm = env.getMasterServices().getAssignmentManager().getTableStateManager();
+ TableStateManager tsm = env.getMasterServices().getTableStateManager();
TableState.State state = tsm.getTableState(tableName);
if(!state.equals(TableState.State.DISABLED)){
LOG.info("Table " + tableName + " isn't disabled;is "+state.name()+"; skipping enable");
@@ -349,7 +349,7 @@ public class EnableTableProcedure
final TableName tableName) throws IOException {
// Set table disabling flag up in zk.
LOG.info("Attempting to enable the table " + tableName);
- env.getMasterServices().getAssignmentManager().getTableStateManager().setTableState(
+ env.getMasterServices().getTableStateManager().setTableState(
tableName,
TableState.State.ENABLING);
}
@@ -494,7 +494,7 @@ public class EnableTableProcedure
final MasterProcedureEnv env,
final TableName tableName) throws IOException {
// Flip the table to Enabled
- env.getMasterServices().getAssignmentManager().getTableStateManager().setTableState(
+ env.getMasterServices().getTableStateManager().setTableState(
tableName,
TableState.State.ENABLED);
LOG.info("Table '" + tableName + "' was successfully enabled.");
http://git-wip-us.apache.org/repos/asf/hbase/blob/513ca348/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterDDLOperationHelper.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterDDLOperationHelper.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterDDLOperationHelper.java
index 19bd015..f2ee97f 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterDDLOperationHelper.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterDDLOperationHelper.java
@@ -77,7 +77,7 @@ public final class MasterDDLOperationHelper {
}
// We only execute this procedure with table online if online schema change config is set.
- if (!env.getMasterServices().getAssignmentManager().getTableStateManager()
+ if (!env.getMasterServices().getTableStateManager()
.isTableState(tableName, TableState.State.DISABLED)
&& !MasterDDLOperationHelper.isOnlineSchemaChangeAllowed(env)) {
throw new TableNotDisabledException(tableName);
http://git-wip-us.apache.org/repos/asf/hbase/blob/513ca348/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyColumnFamilyProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyColumnFamilyProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyColumnFamilyProcedure.java
index 0b7d8a2..ee534e9 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyColumnFamilyProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyColumnFamilyProcedure.java
@@ -319,7 +319,7 @@ public class ModifyColumnFamilyProcedure
*/
private void reOpenAllRegionsIfTableIsOnline(final MasterProcedureEnv env) throws IOException {
// This operation only run when the table is enabled.
- if (!env.getMasterServices().getAssignmentManager().getTableStateManager()
+ if (!env.getMasterServices().getTableStateManager()
.isTableState(getTableName(), TableState.State.ENABLED)) {
return;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/513ca348/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyTableProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyTableProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyTableProcedure.java
index 8f088da..3f76df3 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyTableProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyTableProcedure.java
@@ -298,7 +298,7 @@ public class ModifyTableProcedure
this.unmodifiedHTableDescriptor =
env.getMasterServices().getTableDescriptors().get(getTableName());
- if (env.getMasterServices().getAssignmentManager().getTableStateManager()
+ if (env.getMasterServices().getTableStateManager()
.isTableState(getTableName(), TableState.State.ENABLED)) {
// We only execute this procedure with table online if online schema change config is set.
if (!MasterDDLOperationHelper.isOnlineSchemaChangeAllowed(env)) {
@@ -437,7 +437,7 @@ public class ModifyTableProcedure
*/
private void reOpenAllRegionsIfTableIsOnline(final MasterProcedureEnv env) throws IOException {
// This operation only run when the table is enabled.
- if (!env.getMasterServices().getAssignmentManager().getTableStateManager()
+ if (!env.getMasterServices().getTableStateManager()
.isTableState(getTableName(), TableState.State.ENABLED)) {
return;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/513ca348/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java
index 8ef63fb..0304e38 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java
@@ -48,7 +48,6 @@ import org.apache.hadoop.hbase.client.TableState;
import org.apache.hadoop.hbase.errorhandling.ForeignException;
import org.apache.hadoop.hbase.executor.ExecutorService;
import org.apache.hadoop.hbase.ipc.RpcServer;
-import org.apache.hadoop.hbase.master.AssignmentManager;
import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
import org.apache.hadoop.hbase.master.MasterFileSystem;
import org.apache.hadoop.hbase.master.MasterServices;
@@ -582,15 +581,14 @@ public class SnapshotManager extends MasterProcedureManager implements Stoppable
// if the table is enabled, then have the RS run actually the snapshot work
TableName snapshotTable = TableName.valueOf(snapshot.getTable());
- AssignmentManager assignmentMgr = master.getAssignmentManager();
- if (assignmentMgr.getTableStateManager().isTableState(snapshotTable,
+ if (master.getTableStateManager().isTableState(snapshotTable,
TableState.State.ENABLED)) {
LOG.debug("Table enabled, starting distributed snapshot.");
snapshotEnabledTable(snapshot);
LOG.debug("Started snapshot: " + ClientSnapshotDescriptionUtils.toString(snapshot));
}
// For disabled table, snapshot is created by the master
- else if (assignmentMgr.getTableStateManager().isTableState(snapshotTable,
+ else if (master.getTableStateManager().isTableState(snapshotTable,
TableState.State.DISABLED)) {
LOG.debug("Table is disabled, running snapshot entirely on master.");
snapshotDisabledTable(snapshot);
http://git-wip-us.apache.org/repos/asf/hbase/blob/513ca348/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin1.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin1.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin1.java
index 10dbed0..ce48032 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin1.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin1.java
@@ -249,7 +249,7 @@ public class TestAdmin1 {
this.admin.disableTable(ht.getName());
assertTrue("Table must be disabled.", TEST_UTIL.getHBaseCluster()
- .getMaster().getAssignmentManager().getTableStateManager().isTableState(
+ .getMaster().getTableStateManager().isTableState(
ht.getName(), TableState.State.DISABLED));
assertEquals(TableState.State.DISABLED, getStateFromMeta(table));
@@ -277,7 +277,7 @@ public class TestAdmin1 {
assertTrue(ok);
this.admin.enableTable(table);
assertTrue("Table must be enabled.", TEST_UTIL.getHBaseCluster()
- .getMaster().getAssignmentManager().getTableStateManager().isTableState(
+ .getMaster().getTableStateManager().isTableState(
ht.getName(), TableState.State.ENABLED));
assertEquals(TableState.State.ENABLED, getStateFromMeta(table));
@@ -365,7 +365,7 @@ public class TestAdmin1 {
tables = this.admin.listTables();
assertEquals(numTables + 1, tables.length);
assertTrue("Table must be enabled.",
- TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager().getTableStateManager()
+ TEST_UTIL.getHBaseCluster().getMaster().getTableStateManager()
.isTableState(tableName, TableState.State.ENABLED));
assertEquals(TableState.State.ENABLED, getStateFromMeta(tableName));
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/513ca348/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMaster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMaster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMaster.java
index 99e0e3f..c03afbc 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMaster.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMaster.java
@@ -84,8 +84,7 @@ public class TestMaster {
HMaster m = cluster.getMaster();
try (Table ht = TEST_UTIL.createTable(TABLENAME, FAMILYNAME)) {
- assertTrue(m.assignmentManager.getTableStateManager().isTableState(TABLENAME,
- TableState.State.ENABLED));
+ assertTrue(m.getTableStateManager().isTableState(TABLENAME, TableState.State.ENABLED));
TEST_UTIL.loadTable(ht, FAMILYNAME, false);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/513ca348/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterRestartAfterDisablingTable.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterRestartAfterDisablingTable.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterRestartAfterDisablingTable.java
index d4250e3..b62279e 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterRestartAfterDisablingTable.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterRestartAfterDisablingTable.java
@@ -95,8 +95,8 @@ public class TestMasterRestartAfterDisablingTable {
cluster.hbaseCluster.waitOnMaster(activeMaster);
cluster.waitForActiveAndReadyMaster();
- assertTrue("The table should not be in enabled state", cluster.getMaster()
- .getAssignmentManager().getTableStateManager().isTableState(
+ assertTrue("The table should not be in enabled state",
+ cluster.getMaster().getTableStateManager().isTableState(
TableName.valueOf("tableRestart"), TableState.State.DISABLED,
TableState.State.DISABLING));
log("Enabling table\n");
@@ -111,8 +111,8 @@ public class TestMasterRestartAfterDisablingTable {
assertEquals("The assigned regions were not onlined after master"
+ " switch except for the catalog and namespace tables.",
6, regions.size());
- assertTrue("The table should be in enabled state", cluster.getMaster()
- .getAssignmentManager().getTableStateManager()
+ assertTrue("The table should be in enabled state",
+ cluster.getMaster().getTableStateManager()
.isTableState(TableName.valueOf("tableRestart"), TableState.State.ENABLED));
ht.close();
TEST_UTIL.shutdownMiniCluster();
http://git-wip-us.apache.org/repos/asf/hbase/blob/513ca348/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java
index 430a017..68a22a8 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java
@@ -182,13 +182,13 @@ public class MasterProcedureTestingUtility {
public static void validateTableIsEnabled(final HMaster master, final TableName tableName)
throws IOException {
- TableStateManager tsm = master.getAssignmentManager().getTableStateManager();
+ TableStateManager tsm = master.getTableStateManager();
assertTrue(tsm.getTableState(tableName).equals(TableState.State.ENABLED));
}
public static void validateTableIsDisabled(final HMaster master, final TableName tableName)
throws IOException {
- TableStateManager tsm = master.getAssignmentManager().getTableStateManager();
+ TableStateManager tsm = master.getTableStateManager();
assertTrue(tsm.getTableState(tableName).equals(TableState.State.DISABLED));
}
[16/50] [abbrv] hbase git commit: HBASE-15292 Refined
ZooKeeperWatcher to prevent ZooKeeper's callback while construction (Hiroshi
Ikeda)
Posted by sy...@apache.org.
HBASE-15292 Refined ZooKeeperWatcher to prevent ZooKeeper's callback while construction (Hiroshi Ikeda)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/8b0ce779
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/8b0ce779
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/8b0ce779
Branch: refs/heads/hbase-12439
Commit: 8b0ce77942e99751394cc23a40436d9f20d4254c
Parents: efcf94d
Author: tedyu <yu...@gmail.com>
Authored: Thu May 5 11:14:32 2016 -0700
Committer: tedyu <yu...@gmail.com>
Committed: Thu May 5 11:14:32 2016 -0700
----------------------------------------------------------------------
.../hadoop/hbase/zookeeper/InstancePending.java | 80 ++++++++++++++++++++
.../hadoop/hbase/zookeeper/PendingWatcher.java | 53 +++++++++++++
.../hbase/zookeeper/ZooKeeperWatcher.java | 40 ++--------
.../hbase/zookeeper/TestInstancePending.java | 49 ++++++++++++
4 files changed, 187 insertions(+), 35 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/8b0ce779/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/InstancePending.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/InstancePending.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/InstancePending.java
new file mode 100644
index 0000000..7464dbb
--- /dev/null
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/InstancePending.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.zookeeper;
+
+import java.util.concurrent.CountDownLatch;
+
+/**
+ * Placeholder of an instance which will be accessed by other threads
+ * but is not yet created. Thread safe.
+ */
+class InstancePending<T> {
+ // Based on a subtle part of the Java Language Specification,
+ // in order to avoid a slight overhead of synchronization for each access.
+
+ private final CountDownLatch pendingLatch = new CountDownLatch(1);
+
+ /** Piggybacking on {@code pendingLatch}. */
+ private InstanceHolder<T> instanceHolder;
+
+ private static class InstanceHolder<T> {
+ // The JLS ensures the visibility of a final field and its contents
+ // unless they are exposed to another thread while the construction.
+ final T instance;
+
+ InstanceHolder(T instance) {
+ this.instance = instance;
+ }
+ }
+
+ /**
+ * Returns the instance given by the method {@link #prepare}.
+ * This is an interruptible blocking method
+ * and the interruption flag will be set just before returning if any.
+ */
+ T get() {
+ InstanceHolder<T> instanceHolder;
+ boolean interrupted = false;
+
+ while ((instanceHolder = this.instanceHolder) == null) {
+ try {
+ pendingLatch.await();
+ } catch (InterruptedException e) {
+ interrupted = true;
+ }
+ }
+
+ if (interrupted) {
+ Thread.currentThread().interrupt();
+ }
+ return instanceHolder.instance;
+ }
+
+ /**
+ * Associates the given instance for the method {@link #get}.
+ * This method should be called once, and {@code instance} should be non-null.
+ * This method is expected to call as soon as possible
+ * because the method {@code get} is uninterruptibly blocked until this method is called.
+ */
+ void prepare(T instance) {
+ assert instance != null;
+ instanceHolder = new InstanceHolder<T>(instance);
+ pendingLatch.countDown();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/8b0ce779/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/PendingWatcher.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/PendingWatcher.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/PendingWatcher.java
new file mode 100644
index 0000000..11d0e5d
--- /dev/null
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/PendingWatcher.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.zookeeper;
+
+import org.apache.zookeeper.WatchedEvent;
+import org.apache.zookeeper.Watcher;
+
+/**
+ * Placeholder of a watcher which might be triggered before the instance is not yet created.
+ * <p>
+ * {@code ZooKeeper} starts its event thread within its constructor (and that is an anti-pattern),
+ * and the watcher passed to the constructor might be called back by the event thread
+ * before you get the instance of {@code ZooKeeper} from the constructor.
+ * If your watcher calls methods of {@code ZooKeeper},
+ * pass this placeholder to the constructor of the {@code ZooKeeper},
+ * create your watcher using the instance of {@code ZooKeeper},
+ * and then call the method {@code PendingWatcher.prepare}.
+ */
+class PendingWatcher implements Watcher {
+ private final InstancePending<Watcher> pending = new InstancePending<Watcher>();
+
+ @Override
+ public void process(WatchedEvent event) {
+ pending.get().process(event);
+ }
+
+ /**
+ * Associates the substantial watcher of processing events.
+ * This method should be called once, and {@code watcher} should be non-null.
+ * This method is expected to call as soon as possible
+ * because the event processing, being invoked by the ZooKeeper event thread,
+ * is uninterruptibly blocked until this method is called.
+ */
+ void prepare(Watcher watcher) {
+ pending.prepare(watcher);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/8b0ce779/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java
index 205d397..dcbcf30 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java
@@ -73,7 +73,7 @@ public class ZooKeeperWatcher implements Watcher, Abortable, Closeable {
private String quorum;
// zookeeper connection
- private RecoverableZooKeeper recoverableZooKeeper;
+ private final RecoverableZooKeeper recoverableZooKeeper;
// abortable in case of zk failure
protected Abortable abortable;
@@ -130,8 +130,6 @@ public class ZooKeeperWatcher implements Watcher, Abortable, Closeable {
private final Configuration conf;
- private final Exception constructorCaller;
-
/* A pattern that matches a Kerberos name, borrowed from Hadoop's KerberosName */
private static final Pattern NAME_PATTERN = Pattern.compile("([^/@]*)(/([^/@]*))?@([^/@]*)");
@@ -162,13 +160,6 @@ public class ZooKeeperWatcher implements Watcher, Abortable, Closeable {
Abortable abortable, boolean canCreateBaseZNode)
throws IOException, ZooKeeperConnectionException {
this.conf = conf;
- // Capture a stack trace now. Will print it out later if problem so we can
- // distingush amongst the myriad ZKWs.
- try {
- throw new Exception("ZKW CONSTRUCTOR STACK TRACE FOR DEBUGGING");
- } catch (Exception e) {
- this.constructorCaller = e;
- }
this.quorum = ZKConfig.getZKQuorumServersString(conf);
this.prefix = identifier;
// Identifier will get the sessionid appended later below down when we
@@ -176,7 +167,9 @@ public class ZooKeeperWatcher implements Watcher, Abortable, Closeable {
this.identifier = identifier + "0x0";
this.abortable = abortable;
setNodeNames(conf);
- this.recoverableZooKeeper = ZKUtil.connect(conf, quorum, this, identifier);
+ PendingWatcher pendingWatcher = new PendingWatcher();
+ this.recoverableZooKeeper = ZKUtil.connect(conf, quorum, pendingWatcher, identifier);
+ pendingWatcher.prepare(this);
if (canCreateBaseZNode) {
createBaseZNodes();
}
@@ -650,27 +643,6 @@ public class ZooKeeperWatcher implements Watcher, Abortable, Closeable {
private void connectionEvent(WatchedEvent event) {
switch(event.getState()) {
case SyncConnected:
- // Now, this callback can be invoked before the this.zookeeper is set.
- // Wait a little while.
- long finished = System.currentTimeMillis() +
- this.conf.getLong("hbase.zookeeper.watcher.sync.connected.wait", 2000);
- while (System.currentTimeMillis() < finished) {
- try {
- Thread.sleep(1);
- } catch (InterruptedException e) {
- LOG.warn("Interrupted while sleeping");
- throw new RuntimeException("Interrupted while waiting for" +
- " recoverableZooKeeper is set");
- }
- if (this.recoverableZooKeeper != null) break;
- }
-
- if (this.recoverableZooKeeper == null) {
- LOG.error("ZK is null on connection event -- see stack trace " +
- "for the stack trace when constructor was called on this zkw",
- this.constructorCaller);
- throw new NullPointerException("ZK is null");
- }
this.identifier = this.prefix + "-0x" +
Long.toHexString(this.recoverableZooKeeper.getSessionId());
// Update our identifier. Otherwise ignore.
@@ -759,9 +731,7 @@ public class ZooKeeperWatcher implements Watcher, Abortable, Closeable {
@Override
public void close() {
try {
- if (recoverableZooKeeper != null) {
- recoverableZooKeeper.close();
- }
+ recoverableZooKeeper.close();
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/8b0ce779/hbase-client/src/test/java/org/apache/hadoop/hbase/zookeeper/TestInstancePending.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/zookeeper/TestInstancePending.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/zookeeper/TestInstancePending.java
new file mode 100644
index 0000000..667fed8
--- /dev/null
+++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/zookeeper/TestInstancePending.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.zookeeper;
+
+import java.util.concurrent.atomic.AtomicReference;
+
+import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category(SmallTests.class)
+public class TestInstancePending {
+ @Test(timeout = 1000)
+ public void test() throws Exception {
+ final InstancePending<String> pending = new InstancePending<String>();
+ final AtomicReference<String> getResultRef = new AtomicReference<String>();
+
+ new Thread() {
+ @Override
+ public void run() {
+ getResultRef.set(pending.get());
+ }
+ }.start();
+
+ Thread.sleep(100);
+ Assert.assertNull(getResultRef.get());
+
+ pending.prepare("abc");
+ Thread.sleep(100);
+ Assert.assertEquals("abc", getResultRef.get());
+ }
+}
[35/50] [abbrv] hbase git commit: HBASE-15793 Port over AsyncCall
improvements Signed-off-by: stack
Posted by sy...@apache.org.
HBASE-15793 Port over AsyncCall improvements
Signed-off-by: stack <st...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/fa033b6a
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/fa033b6a
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/fa033b6a
Branch: refs/heads/hbase-12439
Commit: fa033b6a08020282004ec87353f743fc205140ef
Parents: ac31ceb
Author: Jurriaan Mous <ju...@jurmo.us>
Authored: Sat May 7 12:46:58 2016 +0200
Committer: stack <st...@apache.org>
Committed: Sat May 7 10:38:38 2016 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hbase/client/Future.java | 31 ++++++
.../hbase/client/ResponseFutureListener.java | 30 ++++++
.../org/apache/hadoop/hbase/ipc/AsyncCall.java | 87 ++++++++++++----
.../hadoop/hbase/ipc/AsyncRpcChannel.java | 21 +++-
.../hadoop/hbase/ipc/AsyncRpcChannelImpl.java | 50 ++++-----
.../apache/hadoop/hbase/ipc/AsyncRpcClient.java | 101 ++++++++++++-------
.../hadoop/hbase/ipc/IOExceptionConverter.java | 34 +++++++
.../hadoop/hbase/ipc/MessageConverter.java | 47 +++++++++
.../org/apache/hadoop/hbase/ipc/Promise.java | 38 +++++++
9 files changed, 346 insertions(+), 93 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/fa033b6a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Future.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Future.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Future.java
new file mode 100644
index 0000000..1247fd4
--- /dev/null
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Future.java
@@ -0,0 +1,31 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.client;
+
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+
+/**
+ * Promise for responses
+ * @param <V> Value type
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public interface Future<V> extends io.netty.util.concurrent.Future<V> {
+
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/fa033b6a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ResponseFutureListener.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ResponseFutureListener.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ResponseFutureListener.java
new file mode 100644
index 0000000..f23dc8f
--- /dev/null
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ResponseFutureListener.java
@@ -0,0 +1,30 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.client;
+
+import io.netty.util.concurrent.GenericFutureListener;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+
+/**
+ * Specific interface for the Response future listener
+ * @param <V> Value type.
+ */
+@InterfaceAudience.Private
+public interface ResponseFutureListener<V>
+ extends GenericFutureListener<Future<V>> {
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/fa033b6a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncCall.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncCall.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncCall.java
index a5da0dc..3acf280 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncCall.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncCall.java
@@ -19,8 +19,7 @@ package org.apache.hadoop.hbase.ipc;
import com.google.protobuf.Descriptors;
import com.google.protobuf.Message;
-import io.netty.channel.EventLoop;
-import io.netty.util.concurrent.DefaultPromise;
+import java.io.IOException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.CellScanner;
@@ -31,51 +30,72 @@ import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.ExceptionUtil;
import org.apache.hadoop.ipc.RemoteException;
-import java.io.IOException;
-
/**
* Represents an Async Hbase call and its response.
*
* Responses are passed on to its given doneHandler and failures to the rpcController
+ *
+ * @param <T> Type of message returned
+ * @param <M> Message returned in communication to be converted
*/
@InterfaceAudience.Private
-public class AsyncCall extends DefaultPromise<Message> {
+public class AsyncCall<M extends Message, T> extends Promise<T> {
private static final Log LOG = LogFactory.getLog(AsyncCall.class.getName());
final int id;
+ private final AsyncRpcChannelImpl channel;
+
final Descriptors.MethodDescriptor method;
final Message param;
- final PayloadCarryingRpcController controller;
final Message responseDefaultType;
+
+ private final MessageConverter<M,T> messageConverter;
final long startTime;
final long rpcTimeout;
+ private final IOExceptionConverter exceptionConverter;
+
+ // For only the request
+ private final CellScanner cellScanner;
+ private final int priority;
+
final MetricsConnection.CallStats callStats;
/**
* Constructor
*
- * @param eventLoop for call
+ * @param channel which initiated call
* @param connectId connection id
* @param md the method descriptor
* @param param parameters to send to Server
- * @param controller controller for response
+ * @param cellScanner cellScanner containing cells to send as request
* @param responseDefaultType the default response type
+ * @param messageConverter converts the messages to what is the expected output
+ * @param rpcTimeout timeout for this call in ms
+ * @param priority for this request
*/
- public AsyncCall(EventLoop eventLoop, int connectId, Descriptors.MethodDescriptor md, Message
- param, PayloadCarryingRpcController controller, Message responseDefaultType,
+ public AsyncCall(AsyncRpcChannelImpl channel, int connectId, Descriptors.MethodDescriptor
+ md, Message param, CellScanner cellScanner, M responseDefaultType, MessageConverter<M, T>
+ messageConverter, IOExceptionConverter exceptionConverter, long rpcTimeout, int priority,
MetricsConnection.CallStats callStats) {
- super(eventLoop);
+ super(channel.getEventExecutor());
+ this.channel = channel;
this.id = connectId;
this.method = md;
this.param = param;
- this.controller = controller;
this.responseDefaultType = responseDefaultType;
+ this.messageConverter = messageConverter;
+ this.exceptionConverter = exceptionConverter;
+
this.startTime = EnvironmentEdgeManager.currentTime();
- this.rpcTimeout = controller.hasCallTimeout() ? controller.getCallTimeout() : 0;
+ this.rpcTimeout = rpcTimeout;
+
+ this.priority = priority;
+ this.cellScanner = cellScanner;
+
this.callStats = callStats;
}
@@ -101,17 +121,19 @@ public class AsyncCall extends DefaultPromise<Message> {
* @param value to set
* @param cellBlockScanner to set
*/
- public void setSuccess(Message value, CellScanner cellBlockScanner) {
- if (cellBlockScanner != null) {
- controller.setCellScanner(cellBlockScanner);
- }
-
+ public void setSuccess(M value, CellScanner cellBlockScanner) {
if (LOG.isTraceEnabled()) {
long callTime = EnvironmentEdgeManager.currentTime() - startTime;
LOG.trace("Call: " + method.getName() + ", callTime: " + callTime + "ms");
}
- this.setSuccess(value);
+ try {
+ this.setSuccess(
+ this.messageConverter.convert(value, cellBlockScanner)
+ );
+ } catch (IOException e) {
+ this.setFailed(e);
+ }
}
/**
@@ -127,6 +149,10 @@ public class AsyncCall extends DefaultPromise<Message> {
exception = ((RemoteException) exception).unwrapRemoteException();
}
+ if (this.exceptionConverter != null) {
+ exception = this.exceptionConverter.convert(exception);
+ }
+
this.setFailure(exception);
}
@@ -138,4 +164,27 @@ public class AsyncCall extends DefaultPromise<Message> {
public long getRpcTimeout() {
return rpcTimeout;
}
+
+
+ /**
+ * @return Priority for this call
+ */
+ public int getPriority() {
+ return priority;
+ }
+
+ /**
+ * Get the cellScanner for this request.
+ * @return CellScanner
+ */
+ public CellScanner cellScanner() {
+ return cellScanner;
+ }
+
+ @Override
+ public boolean cancel(boolean mayInterupt){
+ this.channel.removePendingCall(this.id);
+ return super.cancel(mayInterupt);
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/fa033b6a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncRpcChannel.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncRpcChannel.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncRpcChannel.java
index 60dc5e4..bd4be5a 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncRpcChannel.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncRpcChannel.java
@@ -21,11 +21,12 @@ import com.google.protobuf.Descriptors;
import com.google.protobuf.Message;
import io.netty.util.concurrent.EventExecutor;
-import io.netty.util.concurrent.Promise;
import java.net.InetSocketAddress;
+import org.apache.hadoop.hbase.CellScanner;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.client.Future;
import org.apache.hadoop.hbase.client.MetricsConnection;
/**
@@ -37,13 +38,23 @@ public interface AsyncRpcChannel {
/**
* Calls method on channel
* @param method to call
- * @param controller to run call with
* @param request to send
+ * @param cellScanner with cells to send
* @param responsePrototype to construct response with
+ * @param messageConverter for the messages to expected result
+ * @param exceptionConverter for converting exceptions
+ * @param rpcTimeout timeout for request
+ * @param priority for request
+ * @param callStats collects stats of the call
+ * @return Promise for the response Message
*/
- Promise<Message> callMethod(final Descriptors.MethodDescriptor method,
- final PayloadCarryingRpcController controller, final Message request,
- final Message responsePrototype, MetricsConnection.CallStats callStats);
+
+ <R extends Message, O> Future<O> callMethod(
+ final Descriptors.MethodDescriptor method,
+ final Message request,final CellScanner cellScanner,
+ R responsePrototype, MessageConverter<R, O> messageConverter, IOExceptionConverter
+ exceptionConverter, long rpcTimeout, int priority, MetricsConnection.CallStats callStats);
+
/**
* Get the EventLoop on which this channel operated
http://git-wip-us.apache.org/repos/asf/hbase/blob/fa033b6a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncRpcChannelImpl.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncRpcChannelImpl.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncRpcChannelImpl.java
index 7cc9e78..5af2354 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncRpcChannelImpl.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncRpcChannelImpl.java
@@ -19,8 +19,6 @@ package org.apache.hadoop.hbase.ipc;
import com.google.protobuf.Descriptors;
import com.google.protobuf.Message;
-import com.google.protobuf.RpcCallback;
-
import io.netty.bootstrap.Bootstrap;
import io.netty.buffer.ByteBuf;
import io.netty.buffer.ByteBufOutputStream;
@@ -32,7 +30,6 @@ import io.netty.handler.codec.LengthFieldBasedFrameDecoder;
import io.netty.util.Timeout;
import io.netty.util.TimerTask;
import io.netty.util.concurrent.GenericFutureListener;
-import io.netty.util.concurrent.Promise;
import java.io.IOException;
import java.net.ConnectException;
@@ -51,8 +48,10 @@ import javax.security.sasl.SaslException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.CellScanner;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.client.Future;
import org.apache.hadoop.hbase.client.MetricsConnection;
import org.apache.hadoop.hbase.exceptions.ConnectionClosingException;
import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
@@ -291,36 +290,25 @@ public class AsyncRpcChannelImpl implements AsyncRpcChannel {
/**
* Calls method on channel
* @param method to call
- * @param controller to run call with
* @param request to send
+ * @param cellScanner with cells to send
* @param responsePrototype to construct response with
+ * @param rpcTimeout timeout for request
+ * @param priority for request
+ * @return Promise for the response Message
*/
- public Promise<Message> callMethod(final Descriptors.MethodDescriptor method,
- final PayloadCarryingRpcController controller, final Message request,
- final Message responsePrototype, MetricsConnection.CallStats callStats) {
- final AsyncCall call = new AsyncCall(channel.eventLoop(), client.callIdCnt.getAndIncrement(),
- method, request, controller, responsePrototype, callStats);
- controller.notifyOnCancel(new RpcCallback<Object>() {
- @Override
- public void run(Object parameter) {
- // TODO: do not need to call AsyncCall.setFailed?
- synchronized (pendingCalls) {
- pendingCalls.remove(call.id);
- }
- }
- });
- // TODO: this should be handled by PayloadCarryingRpcController.
- if (controller.isCanceled()) {
- // To finish if the call was cancelled before we set the notification (race condition)
- call.cancel(true);
- return call;
- }
-
+ public <R extends Message, O> Future<O> callMethod(
+ final Descriptors.MethodDescriptor method,
+ final Message request,final CellScanner cellScanner,
+ R responsePrototype, MessageConverter<R, O> messageConverter, IOExceptionConverter
+ exceptionConverter, long rpcTimeout, int priority, MetricsConnection.CallStats callStats) {
+ final AsyncCall<R, O> call = new AsyncCall<>(this, client.callIdCnt.getAndIncrement(),
+ method, request, cellScanner, responsePrototype, messageConverter, exceptionConverter,
+ rpcTimeout, priority, callStats);
synchronized (pendingCalls) {
if (closed) {
- Promise<Message> promise = channel.eventLoop().newPromise();
- promise.setFailure(new ConnectException());
- return promise;
+ call.setFailure(new ConnectException());
+ return call;
}
pendingCalls.put(call.id, call);
// Add timeout for cleanup if none is present
@@ -398,7 +386,7 @@ public class AsyncRpcChannelImpl implements AsyncRpcChannel {
.setParentId(s.getSpanId()).setTraceId(s.getTraceId()));
}
- ByteBuffer cellBlock = client.buildCellBlock(call.controller.cellScanner());
+ ByteBuffer cellBlock = client.buildCellBlock(call.cellScanner());
if (cellBlock != null) {
final RPCProtos.CellBlockMeta.Builder cellBlockBuilder = RPCProtos.CellBlockMeta
.newBuilder();
@@ -406,8 +394,8 @@ public class AsyncRpcChannelImpl implements AsyncRpcChannel {
requestHeaderBuilder.setCellBlockMeta(cellBlockBuilder.build());
}
// Only pass priority if there one. Let zero be same as no priority.
- if (call.controller.getPriority() != PayloadCarryingRpcController.PRIORITY_UNSET) {
- requestHeaderBuilder.setPriority(call.controller.getPriority());
+ if (call.getPriority() != PayloadCarryingRpcController.PRIORITY_UNSET) {
+ requestHeaderBuilder.setPriority(call.getPriority());
}
RPCProtos.RequestHeader rh = requestHeaderBuilder.build();
http://git-wip-us.apache.org/repos/asf/hbase/blob/fa033b6a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncRpcClient.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncRpcClient.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncRpcClient.java
index 8d9a5b3..2fdc1ec 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncRpcClient.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncRpcClient.java
@@ -17,6 +17,12 @@
*/
package org.apache.hadoop.hbase.ipc;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.protobuf.Descriptors;
+import com.google.protobuf.Message;
+import com.google.protobuf.RpcCallback;
+import com.google.protobuf.RpcChannel;
+import com.google.protobuf.RpcController;
import io.netty.bootstrap.Bootstrap;
import io.netty.channel.Channel;
import io.netty.channel.ChannelInitializer;
@@ -30,9 +36,6 @@ import io.netty.channel.socket.nio.NioSocketChannel;
import io.netty.util.HashedWheelTimer;
import io.netty.util.Timeout;
import io.netty.util.TimerTask;
-import io.netty.util.concurrent.Future;
-import io.netty.util.concurrent.GenericFutureListener;
-import io.netty.util.concurrent.Promise;
import java.io.IOException;
import java.net.InetSocketAddress;
@@ -52,7 +55,9 @@ import org.apache.hadoop.hbase.HBaseInterfaceAudience;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.client.Future;
import org.apache.hadoop.hbase.client.MetricsConnection;
+import org.apache.hadoop.hbase.client.ResponseFutureListener;
import org.apache.hadoop.hbase.security.User;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.JVM;
@@ -60,13 +65,6 @@ import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.hbase.util.PoolMap;
import org.apache.hadoop.hbase.util.Threads;
-import com.google.common.annotations.VisibleForTesting;
-import com.google.protobuf.Descriptors;
-import com.google.protobuf.Message;
-import com.google.protobuf.RpcCallback;
-import com.google.protobuf.RpcChannel;
-import com.google.protobuf.RpcController;
-
/**
* Netty client for the requests and responses
*/
@@ -242,7 +240,18 @@ public class AsyncRpcClient extends AbstractRpcClient {
}
final AsyncRpcChannel connection = createRpcChannel(md.getService().getName(), addr, ticket);
- Promise<Message> promise = connection.callMethod(md, pcrc, param, returnType, callStats);
+ final Future<Message> promise = connection.callMethod(md, param, pcrc.cellScanner(), returnType,
+ getMessageConverterWithRpcController(pcrc), null, pcrc.getCallTimeout(), pcrc.getPriority(),
+ callStats);
+
+ pcrc.notifyOnCancel(new RpcCallback<Object>() {
+ @Override
+ public void run(Object parameter) {
+ // Will automatically fail the promise with CancellationException
+ promise.cancel(true);
+ }
+ });
+
long timeout = pcrc.hasCallTimeout() ? pcrc.getCallTimeout() : 0;
try {
Message response = timeout > 0 ? promise.get(timeout, TimeUnit.MILLISECONDS) : promise.get();
@@ -259,6 +268,18 @@ public class AsyncRpcClient extends AbstractRpcClient {
}
}
+ private MessageConverter<Message, Message> getMessageConverterWithRpcController(
+ final PayloadCarryingRpcController pcrc) {
+ return new
+ MessageConverter<Message, Message>() {
+ @Override
+ public Message convert(Message msg, CellScanner cellScanner) {
+ pcrc.setCellScanner(cellScanner);
+ return msg;
+ }
+ };
+ }
+
/**
* Call method async
*/
@@ -269,42 +290,46 @@ public class AsyncRpcClient extends AbstractRpcClient {
try {
connection = createRpcChannel(md.getService().getName(), addr, ticket);
final MetricsConnection.CallStats cs = MetricsConnection.newCallStats();
- GenericFutureListener<Future<Message>> listener =
- new GenericFutureListener<Future<Message>>() {
- @Override
- public void operationComplete(Future<Message> future) throws Exception {
- cs.setCallTimeMs(EnvironmentEdgeManager.currentTime() - cs.getStartTime());
- if (metrics != null) {
- metrics.updateRpc(md, param, cs);
- }
- if (LOG.isTraceEnabled()) {
- LOG.trace("Call: " + md.getName() + ", callTime: " + cs.getCallTimeMs() + "ms");
+
+ ResponseFutureListener<Message> listener =
+ new ResponseFutureListener<Message>() {
+ @Override
+ public void operationComplete(Future<Message> future) throws Exception {
+ cs.setCallTimeMs(EnvironmentEdgeManager.currentTime() - cs.getStartTime());
+ if (metrics != null) {
+ metrics.updateRpc(md, param, cs);
+ }
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("Call: " + md.getName() + ", callTime: " + cs.getCallTimeMs() + "ms");
+ }
+ if (!future.isSuccess()) {
+ Throwable cause = future.cause();
+ if (cause instanceof IOException) {
+ pcrc.setFailed((IOException) cause);
+ } else {
+ pcrc.setFailed(new IOException(cause));
}
- if (!future.isSuccess()) {
- Throwable cause = future.cause();
+ } else {
+ try {
+ done.run(future.get());
+ } catch (ExecutionException e) {
+ Throwable cause = e.getCause();
if (cause instanceof IOException) {
pcrc.setFailed((IOException) cause);
} else {
pcrc.setFailed(new IOException(cause));
}
- } else {
- try {
- done.run(future.get());
- } catch (ExecutionException e) {
- Throwable cause = e.getCause();
- if (cause instanceof IOException) {
- pcrc.setFailed((IOException) cause);
- } else {
- pcrc.setFailed(new IOException(cause));
- }
- } catch (InterruptedException e) {
- pcrc.setFailed(new IOException(e));
- }
+ } catch (InterruptedException e) {
+ pcrc.setFailed(new IOException(e));
}
}
- };
+ }
+ };
cs.setStartTime(EnvironmentEdgeManager.currentTime());
- connection.callMethod(md, pcrc, param, returnType, cs).addListener(listener);
+ connection.callMethod(md, param, pcrc.cellScanner(), returnType,
+ getMessageConverterWithRpcController(pcrc), null,
+ pcrc.getCallTimeout(), pcrc.getPriority(), cs)
+ .addListener(listener);
} catch (StoppedRpcClientException|FailedServerException e) {
pcrc.setFailed(e);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/fa033b6a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/IOExceptionConverter.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/IOExceptionConverter.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/IOExceptionConverter.java
new file mode 100644
index 0000000..09dda09
--- /dev/null
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/IOExceptionConverter.java
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.ipc;
+
+import java.io.IOException;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+
+/**
+ * Converts exceptions to other exceptions
+ */
+@InterfaceAudience.Private
+public interface IOExceptionConverter {
+ /**
+ * Converts given IOException
+ * @param e exception to convert
+ * @return converted IOException
+ */
+ IOException convert(IOException e);
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/fa033b6a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/MessageConverter.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/MessageConverter.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/MessageConverter.java
new file mode 100644
index 0000000..527ac95
--- /dev/null
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/MessageConverter.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.ipc;
+
+import com.google.protobuf.Message;
+import java.io.IOException;
+import org.apache.hadoop.hbase.CellScanner;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+
+/**
+ * Interface to convert Messages to specific types
+ * @param <M> Message Type to convert
+ * @param <O> Output Type
+ */
+@InterfaceAudience.Private
+public interface MessageConverter<M,O> {
+ /**
+ * Converts Message to Output
+ * @param msg to convert
+ * @param cellScanner to use for conversion
+ * @return Output
+ * @throws IOException if message could not be converted to response
+ */
+ O convert(M msg, CellScanner cellScanner) throws IOException;
+
+ MessageConverter<Message,Message> NO_CONVERTER = new MessageConverter<Message, Message>() {
+ @Override
+ public Message convert(Message msg, CellScanner cellScanner) throws IOException {
+ return null;
+ }
+ };
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/fa033b6a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/Promise.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/Promise.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/Promise.java
new file mode 100644
index 0000000..0d05db8
--- /dev/null
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/Promise.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.ipc;
+
+import io.netty.util.concurrent.DefaultPromise;
+import io.netty.util.concurrent.EventExecutor;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.client.Future;
+
+/**
+ * Abstract response promise
+ * @param <T> Type of result contained in Promise
+ */
+@InterfaceAudience.Private
+public class Promise<T> extends DefaultPromise<T> implements Future<T> {
+ /**
+ * Constructor
+ * @param eventLoop to handle events on
+ */
+ public Promise(EventExecutor eventLoop) {
+ super(eventLoop);
+ }
+}
[25/50] [abbrv] hbase git commit: HBASE-15669 HFile size is not
considered correctly in a replication request
Posted by sy...@apache.org.
HBASE-15669 HFile size is not considered correctly in a replication request
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/34e9a6ff
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/34e9a6ff
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/34e9a6ff
Branch: refs/heads/hbase-12439
Commit: 34e9a6ff301f40aa3f6ce33ac1b86f9e50fa6694
Parents: bec81b1
Author: Ashish Singhi <as...@apache.org>
Authored: Fri May 6 17:26:17 2016 +0530
Committer: Ashish Singhi <as...@apache.org>
Committed: Fri May 6 17:26:17 2016 +0530
----------------------------------------------------------------------
.../hadoop/hbase/protobuf/ProtobufUtil.java | 12 +-
.../hbase/protobuf/generated/WALProtos.java | 159 ++++++++++++++++---
hbase-protocol/src/main/protobuf/WAL.proto | 1 +
.../hadoop/hbase/regionserver/HRegion.java | 18 ++-
.../regionserver/ReplicationSource.java | 44 ++++-
.../regionserver/TestReplicationSink.java | 4 +-
.../TestReplicationSourceManager.java | 25 ++-
7 files changed, 229 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/34e9a6ff/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
index 50a4920..62dfd45 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
@@ -3063,13 +3063,16 @@ public final class ProtobufUtil {
* @param tableName The tableName into which the bulk load is being imported into.
* @param encodedRegionName Encoded region name of the region which is being bulk loaded.
* @param storeFiles A set of store files of a column family are bulk loaded.
+ * @param storeFilesSize Map of store files and their lengths
* @param bulkloadSeqId sequence ID (by a force flush) used to create bulk load hfile
* name
* @return The WAL log marker for bulk loads.
*/
public static WALProtos.BulkLoadDescriptor toBulkLoadDescriptor(TableName tableName,
- ByteString encodedRegionName, Map<byte[], List<Path>> storeFiles, long bulkloadSeqId) {
- BulkLoadDescriptor.Builder desc = BulkLoadDescriptor.newBuilder()
+ ByteString encodedRegionName, Map<byte[], List<Path>> storeFiles,
+ Map<String, Long> storeFilesSize, long bulkloadSeqId) {
+ BulkLoadDescriptor.Builder desc =
+ BulkLoadDescriptor.newBuilder()
.setTableName(ProtobufUtil.toProtoTableName(tableName))
.setEncodedRegionName(encodedRegionName).setBulkloadSeqNum(bulkloadSeqId);
@@ -3078,7 +3081,10 @@ public final class ProtobufUtil {
.setFamilyName(ByteStringer.wrap(entry.getKey()))
.setStoreHomeDir(Bytes.toString(entry.getKey())); // relative to region
for (Path path : entry.getValue()) {
- builder.addStoreFile(path.getName());
+ String name = path.getName();
+ builder.addStoreFile(name);
+ Long size = storeFilesSize.get(name) == null ? (Long) 0L : storeFilesSize.get(name);
+ builder.setStoreFileSize(size);
}
desc.addStores(builder);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/34e9a6ff/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/WALProtos.java
----------------------------------------------------------------------
diff --git a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/WALProtos.java b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/WALProtos.java
index d74688e..6252d51 100644
--- a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/WALProtos.java
+++ b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/WALProtos.java
@@ -7821,6 +7821,24 @@ public final class WALProtos {
*/
com.google.protobuf.ByteString
getStoreFileBytes(int index);
+
+ // optional uint64 store_file_size = 4;
+ /**
+ * <code>optional uint64 store_file_size = 4;</code>
+ *
+ * <pre>
+ * size of store file
+ * </pre>
+ */
+ boolean hasStoreFileSize();
+ /**
+ * <code>optional uint64 store_file_size = 4;</code>
+ *
+ * <pre>
+ * size of store file
+ * </pre>
+ */
+ long getStoreFileSize();
}
/**
* Protobuf type {@code hbase.pb.StoreDescriptor}
@@ -7891,6 +7909,11 @@ public final class WALProtos {
storeFile_.add(input.readBytes());
break;
}
+ case 32: {
+ bitField0_ |= 0x00000004;
+ storeFileSize_ = input.readUInt64();
+ break;
+ }
}
}
} catch (com.google.protobuf.InvalidProtocolBufferException e) {
@@ -8051,10 +8074,35 @@ public final class WALProtos {
return storeFile_.getByteString(index);
}
+ // optional uint64 store_file_size = 4;
+ public static final int STORE_FILE_SIZE_FIELD_NUMBER = 4;
+ private long storeFileSize_;
+ /**
+ * <code>optional uint64 store_file_size = 4;</code>
+ *
+ * <pre>
+ * size of store file
+ * </pre>
+ */
+ public boolean hasStoreFileSize() {
+ return ((bitField0_ & 0x00000004) == 0x00000004);
+ }
+ /**
+ * <code>optional uint64 store_file_size = 4;</code>
+ *
+ * <pre>
+ * size of store file
+ * </pre>
+ */
+ public long getStoreFileSize() {
+ return storeFileSize_;
+ }
+
private void initFields() {
familyName_ = com.google.protobuf.ByteString.EMPTY;
storeHomeDir_ = "";
storeFile_ = com.google.protobuf.LazyStringArrayList.EMPTY;
+ storeFileSize_ = 0L;
}
private byte memoizedIsInitialized = -1;
public final boolean isInitialized() {
@@ -8085,6 +8133,9 @@ public final class WALProtos {
for (int i = 0; i < storeFile_.size(); i++) {
output.writeBytes(3, storeFile_.getByteString(i));
}
+ if (((bitField0_ & 0x00000004) == 0x00000004)) {
+ output.writeUInt64(4, storeFileSize_);
+ }
getUnknownFields().writeTo(output);
}
@@ -8111,6 +8162,10 @@ public final class WALProtos {
size += dataSize;
size += 1 * getStoreFileList().size();
}
+ if (((bitField0_ & 0x00000004) == 0x00000004)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeUInt64Size(4, storeFileSize_);
+ }
size += getUnknownFields().getSerializedSize();
memoizedSerializedSize = size;
return size;
@@ -8146,6 +8201,11 @@ public final class WALProtos {
}
result = result && getStoreFileList()
.equals(other.getStoreFileList());
+ result = result && (hasStoreFileSize() == other.hasStoreFileSize());
+ if (hasStoreFileSize()) {
+ result = result && (getStoreFileSize()
+ == other.getStoreFileSize());
+ }
result = result &&
getUnknownFields().equals(other.getUnknownFields());
return result;
@@ -8171,6 +8231,10 @@ public final class WALProtos {
hash = (37 * hash) + STORE_FILE_FIELD_NUMBER;
hash = (53 * hash) + getStoreFileList().hashCode();
}
+ if (hasStoreFileSize()) {
+ hash = (37 * hash) + STORE_FILE_SIZE_FIELD_NUMBER;
+ hash = (53 * hash) + hashLong(getStoreFileSize());
+ }
hash = (29 * hash) + getUnknownFields().hashCode();
memoizedHashCode = hash;
return hash;
@@ -8286,6 +8350,8 @@ public final class WALProtos {
bitField0_ = (bitField0_ & ~0x00000002);
storeFile_ = com.google.protobuf.LazyStringArrayList.EMPTY;
bitField0_ = (bitField0_ & ~0x00000004);
+ storeFileSize_ = 0L;
+ bitField0_ = (bitField0_ & ~0x00000008);
return this;
}
@@ -8328,6 +8394,10 @@ public final class WALProtos {
bitField0_ = (bitField0_ & ~0x00000004);
}
result.storeFile_ = storeFile_;
+ if (((from_bitField0_ & 0x00000008) == 0x00000008)) {
+ to_bitField0_ |= 0x00000004;
+ }
+ result.storeFileSize_ = storeFileSize_;
result.bitField0_ = to_bitField0_;
onBuilt();
return result;
@@ -8362,6 +8432,9 @@ public final class WALProtos {
}
onChanged();
}
+ if (other.hasStoreFileSize()) {
+ setStoreFileSize(other.getStoreFileSize());
+ }
this.mergeUnknownFields(other.getUnknownFields());
return this;
}
@@ -8660,6 +8733,55 @@ public final class WALProtos {
return this;
}
+ // optional uint64 store_file_size = 4;
+ private long storeFileSize_ ;
+ /**
+ * <code>optional uint64 store_file_size = 4;</code>
+ *
+ * <pre>
+ * size of store file
+ * </pre>
+ */
+ public boolean hasStoreFileSize() {
+ return ((bitField0_ & 0x00000008) == 0x00000008);
+ }
+ /**
+ * <code>optional uint64 store_file_size = 4;</code>
+ *
+ * <pre>
+ * size of store file
+ * </pre>
+ */
+ public long getStoreFileSize() {
+ return storeFileSize_;
+ }
+ /**
+ * <code>optional uint64 store_file_size = 4;</code>
+ *
+ * <pre>
+ * size of store file
+ * </pre>
+ */
+ public Builder setStoreFileSize(long value) {
+ bitField0_ |= 0x00000008;
+ storeFileSize_ = value;
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>optional uint64 store_file_size = 4;</code>
+ *
+ * <pre>
+ * size of store file
+ * </pre>
+ */
+ public Builder clearStoreFileSize() {
+ bitField0_ = (bitField0_ & ~0x00000008);
+ storeFileSize_ = 0L;
+ onChanged();
+ return this;
+ }
+
// @@protoc_insertion_point(builder_scope:hbase.pb.StoreDescriptor)
}
@@ -11877,24 +11999,25 @@ public final class WALProtos {
"ome_dir\030\002 \002(\t\022\024\n\014flush_output\030\003 \003(\t\"S\n\013F" +
"lushAction\022\017\n\013START_FLUSH\020\000\022\020\n\014COMMIT_FL" +
"USH\020\001\022\017\n\013ABORT_FLUSH\020\002\022\020\n\014CANNOT_FLUSH\020\003",
- "\"R\n\017StoreDescriptor\022\023\n\013family_name\030\001 \002(\014" +
+ "\"k\n\017StoreDescriptor\022\023\n\013family_name\030\001 \002(\014" +
"\022\026\n\016store_home_dir\030\002 \002(\t\022\022\n\nstore_file\030\003" +
- " \003(\t\"\237\001\n\022BulkLoadDescriptor\022\'\n\ntable_nam" +
- "e\030\001 \002(\0132\023.hbase.pb.TableName\022\033\n\023encoded_" +
- "region_name\030\002 \002(\014\022)\n\006stores\030\003 \003(\0132\031.hbas" +
- "e.pb.StoreDescriptor\022\030\n\020bulkload_seq_num" +
- "\030\004 \002(\003\"\272\002\n\025RegionEventDescriptor\022=\n\neven" +
- "t_type\030\001 \002(\0162).hbase.pb.RegionEventDescr" +
- "iptor.EventType\022\022\n\ntable_name\030\002 \002(\014\022\033\n\023e" +
- "ncoded_region_name\030\003 \002(\014\022\033\n\023log_sequence",
- "_number\030\004 \001(\004\022)\n\006stores\030\005 \003(\0132\031.hbase.pb" +
- ".StoreDescriptor\022$\n\006server\030\006 \001(\0132\024.hbase" +
- ".pb.ServerName\022\023\n\013region_name\030\007 \001(\014\".\n\tE" +
- "ventType\022\017\n\013REGION_OPEN\020\000\022\020\n\014REGION_CLOS" +
- "E\020\001\"\014\n\nWALTrailer*F\n\tScopeType\022\033\n\027REPLIC" +
- "ATION_SCOPE_LOCAL\020\000\022\034\n\030REPLICATION_SCOPE" +
- "_GLOBAL\020\001B?\n*org.apache.hadoop.hbase.pro" +
- "tobuf.generatedB\tWALProtosH\001\210\001\000\240\001\001"
+ " \003(\t\022\027\n\017store_file_size\030\004 \001(\004\"\237\001\n\022BulkLo" +
+ "adDescriptor\022\'\n\ntable_name\030\001 \002(\0132\023.hbase" +
+ ".pb.TableName\022\033\n\023encoded_region_name\030\002 \002" +
+ "(\014\022)\n\006stores\030\003 \003(\0132\031.hbase.pb.StoreDescr" +
+ "iptor\022\030\n\020bulkload_seq_num\030\004 \002(\003\"\272\002\n\025Regi" +
+ "onEventDescriptor\022=\n\nevent_type\030\001 \002(\0162)." +
+ "hbase.pb.RegionEventDescriptor.EventType" +
+ "\022\022\n\ntable_name\030\002 \002(\014\022\033\n\023encoded_region_n",
+ "ame\030\003 \002(\014\022\033\n\023log_sequence_number\030\004 \001(\004\022)" +
+ "\n\006stores\030\005 \003(\0132\031.hbase.pb.StoreDescripto" +
+ "r\022$\n\006server\030\006 \001(\0132\024.hbase.pb.ServerName\022" +
+ "\023\n\013region_name\030\007 \001(\014\".\n\tEventType\022\017\n\013REG" +
+ "ION_OPEN\020\000\022\020\n\014REGION_CLOSE\020\001\"\014\n\nWALTrail" +
+ "er*F\n\tScopeType\022\033\n\027REPLICATION_SCOPE_LOC" +
+ "AL\020\000\022\034\n\030REPLICATION_SCOPE_GLOBAL\020\001B?\n*or" +
+ "g.apache.hadoop.hbase.protobuf.generated" +
+ "B\tWALProtosH\001\210\001\000\240\001\001"
};
com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner =
new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() {
@@ -11942,7 +12065,7 @@ public final class WALProtos {
internal_static_hbase_pb_StoreDescriptor_fieldAccessorTable = new
com.google.protobuf.GeneratedMessage.FieldAccessorTable(
internal_static_hbase_pb_StoreDescriptor_descriptor,
- new java.lang.String[] { "FamilyName", "StoreHomeDir", "StoreFile", });
+ new java.lang.String[] { "FamilyName", "StoreHomeDir", "StoreFile", "StoreFileSize", });
internal_static_hbase_pb_BulkLoadDescriptor_descriptor =
getDescriptor().getMessageTypes().get(6);
internal_static_hbase_pb_BulkLoadDescriptor_fieldAccessorTable = new
http://git-wip-us.apache.org/repos/asf/hbase/blob/34e9a6ff/hbase-protocol/src/main/protobuf/WAL.proto
----------------------------------------------------------------------
diff --git a/hbase-protocol/src/main/protobuf/WAL.proto b/hbase-protocol/src/main/protobuf/WAL.proto
index cb9bd8f..aeb8a16 100644
--- a/hbase-protocol/src/main/protobuf/WAL.proto
+++ b/hbase-protocol/src/main/protobuf/WAL.proto
@@ -132,6 +132,7 @@ message StoreDescriptor {
required bytes family_name = 1;
required string store_home_dir = 2; //relative to region dir
repeated string store_file = 3; // relative to store dir
+ optional uint64 store_file_size = 4; // size of store file
}
/**
http://git-wip-us.apache.org/repos/asf/hbase/blob/34e9a6ff/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 4532317..e7a99a9 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -5315,6 +5315,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
BulkLoadListener bulkLoadListener) throws IOException {
long seqId = -1;
Map<byte[], List<Path>> storeFiles = new TreeMap<byte[], List<Path>>(Bytes.BYTES_COMPARATOR);
+ Map<String, Long> storeFilesSizes = new HashMap<String, Long>();
Preconditions.checkNotNull(familyPaths);
// we need writeLock for multi-family bulk load
startBulkRegionOperation(hasMultipleColumnFamilies(familyPaths));
@@ -5397,6 +5398,16 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
}
Path commitedStoreFile = store.bulkLoadHFile(finalPath, seqId);
+ // Note the size of the store file
+ try {
+ FileSystem fs = commitedStoreFile.getFileSystem(baseConf);
+ storeFilesSizes.put(commitedStoreFile.getName(), fs.getFileStatus(commitedStoreFile)
+ .getLen());
+ } catch (IOException e) {
+ LOG.warn("Failed to find the size of hfile " + commitedStoreFile);
+ storeFilesSizes.put(commitedStoreFile.getName(), 0L);
+ }
+
if(storeFiles.containsKey(familyName)) {
storeFiles.get(familyName).add(commitedStoreFile);
} else {
@@ -5431,9 +5442,10 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
if (wal != null && !storeFiles.isEmpty()) {
// Write a bulk load event for hfiles that are loaded
try {
- WALProtos.BulkLoadDescriptor loadDescriptor = ProtobufUtil.toBulkLoadDescriptor(
- this.getRegionInfo().getTable(),
- ByteStringer.wrap(this.getRegionInfo().getEncodedNameAsBytes()), storeFiles, seqId);
+ WALProtos.BulkLoadDescriptor loadDescriptor =
+ ProtobufUtil.toBulkLoadDescriptor(this.getRegionInfo().getTable(),
+ ByteStringer.wrap(this.getRegionInfo().getEncodedNameAsBytes()), storeFiles,
+ storeFilesSizes, seqId);
WALUtil.writeBulkLoadMarkerAndSync(this.wal, this.getReplicationScope(), getRegionInfo(),
loadDescriptor, mvcc);
} catch (IOException ioe) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/34e9a6ff/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
index 7e58e41..0f956c8 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
@@ -712,6 +712,7 @@ public class ReplicationSource extends Thread
currentNbOperations += countDistinctRowKeys(edit);
entries.add(entry);
currentSize += entry.getEdit().heapSize();
+ currentSize += calculateTotalSizeOfStoreFiles(edit);
} else {
metrics.incrLogEditsFiltered();
}
@@ -738,6 +739,35 @@ public class ReplicationSource extends Thread
return seenEntries == 0 && processEndOfFile();
}
+ /**
+ * Calculate the total size of all the store files
+ * @param edit edit to count row keys from
+ * @return the total size of the store files
+ */
+ private int calculateTotalSizeOfStoreFiles(WALEdit edit) {
+ List<Cell> cells = edit.getCells();
+ int totalStoreFilesSize = 0;
+
+ int totalCells = edit.size();
+ for (int i = 0; i < totalCells; i++) {
+ if (CellUtil.matchingQualifier(cells.get(i), WALEdit.BULK_LOAD)) {
+ try {
+ BulkLoadDescriptor bld = WALEdit.getBulkLoadDescriptor(cells.get(i));
+ List<StoreDescriptor> stores = bld.getStoresList();
+ int totalStores = stores.size();
+ for (int j = 0; j < totalStores; j++) {
+ totalStoreFilesSize += stores.get(j).getStoreFileSize();
+ }
+ } catch (IOException e) {
+ LOG.error("Failed to deserialize bulk load entry from wal edit. "
+ + "Size of HFiles part of cell will not be considered in replication "
+ + "request size calculation.", e);
+ }
+ }
+ }
+ return totalStoreFilesSize;
+ }
+
private void cleanUpHFileRefs(WALEdit edit) throws IOException {
String peerId = peerClusterZnode;
if (peerId.contains("-")) {
@@ -746,12 +776,14 @@ public class ReplicationSource extends Thread
peerId = peerClusterZnode.split("-")[0];
}
List<Cell> cells = edit.getCells();
- for (int i = 0; i < cells.size(); i++) {
+ int totalCells = cells.size();
+ for (int i = 0; i < totalCells; i++) {
Cell cell = cells.get(i);
if (CellUtil.matchingQualifier(cell, WALEdit.BULK_LOAD)) {
BulkLoadDescriptor bld = WALEdit.getBulkLoadDescriptor(cell);
List<StoreDescriptor> stores = bld.getStoresList();
- for (int j = 0; j < stores.size(); j++) {
+ int totalStores = stores.size();
+ for (int j = 0; j < totalStores; j++) {
List<String> storeFileList = stores.get(j).getStoreFileList();
manager.cleanUpHFileRefs(peerId, storeFileList);
metrics.decrSizeOfHFileRefsQueue(storeFileList.size());
@@ -934,18 +966,20 @@ public class ReplicationSource extends Thread
int totalHFileEntries = 0;
Cell lastCell = cells.get(0);
- for (int i = 0; i < edit.size(); i++) {
+ int totalCells = edit.size();
+ for (int i = 0; i < totalCells; i++) {
// Count HFiles to be replicated
if (CellUtil.matchingQualifier(cells.get(i), WALEdit.BULK_LOAD)) {
try {
BulkLoadDescriptor bld = WALEdit.getBulkLoadDescriptor(cells.get(i));
List<StoreDescriptor> stores = bld.getStoresList();
- for (int j = 0; j < stores.size(); j++) {
+ int totalStores = stores.size();
+ for (int j = 0; j < totalStores; j++) {
totalHFileEntries += stores.get(j).getStoreFileList().size();
}
} catch (IOException e) {
LOG.error("Failed to deserialize bulk load entry from wal edit. "
- + "This its hfiles count will not be added into metric.");
+ + "Then its hfiles count will not be added into metric.");
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/34e9a6ff/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSink.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSink.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSink.java
index f08d2bf..1e80548 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSink.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSink.java
@@ -287,6 +287,7 @@ public class TestReplicationSink {
}
List<Integer> numberList = new ArrayList<>(numbers);
Collections.sort(numberList);
+ Map<String, Long> storeFilesSize = new HashMap<String, Long>(1);
// 2. Create 25 hfiles
Configuration conf = TEST_UTIL.getConfiguration();
@@ -297,6 +298,7 @@ public class TestReplicationSink {
HFileTestUtil.createHFile(conf, fs, hfilePath, FAM_NAME1, FAM_NAME1,
Bytes.toBytes(numbersItr.next()), Bytes.toBytes(numbersItr.next()), numRows);
p.add(hfilePath);
+ storeFilesSize.put(hfilePath.getName(), fs.getFileStatus(hfilePath).getLen());
}
// 3. Create a BulkLoadDescriptor and a WALEdit
@@ -310,7 +312,7 @@ public class TestReplicationSink {
HRegionInfo regionInfo = l.getAllRegionLocations().get(0).getRegionInfo();
loadDescriptor =
ProtobufUtil.toBulkLoadDescriptor(TABLE_NAME1,
- ByteStringer.wrap(regionInfo.getEncodedNameAsBytes()), storeFiles, 1);
+ ByteStringer.wrap(regionInfo.getEncodedNameAsBytes()), storeFiles, storeFilesSize, 1);
edit = WALEdit.createBulkLoadEvent(regionInfo, loadDescriptor);
}
List<WALEntry> entries = new ArrayList<WALEntry>(1);
http://git-wip-us.apache.org/repos/asf/hbase/blob/34e9a6ff/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java
index fb8cfa0..9e950d2 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java
@@ -23,6 +23,7 @@ import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
+import java.io.IOException;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.Collection;
@@ -479,16 +480,32 @@ public class TestReplicationSourceManager {
private WALEdit getBulkLoadWALEdit(NavigableMap<byte[], Integer> scope) {
// 1. Create store files for the families
Map<byte[], List<Path>> storeFiles = new HashMap<>(1);
+ Map<String, Long> storeFilesSize = new HashMap<>(1);
List<Path> p = new ArrayList<>(1);
- p.add(new Path(Bytes.toString(f1)));
+ Path hfilePath1 = new Path(Bytes.toString(f1));
+ p.add(hfilePath1);
+ try {
+ storeFilesSize.put(hfilePath1.getName(), fs.getFileStatus(hfilePath1).getLen());
+ } catch (IOException e) {
+ LOG.debug("Failed to calculate the size of hfile " + hfilePath1);
+ storeFilesSize.put(hfilePath1.getName(), 0L);
+ }
storeFiles.put(f1, p);
scope.put(f1, 1);
p = new ArrayList<>(1);
- p.add(new Path(Bytes.toString(f2)));
+ Path hfilePath2 = new Path(Bytes.toString(f2));
+ p.add(hfilePath2);
+ try {
+ storeFilesSize.put(hfilePath2.getName(), fs.getFileStatus(hfilePath2).getLen());
+ } catch (IOException e) {
+ LOG.debug("Failed to calculate the size of hfile " + hfilePath2);
+ storeFilesSize.put(hfilePath2.getName(), 0L);
+ }
storeFiles.put(f2, p);
// 2. Create bulk load descriptor
- BulkLoadDescriptor desc = ProtobufUtil.toBulkLoadDescriptor(hri.getTable(),
- ByteStringer.wrap(hri.getEncodedNameAsBytes()), storeFiles, 1);
+ BulkLoadDescriptor desc =
+ ProtobufUtil.toBulkLoadDescriptor(hri.getTable(),
+ ByteStringer.wrap(hri.getEncodedNameAsBytes()), storeFiles, storeFilesSize, 1);
// 3. create bulk load wal edit event
WALEdit logEdit = WALEdit.createBulkLoadEvent(hri, desc);
[49/50] [abbrv] hbase git commit: HBASE-15799 Two Shell
'close_region' Example Syntaxes Don't Work (Matt Warhaftig)
Posted by sy...@apache.org.
HBASE-15799 Two Shell 'close_region' Example Syntaxes Don't Work (Matt Warhaftig)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/3b74b6f3
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/3b74b6f3
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/3b74b6f3
Branch: refs/heads/hbase-12439
Commit: 3b74b6f329ad6ebc0d2d2548a7e1290297674529
Parents: 8604f9e
Author: tedyu <yu...@gmail.com>
Authored: Mon May 9 20:58:55 2016 -0700
Committer: tedyu <yu...@gmail.com>
Committed: Mon May 9 20:58:55 2016 -0700
----------------------------------------------------------------------
hbase-shell/src/main/ruby/hbase/admin.rb | 6 ++--
.../main/ruby/shell/commands/close_region.rb | 1 -
hbase-shell/src/test/ruby/hbase/admin_test.rb | 29 ++++++++++++++++++++
3 files changed, 33 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/3b74b6f3/hbase-shell/src/main/ruby/hbase/admin.rb
----------------------------------------------------------------------
diff --git a/hbase-shell/src/main/ruby/hbase/admin.rb b/hbase-shell/src/main/ruby/hbase/admin.rb
index 7ea315f..88a6598 100644
--- a/hbase-shell/src/main/ruby/hbase/admin.rb
+++ b/hbase-shell/src/main/ruby/hbase/admin.rb
@@ -433,8 +433,10 @@ module Hbase
# If server name is nil, we presume region_name is full region name (HRegionInfo.getRegionName).
# If server name is not nil, we presume it is the region's encoded name (HRegionInfo.getEncodedName)
def close_region(region_name, server)
- if (server == nil || !closeEncodedRegion?(region_name, server))
- @admin.closeRegion(region_name, server)
+ if (region_name.end_with? ".")
+ @admin.closeRegion(region_name, server)
+ else
+ closeEncodedRegion?(region_name, server)
end
end
http://git-wip-us.apache.org/repos/asf/hbase/blob/3b74b6f3/hbase-shell/src/main/ruby/shell/commands/close_region.rb
----------------------------------------------------------------------
diff --git a/hbase-shell/src/main/ruby/shell/commands/close_region.rb b/hbase-shell/src/main/ruby/shell/commands/close_region.rb
index 9e2900c..ef1c99c 100644
--- a/hbase-shell/src/main/ruby/shell/commands/close_region.rb
+++ b/hbase-shell/src/main/ruby/shell/commands/close_region.rb
@@ -46,7 +46,6 @@ Examples:
hbase> close_region 'REGIONNAME'
hbase> close_region 'REGIONNAME', 'SERVER_NAME'
- hbase> close_region 'ENCODED_REGIONNAME'
hbase> close_region 'ENCODED_REGIONNAME', 'SERVER_NAME'
EOF
end
http://git-wip-us.apache.org/repos/asf/hbase/blob/3b74b6f3/hbase-shell/src/test/ruby/hbase/admin_test.rb
----------------------------------------------------------------------
diff --git a/hbase-shell/src/test/ruby/hbase/admin_test.rb b/hbase-shell/src/test/ruby/hbase/admin_test.rb
index 54f7418..50a65d0 100644
--- a/hbase-shell/src/test/ruby/hbase/admin_test.rb
+++ b/hbase-shell/src/test/ruby/hbase/admin_test.rb
@@ -238,6 +238,35 @@ module Hbase
end
end
+ # Simple administration methods tests
+ class AdminRegionTest < Test::Unit::TestCase
+ include TestHelpers
+ def setup
+ setup_hbase
+ # Create test table if it does not exist
+ @test_name = "hbase_shell_tests_table"
+ drop_test_table(@test_name)
+ create_test_table(@test_name)
+ end
+
+ def teardown
+ shutdown
+ end
+
+ define_test "close_region should allow encoded & non-encoded region names" do
+ region = admin.locate_region(@test_name, '')
+ serverName = region.getServerName().getServerName()
+ regionName = region.getRegionInfo().getRegionNameAsString()
+ encodedRegionName = region.getRegionInfo().getEncodedName()
+
+ # Close region with just region name.
+ admin.close_region(regionName, nil)
+ # Close region with region name and server.
+ admin.close_region(regionName, serverName)
+ admin.close_region(encodedRegionName, serverName)
+ end
+ end
+
# Simple administration methods tests
class AdminAlterTableTest < Test::Unit::TestCase
include TestHelpers
[10/50] [abbrv] hbase git commit: HBASE-15760
TestBlockEvictionFromClient#testParallelGetsAndScanWithWrappedRegionScanner
fails in master branch (Ram)
Posted by sy...@apache.org.
HBASE-15760 TestBlockEvictionFromClient#testParallelGetsAndScanWithWrappedRegionScanner
fails in master branch (Ram)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/809ed326
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/809ed326
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/809ed326
Branch: refs/heads/hbase-12439
Commit: 809ed326f0f39698c9ee7ed8847440ba89e0f5ce
Parents: a8a2c51
Author: Ramkrishna <ra...@intel.com>
Authored: Wed May 4 22:07:11 2016 +0530
Committer: Ramkrishna <ra...@intel.com>
Committed: Wed May 4 22:07:11 2016 +0530
----------------------------------------------------------------------
.../client/TestBlockEvictionFromClient.java | 26 ++++++++++----------
1 file changed, 13 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/809ed326/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestBlockEvictionFromClient.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestBlockEvictionFromClient.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestBlockEvictionFromClient.java
index d3f718b..f94ed2f 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestBlockEvictionFromClient.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestBlockEvictionFromClient.java
@@ -200,7 +200,7 @@ public class TestBlockEvictionFromClient {
// Create three sets of scan
ScanThread[] scanThreads = initiateScan(table, false);
Thread.sleep(100);
- checkForBlockEviction(cache, false, false, false);
+ checkForBlockEviction(cache, false, false);
for (ScanThread thread : scanThreads) {
thread.join();
}
@@ -284,16 +284,16 @@ public class TestBlockEvictionFromClient {
ScanThread[] scanThreads = initiateScan(table, false);
// Create three sets of gets
GetThread[] getThreads = initiateGet(table, false, false);
- checkForBlockEviction(cache, false, false, false);
+ checkForBlockEviction(cache, false, false);
CustomInnerRegionObserver.waitForGets.set(false);
- checkForBlockEviction(cache, false, false, false);
+ checkForBlockEviction(cache, false, false);
for (GetThread thread : getThreads) {
thread.join();
}
// Verify whether the gets have returned the blocks that it had
CustomInnerRegionObserver.waitForGets.set(true);
// giving some time for the block to be decremented
- checkForBlockEviction(cache, true, false, false);
+ checkForBlockEviction(cache, true, false);
getLatch.countDown();
for (ScanThread thread : scanThreads) {
thread.join();
@@ -302,7 +302,7 @@ public class TestBlockEvictionFromClient {
// Check with either true or false
CustomInnerRegionObserver.waitForGets.set(false);
// The scan should also have released the blocks by now
- checkForBlockEviction(cache, true, true, false);
+ checkForBlockEviction(cache, true, true);
} finally {
if (table != null) {
table.close();
@@ -360,7 +360,7 @@ public class TestBlockEvictionFromClient {
// Verify whether the gets have returned the blocks that it had
CustomInnerRegionObserver.waitForGets.set(true);
// giving some time for the block to be decremented
- checkForBlockEviction(cache, true, false, false);
+ checkForBlockEviction(cache, true, false);
getLatch.countDown();
System.out.println("Gets should have returned the bloks");
} finally {
@@ -450,7 +450,7 @@ public class TestBlockEvictionFromClient {
// Verify whether the gets have returned the blocks that it had
CustomInnerRegionObserver.waitForGets.set(true);
// giving some time for the block to be decremented
- checkForBlockEviction(cache, true, false, false);
+ checkForBlockEviction(cache, true, false);
getLatch.countDown();
System.out.println("Gets should have returned the bloks");
} finally {
@@ -545,7 +545,7 @@ public class TestBlockEvictionFromClient {
// Verify whether the gets have returned the blocks that it had
CustomInnerRegionObserver.waitForGets.set(true);
// giving some time for the block to be decremented
- checkForBlockEviction(cache, true, false, false);
+ checkForBlockEviction(cache, true, false);
getLatch.countDown();
System.out.println("Gets should have returned the bloks");
} finally {
@@ -766,7 +766,7 @@ public class TestBlockEvictionFromClient {
thread.join();
}
// giving some time for the block to be decremented
- checkForBlockEviction(cache, true, false, false);
+ checkForBlockEviction(cache, true, false);
} finally {
if (table != null) {
table.close();
@@ -830,7 +830,7 @@ public class TestBlockEvictionFromClient {
// giving some time for the block to be decremented
Thread.sleep(100);
CustomInnerRegionObserver.waitForGets.set(false);
- checkForBlockEviction(cache, false, false, true);
+ checkForBlockEviction(cache, false, false);
// countdown the latch
CustomInnerRegionObserver.getCdl().get().countDown();
for (GetThread thread : getThreads) {
@@ -1274,8 +1274,8 @@ public class TestBlockEvictionFromClient {
return multiGetThreads;
}
- private void checkForBlockEviction(BlockCache cache, boolean getClosed, boolean expectOnlyZero,
- boolean wrappedCp) throws InterruptedException {
+ private void checkForBlockEviction(BlockCache cache, boolean getClosed, boolean expectOnlyZero)
+ throws InterruptedException {
int counter = NO_OF_THREADS;
if (CustomInnerRegionObserver.waitForGets.get()) {
// Because only one row is selected, it has only 2 blocks
@@ -1325,7 +1325,7 @@ public class TestBlockEvictionFromClient {
assertTrue(refCount == 0);
}
if (refCount != 0) {
- if (getLatch == null || wrappedCp) {
+ if (getLatch == null) {
assertEquals(refCount, CustomInnerRegionObserver.countOfNext.get());
} else {
assertEquals(refCount, CustomInnerRegionObserver.countOfNext.get() + (NO_OF_THREADS));
[11/50] [abbrv] hbase git commit: HBASE-15729 Remove old JDiff
wrapper scripts in dev-support
Posted by sy...@apache.org.
HBASE-15729 Remove old JDiff wrapper scripts in dev-support
Signed-off-by: Sean Busbey <bu...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/fa9fdfee
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/fa9fdfee
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/fa9fdfee
Branch: refs/heads/hbase-12439
Commit: fa9fdfee2d94c94cc67e7129a0ff5edef5e5f69b
Parents: 809ed32
Author: Dima Spivak <ds...@cloudera.com>
Authored: Wed Apr 27 15:53:05 2016 -0700
Committer: Sean Busbey <bu...@apache.org>
Committed: Wed May 4 13:28:17 2016 -0500
----------------------------------------------------------------------
.../hbase_jdiff_acrossSingularityTemplate.xml | 59 -----
.../hbase_jdiff_afterSingularityTemplate.xml | 66 -----
dev-support/hbase_jdiff_template.xml | 53 ----
dev-support/jdiffHBasePublicAPI.sh | 258 -------------------
dev-support/jdiffHBasePublicAPI_common.sh | 76 ------
5 files changed, 512 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/fa9fdfee/dev-support/hbase_jdiff_acrossSingularityTemplate.xml
----------------------------------------------------------------------
diff --git a/dev-support/hbase_jdiff_acrossSingularityTemplate.xml b/dev-support/hbase_jdiff_acrossSingularityTemplate.xml
deleted file mode 100644
index e8c5867..0000000
--- a/dev-support/hbase_jdiff_acrossSingularityTemplate.xml
+++ /dev/null
@@ -1,59 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
--->
-
-<project name="jdiff" default="jdiff_report" basedir="JDIFF_FOLDER">
-
-<description>
-Creates a report in the directory "hbase_jdiff_report-p-PREVIOUS_BRANCH-c-CURRENT_BRANCH" of the default jdiff report folder.
-This defaults to /tmp/jdiff but can optionally be specified by export $JDIFF_WORKING_DIRECTORY.
-</description>
-
-<target name="jdiff_report" depends="">
- <!-- This variable is used by the Ant JDiff task. Set it to the
- location where the jdiff package was unpacked -->
- <property name="JDIFF_HOME" value="JDIFF_HOME_NAME" />
- <property name="OLD_BRANCH" value="OLD_BRANCH_NAME"/>
- <property name="NEW_BRANCH" value="NEW_BRANCH_NAME"/>
-
- <taskdef name="jdiff"
- classname="jdiff.JDiffAntTask"
- classpath="${JDIFF_HOME}/antjdiff.jar" />
- <jdiff destdir="hbase_jdiff_report"
- verbose="off"
- stats="on">
- <old name="V1">
- <!-- This is the location of your old source files -->
- <dirset dir="${OLD_BRANCH}/src/main/java" includes="**" />
- </old>
- <new name="V2">
- <!-- This is the location of your new source files -->
- <dirset dir="${NEW_BRANCH}/hbase-server/src/main/java" includes="**" />
- <dirset dir="${NEW_BRANCH}/hbase-client/src/main/java" includes="**" />
- <dirset dir="${NEW_BRANCH}/hbase-common/src/main/java" includes="**" />
- <dirset dir="${NEW_BRANCH}/hbase-protocol/src/main/java" includes="**" />
- <dirset dir="${NEW_BRANCH}/hbase-it/src/test/java" includes="**" />
- <dirset dir="${NEW_BRANCH}/hbase-hadoop-compat/src/main/java" includes="**" />
- <dirset dir="${NEW_BRANCH}/hbase-hadoop2-compat/src/main/java" includes="**" />
- </new>
- </jdiff>
-</target>
-
-</project>
-
http://git-wip-us.apache.org/repos/asf/hbase/blob/fa9fdfee/dev-support/hbase_jdiff_afterSingularityTemplate.xml
----------------------------------------------------------------------
diff --git a/dev-support/hbase_jdiff_afterSingularityTemplate.xml b/dev-support/hbase_jdiff_afterSingularityTemplate.xml
deleted file mode 100644
index 6c4cd93..0000000
--- a/dev-support/hbase_jdiff_afterSingularityTemplate.xml
+++ /dev/null
@@ -1,66 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
--->
-
-<project name="jdiff" default="jdiff_report" basedir="JDIFF_FOLDER">
-
-<description>
-Creates a report in the directory "hbase_jdiff_report-p-PREVIOUS_BRANCH-c-CURRENT_BRANCH" of the default jdiff report folder.
-This defaults to /tmp/jdiff but can optionally be specified by export $JDIFF_WORKING_DIRECTORY.
-</description>
-
-<target name="jdiff_report" depends="">
- <!-- This variable is used by the Ant JDiff task. Set it to the
- location where the jdiff package was unpacked -->
- <property name="JDIFF_HOME" value="JDIFF_HOME_NAME" />
- <property name="OLD_BRANCH" value="OLD_BRANCH_NAME"/>
- <property name="NEW_BRANCH" value="NEW_BRANCH_NAME"/>
-
- <taskdef name="jdiff"
- classname="jdiff.JDiffAntTask"
- classpath="${JDIFF_HOME}/antjdiff.jar" />
- <jdiff destdir="hbase_jdiff_report"
- verbose="off"
- stats="on">
- <old name="V1">
- <!-- This is the location of your old source files -->
- <dirset dir="${OLD_BRANCH}/hbase-server/src/main/java" includes="**" />
- <dirset dir="${OLD_BRANCH}/hbase-client/src/main/java" includes="**" />
- <dirset dir="${OLD_BRANCH}/hbase-common/src/main/java" includes="**" />
- <dirset dir="${OLD_BRANCH}/hbase-protocol/src/main/java" includes="**" />
- <dirset dir="${OLD_BRANCH}/hbase-it/src/test/java" includes="**" />
- <dirset dir="${OLD_BRANCH}/hbase-hadoop-compat/src/main/java" includes="**" />
- <dirset dir="${OLD_BRANCH}/hbase-hadoop2-compat/src/main/java" includes="**" />
-
- </old>
- <new name="V2">
- <!-- This is the location of your new source files -->
- <dirset dir="${NEW_BRANCH}/hbase-server/src/main/java" includes="**" />
- <dirset dir="${NEW_BRANCH}/hbase-client/src/main/java" includes="**" />
- <dirset dir="${NEW_BRANCH}/hbase-common/src/main/java" includes="**" />
- <dirset dir="${NEW_BRANCH}/hbase-protocol/src/main/java" includes="**" />
- <dirset dir="${NEW_BRANCH}/hbase-it/src/test/java" includes="**" />
- <dirset dir="${NEW_BRANCH}/hbase-hadoop-compat/src/main/java" includes="**" />
- <dirset dir="${NEW_BRANCH}/hbase-hadoop2-compat/src/main/java" includes="**" />
- </new>
- </jdiff>
-</target>
-
-</project>
-
http://git-wip-us.apache.org/repos/asf/hbase/blob/fa9fdfee/dev-support/hbase_jdiff_template.xml
----------------------------------------------------------------------
diff --git a/dev-support/hbase_jdiff_template.xml b/dev-support/hbase_jdiff_template.xml
deleted file mode 100644
index 21fe8ed..0000000
--- a/dev-support/hbase_jdiff_template.xml
+++ /dev/null
@@ -1,53 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
--->
-
-<project name="jdiff" default="jdiff_report" basedir="JDIFF_FOLDER">
-
-<description>
-Creates a report in the directory "hbase_jdiff_report-p-PREVIOUS_BRANCH-c-CURRENT_BRANCH" of the default jdiff report folder.
-This defaults to /tmp/jdiff but can optionally be specified by export $JDIFF_WORKING_DIRECTORY.
-</description>
-
-<target name="jdiff_report" depends="">
- <!-- This variable is used by the Ant JDiff task. Set it to the
- location where the jdiff package was unpacked -->
- <property name="JDIFF_HOME" value="JDIFF_HOME_NAME" />
- <property name="OLD_BRANCH" value="OLD_BRANCH_NAME"/>
- <property name="NEW_BRANCH" value="NEW_BRANCH_NAME"/>
-
- <taskdef name="jdiff"
- classname="jdiff.JDiffAntTask"
- classpath="${JDIFF_HOME}/antjdiff.jar" />
- <jdiff destdir="hbase_jdiff_report"
- verbose="off"
- stats="on">
- <old name="V1">
- <!-- This is the location of your old source files -->
- <dirset dir="${OLD_BRANCH}/src/main/java" includes="**" />
- </old>
- <new name="V2">
- <!-- This is the location of your new source files -->
- <dirset dir="${NEW_BRANCH}/src/main/java" includes="**" />
- </new>
- </jdiff>
-</target>
-
-</project>
-
http://git-wip-us.apache.org/repos/asf/hbase/blob/fa9fdfee/dev-support/jdiffHBasePublicAPI.sh
----------------------------------------------------------------------
diff --git a/dev-support/jdiffHBasePublicAPI.sh b/dev-support/jdiffHBasePublicAPI.sh
deleted file mode 100755
index 4a18dbf..0000000
--- a/dev-support/jdiffHBasePublicAPI.sh
+++ /dev/null
@@ -1,258 +0,0 @@
-#!/bin/bash
-set -e
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-################################################ ABOUT JDIFF #######################################################
-#
-# What is JDiff? JDiff is a tool for comparing the public APIs of two separate Java codebases. Like diff, it will
-# give additions, changes, and removals. It will output an HTML report with the information.
-# To learn more, visit http://javadiff.sourceforge.net/.
-# JDiff is licensed under LGPL.
-
-############################################# QUICK-START EXAMPLE ##################################################
-#
-# Suppose we wanted to see the API diffs between HBase 0.92 and HBase 0.94. We could use this tool like so:
-# > ./jdiffHBasePublicAPI.sh https://github.com/apache/hbase.git 0.92 https://github.com/apache/hbase.git 0.94
-#
-# This would generate a report in the local folder /tmp/jdiff/hbase_jdiff_report-p-0.92-c-0.94/
-# To view the report, simply examine /tmp/jdiff/hbase_jdiff_report-p-0.92-c-0.94/changes.html in your choice of
-# browser.
-#
-# Note that this works because 0.92 and 0.94 have the source directory structure that is specified in the
-# hbase_jdiff_template.xml file. To compare 0.95 to 0.96, which have the post-singularity structure, two other
-# template files (included) are used. The formats are autoated and is all taken care of automatically by the script.
-#
-# On a local machine, JDiff reports have taken ~20-30 minutes to run. On Jenkins, it has taken over 35 minutes
-# in some cases. Your mileage may vary. Trunk and 0.95 take more time than 0.92 and 0.94.
-#
-#
-############################################ SPECIFYING A LOCAL REPO ###############################################
-#
-# The JDiff tool also works with local code. Instead of specifying a repo and a branch, you can specifying the
-# absolute path of the ./hbase folder and a name for code (e.g. experimental_94).
-#
-# A local repo can be specified for none, one, or both of the sources.
-#
-############################################### EXAMPLE USE CASES ##################################################
-#
-# Example 1: Generate a report to check if potential change doesn't break API compatibility with Apache HBase 0.94
-#
-# In this case, you could compare the version you are using against a repo branch where your changes are.
-# > ./jdiffHBasePublicAPI.sh https://github.com/apache/hbase.git 0.94 https://github.com/MY_REPO/hbase.git 0.94
-#
-# Example 2: Generate a report to check if two branches of the same repo have any public API incompatibilities
-# > ./jdiffHBasePublicAPI.sh https://github.com/MY_REPO/hbase.git $BRANCH_1 \
-# > https://github.com/MY_REPO/hbase.git $BRANCH_2
-#
-# Example 3: Have Example 1 done in a special directory in the user's home folder
-#
-# > export JDIFF_WORKING_DIRECTORY=~/jdiff_reports
-# > ./jdiffHBasePublicAPI.sh https://github.com/apache/hbase.git 0.94 https://github.com/MY_REPO/hbase.git 0.94
-#
-# Example 4: Check the API diff of a local change against an existing repo branch.
-# > ./jdiffHBasePublicAPI.sh https://github.com/apache/hbase.git 0.95 /home/aleks/exp_hbase/hbase experiment_95
-#
-# Example 5: Compare two local repos for public API changes
-# > ./jdiffHBasePublicAPI.sh /home/aleks/stable_hbase/hbase stable_95 /home/aleks/exp_hbase/hbase experiment_95
-#
-#
-################################################## NOTE ON USAGE ###################################################
-#
-# 1. When using this tool, please specify the initial version first and the current version second. The semantics
-# do not make sense otherwise. For example: jdiff 94 95 is good. jdiff 95 94 is bad
-#
-############################################# READING A JDIFF REPORT ###############################################
-#
-# The purpose of the JDiff report is show things that have changed between two versions of the public API. A user
-# would use this report to determine if committing a change would cause existing API clients to break. To do so,
-# there are specific things that one should look for in the report.
-#
-# 1. Identify the classes that constitute the public API. An example in 0.94 might be all classes in
-# org.apache.hadoop.hbase.client.*
-# 2. After identifying those classes, go through each one and look for offending changes.
-# Those may include, but are not limited to:
-# 1. Removed methods
-# 2. Changed methods (including changes in return type and exception types)
-# 3. Methods added to interfaces
-# 4. Changed class inheritence information (may in innocuous but definitely worth validating)
-# 5. Removed or renamed public static member variables and constants
-# 6. Removed or renamed packages
-# 7. Class moved to a different package
-
-########################################### SETTING THE JDIFF WORKING DIRECTORY ####################################
-#
-# By default, the working environment of jdiff is /tmp/jdiff. However, sometimes it is nice to have it place reports
-# and temp files elsewhere. In that case, please export JDIFF_WORKING_DIRECTORY into the bash environment and this
-# script will pick that up and use it.
-#
-
-scriptDirectory=$(dirname ${BASH_SOURCE[0]})
-x=`echo $scriptDirectory | sed "s{\.{{g"`
-DEV_SUPPORT_HOME="`pwd`$x"
-. $scriptDirectory/jdiffHBasePublicAPI_common.sh
-
-EXPECTED_ARGS=4
-
-if [[ "$#" -ne "$EXPECTED_ARGS" ]]; then
- echo "This tool expects $EXPECTED_ARGS arguments, but received $#. Please check your command and try again.";
- echo "Usage: $0 <repoUrl or local source directory 1> <branch or source name 1> <repoUrl or local source directory 2> <branch or source name 2>"
- exit 1;
-fi
-
-echo "JDiff evaluation beginning:";
-isGitRepo $1
-FIRST_SOURCE_TYPE=$INPUT_FORMAT;
-isGitRepo $3
-SECOND_SOURCE_TYPE=$INPUT_FORMAT;
-
-PREVIOUS_BRANCH=$2 ## We will still call it a branch even if it's not from a git repo.
-CURRENT_BRANCH=$4
-
-echo "We are going to compare source 1 which is a $FIRST_SOURCE_TYPE and source 2, which is a $SECOND_SOURCE_TYPE"
-
-
-# Check that if either source is from a git repo, that the name is reasonable.
-if [[ "$FIRST_SOURCE_TYPE" = "git_repo" ]]; then
-
- git check-ref-format --branch $2
-fi
-
-if [[ "$SECOND_SOURCE_TYPE" = "git_repo" ]]; then
-
- git check-ref-format --branch $4
-fi
-
-#If the JDIFF_WORKING_DIRECTORY is set, then we will output the report there. Otherwise, to the default location
-if [[ "$JDIFF_WORKING_DIRECTORY" = "" ]]; then
-
- JDIFF_WORKING_DIRECTORY=/tmp/jdiff
- echo "JDIFF_WORKING_DIRECTORY not set. That's not an issue. We will default it to $JDIFF_WORKING_DIRECTORY."
-else
- echo "JDIFF_WORKING_DIRECTORY set to $JDIFF_WORKING_DIRECTORY";
-fi
-mkdir -p $JDIFF_WORKING_DIRECTORY
-
-# We will need this to reference the template we want to use
-cd $JDIFF_WORKING_DIRECTORY
-scenario_template_name=hbase_jdiff_p-$PREVIOUS_BRANCH-c-$CURRENT_BRANCH.xml
-
-
-# Pull down JDiff tool and unpack it
-if [ ! -d jdiff-1.1.1-with-incompatible-option ]; then
- curl -OL http://cloud.github.com/downloads/tomwhite/jdiff/jdiff-1.1.1-with-incompatible-option.zip
- unzip jdiff-1.1.1-with-incompatible-option.zip
-fi
-
-JDIFF_HOME=`pwd`/jdiff-1.1.1-with-incompatible-option
-cd $JDIFF_WORKING_DIRECTORY
-
-# Pull down sources if necessary. Note that references to previous change are prefaced with p- in order to avoid collission of branch names
-if [[ "$FIRST_SOURCE_TYPE" = "git_repo" ]]; then
-
- PREVIOUS_REPO=$1
- rm -rf p-$PREVIOUS_BRANCH
- mkdir -p p-$PREVIOUS_BRANCH
- cd p-$PREVIOUS_BRANCH
- git clone --depth 1 --branch $PREVIOUS_BRANCH $PREVIOUS_REPO
- cd $JDIFF_WORKING_DIRECTORY
- HBASE_1_HOME=`pwd`/p-$PREVIOUS_BRANCH/hbase
-else
- HBASE_1_HOME=$1
-fi
-
-echo "HBASE_1_HOME set to $HBASE_1_HOME"
-echo "In HBASE_1_HOME, we have"
-ls -la $HBASE_1_HOME
-
-if [[ "$SECOND_SOURCE_TYPE" = "git_repo" ]]; then
- CURRENT_REPO=$3
- rm -rf $JDIFF_WORKING_DIRECTORY/c-$CURRENT_BRANCH
- mkdir -p $JDIFF_WORKING_DIRECTORY/c-$CURRENT_BRANCH
- cd $JDIFF_WORKING_DIRECTORY/c-$CURRENT_BRANCH
- git clone --depth 1 --branch $CURRENT_BRANCH $CURRENT_REPO
- cd $JDIFF_WORKING_DIRECTORY
- HBASE_2_HOME=`pwd`/c-$CURRENT_BRANCH/hbase
-else
- HBASE_2_HOME=$3
-fi
-
-echo "HBASE_2_HOME set to $HBASE_2_HOME"
-echo "In HBASE_2_HOME, we have"
-ls -la $HBASE_2_HOME
-
-# Next step is to pull down the proper template based on the directory structure
-isNewFormat $HBASE_1_HOME
-export P_FORMAT=$BRANCH_FORMAT
-
-isNewFormat $HBASE_2_HOME
-export C_FORMAT=$BRANCH_FORMAT
-
-if [[ "$C_FORMAT" = "new" ]]; then
-
- if [[ "$P_FORMAT" = "new" ]]; then
- templateFile=$DEV_SUPPORT_HOME/hbase_jdiff_afterSingularityTemplate.xml
- echo "Previous format is of the new style. We'll be using template $templateFile";
- else
- templateFile=$DEV_SUPPORT_HOME/hbase_jdiff_acrossSingularityTemplate.xml
- echo "Previous format is of the old style. We'll be using template $templateFile";
- fi
-
-else
-
- if [[ "P_FORMAT" != "old" ]]; then
- echo "When using this tool, please specify the initial version first and the current version second. They should be in ascending chronological order.
- The semantics do not make sense otherwise. For example: jdiff 94 95 is good. jdiff 95 94 is bad."
- echo "Exiting the script."
- exit 5;
- fi
- templateFile=$DEV_SUPPORT_HOME/hbase_jdiff_template.xml
- echo "Both formats are using the 94 and earlier style directory format. We'll be using template $templateFile"
-fi
-
-cp $templateFile $JDIFF_WORKING_DIRECTORY/$scenario_template_name
-
-### Configure the jdiff script
-
-### Note that PREVIOUS_BRANCH and CURRENT_BRANCH will be the absolute locations of the source.
-echo "Configuring the jdiff script"
-
-# Extension to -i is done to support in-place editing on GNU sed and BSD sed.
-sed -i.tmp "s]hbase_jdiff_report]hbase_jdiff_report-p-$PREVIOUS_BRANCH-c-$CURRENT_BRANCH]g" \
- $JDIFF_WORKING_DIRECTORY/$scenario_template_name
-sed -i.tmp "s]JDIFF_HOME_NAME]$JDIFF_HOME]g" \
- $JDIFF_WORKING_DIRECTORY/$scenario_template_name
-sed -i.tmp "s]OLD_BRANCH_NAME]$HBASE_1_HOME]g" \
- $JDIFF_WORKING_DIRECTORY/$scenario_template_name
-sed -i.tmp "s]NEW_BRANCH_NAME]$HBASE_2_HOME]g" \
- $JDIFF_WORKING_DIRECTORY/$scenario_template_name
-
-sed -i.tmp "s]V1]$PREVIOUS_BRANCH]g" \
- $JDIFF_WORKING_DIRECTORY/$scenario_template_name
-sed -i.tmp "s]V2]$CURRENT_BRANCH]g" \
- $JDIFF_WORKING_DIRECTORY/$scenario_template_name
-
-sed -i.tmp "s]JDIFF_FOLDER]$JDIFF_WORKING_DIRECTORY]g" \
- $JDIFF_WORKING_DIRECTORY/$scenario_template_name
-
-echo "Running jdiff";
-ls -la $JDIFF_WORKING_DIRECTORY;
-ant -f $JDIFF_WORKING_DIRECTORY/$scenario_template_name;
-
-echo "jdiff operation complete. Report placed into $JDIFF_WORKING_DIRECTORY/hbase_jdiff_report-p-$PREVIOUS_BRANCH-c-$CURRENT_BRANCH/changes.html";
-
http://git-wip-us.apache.org/repos/asf/hbase/blob/fa9fdfee/dev-support/jdiffHBasePublicAPI_common.sh
----------------------------------------------------------------------
diff --git a/dev-support/jdiffHBasePublicAPI_common.sh b/dev-support/jdiffHBasePublicAPI_common.sh
deleted file mode 100755
index 1cc9954..0000000
--- a/dev-support/jdiffHBasePublicAPI_common.sh
+++ /dev/null
@@ -1,76 +0,0 @@
-#!/bin/bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-##########################################################################################################################
-#
-### Purpose: To describe whether the directory specified has the old directory format or the new directory format
-### Usage: This function takes one argument: The directory in question
-### It will set the temporary variable BRANCH_FORMAT. This variable can change with every call, so it is up to the user to
-### store it into something else as soon as the function exists
-### Example:
-### > isNewFormat ./myDevDir/testing/branch/hbase
-isNewFormat() {
-
- echo "Determining if directory $1 is of the 0.94 and before OR 0.95 and after versions";
- if [[ "$1" = "" ]]; then
- echo "Directory not specified. Exiting";
- fi
- echo "First, check that $1 exists";
- if [[ -d $1 ]]; then
- echo "Directory $1 exists"
- else
- echo "Directory $1 does not exist. Exiting";
- exit 1;
- fi
-
- if [[ -d "$1/hbase-server" ]]; then
-
- echo "The directory $1/hbase-server exists so this is of the new format";
- export BRANCH_FORMAT=new;
-
- else
- echo "The directory $1/hbase-server does not exist. Therefore, this is of the old format";
- export BRANCH_FORMAT=old;
- fi
-}
-
-### Purpose: To describe whether the argument specified is a git repo or a local directory
-### Usage: This function takes one argument: The directory in question
-### It will set the temporary variable INPUT_FORMAT. This variable can change with every call, so it is up to the user to
-### store it into something else as soon as the function exists
-### Example:
-### > isGitRepo ./myDevDir/testing/branch/hbase
-
-isGitRepo() {
-
- echo "Determining if this is a local directory or a git repo.";
- if [[ "$1" = "" ]]; then
- echo "No value specified for repo or directory. Exiting."
- exit 1;
- fi
-
- if [[ `echo $1 | grep 'http://'` || `echo $1 | grep 'https://'` || `echo $1 | grep 'git://'` ]]; then
- echo "Looks like $1 is a git repo";
- export INPUT_FORMAT=git_repo
- else
- echo "$1 is a local directory";
- export INPUT_FORMAT=local_directory
- fi
-
-
-}
[05/50] [abbrv] hbase git commit: HBASE-15742 Reduce allocation of
objects in metrics (Phil Yang)
Posted by sy...@apache.org.
HBASE-15742 Reduce allocation of objects in metrics (Phil Yang)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/f76ffb7f
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/f76ffb7f
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/f76ffb7f
Branch: refs/heads/hbase-12439
Commit: f76ffb7f38f003c4e01044eddc5e1d0d970a7efd
Parents: 082d193
Author: tedyu <yu...@gmail.com>
Authored: Tue May 3 08:58:00 2016 -0700
Committer: tedyu <yu...@gmail.com>
Committed: Tue May 3 08:58:00 2016 -0700
----------------------------------------------------------------------
.../hbase/ipc/MetricsHBaseServerSourceImpl.java | 2 +-
.../master/MetricsMasterProcSourceImpl.java | 2 +-
.../hbase/master/MetricsMasterSourceImpl.java | 2 +-
.../MetricsStochasticBalancerSourceImpl.java | 2 +-
.../apache/hadoop/hbase/metrics/Interns.java | 104 +++++++++++++++++++
.../hadoop/hbase/metrics/MetricsInfoImpl.java | 65 ++++++++++++
.../MetricsRegionAggregateSourceImpl.java | 2 +-
.../MetricsRegionServerSourceImpl.java | 2 +-
.../regionserver/MetricsRegionSourceImpl.java | 2 +-
.../MetricsTableAggregateSourceImpl.java | 2 +-
.../regionserver/MetricsTableSourceImpl.java | 2 +-
.../metrics2/lib/DynamicMetricsRegistry.java | 1 +
.../hadoop/metrics2/lib/MutableHistogram.java | 57 +++++++---
.../metrics2/lib/MutableRangeHistogram.java | 1 +
14 files changed, 225 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/f76ffb7f/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSourceImpl.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSourceImpl.java
index c466564..c72641d 100644
--- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSourceImpl.java
+++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSourceImpl.java
@@ -21,10 +21,10 @@ package org.apache.hadoop.hbase.ipc;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.metrics.BaseSourceImpl;
+import org.apache.hadoop.hbase.metrics.Interns;
import org.apache.hadoop.metrics2.MetricHistogram;
import org.apache.hadoop.metrics2.MetricsCollector;
import org.apache.hadoop.metrics2.MetricsRecordBuilder;
-import org.apache.hadoop.metrics2.lib.Interns;
import org.apache.hadoop.metrics2.lib.MutableFastCounter;
@InterfaceAudience.Private
http://git-wip-us.apache.org/repos/asf/hbase/blob/f76ffb7f/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterProcSourceImpl.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterProcSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterProcSourceImpl.java
index 0375e37..2b56ead 100644
--- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterProcSourceImpl.java
+++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterProcSourceImpl.java
@@ -20,9 +20,9 @@ package org.apache.hadoop.hbase.master;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.metrics.BaseSourceImpl;
+import org.apache.hadoop.hbase.metrics.Interns;
import org.apache.hadoop.metrics2.MetricsCollector;
import org.apache.hadoop.metrics2.MetricsRecordBuilder;
-import org.apache.hadoop.metrics2.lib.Interns;
/**
* Hadoop2 implementation of MetricsMasterSource.
http://git-wip-us.apache.org/repos/asf/hbase/blob/f76ffb7f/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSourceImpl.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSourceImpl.java
index ea21967..fc859e5 100644
--- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSourceImpl.java
+++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSourceImpl.java
@@ -20,9 +20,9 @@ package org.apache.hadoop.hbase.master;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.metrics.BaseSourceImpl;
+import org.apache.hadoop.hbase.metrics.Interns;
import org.apache.hadoop.metrics2.MetricsCollector;
import org.apache.hadoop.metrics2.MetricsRecordBuilder;
-import org.apache.hadoop.metrics2.lib.Interns;
import org.apache.hadoop.metrics2.lib.MutableFastCounter;
/**
http://git-wip-us.apache.org/repos/asf/hbase/blob/f76ffb7f/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/balancer/MetricsStochasticBalancerSourceImpl.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/balancer/MetricsStochasticBalancerSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/balancer/MetricsStochasticBalancerSourceImpl.java
index ded0a0c..f658a27 100644
--- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/balancer/MetricsStochasticBalancerSourceImpl.java
+++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/balancer/MetricsStochasticBalancerSourceImpl.java
@@ -23,9 +23,9 @@ import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.metrics.Interns;
import org.apache.hadoop.metrics2.MetricsCollector;
import org.apache.hadoop.metrics2.MetricsRecordBuilder;
-import org.apache.hadoop.metrics2.lib.Interns;
@InterfaceAudience.Private
public class MetricsStochasticBalancerSourceImpl extends MetricsBalancerSourceImpl implements
http://git-wip-us.apache.org/repos/asf/hbase/blob/f76ffb7f/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/metrics/Interns.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/metrics/Interns.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/metrics/Interns.java
new file mode 100644
index 0000000..7905561
--- /dev/null
+++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/metrics/Interns.java
@@ -0,0 +1,104 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.metrics;
+
+
+
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.metrics2.MetricsInfo;
+import org.apache.hadoop.metrics2.MetricsTag;
+
+
+/**
+ * Helpers to create interned metrics info
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public final class Interns {
+
+ private static LoadingCache<String, ConcurrentHashMap<String, MetricsInfo>> infoCache =
+ CacheBuilder.newBuilder().expireAfterAccess(1, TimeUnit.DAYS)
+ .build(new CacheLoader<String, ConcurrentHashMap<String, MetricsInfo>>() {
+ public ConcurrentHashMap<String, MetricsInfo> load(String key) {
+ return new ConcurrentHashMap<String, MetricsInfo>();
+ }
+ });
+ private static LoadingCache<MetricsInfo, ConcurrentHashMap<String, MetricsTag>> tagCache =
+ CacheBuilder.newBuilder().expireAfterAccess(1, TimeUnit.DAYS)
+ .build(new CacheLoader<MetricsInfo, ConcurrentHashMap<String, MetricsTag>>() {
+ public ConcurrentHashMap<String, MetricsTag> load(MetricsInfo key) {
+ return new ConcurrentHashMap<String, MetricsTag>();
+ }
+ });
+
+ private Interns(){}
+
+ /**
+ * Get a metric info object
+ *
+ * @return an interned metric info object
+ */
+ public static MetricsInfo info(String name, String description) {
+ Map<String, MetricsInfo> map = infoCache.getUnchecked(name);
+ MetricsInfo info = map.get(description);
+ if (info == null) {
+ info = new MetricsInfoImpl(name, description);
+ map.put(description, info);
+ }
+ return info;
+ }
+
+ /**
+ * Get a metrics tag
+ *
+ * @param info of the tag
+ * @param value of the tag
+ * @return an interned metrics tag
+ */
+ public static MetricsTag tag(MetricsInfo info, String value) {
+ Map<String, MetricsTag> map = tagCache.getUnchecked(info);
+ MetricsTag tag = map.get(value);
+ if (tag == null) {
+ tag = new MetricsTag(info, value);
+ map.put(value, tag);
+ }
+ return tag;
+ }
+
+ /**
+ * Get a metrics tag
+ *
+ * @param name of the tag
+ * @param description of the tag
+ * @param value of the tag
+ * @return an interned metrics tag
+ */
+ public static MetricsTag tag(String name, String description, String value) {
+ return tag(info(name, description), value);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/f76ffb7f/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/metrics/MetricsInfoImpl.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/metrics/MetricsInfoImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/metrics/MetricsInfoImpl.java
new file mode 100644
index 0000000..73dc459
--- /dev/null
+++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/metrics/MetricsInfoImpl.java
@@ -0,0 +1,65 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.metrics;
+
+import com.google.common.base.Objects;
+import com.google.common.base.Preconditions;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.metrics2.MetricsInfo;
+
+/**
+ * Making implementing metric info a little easier
+ */
+@InterfaceAudience.Private
+class MetricsInfoImpl implements MetricsInfo {
+ private final String name, description;
+
+ MetricsInfoImpl(String name, String description) {
+ this.name = Preconditions.checkNotNull(name, "name");
+ this.description = Preconditions.checkNotNull(description, "description");
+ }
+
+ @Override public String name() {
+ return name;
+ }
+
+ @Override public String description() {
+ return description;
+ }
+
+ @Override public boolean equals(Object obj) {
+ if (obj instanceof MetricsInfo) {
+ MetricsInfo other = (MetricsInfo) obj;
+ return Objects.equal(name, other.name()) &&
+ Objects.equal(description, other.description());
+ }
+ return false;
+ }
+
+ @Override public int hashCode() {
+ return Objects.hashCode(name, description);
+ }
+
+ @Override public String toString() {
+ return Objects.toStringHelper(this)
+ .add("name", name).add("description", description)
+ .toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/f76ffb7f/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionAggregateSourceImpl.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionAggregateSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionAggregateSourceImpl.java
index 1835f6b..c269c40 100644
--- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionAggregateSourceImpl.java
+++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionAggregateSourceImpl.java
@@ -27,10 +27,10 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.metrics.BaseSourceImpl;
+import org.apache.hadoop.hbase.metrics.Interns;
import org.apache.hadoop.metrics2.MetricsCollector;
import org.apache.hadoop.metrics2.MetricsRecordBuilder;
import org.apache.hadoop.metrics2.impl.JmxCacheBuster;
-import org.apache.hadoop.metrics2.lib.Interns;
import org.apache.hadoop.metrics2.lib.MetricsExecutorImpl;
@InterfaceAudience.Private
http://git-wip-us.apache.org/repos/asf/hbase/blob/f76ffb7f/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
index 965fc66..b412fd1 100644
--- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
+++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
@@ -20,10 +20,10 @@ package org.apache.hadoop.hbase.regionserver;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.metrics.BaseSourceImpl;
+import org.apache.hadoop.hbase.metrics.Interns;
import org.apache.hadoop.metrics2.MetricHistogram;
import org.apache.hadoop.metrics2.MetricsCollector;
import org.apache.hadoop.metrics2.MetricsRecordBuilder;
-import org.apache.hadoop.metrics2.lib.Interns;
import org.apache.hadoop.metrics2.lib.MutableFastCounter;
/**
http://git-wip-us.apache.org/repos/asf/hbase/blob/f76ffb7f/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java
index 05f1126..de46ac7 100644
--- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java
+++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java
@@ -23,10 +23,10 @@ import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.metrics.Interns;
import org.apache.hadoop.metrics2.MetricHistogram;
import org.apache.hadoop.metrics2.MetricsRecordBuilder;
import org.apache.hadoop.metrics2.lib.DynamicMetricsRegistry;
-import org.apache.hadoop.metrics2.lib.Interns;
import org.apache.hadoop.metrics2.lib.MutableFastCounter;
@InterfaceAudience.Private
http://git-wip-us.apache.org/repos/asf/hbase/blob/f76ffb7f/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableAggregateSourceImpl.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableAggregateSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableAggregateSourceImpl.java
index fba6e72..b46a21f 100644
--- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableAggregateSourceImpl.java
+++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableAggregateSourceImpl.java
@@ -24,9 +24,9 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.metrics.BaseSourceImpl;
+import org.apache.hadoop.hbase.metrics.Interns;
import org.apache.hadoop.metrics2.MetricsCollector;
import org.apache.hadoop.metrics2.MetricsRecordBuilder;
-import org.apache.hadoop.metrics2.lib.Interns;
@InterfaceAudience.Private
public class MetricsTableAggregateSourceImpl extends BaseSourceImpl
http://git-wip-us.apache.org/repos/asf/hbase/blob/f76ffb7f/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableSourceImpl.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableSourceImpl.java
index 33a807f..5d1dd79 100644
--- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableSourceImpl.java
+++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableSourceImpl.java
@@ -24,9 +24,9 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.metrics.Interns;
import org.apache.hadoop.metrics2.MetricsRecordBuilder;
import org.apache.hadoop.metrics2.lib.DynamicMetricsRegistry;
-import org.apache.hadoop.metrics2.lib.Interns;
@InterfaceAudience.Private
public class MetricsTableSourceImpl implements MetricsTableSource {
http://git-wip-us.apache.org/repos/asf/hbase/blob/f76ffb7f/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/DynamicMetricsRegistry.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/DynamicMetricsRegistry.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/DynamicMetricsRegistry.java
index 1f3da12..977536a 100644
--- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/DynamicMetricsRegistry.java
+++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/DynamicMetricsRegistry.java
@@ -24,6 +24,7 @@ import java.util.concurrent.ConcurrentMap;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.metrics.Interns;
import org.apache.hadoop.metrics2.MetricsException;
import org.apache.hadoop.metrics2.MetricsInfo;
import org.apache.hadoop.metrics2.MetricsRecordBuilder;
http://git-wip-us.apache.org/repos/asf/hbase/blob/f76ffb7f/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableHistogram.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableHistogram.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableHistogram.java
index 5b4a294..0bc7e9c 100644
--- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableHistogram.java
+++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableHistogram.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.metrics2.lib;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.metrics.Interns;
import org.apache.hadoop.hbase.util.Counter;
import org.apache.hadoop.hbase.util.FastLongHistogram;
import org.apache.hadoop.metrics2.MetricHistogram;
@@ -40,6 +41,20 @@ public class MutableHistogram extends MutableMetric implements MetricHistogram {
protected final String desc;
protected final Counter counter = new Counter(0);
+ private boolean metricsInfoStringInited = false;
+ private String NUM_OPS_METRIC;
+ private String MIN_METRIC;
+ private String MAX_METRIC;
+ private String MEAN_METRIC;
+ private String MEDIAN_METRIC;
+ private String TWENTY_FIFTH_PERCENTILE_METRIC;
+ private String SEVENTY_FIFTH_PERCENTILE_METRIC;
+ private String NINETIETH_PERCENTILE_METRIC;
+ private String NINETY_FIFTH_PERCENTILE_METRIC;
+ private String NINETY_EIGHTH_PERCENTILE_METRIC;
+ private String NINETY_NINETH_PERCENTILE_METRIC;
+ private String NINETY_NINE_POINT_NINETH_PERCENTILE_METRIC;
+
public MutableHistogram(MetricsInfo info) {
this(info.name(), info.description());
}
@@ -70,29 +85,47 @@ public class MutableHistogram extends MutableMetric implements MetricHistogram {
protected void updateSnapshotMetrics(MetricsRecordBuilder metricsRecordBuilder,
FastLongHistogram histo) {
- metricsRecordBuilder.addCounter(Interns.info(name + NUM_OPS_METRIC_NAME, desc), counter.get());
- metricsRecordBuilder.addGauge(Interns.info(name + MIN_METRIC_NAME, desc), histo.getMin());
- metricsRecordBuilder.addGauge(Interns.info(name + MAX_METRIC_NAME, desc), histo.getMax());
- metricsRecordBuilder.addGauge(Interns.info(name + MEAN_METRIC_NAME, desc), histo.getMean());
+ if (!metricsInfoStringInited) {
+ NUM_OPS_METRIC = name + NUM_OPS_METRIC_NAME;
+ MIN_METRIC = name + MIN_METRIC_NAME;
+ MAX_METRIC = name + MAX_METRIC_NAME;
+ MEAN_METRIC = name + MEAN_METRIC_NAME;
+ MEDIAN_METRIC = name + MEDIAN_METRIC_NAME;
+ TWENTY_FIFTH_PERCENTILE_METRIC = name + TWENTY_FIFTH_PERCENTILE_METRIC_NAME;
+ SEVENTY_FIFTH_PERCENTILE_METRIC = name + SEVENTY_FIFTH_PERCENTILE_METRIC_NAME;
+ NINETIETH_PERCENTILE_METRIC = name + NINETIETH_PERCENTILE_METRIC_NAME;
+ NINETY_FIFTH_PERCENTILE_METRIC = name + NINETY_FIFTH_PERCENTILE_METRIC_NAME;
+ NINETY_EIGHTH_PERCENTILE_METRIC = name + NINETY_EIGHTH_PERCENTILE_METRIC_NAME;
+ NINETY_NINETH_PERCENTILE_METRIC = name + NINETY_NINETH_PERCENTILE_METRIC_NAME;
+ NINETY_NINE_POINT_NINETH_PERCENTILE_METRIC = name +
+ NINETY_NINE_POINT_NINETH_PERCENTILE_METRIC_NAME;
+
+ metricsInfoStringInited = true;
+ }
+
+ metricsRecordBuilder.addCounter(Interns.info(NUM_OPS_METRIC, desc), counter.get());
+ metricsRecordBuilder.addGauge(Interns.info(MIN_METRIC, desc), histo.getMin());
+ metricsRecordBuilder.addGauge(Interns.info(MAX_METRIC, desc), histo.getMax());
+ metricsRecordBuilder.addGauge(Interns.info(MEAN_METRIC, desc), histo.getMean());
long[] percentiles = histo.getQuantiles();
- metricsRecordBuilder.addGauge(Interns.info(name + TWENTY_FIFTH_PERCENTILE_METRIC_NAME, desc),
+ metricsRecordBuilder.addGauge(Interns.info(TWENTY_FIFTH_PERCENTILE_METRIC, desc),
percentiles[0]);
- metricsRecordBuilder.addGauge(Interns.info(name + MEDIAN_METRIC_NAME, desc),
+ metricsRecordBuilder.addGauge(Interns.info(MEDIAN_METRIC, desc),
percentiles[1]);
- metricsRecordBuilder.addGauge(Interns.info(name + SEVENTY_FIFTH_PERCENTILE_METRIC_NAME, desc),
+ metricsRecordBuilder.addGauge(Interns.info(SEVENTY_FIFTH_PERCENTILE_METRIC, desc),
percentiles[2]);
- metricsRecordBuilder.addGauge(Interns.info(name + NINETIETH_PERCENTILE_METRIC_NAME, desc),
+ metricsRecordBuilder.addGauge(Interns.info(NINETIETH_PERCENTILE_METRIC, desc),
percentiles[3]);
- metricsRecordBuilder.addGauge(Interns.info(name + NINETY_FIFTH_PERCENTILE_METRIC_NAME, desc),
+ metricsRecordBuilder.addGauge(Interns.info(NINETY_FIFTH_PERCENTILE_METRIC, desc),
percentiles[4]);
- metricsRecordBuilder.addGauge(Interns.info(name + NINETY_EIGHTH_PERCENTILE_METRIC_NAME, desc),
+ metricsRecordBuilder.addGauge(Interns.info(NINETY_EIGHTH_PERCENTILE_METRIC, desc),
percentiles[5]);
- metricsRecordBuilder.addGauge(Interns.info(name + NINETY_NINETH_PERCENTILE_METRIC_NAME, desc),
+ metricsRecordBuilder.addGauge(Interns.info(NINETY_NINETH_PERCENTILE_METRIC, desc),
percentiles[6]);
metricsRecordBuilder.addGauge(
- Interns.info(name + NINETY_NINE_POINT_NINETH_PERCENTILE_METRIC_NAME, desc),
+ Interns.info(NINETY_NINE_POINT_NINETH_PERCENTILE_METRIC, desc),
percentiles[7]);
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/f76ffb7f/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableRangeHistogram.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableRangeHistogram.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableRangeHistogram.java
index 13187af..94bcdaa 100644
--- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableRangeHistogram.java
+++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableRangeHistogram.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.metrics2.lib;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.metrics.Interns;
import org.apache.hadoop.hbase.util.FastLongHistogram;
import org.apache.hadoop.metrics2.MetricHistogram;
import org.apache.hadoop.metrics2.MetricsInfo;
[04/50] [abbrv] hbase git commit: HBASE-15675 Add more details about
region on table.jsp
Posted by sy...@apache.org.
HBASE-15675 Add more details about region on table.jsp
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/082d1934
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/082d1934
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/082d1934
Branch: refs/heads/hbase-12439
Commit: 082d193455125f451d3fc1a63146ca024899dd0f
Parents: c77a153
Author: Yu Li <li...@apache.org>
Authored: Tue May 3 12:00:09 2016 +0800
Committer: Yu Li <li...@apache.org>
Committed: Tue May 3 20:18:48 2016 +0800
----------------------------------------------------------------------
.../resources/hbase-webapps/master/table.jsp | 361 +++++++++++++++++--
1 file changed, 335 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/082d1934/hbase-server/src/main/resources/hbase-webapps/master/table.jsp
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/resources/hbase-webapps/master/table.jsp b/hbase-server/src/main/resources/hbase-webapps/master/table.jsp
index 693a663..ee2a7ba 100644
--- a/hbase-server/src/main/resources/hbase-webapps/master/table.jsp
+++ b/hbase-server/src/main/resources/hbase-webapps/master/table.jsp
@@ -19,13 +19,19 @@
--%>
<%@ page contentType="text/html;charset=UTF-8"
import="static org.apache.commons.lang.StringEscapeUtils.escapeXml"
+ import="com.google.protobuf.ByteString"
+ import="java.util.ArrayList"
import="java.util.TreeMap"
import="java.util.List"
+ import="java.util.LinkedHashMap"
import="java.util.Map"
import="java.util.Set"
import="java.util.Collection"
+ import="java.util.Collections"
+ import="java.util.Comparator"
import="org.owasp.esapi.ESAPI"
import="org.apache.hadoop.conf.Configuration"
+ import="org.apache.hadoop.util.StringUtils"
import="org.apache.hadoop.hbase.client.HTable"
import="org.apache.hadoop.hbase.client.Admin"
import="org.apache.hadoop.hbase.client.CompactionState"
@@ -40,6 +46,8 @@
import="org.apache.hadoop.hbase.zookeeper.MetaTableLocator"
import="org.apache.hadoop.hbase.util.Bytes"
import="org.apache.hadoop.hbase.util.FSUtils"
+ import="org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos"
+ import="org.apache.hadoop.hbase.protobuf.generated.HBaseProtos"
import="org.apache.hadoop.hbase.TableName"
import="org.apache.hadoop.hbase.HColumnDescriptor"
import="org.apache.hadoop.hbase.client.RegionReplicaUtil"
@@ -51,6 +59,11 @@
MetaTableLocator metaTableLocator = new MetaTableLocator();
String fqtn = request.getParameter("name");
+ String sortKey = request.getParameter("sort");
+ String reverse = request.getParameter("reverse");
+ final boolean reverseOrder = (reverse==null||!reverse.equals("false"));
+ String showWholeKey = request.getParameter("showwhole");
+ final boolean showWhole = (showWholeKey!=null && showWholeKey.equals("true"));
HTable table = null;
String tableHeader;
boolean withReplica = false;
@@ -130,10 +143,10 @@ if ( fqtn != null ) {
try {
table = (HTable) master.getConnection().getTable(TableName.valueOf(fqtn));
if (table.getTableDescriptor().getRegionReplication() > 1) {
- tableHeader = "<h2>Table Regions</h2><table class=\"table table-striped\" style=\"table-layout: fixed; word-wrap: break-word;\"><tr><th style=\"width:22%\">Name</th><th>Region Server</th><th style=\"width:22%\">Start Key</th><th style=\"width:22%\">End Key</th><th>Locality</th><th>Requests</th><th>ReplicaID</th></tr>";
+ tableHeader = "<h2>Table Regions</h2><table class=\"table table-striped\" style=\"table-layout: fixed; word-wrap: break-word;\"><tr><th>Name</th><th>Region Server</th><th>ReadRequests</th><th>WriteRequests</th><th>StorefileSize</th><th>Num.Storefiles</th><th>MemSize</th><th>Locality</th><th>Start Key</th><th>End Key</th><th>ReplicaID</th></tr>";
withReplica = true;
} else {
- tableHeader = "<h2>Table Regions</h2><table class=\"table table-striped\" style=\"table-layout: fixed; word-wrap: break-word;\"><tr><th style=\"width:22%\">Name</th><th>Region Server</th><th style=\"width:22%\">Start Key</th><th style=\"width:22%\">End Key</th><th>Locality</th><th>Requests</th></tr>";
+ tableHeader = "<h2>Table Regions</h2><table class=\"table table-striped\" style=\"table-layout: fixed; word-wrap: break-word;\"><tr><th>Name</th><th>Region Server</th><th>ReadRequests</th><th>WriteRequests</th><th>StorefileSize</th><th>Num.Storefiles</th><th>MemSize</th><th>Locality</th><th>Start Key</th><th>End Key</th></tr>";
}
if ( !readOnly && action != null ) {
%>
@@ -187,16 +200,42 @@ if ( fqtn != null ) {
HRegionInfo.FIRST_META_REGIONINFO, j);
ServerName metaLocation = metaTableLocator.waitMetaRegionLocation(master.getZooKeeper(), j, 1);
for (int i = 0; i < 1; i++) {
- String url = "//" + metaLocation.getHostname() + ":" +
- master.getRegionServerInfoPort(metaLocation) + "/";
+ String url = "";
+ String readReq = "N/A";
+ String writeReq = "N/A";
+ String fileSize = "N/A";
+ String fileCount = "N/A";
+ String memSize = "N/A";
+ float locality = 0.0f;
+
+ if (metaLocation != null) {
+ ServerLoad sl = master.getServerManager().getLoad(metaLocation);
+ url = "//" + metaLocation.getHostname() + ":" + master.getRegionServerInfoPort(metaLocation) + "/";
+ if (sl != null) {
+ Map<byte[], RegionLoad> map = sl.getRegionsLoad();
+ if (map.containsKey(meta.getRegionName())) {
+ RegionLoad load = map.get(meta.getRegionName());
+ readReq = String.format("%,1d", load.getReadRequestsCount());
+ writeReq = String.format("%,1d", load.getWriteRequestsCount());
+ fileSize = StringUtils.byteDesc(load.getStorefileSizeMB()*1024l*1024);
+ fileCount = String.format("%,1d", load.getStorefiles());
+ memSize = StringUtils.byteDesc(load.getMemStoreSizeMB()*1024l*1024);
+ locality = load.getDataLocality();
+ }
+ }
+ }
%>
<tr>
<td><%= escapeXml(meta.getRegionNameAsString()) %></td>
<td><a href="<%= url %>"><%= metaLocation.getHostname().toString() + ":" + master.getRegionServerInfoPort(metaLocation) %></a></td>
+ <td><%= readReq%></td>
+ <td><%= writeReq%></td>
+ <td><%= fileSize%></td>
+ <td><%= fileCount%></td>
+ <td><%= memSize%></td>
+ <td><%= locality%></td>
<td><%= escapeXml(Bytes.toString(meta.getStartKey())) %></td>
<td><%= escapeXml(Bytes.toString(meta.getEndKey())) %></td>
- <td>-</td>
- <td>-</td>
</tr>
<% } %>
<%} %>
@@ -280,27 +319,265 @@ if ( fqtn != null ) {
<% } %>
</table>
<%
+ long totalReadReq = 0;
+ long totalWriteReq = 0;
+ long totalSize = 0;
+ long totalStoreFileCount = 0;
+ long totalMemSize = 0;
+ String urlRegionServer = null;
Map<ServerName, Integer> regDistribution = new TreeMap<ServerName, Integer>();
Map<ServerName, Integer> primaryRegDistribution = new TreeMap<ServerName, Integer>();
List<HRegionLocation> regions = r.getAllRegionLocations();
- if(regions != null && regions.size() > 0) { %>
-<%= tableHeader %>
-<%
+ Map<HRegionInfo, RegionLoad> regionsToLoad = new LinkedHashMap<HRegionInfo, RegionLoad>();
+ Map<HRegionInfo, ServerName> regionsToServer = new LinkedHashMap<HRegionInfo, ServerName>();
for (HRegionLocation hriEntry : regions) {
HRegionInfo regionInfo = hriEntry.getRegionInfo();
ServerName addr = hriEntry.getServerName();
- long req = 0;
- float locality = 0.0f;
- String urlRegionServer = null;
+ regionsToServer.put(regionInfo, addr);
if (addr != null) {
ServerLoad sl = master.getServerManager().getLoad(addr);
if (sl != null) {
Map<byte[], RegionLoad> map = sl.getRegionsLoad();
- if (map.containsKey(regionInfo.getRegionName())) {
- req = map.get(regionInfo.getRegionName()).getRequestsCount();
- locality = map.get(regionInfo.getRegionName()).getDataLocality();
+ RegionLoad regionload = map.get(regionInfo.getRegionName());
+ regionsToLoad.put(regionInfo, regionload);
+ if(regionload != null) {
+ totalReadReq += regionload.getReadRequestsCount();
+ totalWriteReq += regionload.getWriteRequestsCount();
+ totalSize += regionload.getStorefileSizeMB();
+ totalStoreFileCount += regionload.getStorefiles();
+ totalMemSize += regionload.getMemStoreSizeMB();
+ } else {
+ RegionLoad load0 = new RegionLoad(ClusterStatusProtos.RegionLoad.newBuilder().setRegionSpecifier(HBaseProtos.RegionSpecifier.newBuilder().setValue(ByteString.copyFrom(regionInfo.getRegionName())).build()).build());
+ regionsToLoad.put(regionInfo, load0);
}
+ }else{
+ RegionLoad load0 = new RegionLoad(ClusterStatusProtos.RegionLoad.newBuilder().setRegionSpecifier(HBaseProtos.RegionSpecifier.newBuilder().setValue(ByteString.copyFrom(regionInfo.getRegionName())).build()).build());
+ regionsToLoad.put(regionInfo, load0);
+ }
+ }else{
+ RegionLoad load0 = new RegionLoad(ClusterStatusProtos.RegionLoad.newBuilder().setRegionSpecifier(HBaseProtos.RegionSpecifier.newBuilder().setValue(ByteString.copyFrom(regionInfo.getRegionName())).build()).build());
+ regionsToLoad.put(regionInfo, load0);
+ }
+ }
+
+ if(regions != null && regions.size() > 0) { %>
+<h2>Table Regions</h2>
+Sort As
+<select id="sel" style="margin-right: 10px">
+<option value="regionName">RegionName</option>
+<option value="readrequest">ReadRequest</option>
+<option value="writerequest">WriteRequest</option>
+<option value="size">StorefileSize</option>
+<option value="filecount">Num.Storefiles</option>
+<option value="memstore">MemstoreSize</option>
+<option value="locality">Locality</option>
+</select>
+Ascending<input type="checkbox" id="ascending" value="Ascending" style="margin-right:10px">
+ShowDetailName&Start/End Key<input type="checkbox" id="showWhole" style="margin-right:10px">
+<input type="button" id="submit" value="Reorder" onClick="reloadAsSort()" style="font-size: 12pt; width: 5em; margin-bottom: 5px" class="btn">
+<p>
+
+<table class="table table-striped">
+<tr>
+<th>Name(<%= String.format("%,1d", regions.size())%>)</th>
+<th>Region Server</th>
+<th>ReadRequests<br>(<%= String.format("%,1d", totalReadReq)%>)</th>
+<th>WriteRequests<br>(<%= String.format("%,1d", totalWriteReq)%>)</th>
+<th>StorefileSize<br>(<%= StringUtils.byteDesc(totalSize*1024l*1024)%>)</th>
+<th>Num.Storefiles<br>(<%= String.format("%,1d", totalStoreFileCount)%>)</th>
+<th>MemSize<br>(<%= StringUtils.byteDesc(totalMemSize*1024l*1024)%>)</th>
+<th>Locality</th>
+<th>Start Key</th>
+<th>End Key</th>
+<%
+ if (withReplica) {
+%>
+<th>ReplicaID</th>
+<%
+ }
+%>
+</tr>
+
+<%
+ List<Map.Entry<HRegionInfo, RegionLoad>> entryList = new ArrayList<Map.Entry<HRegionInfo, RegionLoad>>(regionsToLoad.entrySet());
+ if(sortKey != null) {
+ if (sortKey.equals("readrequest")) {
+ Collections.sort(entryList,
+ new Comparator<Map.Entry<HRegionInfo, RegionLoad>>() {
+ public int compare(
+ Map.Entry<HRegionInfo, RegionLoad> entry1,
+ Map.Entry<HRegionInfo, RegionLoad> entry2) {
+ if (entry1 == null || entry1.getValue() == null) {
+ return -1;
+ } else if (entry2 == null || entry2.getValue() == null) {
+ return 1;
+ }
+ int result = 0;
+ if (entry1.getValue().getReadRequestsCount() < entry2.getValue().getReadRequestsCount()) {
+ result = -1;
+ } else if (entry1.getValue().getReadRequestsCount() > entry2.getValue().getReadRequestsCount()) {
+ result = 1;
+ }
+ if (reverseOrder) {
+ result = -1 * result;
+ }
+ return result;
+ }
+ });
+ } else if (sortKey.equals("writerequest")) {
+ Collections.sort(entryList,
+ new Comparator<Map.Entry<HRegionInfo, RegionLoad>>() {
+ public int compare(
+ Map.Entry<HRegionInfo, RegionLoad> entry1,
+ Map.Entry<HRegionInfo, RegionLoad> entry2) {
+ if (entry1 == null || entry1.getValue() == null) {
+ return -1;
+ } else if (entry2 == null || entry2.getValue() == null) {
+ return 1;
+ }
+ int result = 0;
+ if (entry1.getValue().getWriteRequestsCount() < entry2.getValue()
+ .getWriteRequestsCount()) {
+ result = -1;
+ } else if (entry1.getValue().getWriteRequestsCount() > entry2.getValue()
+ .getWriteRequestsCount()) {
+ result = 1;
+ }
+ if (reverseOrder) {
+ result = -1 * result;
+ }
+ return result;
+ }
+ });
+ } else if (sortKey.equals("size")) {
+ Collections.sort(entryList,
+ new Comparator<Map.Entry<HRegionInfo, RegionLoad>>() {
+ public int compare(
+ Map.Entry<HRegionInfo, RegionLoad> entry1,
+ Map.Entry<HRegionInfo, RegionLoad> entry2) {
+ if (entry1 == null || entry1.getValue() == null) {
+ return -1;
+ } else if (entry2 == null || entry2.getValue() == null) {
+ return 1;
+ }
+ int result = 0;
+ if (entry1.getValue().getStorefileSizeMB() < entry2.getValue()
+ .getStorefileSizeMB()) {
+ result = -1;
+ } else if (entry1.getValue().getStorefileSizeMB() > entry2
+ .getValue().getStorefileSizeMB()) {
+ result = 1;
+ }
+ if (reverseOrder) {
+ result = -1 * result;
+ }
+ return result;
+ }
+ });
+ } else if (sortKey.equals("filecount")) {
+ Collections.sort(entryList,
+ new Comparator<Map.Entry<HRegionInfo, RegionLoad>>() {
+ public int compare(
+ Map.Entry<HRegionInfo, RegionLoad> entry1,
+ Map.Entry<HRegionInfo, RegionLoad> entry2) {
+ if (entry1 == null || entry1.getValue() == null) {
+ return -1;
+ } else if (entry2 == null || entry2.getValue() == null) {
+ return 1;
+ }
+ int result = 0;
+ if (entry1.getValue().getStorefiles() < entry2.getValue()
+ .getStorefiles()) {
+ result = -1;
+ } else if (entry1.getValue().getStorefiles() > entry2.getValue()
+ .getStorefiles()) {
+ result = 1;
+ }
+ if (reverseOrder) {
+ result = -1 * result;
+ }
+ return result;
+ }
+ });
+ } else if (sortKey.equals("memstore")) {
+ Collections.sort(entryList,
+ new Comparator<Map.Entry<HRegionInfo, RegionLoad>>() {
+ public int compare(
+ Map.Entry<HRegionInfo, RegionLoad> entry1,
+ Map.Entry<HRegionInfo, RegionLoad> entry2) {
+ if (entry1 == null || entry1.getValue()==null) {
+ return -1;
+ } else if (entry2 == null || entry2.getValue()==null) {
+ return 1;
+ }
+ int result = 0;
+ if (entry1.getValue().getMemStoreSizeMB() < entry2.getValue()
+ .getMemStoreSizeMB()) {
+ result = -1;
+ } else if (entry1.getValue().getMemStoreSizeMB() > entry2
+ .getValue().getMemStoreSizeMB()) {
+ result = 1;
+ }
+ if (reverseOrder) {
+ result = -1 * result;
+ }
+ return result;
+ }
+ });
+ } else if (sortKey.equals("locality")) {
+ Collections.sort(entryList,
+ new Comparator<Map.Entry<HRegionInfo, RegionLoad>>() {
+ public int compare(
+ Map.Entry<HRegionInfo, RegionLoad> entry1,
+ Map.Entry<HRegionInfo, RegionLoad> entry2) {
+ if (entry1 == null || entry1.getValue()==null) {
+ return -1;
+ } else if (entry2 == null || entry2.getValue()==null) {
+ return 1;
+ }
+ int result = 0;
+ if (entry1.getValue().getDataLocality() < entry2.getValue()
+ .getDataLocality()) {
+ result = -1;
+ } else if (entry1.getValue().getDataLocality() > entry2
+ .getValue().getDataLocality()) {
+ result = 1;
+ }
+ if (reverseOrder) {
+ result = -1 * result;
+ }
+ return result;
+ }
+ });
+ }
+ }
+
+ for (Map.Entry<HRegionInfo, RegionLoad> hriEntry : entryList) {
+ HRegionInfo regionInfo = hriEntry.getKey();
+ ServerName addr = regionsToServer.get(regionInfo);
+ RegionLoad load = hriEntry.getValue();
+ String readReq = "N/A";
+ String writeReq = "N/A";
+ String regionSize = "N/A";
+ String fileCount = "N/A";
+ String memSize = "N/A";
+ float locality = 0.0f;
+ if(load != null) {
+ readReq = String.format("%,1d", load.getReadRequestsCount());
+ writeReq = String.format("%,1d", load.getWriteRequestsCount());
+ regionSize = StringUtils.byteDesc(load.getStorefileSizeMB()*1024l*1024);
+ fileCount = String.format("%,1d", load.getStorefiles());
+ memSize = StringUtils.byteDesc(load.getMemStoreSizeMB()*1024l*1024);
+ locality = load.getDataLocality();
+ }
+
+ if (addr != null) {
+ ServerLoad sl = master.getServerManager().getLoad(addr);
+ // This port might be wrong if RS actually ended up using something else.
+ urlRegionServer =
+ "//" + addr.getHostname() + ":" + master.getRegionServerInfoPort(addr) + "/";
+ if(sl != null) {
Integer i = regDistribution.get(addr);
if (null == i) i = Integer.valueOf(0);
regDistribution.put(addr, i + 1);
@@ -313,14 +590,12 @@ if ( fqtn != null ) {
}
%>
<tr>
- <td><%= escapeXml(Bytes.toStringBinary(HRegionInfo.getRegionNameForDisplay(regionInfo,
- conf))) %></td>
+ <td><%= escapeXml(showWhole?Bytes.toStringBinary(regionInfo.getRegionName()):regionInfo.getEncodedName()) %></td>
<%
- if (addr != null) {
- String url = "//" + addr.getHostname() + ":" + master.getRegionServerInfoPort(addr) + "/rs-status";
+ if (urlRegionServer != null) {
%>
<td>
- <a href="<%= url %>"><%= addr.getHostname().toString() + ":" + addr.getPort() %></a>
+ <a href="<%= urlRegionServer %>"><%= addr.getHostname().toString() + ":" + master.getRegionServerInfoPort(addr) %></a>
</td>
<%
} else {
@@ -329,12 +604,14 @@ if ( fqtn != null ) {
<%
}
%>
- <td><%= escapeXml(Bytes.toStringBinary(HRegionInfo.getStartKeyForDisplay(regionInfo,
- conf))) %></td>
- <td><%= escapeXml(Bytes.toStringBinary(HRegionInfo.getEndKeyForDisplay(regionInfo,
- conf))) %></td>
+ <td><%= readReq%></td>
+ <td><%= writeReq%></td>
+ <td><%= regionSize%></td>
+ <td><%= fileCount%></td>
+ <td><%= memSize%></td>
<td><%= locality%></td>
- <td><%= req%></td>
+ <td><%= escapeXml(showWhole?Bytes.toStringBinary(regionInfo.getStartKey()):"-")%></td>
+ <td><%= escapeXml(showWhole?Bytes.toStringBinary(regionInfo.getEndKey()):"-")%></td>
<%
if (withReplica) {
%>
@@ -363,7 +640,7 @@ if (withReplica) {
String url = "//" + addr.getHostname() + ":" + master.getRegionServerInfoPort(addr) + "/";
%>
<tr>
- <td><a href="<%= url %>"><%= addr.getHostname().toString() + ":" + addr.getPort() %></a></td>
+ <td><a href="<%= url %>"><%= addr.getHostname().toString() + ":" + master.getRegionServerInfoPort(addr) %></a></td>
<td><%= rdEntry.getValue()%></td>
<%
if (withReplica) {
@@ -465,3 +742,35 @@ Actions:
</body>
</html>
+
+<script>
+var index=0;
+var sortKeyValue='<%= sortKey %>';
+if(sortKeyValue=="readrequest")index=1;
+else if(sortKeyValue=="writerequest")index=2;
+else if(sortKeyValue=="size")index=3;
+else if(sortKeyValue=="filecount")index=4;
+else if(sortKeyValue=="memstore")index=5;
+else if(sortKeyValue=="locality")index=6;
+document.getElementById("sel").selectedIndex=index;
+
+var reverse='<%= reverseOrder %>';
+if(reverse=='false')document.getElementById("ascending").checked=true;
+
+var showWhole='<%= showWhole %>';
+if(showWhole=='true')document.getElementById("showWhole").checked=true;
+
+function reloadAsSort(){
+ var url="?name="+'<%= fqtn %>';
+ if(document.getElementById("sel").selectedIndex>0){
+ url=url+"&sort="+document.getElementById("sel").value;
+ }
+ if(document.getElementById("ascending").checked){
+ url=url+"&reverse=false";
+ }
+ if(document.getElementById("showWhole").checked){
+ url=url+"&showwhole=true";
+ }
+ location.href=url;
+}
+</script>
[12/50] [abbrv] hbase git commit: HBASE-15613 TestNamespaceCommand
times out
Posted by sy...@apache.org.
HBASE-15613 TestNamespaceCommand times out
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/29eb3e8b
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/29eb3e8b
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/29eb3e8b
Branch: refs/heads/hbase-12439
Commit: 29eb3e8bccb1dcea3cd39c055e175b21b4d33a26
Parents: fa9fdfe
Author: Enis Soztutar <en...@apache.org>
Authored: Wed May 4 11:37:03 2016 -0700
Committer: Enis Soztutar <en...@apache.org>
Committed: Wed May 4 11:37:03 2016 -0700
----------------------------------------------------------------------
hbase-server/src/test/resources/hbase-site.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/29eb3e8b/hbase-server/src/test/resources/hbase-site.xml
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/resources/hbase-site.xml b/hbase-server/src/test/resources/hbase-site.xml
index bca90a3..1b429b5 100644
--- a/hbase-server/src/test/resources/hbase-site.xml
+++ b/hbase-server/src/test/resources/hbase-site.xml
@@ -52,7 +52,7 @@
</property>
<property>
<name>hbase.regionserver.metahandler.count</name>
- <value>5</value>
+ <value>6</value>
</property>
<property>
<name>hbase.ipc.server.read.threadpool.size</name>
[29/50] [abbrv] hbase git commit: HBASE-15745 Refactor RPC classes to
better accept async changes
Posted by sy...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase/blob/56358a0f/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RegionServerCoprocessorRpcChannel.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RegionServerCoprocessorRpcChannel.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RegionServerCoprocessorRpcChannel.java
index fba0373..6ffb579 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RegionServerCoprocessorRpcChannel.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RegionServerCoprocessorRpcChannel.java
@@ -36,7 +36,7 @@ import com.google.protobuf.RpcController;
* @see org.apache.hadoop.hbase.client.HBaseAdmin#coprocessorService(ServerName)
*/
@InterfaceAudience.Private
-public class RegionServerCoprocessorRpcChannel extends CoprocessorRpcChannel {
+public class RegionServerCoprocessorRpcChannel extends SyncCoprocessorRpcChannel {
private static final Log LOG = LogFactory.getLog(RegionServerCoprocessorRpcChannel.class);
private final ClusterConnection connection;
private final ServerName serverName;
http://git-wip-us.apache.org/repos/asf/hbase/blob/56358a0f/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/SyncCoprocessorRpcChannel.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/SyncCoprocessorRpcChannel.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/SyncCoprocessorRpcChannel.java
new file mode 100644
index 0000000..af8ddd4
--- /dev/null
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/SyncCoprocessorRpcChannel.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.ipc;
+
+import com.google.protobuf.Descriptors;
+import com.google.protobuf.Message;
+import com.google.protobuf.RpcCallback;
+import com.google.protobuf.RpcController;
+import com.google.protobuf.ServiceException;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+import org.apache.hadoop.hbase.protobuf.ResponseConverter;
+
+/**
+ * Base class which provides clients with an RPC connection to
+ * call coprocessor endpoint {@link com.google.protobuf.Service}s.
+ * Note that clients should not use this class directly, except through
+ * {@link org.apache.hadoop.hbase.client.HTableInterface#coprocessorService(byte[])}.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public abstract class SyncCoprocessorRpcChannel implements CoprocessorRpcChannel {
+ private static final Log LOG = LogFactory.getLog(SyncCoprocessorRpcChannel.class);
+
+ @Override
+ @InterfaceAudience.Private
+ public void callMethod(Descriptors.MethodDescriptor method,
+ RpcController controller,
+ Message request, Message responsePrototype,
+ RpcCallback<Message> callback) {
+ Message response = null;
+ try {
+ response = callExecService(controller, method, request, responsePrototype);
+ } catch (IOException ioe) {
+ LOG.warn("Call failed on IOException", ioe);
+ ResponseConverter.setControllerException(controller, ioe);
+ }
+ if (callback != null) {
+ callback.run(response);
+ }
+ }
+
+ @Override
+ @InterfaceAudience.Private
+ public Message callBlockingMethod(Descriptors.MethodDescriptor method,
+ RpcController controller,
+ Message request, Message responsePrototype)
+ throws ServiceException {
+ try {
+ return callExecService(controller, method, request, responsePrototype);
+ } catch (IOException ioe) {
+ throw new ServiceException("Error calling method "+method.getFullName(), ioe);
+ }
+ }
+
+ protected abstract Message callExecService(RpcController controller,
+ Descriptors.MethodDescriptor method, Message request, Message responsePrototype)
+ throws IOException;
+}
[03/50] [abbrv] hbase git commit: HBASE-15752 ClassNotFoundException
is encountered when custom WAL codec is not found in WALPlayer job
Posted by sy...@apache.org.
HBASE-15752 ClassNotFoundException is encountered when custom WAL codec is not found in WALPlayer job
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/c77a1530
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/c77a1530
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/c77a1530
Branch: refs/heads/hbase-12439
Commit: c77a153001a59954481929b629a4f78492b3da46
Parents: c06a976
Author: tedyu <yu...@gmail.com>
Authored: Tue May 3 01:57:25 2016 -0700
Committer: tedyu <yu...@gmail.com>
Committed: Tue May 3 01:57:25 2016 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java | 7 +++++++
.../apache/hadoop/hbase/regionserver/wal/WALCellCodec.java | 2 +-
2 files changed, 8 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/c77a1530/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java
index 9d9cee0..d36721e 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java
@@ -44,6 +44,7 @@ import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.RegionLocator;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.regionserver.wal.WALCellCodec;
import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.wal.WALKey;
@@ -292,6 +293,12 @@ public class WALPlayer extends Configured implements Tool {
// No reducers.
job.setNumReduceTasks(0);
}
+ String codecCls = WALCellCodec.getWALCellCodecClass(conf);
+ try {
+ TableMapReduceUtil.addDependencyJars(conf, Class.forName(codecCls));
+ } catch (Exception e) {
+ throw new IOException("Cannot determine wal codec class " + codecCls, e);
+ }
return job;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/c77a1530/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALCellCodec.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALCellCodec.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALCellCodec.java
index 6b89e89..87850aa 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALCellCodec.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALCellCodec.java
@@ -82,7 +82,7 @@ public class WALCellCodec implements Codec {
this.compression = compression;
}
- static String getWALCellCodecClass(Configuration conf) {
+ public static String getWALCellCodecClass(Configuration conf) {
return conf.get(WAL_CELL_CODEC_CLASS_KEY, WALCellCodec.class.getName());
}
[13/50] [abbrv] hbase git commit: HBASE-15741 Provide backward
compatibility for HBase coprocessor service names
Posted by sy...@apache.org.
HBASE-15741 Provide backward compatibility for HBase coprocessor service names
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/d23d600d
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/d23d600d
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/d23d600d
Branch: refs/heads/hbase-12439
Commit: d23d600d768cd73125d5be62b7b8e3fa6fac5761
Parents: 29eb3e8
Author: Gary Helmling <ga...@apache.org>
Authored: Mon May 2 13:22:43 2016 -0700
Committer: Gary Helmling <ga...@apache.org>
Committed: Wed May 4 12:09:57 2016 -0700
----------------------------------------------------------------------
.../hadoop/hbase/ipc/CoprocessorRpcUtils.java | 72 ++++++++++++++++
.../hbase/ipc/MasterCoprocessorRpcChannel.java | 7 +-
.../hbase/ipc/RegionCoprocessorRpcChannel.java | 7 +-
.../ipc/RegionServerCoprocessorRpcChannel.java | 6 +-
.../org/apache/hadoop/hbase/master/HMaster.java | 10 ++-
.../hadoop/hbase/regionserver/HRegion.java | 10 ++-
.../hbase/regionserver/HRegionServer.java | 10 ++-
.../DummyRegionServerEndpointProtos.java | 88 ++++++++++----------
.../hbase/ipc/TestCoprocessorRpcUtils.java | 44 ++++++++++
.../protobuf/DummyRegionServerEndpoint.proto | 3 +
10 files changed, 185 insertions(+), 72 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/d23d600d/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/CoprocessorRpcUtils.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/CoprocessorRpcUtils.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/CoprocessorRpcUtils.java
new file mode 100644
index 0000000..60044d4
--- /dev/null
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/CoprocessorRpcUtils.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.hbase.ipc;
+
+import com.google.protobuf.Descriptors;
+import com.google.protobuf.Message;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
+import org.apache.hadoop.hbase.util.ByteStringer;
+
+/**
+ * Utilities for handling coprocessor service calls.
+ */
+@InterfaceAudience.Private
+public final class CoprocessorRpcUtils {
+ /**
+ * We assume that all HBase protobuf services share a common package name
+ * (defined in the .proto files).
+ */
+ private static String hbaseServicePackage;
+ static {
+ Descriptors.ServiceDescriptor clientService = ClientProtos.ClientService.getDescriptor();
+ hbaseServicePackage = clientService.getFullName()
+ .substring(0, clientService.getFullName().lastIndexOf(clientService.getName()));
+ }
+
+ private CoprocessorRpcUtils() {
+ // private for utility class
+ }
+
+ /**
+ * Returns the name to use for coprocessor service calls. For core HBase services
+ * (in the hbase.pb protobuf package), this returns the unqualified name in order to provide
+ * backward compatibility across the package name change. For all other services,
+ * the fully-qualified service name is used.
+ */
+ public static String getServiceName(Descriptors.ServiceDescriptor service) {
+ if (service.getFullName().startsWith(hbaseServicePackage)) {
+ return service.getName();
+ }
+ return service.getFullName();
+ }
+
+ /**
+ * Returns a service call instance for the given coprocessor request.
+ */
+ public static ClientProtos.CoprocessorServiceCall buildServiceCall(byte[] row,
+ Descriptors.MethodDescriptor method, Message request) {
+ return ClientProtos.CoprocessorServiceCall.newBuilder()
+ .setRow(ByteStringer.wrap(row))
+ .setServiceName(CoprocessorRpcUtils.getServiceName(method.getService()))
+ .setMethodName(method.getName())
+ .setRequest(request.toByteString()).build();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/d23d600d/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/MasterCoprocessorRpcChannel.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/MasterCoprocessorRpcChannel.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/MasterCoprocessorRpcChannel.java
index 6e59972..ac6a022 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/MasterCoprocessorRpcChannel.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/MasterCoprocessorRpcChannel.java
@@ -28,7 +28,6 @@ import org.apache.hadoop.hbase.client.ClusterConnection;
import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.CoprocessorServiceResponse;
-import org.apache.hadoop.hbase.util.ByteStringer;
import com.google.protobuf.Descriptors;
import com.google.protobuf.Message;
@@ -61,11 +60,7 @@ public class MasterCoprocessorRpcChannel extends CoprocessorRpcChannel{
}
final ClientProtos.CoprocessorServiceCall call =
- ClientProtos.CoprocessorServiceCall.newBuilder()
- .setRow(ByteStringer.wrap(HConstants.EMPTY_BYTE_ARRAY))
- .setServiceName(method.getService().getFullName())
- .setMethodName(method.getName())
- .setRequest(request.toByteString()).build();
+ CoprocessorRpcUtils.buildServiceCall(HConstants.EMPTY_BYTE_ARRAY, method, request);
// TODO: Are we retrying here? Does not seem so. We should use RetryingRpcCaller
CoprocessorServiceResponse result = ProtobufUtil.execService(controller,
http://git-wip-us.apache.org/repos/asf/hbase/blob/d23d600d/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RegionCoprocessorRpcChannel.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RegionCoprocessorRpcChannel.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RegionCoprocessorRpcChannel.java
index 321dd62..0f06af1 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RegionCoprocessorRpcChannel.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RegionCoprocessorRpcChannel.java
@@ -30,7 +30,6 @@ import org.apache.hadoop.hbase.client.RpcRetryingCallerFactory;
import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.CoprocessorServiceResponse;
-import org.apache.hadoop.hbase.util.ByteStringer;
import org.apache.hadoop.hbase.util.Bytes;
import com.google.protobuf.Descriptors;
@@ -83,11 +82,7 @@ public class RegionCoprocessorRpcChannel extends CoprocessorRpcChannel{
? rpcControllerFactory.newController() : controller;
final ClientProtos.CoprocessorServiceCall call =
- ClientProtos.CoprocessorServiceCall.newBuilder()
- .setRow(ByteStringer.wrap(row))
- .setServiceName(method.getService().getFullName())
- .setMethodName(method.getName())
- .setRequest(request.toByteString()).build();
+ CoprocessorRpcUtils.buildServiceCall(row, method, request);
RegionServerCallable<CoprocessorServiceResponse> callable =
new RegionServerCallable<CoprocessorServiceResponse>(connection, table, row) {
@Override
http://git-wip-us.apache.org/repos/asf/hbase/blob/d23d600d/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RegionServerCoprocessorRpcChannel.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RegionServerCoprocessorRpcChannel.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RegionServerCoprocessorRpcChannel.java
index 24d2de4..fba0373 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RegionServerCoprocessorRpcChannel.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RegionServerCoprocessorRpcChannel.java
@@ -22,7 +22,6 @@ import org.apache.hadoop.hbase.client.ClusterConnection;
import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.CoprocessorServiceResponse;
-import org.apache.hadoop.hbase.util.ByteStringer;
import com.google.protobuf.Descriptors;
import com.google.protobuf.Message;
@@ -55,10 +54,7 @@ public class RegionServerCoprocessorRpcChannel extends CoprocessorRpcChannel {
LOG.trace("Call: " + method.getName() + ", " + request.toString());
}
final ClientProtos.CoprocessorServiceCall call =
- ClientProtos.CoprocessorServiceCall.newBuilder()
- .setRow(ByteStringer.wrap(HConstants.EMPTY_BYTE_ARRAY))
- .setServiceName(method.getService().getFullName()).setMethodName(method.getName())
- .setRequest(request.toByteString()).build();
+ CoprocessorRpcUtils.buildServiceCall(HConstants.EMPTY_BYTE_ARRAY, method, request);
// TODO: Are we retrying here? Does not seem so. We should use RetryingRpcCaller
CoprocessorServiceResponse result =
http://git-wip-us.apache.org/repos/asf/hbase/blob/d23d600d/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index d7f7c18..ff3d792 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -91,6 +91,7 @@ import org.apache.hadoop.hbase.coprocessor.BypassCoprocessorException;
import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
import org.apache.hadoop.hbase.exceptions.DeserializationException;
import org.apache.hadoop.hbase.executor.ExecutorType;
+import org.apache.hadoop.hbase.ipc.CoprocessorRpcUtils;
import org.apache.hadoop.hbase.ipc.RpcServer;
import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException;
import org.apache.hadoop.hbase.master.MasterRpcServices.BalanceSwitchMode;
@@ -2419,16 +2420,17 @@ public class HMaster extends HRegionServer implements MasterServices {
* No stacking of instances is allowed for a single service name
*/
Descriptors.ServiceDescriptor serviceDesc = instance.getDescriptorForType();
- if (coprocessorServiceHandlers.containsKey(serviceDesc.getFullName())) {
- LOG.error("Coprocessor service "+serviceDesc.getFullName()+
+ String serviceName = CoprocessorRpcUtils.getServiceName(serviceDesc);
+ if (coprocessorServiceHandlers.containsKey(serviceName)) {
+ LOG.error("Coprocessor service "+serviceName+
" already registered, rejecting request from "+instance
);
return false;
}
- coprocessorServiceHandlers.put(serviceDesc.getFullName(), instance);
+ coprocessorServiceHandlers.put(serviceName, instance);
if (LOG.isDebugEnabled()) {
- LOG.debug("Registered master coprocessor service: service="+serviceDesc.getFullName());
+ LOG.debug("Registered master coprocessor service: service="+serviceName);
}
return true;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/d23d600d/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 5676e15..4532317 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -123,6 +123,7 @@ import org.apache.hadoop.hbase.io.HeapSize;
import org.apache.hadoop.hbase.io.TimeRange;
import org.apache.hadoop.hbase.io.hfile.HFile;
import org.apache.hadoop.hbase.ipc.CallerDisconnectedException;
+import org.apache.hadoop.hbase.ipc.CoprocessorRpcUtils;
import org.apache.hadoop.hbase.ipc.RpcCallContext;
import org.apache.hadoop.hbase.ipc.RpcServer;
import org.apache.hadoop.hbase.mob.MobUtils;
@@ -7428,18 +7429,19 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
* No stacking of instances is allowed for a single service name
*/
Descriptors.ServiceDescriptor serviceDesc = instance.getDescriptorForType();
- if (coprocessorServiceHandlers.containsKey(serviceDesc.getFullName())) {
- LOG.error("Coprocessor service " + serviceDesc.getFullName() +
+ String serviceName = CoprocessorRpcUtils.getServiceName(serviceDesc);
+ if (coprocessorServiceHandlers.containsKey(serviceName)) {
+ LOG.error("Coprocessor service " + serviceName +
" already registered, rejecting request from " + instance
);
return false;
}
- coprocessorServiceHandlers.put(serviceDesc.getFullName(), instance);
+ coprocessorServiceHandlers.put(serviceName, instance);
if (LOG.isDebugEnabled()) {
LOG.debug("Registered coprocessor service: region=" +
Bytes.toStringBinary(getRegionInfo().getRegionName()) +
- " service=" + serviceDesc.getFullName());
+ " service=" + serviceName);
}
return true;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/d23d600d/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 06a2aff..5b311b1 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -96,6 +96,7 @@ import org.apache.hadoop.hbase.fs.HFileSystem;
import org.apache.hadoop.hbase.http.InfoServer;
import org.apache.hadoop.hbase.io.hfile.CacheConfig;
import org.apache.hadoop.hbase.io.hfile.HFile;
+import org.apache.hadoop.hbase.ipc.CoprocessorRpcUtils;
import org.apache.hadoop.hbase.ipc.RpcClient;
import org.apache.hadoop.hbase.ipc.RpcClientFactory;
import org.apache.hadoop.hbase.ipc.RpcControllerFactory;
@@ -696,15 +697,16 @@ public class HRegionServer extends HasThread implements
* No stacking of instances is allowed for a single service name
*/
Descriptors.ServiceDescriptor serviceDesc = instance.getDescriptorForType();
- if (coprocessorServiceHandlers.containsKey(serviceDesc.getFullName())) {
- LOG.error("Coprocessor service " + serviceDesc.getFullName()
+ String serviceName = CoprocessorRpcUtils.getServiceName(serviceDesc);
+ if (coprocessorServiceHandlers.containsKey(serviceName)) {
+ LOG.error("Coprocessor service " + serviceName
+ " already registered, rejecting request from " + instance);
return false;
}
- coprocessorServiceHandlers.put(serviceDesc.getFullName(), instance);
+ coprocessorServiceHandlers.put(serviceName, instance);
if (LOG.isDebugEnabled()) {
- LOG.debug("Registered regionserver coprocessor service: service=" + serviceDesc.getFullName());
+ LOG.debug("Registered regionserver coprocessor service: service=" + serviceName);
}
return true;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/d23d600d/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/protobuf/generated/DummyRegionServerEndpointProtos.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/protobuf/generated/DummyRegionServerEndpointProtos.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/protobuf/generated/DummyRegionServerEndpointProtos.java
index 12ce551..2ad3c59 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/protobuf/generated/DummyRegionServerEndpointProtos.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/protobuf/generated/DummyRegionServerEndpointProtos.java
@@ -1,5 +1,5 @@
// Generated by the protocol buffer compiler. DO NOT EDIT!
-// source: DummyRegionServerEndpoint.proto
+// source: hbase-server/src/test/protobuf/DummyRegionServerEndpoint.proto
package org.apache.hadoop.hbase.coprocessor.protobuf.generated;
@@ -12,7 +12,7 @@ public final class DummyRegionServerEndpointProtos {
extends com.google.protobuf.MessageOrBuilder {
}
/**
- * Protobuf type {@code DummyRequest}
+ * Protobuf type {@code hbase.test.pb.DummyRequest}
*/
public static final class DummyRequest extends
com.google.protobuf.GeneratedMessage
@@ -75,12 +75,12 @@ public final class DummyRegionServerEndpointProtos {
}
public static final com.google.protobuf.Descriptors.Descriptor
getDescriptor() {
- return org.apache.hadoop.hbase.coprocessor.protobuf.generated.DummyRegionServerEndpointProtos.internal_static_DummyRequest_descriptor;
+ return org.apache.hadoop.hbase.coprocessor.protobuf.generated.DummyRegionServerEndpointProtos.internal_static_hbase_test_pb_DummyRequest_descriptor;
}
protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
internalGetFieldAccessorTable() {
- return org.apache.hadoop.hbase.coprocessor.protobuf.generated.DummyRegionServerEndpointProtos.internal_static_DummyRequest_fieldAccessorTable
+ return org.apache.hadoop.hbase.coprocessor.protobuf.generated.DummyRegionServerEndpointProtos.internal_static_hbase_test_pb_DummyRequest_fieldAccessorTable
.ensureFieldAccessorsInitialized(
org.apache.hadoop.hbase.coprocessor.protobuf.generated.DummyRegionServerEndpointProtos.DummyRequest.class, org.apache.hadoop.hbase.coprocessor.protobuf.generated.DummyRegionServerEndpointProtos.DummyRequest.Builder.class);
}
@@ -231,19 +231,19 @@ public final class DummyRegionServerEndpointProtos {
return builder;
}
/**
- * Protobuf type {@code DummyRequest}
+ * Protobuf type {@code hbase.test.pb.DummyRequest}
*/
public static final class Builder extends
com.google.protobuf.GeneratedMessage.Builder<Builder>
implements org.apache.hadoop.hbase.coprocessor.protobuf.generated.DummyRegionServerEndpointProtos.DummyRequestOrBuilder {
public static final com.google.protobuf.Descriptors.Descriptor
getDescriptor() {
- return org.apache.hadoop.hbase.coprocessor.protobuf.generated.DummyRegionServerEndpointProtos.internal_static_DummyRequest_descriptor;
+ return org.apache.hadoop.hbase.coprocessor.protobuf.generated.DummyRegionServerEndpointProtos.internal_static_hbase_test_pb_DummyRequest_descriptor;
}
protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
internalGetFieldAccessorTable() {
- return org.apache.hadoop.hbase.coprocessor.protobuf.generated.DummyRegionServerEndpointProtos.internal_static_DummyRequest_fieldAccessorTable
+ return org.apache.hadoop.hbase.coprocessor.protobuf.generated.DummyRegionServerEndpointProtos.internal_static_hbase_test_pb_DummyRequest_fieldAccessorTable
.ensureFieldAccessorsInitialized(
org.apache.hadoop.hbase.coprocessor.protobuf.generated.DummyRegionServerEndpointProtos.DummyRequest.class, org.apache.hadoop.hbase.coprocessor.protobuf.generated.DummyRegionServerEndpointProtos.DummyRequest.Builder.class);
}
@@ -277,7 +277,7 @@ public final class DummyRegionServerEndpointProtos {
public com.google.protobuf.Descriptors.Descriptor
getDescriptorForType() {
- return org.apache.hadoop.hbase.coprocessor.protobuf.generated.DummyRegionServerEndpointProtos.internal_static_DummyRequest_descriptor;
+ return org.apache.hadoop.hbase.coprocessor.protobuf.generated.DummyRegionServerEndpointProtos.internal_static_hbase_test_pb_DummyRequest_descriptor;
}
public org.apache.hadoop.hbase.coprocessor.protobuf.generated.DummyRegionServerEndpointProtos.DummyRequest getDefaultInstanceForType() {
@@ -335,7 +335,7 @@ public final class DummyRegionServerEndpointProtos {
return this;
}
- // @@protoc_insertion_point(builder_scope:DummyRequest)
+ // @@protoc_insertion_point(builder_scope:hbase.test.pb.DummyRequest)
}
static {
@@ -343,7 +343,7 @@ public final class DummyRegionServerEndpointProtos {
defaultInstance.initFields();
}
- // @@protoc_insertion_point(class_scope:DummyRequest)
+ // @@protoc_insertion_point(class_scope:hbase.test.pb.DummyRequest)
}
public interface DummyResponseOrBuilder
@@ -365,7 +365,7 @@ public final class DummyRegionServerEndpointProtos {
getValueBytes();
}
/**
- * Protobuf type {@code DummyResponse}
+ * Protobuf type {@code hbase.test.pb.DummyResponse}
*/
public static final class DummyResponse extends
com.google.protobuf.GeneratedMessage
@@ -434,12 +434,12 @@ public final class DummyRegionServerEndpointProtos {
}
public static final com.google.protobuf.Descriptors.Descriptor
getDescriptor() {
- return org.apache.hadoop.hbase.coprocessor.protobuf.generated.DummyRegionServerEndpointProtos.internal_static_DummyResponse_descriptor;
+ return org.apache.hadoop.hbase.coprocessor.protobuf.generated.DummyRegionServerEndpointProtos.internal_static_hbase_test_pb_DummyResponse_descriptor;
}
protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
internalGetFieldAccessorTable() {
- return org.apache.hadoop.hbase.coprocessor.protobuf.generated.DummyRegionServerEndpointProtos.internal_static_DummyResponse_fieldAccessorTable
+ return org.apache.hadoop.hbase.coprocessor.protobuf.generated.DummyRegionServerEndpointProtos.internal_static_hbase_test_pb_DummyResponse_fieldAccessorTable
.ensureFieldAccessorsInitialized(
org.apache.hadoop.hbase.coprocessor.protobuf.generated.DummyRegionServerEndpointProtos.DummyResponse.class, org.apache.hadoop.hbase.coprocessor.protobuf.generated.DummyRegionServerEndpointProtos.DummyResponse.Builder.class);
}
@@ -655,19 +655,19 @@ public final class DummyRegionServerEndpointProtos {
return builder;
}
/**
- * Protobuf type {@code DummyResponse}
+ * Protobuf type {@code hbase.test.pb.DummyResponse}
*/
public static final class Builder extends
com.google.protobuf.GeneratedMessage.Builder<Builder>
implements org.apache.hadoop.hbase.coprocessor.protobuf.generated.DummyRegionServerEndpointProtos.DummyResponseOrBuilder {
public static final com.google.protobuf.Descriptors.Descriptor
getDescriptor() {
- return org.apache.hadoop.hbase.coprocessor.protobuf.generated.DummyRegionServerEndpointProtos.internal_static_DummyResponse_descriptor;
+ return org.apache.hadoop.hbase.coprocessor.protobuf.generated.DummyRegionServerEndpointProtos.internal_static_hbase_test_pb_DummyResponse_descriptor;
}
protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
internalGetFieldAccessorTable() {
- return org.apache.hadoop.hbase.coprocessor.protobuf.generated.DummyRegionServerEndpointProtos.internal_static_DummyResponse_fieldAccessorTable
+ return org.apache.hadoop.hbase.coprocessor.protobuf.generated.DummyRegionServerEndpointProtos.internal_static_hbase_test_pb_DummyResponse_fieldAccessorTable
.ensureFieldAccessorsInitialized(
org.apache.hadoop.hbase.coprocessor.protobuf.generated.DummyRegionServerEndpointProtos.DummyResponse.class, org.apache.hadoop.hbase.coprocessor.protobuf.generated.DummyRegionServerEndpointProtos.DummyResponse.Builder.class);
}
@@ -703,7 +703,7 @@ public final class DummyRegionServerEndpointProtos {
public com.google.protobuf.Descriptors.Descriptor
getDescriptorForType() {
- return org.apache.hadoop.hbase.coprocessor.protobuf.generated.DummyRegionServerEndpointProtos.internal_static_DummyResponse_descriptor;
+ return org.apache.hadoop.hbase.coprocessor.protobuf.generated.DummyRegionServerEndpointProtos.internal_static_hbase_test_pb_DummyResponse_descriptor;
}
public org.apache.hadoop.hbase.coprocessor.protobuf.generated.DummyRegionServerEndpointProtos.DummyResponse getDefaultInstanceForType() {
@@ -852,7 +852,7 @@ public final class DummyRegionServerEndpointProtos {
return this;
}
- // @@protoc_insertion_point(builder_scope:DummyResponse)
+ // @@protoc_insertion_point(builder_scope:hbase.test.pb.DummyResponse)
}
static {
@@ -860,11 +860,11 @@ public final class DummyRegionServerEndpointProtos {
defaultInstance.initFields();
}
- // @@protoc_insertion_point(class_scope:DummyResponse)
+ // @@protoc_insertion_point(class_scope:hbase.test.pb.DummyResponse)
}
/**
- * Protobuf service {@code DummyService}
+ * Protobuf service {@code hbase.test.pb.DummyService}
*/
public static abstract class DummyService
implements com.google.protobuf.Service {
@@ -872,7 +872,7 @@ public final class DummyRegionServerEndpointProtos {
public interface Interface {
/**
- * <code>rpc dummyCall(.DummyRequest) returns (.DummyResponse);</code>
+ * <code>rpc dummyCall(.hbase.test.pb.DummyRequest) returns (.hbase.test.pb.DummyResponse);</code>
*/
public abstract void dummyCall(
com.google.protobuf.RpcController controller,
@@ -880,7 +880,7 @@ public final class DummyRegionServerEndpointProtos {
com.google.protobuf.RpcCallback<org.apache.hadoop.hbase.coprocessor.protobuf.generated.DummyRegionServerEndpointProtos.DummyResponse> done);
/**
- * <code>rpc dummyThrow(.DummyRequest) returns (.DummyResponse);</code>
+ * <code>rpc dummyThrow(.hbase.test.pb.DummyRequest) returns (.hbase.test.pb.DummyResponse);</code>
*/
public abstract void dummyThrow(
com.google.protobuf.RpcController controller,
@@ -979,7 +979,7 @@ public final class DummyRegionServerEndpointProtos {
}
/**
- * <code>rpc dummyCall(.DummyRequest) returns (.DummyResponse);</code>
+ * <code>rpc dummyCall(.hbase.test.pb.DummyRequest) returns (.hbase.test.pb.DummyResponse);</code>
*/
public abstract void dummyCall(
com.google.protobuf.RpcController controller,
@@ -987,7 +987,7 @@ public final class DummyRegionServerEndpointProtos {
com.google.protobuf.RpcCallback<org.apache.hadoop.hbase.coprocessor.protobuf.generated.DummyRegionServerEndpointProtos.DummyResponse> done);
/**
- * <code>rpc dummyThrow(.DummyRequest) returns (.DummyResponse);</code>
+ * <code>rpc dummyThrow(.hbase.test.pb.DummyRequest) returns (.hbase.test.pb.DummyResponse);</code>
*/
public abstract void dummyThrow(
com.google.protobuf.RpcController controller,
@@ -1163,19 +1163,19 @@ public final class DummyRegionServerEndpointProtos {
}
- // @@protoc_insertion_point(class_scope:DummyService)
+ // @@protoc_insertion_point(class_scope:hbase.test.pb.DummyService)
}
private static com.google.protobuf.Descriptors.Descriptor
- internal_static_DummyRequest_descriptor;
+ internal_static_hbase_test_pb_DummyRequest_descriptor;
private static
com.google.protobuf.GeneratedMessage.FieldAccessorTable
- internal_static_DummyRequest_fieldAccessorTable;
+ internal_static_hbase_test_pb_DummyRequest_fieldAccessorTable;
private static com.google.protobuf.Descriptors.Descriptor
- internal_static_DummyResponse_descriptor;
+ internal_static_hbase_test_pb_DummyResponse_descriptor;
private static
com.google.protobuf.GeneratedMessage.FieldAccessorTable
- internal_static_DummyResponse_fieldAccessorTable;
+ internal_static_hbase_test_pb_DummyResponse_fieldAccessorTable;
public static com.google.protobuf.Descriptors.FileDescriptor
getDescriptor() {
@@ -1185,31 +1185,33 @@ public final class DummyRegionServerEndpointProtos {
descriptor;
static {
java.lang.String[] descriptorData = {
- "\n\037DummyRegionServerEndpoint.proto\"\016\n\014Dum" +
- "myRequest\"\036\n\rDummyResponse\022\r\n\005value\030\001 \002(" +
- "\t2g\n\014DummyService\022*\n\tdummyCall\022\r.DummyRe" +
- "quest\032\016.DummyResponse\022+\n\ndummyThrow\022\r.Du" +
- "mmyRequest\032\016.DummyResponseB_\n6org.apache" +
- ".hadoop.hbase.coprocessor.protobuf.gener" +
- "atedB\037DummyRegionServerEndpointProtos\210\001\001" +
- "\240\001\001"
+ "\n>hbase-server/src/test/protobuf/DummyRe" +
+ "gionServerEndpoint.proto\022\rhbase.test.pb\"" +
+ "\016\n\014DummyRequest\"\036\n\rDummyResponse\022\r\n\005valu" +
+ "e\030\001 \002(\t2\237\001\n\014DummyService\022F\n\tdummyCall\022\033." +
+ "hbase.test.pb.DummyRequest\032\034.hbase.test." +
+ "pb.DummyResponse\022G\n\ndummyThrow\022\033.hbase.t" +
+ "est.pb.DummyRequest\032\034.hbase.test.pb.Dumm" +
+ "yResponseB_\n6org.apache.hadoop.hbase.cop" +
+ "rocessor.protobuf.generatedB\037DummyRegion" +
+ "ServerEndpointProtos\210\001\001\240\001\001"
};
com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner =
new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() {
public com.google.protobuf.ExtensionRegistry assignDescriptors(
com.google.protobuf.Descriptors.FileDescriptor root) {
descriptor = root;
- internal_static_DummyRequest_descriptor =
+ internal_static_hbase_test_pb_DummyRequest_descriptor =
getDescriptor().getMessageTypes().get(0);
- internal_static_DummyRequest_fieldAccessorTable = new
+ internal_static_hbase_test_pb_DummyRequest_fieldAccessorTable = new
com.google.protobuf.GeneratedMessage.FieldAccessorTable(
- internal_static_DummyRequest_descriptor,
+ internal_static_hbase_test_pb_DummyRequest_descriptor,
new java.lang.String[] { });
- internal_static_DummyResponse_descriptor =
+ internal_static_hbase_test_pb_DummyResponse_descriptor =
getDescriptor().getMessageTypes().get(1);
- internal_static_DummyResponse_fieldAccessorTable = new
+ internal_static_hbase_test_pb_DummyResponse_fieldAccessorTable = new
com.google.protobuf.GeneratedMessage.FieldAccessorTable(
- internal_static_DummyResponse_descriptor,
+ internal_static_hbase_test_pb_DummyResponse_descriptor,
new java.lang.String[] { "Value", });
return null;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/d23d600d/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestCoprocessorRpcUtils.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestCoprocessorRpcUtils.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestCoprocessorRpcUtils.java
new file mode 100644
index 0000000..82b0341
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestCoprocessorRpcUtils.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.ipc;
+
+import static org.junit.Assert.assertEquals;
+
+import com.google.protobuf.Descriptors;
+import org.apache.hadoop.hbase.coprocessor.protobuf.generated.DummyRegionServerEndpointProtos;
+import org.apache.hadoop.hbase.protobuf.generated.AuthenticationProtos;
+import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category(SmallTests.class)
+public class TestCoprocessorRpcUtils {
+ @Test
+ public void testServiceName() throws Exception {
+ // verify that we de-namespace build in HBase rpc services
+ Descriptors.ServiceDescriptor authService =
+ AuthenticationProtos.AuthenticationService.getDescriptor();
+ assertEquals(authService.getName(), CoprocessorRpcUtils.getServiceName(authService));
+
+ // non-hbase rpc services should remain fully qualified
+ Descriptors.ServiceDescriptor dummyService =
+ DummyRegionServerEndpointProtos.DummyService.getDescriptor();
+ assertEquals(dummyService.getFullName(), CoprocessorRpcUtils.getServiceName(dummyService));
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/d23d600d/hbase-server/src/test/protobuf/DummyRegionServerEndpoint.proto
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/protobuf/DummyRegionServerEndpoint.proto b/hbase-server/src/test/protobuf/DummyRegionServerEndpoint.proto
index cc76825..539f7da 100644
--- a/hbase-server/src/test/protobuf/DummyRegionServerEndpoint.proto
+++ b/hbase-server/src/test/protobuf/DummyRegionServerEndpoint.proto
@@ -15,6 +15,9 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
+package hbase.test.pb;
+
// Coprocessor test
option java_package = "org.apache.hadoop.hbase.coprocessor.protobuf.generated";
option java_outer_classname = "DummyRegionServerEndpointProtos";
[20/50] [abbrv] hbase git commit: HBASE-15768 fix capitalization of
ZooKeeper usage
Posted by sy...@apache.org.
HBASE-15768 fix capitalization of ZooKeeper usage
Signed-off-by: Sean Busbey <bu...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/0bf065a5
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/0bf065a5
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/0bf065a5
Branch: refs/heads/hbase-12439
Commit: 0bf065a5d52222f1a53052aaee21ee501e9d6e64
Parents: b7ce55f
Author: Alex Moundalexis <al...@cloudera.com>
Authored: Wed May 4 15:17:43 2016 -0500
Committer: Sean Busbey <bu...@apache.org>
Committed: Thu May 5 15:35:44 2016 -0500
----------------------------------------------------------------------
bin/hbase | 2 +-
bin/hbase.cmd | 2 +-
conf/hbase-env.cmd | 2 +-
conf/hbase-env.sh | 2 +-
.../archetypes/exemplars/client/HelloHBase.java | 4 +--
.../exemplars/shaded_client/HelloHBase.java | 4 +--
.../hadoop/hbase/client/ZooKeeperRegistry.java | 4 +--
.../replication/ReplicationPeersZKImpl.java | 2 +-
.../replication/ReplicationQueuesZKImpl.java | 2 +-
.../replication/ReplicationTrackerZKImpl.java | 2 +-
.../hbase/zookeeper/RecoverableZooKeeper.java | 6 ++--
.../apache/hadoop/hbase/zookeeper/ZKUtil.java | 4 +--
.../hbase/master/MetricsMasterSource.java | 2 +-
.../hbase/master/MetricsMasterWrapper.java | 4 +--
.../regionserver/MetricsRegionServerSource.java | 2 +-
.../MetricsRegionServerWrapper.java | 4 +--
.../hadoop/hbase/DistributedHBaseCluster.java | 6 ++--
.../hadoop/hbase/HBaseClusterManager.java | 2 +-
.../test/IntegrationTestZKAndFSPermissions.java | 2 +-
.../hbase/tmpl/master/MasterStatusTmpl.jamon | 4 +--
.../hbase/tmpl/regionserver/RSStatusTmpl.jamon | 2 +-
.../apache/hadoop/hbase/ZKNamespaceManager.java | 2 +-
.../org/apache/hadoop/hbase/ZNodeClearer.java | 2 +-
.../ZkSplitLogWorkerCoordination.java | 2 +-
.../procedure/CreateNamespaceProcedure.java | 6 ++--
.../procedure/DeleteNamespaceProcedure.java | 6 ++--
.../procedure/ModifyNamespaceProcedure.java | 6 ++--
.../hadoop/hbase/mob/mapreduce/SweepJob.java | 2 +-
.../mob/mapreduce/SweepJobNodeTracker.java | 2 +-
.../hadoop/hbase/regionserver/HMobStore.java | 2 +-
.../regionserver/ReplicationSyncUp.java | 2 +-
.../security/access/ZKPermissionWatcher.java | 6 ++--
.../token/AuthenticationTokenSecretManager.java | 6 ++--
.../visibility/ZKVisibilityLabelWatcher.java | 4 +--
.../org/apache/hadoop/hbase/util/HBaseFsck.java | 2 +-
.../zookeeper/lock/ZKInterProcessLockBase.java | 2 +-
.../main/resources/hbase-webapps/master/zk.jsp | 2 +-
.../hbase/TestIPv6NIOServerSocketChannel.java | 2 +-
...estMasterCoprocessorExceptionWithRemove.java | 2 +-
.../hbase/filter/FilterTestingCluster.java | 2 +-
.../hadoop/hbase/filter/TestFilterWrapper.java | 2 +-
.../hbase/zookeeper/TestZooKeeperACL.java | 8 +++---
src/main/asciidoc/_chapters/configuration.adoc | 8 +++---
.../asciidoc/_chapters/getting_started.adoc | 4 +--
src/main/asciidoc/_chapters/ops_mgt.adoc | 2 +-
src/main/asciidoc/_chapters/zookeeper.adoc | 30 ++++++++++----------
46 files changed, 89 insertions(+), 89 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/0bf065a5/bin/hbase
----------------------------------------------------------------------
diff --git a/bin/hbase b/bin/hbase
index be519ac..1653c5a 100755
--- a/bin/hbase
+++ b/bin/hbase
@@ -92,7 +92,7 @@ if [ $# = 0 ]; then
echo " zkcli Run the ZooKeeper shell"
echo " master Run an HBase HMaster node"
echo " regionserver Run an HBase HRegionServer node"
- echo " zookeeper Run a Zookeeper server"
+ echo " zookeeper Run a ZooKeeper server"
echo " rest Run an HBase REST server"
echo " thrift Run the HBase Thrift server"
echo " thrift2 Run the HBase Thrift2 server"
http://git-wip-us.apache.org/repos/asf/hbase/blob/0bf065a5/bin/hbase.cmd
----------------------------------------------------------------------
diff --git a/bin/hbase.cmd b/bin/hbase.cmd
index d5f39a4..57a0cb4 100644
--- a/bin/hbase.cmd
+++ b/bin/hbase.cmd
@@ -423,7 +423,7 @@ goto :eof
echo zkcli Run the ZooKeeper shell
echo master Run an HBase HMaster node
echo regionserver Run an HBase HRegionServer node
- echo zookeeper Run a Zookeeper server
+ echo zookeeper Run a ZooKeeper server
echo rest Run an HBase REST server
echo thrift Run the HBase Thrift server
echo thrift2 Run the HBase Thrift2 server
http://git-wip-us.apache.org/repos/asf/hbase/blob/0bf065a5/conf/hbase-env.cmd
----------------------------------------------------------------------
diff --git a/conf/hbase-env.cmd b/conf/hbase-env.cmd
index a40be8e..eb31b1b 100644
--- a/conf/hbase-env.cmd
+++ b/conf/hbase-env.cmd
@@ -83,5 +83,5 @@ set HBASE_REGIONSERVER_OPTS=%HBASE_REGIONSERVER_OPTS% "-XX:PermSize=128m" "-XX:M
@rem otherwise arrive faster than the master can service them.
@rem set HBASE_SLAVE_SLEEP=0.1
-@rem Tell HBase whether it should manage it's own instance of Zookeeper or not.
+@rem Tell HBase whether it should manage it's own instance of ZooKeeper or not.
@rem set HBASE_MANAGES_ZK=true
http://git-wip-us.apache.org/repos/asf/hbase/blob/0bf065a5/conf/hbase-env.sh
----------------------------------------------------------------------
diff --git a/conf/hbase-env.sh b/conf/hbase-env.sh
index 5ce959b..c45e7a5 100644
--- a/conf/hbase-env.sh
+++ b/conf/hbase-env.sh
@@ -124,7 +124,7 @@ export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS -XX:PermSize=128m -XX:M
# otherwise arrive faster than the master can service them.
# export HBASE_SLAVE_SLEEP=0.1
-# Tell HBase whether it should manage it's own instance of Zookeeper or not.
+# Tell HBase whether it should manage it's own instance of ZooKeeper or not.
# export HBASE_MANAGES_ZK=true
# The default log rolling policy is RFA, where the log file is rolled as per the size defined for the
http://git-wip-us.apache.org/repos/asf/hbase/blob/0bf065a5/hbase-archetypes/hbase-client-project/src/main/java/org/apache/hbase/archetypes/exemplars/client/HelloHBase.java
----------------------------------------------------------------------
diff --git a/hbase-archetypes/hbase-client-project/src/main/java/org/apache/hbase/archetypes/exemplars/client/HelloHBase.java b/hbase-archetypes/hbase-client-project/src/main/java/org/apache/hbase/archetypes/exemplars/client/HelloHBase.java
index c5b1b6a..e327541 100644
--- a/hbase-archetypes/hbase-client-project/src/main/java/org/apache/hbase/archetypes/exemplars/client/HelloHBase.java
+++ b/hbase-archetypes/hbase-client-project/src/main/java/org/apache/hbase/archetypes/exemplars/client/HelloHBase.java
@@ -62,14 +62,14 @@ public final class HelloHBase {
/**
* ConnectionFactory#createConnection() automatically looks for
* hbase-site.xml (HBase configuration parameters) on the system's
- * CLASSPATH, to enable creation of Connection to HBase via Zookeeper.
+ * CLASSPATH, to enable creation of Connection to HBase via ZooKeeper.
*/
try (Connection connection = ConnectionFactory.createConnection();
Admin admin = connection.getAdmin()) {
admin.getClusterStatus(); // assure connection successfully established
System.out.println("\n*** Hello HBase! -- Connection has been "
- + "established via Zookeeper!!\n");
+ + "established via ZooKeeper!!\n");
createNamespaceAndTable(admin);
http://git-wip-us.apache.org/repos/asf/hbase/blob/0bf065a5/hbase-archetypes/hbase-shaded-client-project/src/main/java/org/apache/hbase/archetypes/exemplars/shaded_client/HelloHBase.java
----------------------------------------------------------------------
diff --git a/hbase-archetypes/hbase-shaded-client-project/src/main/java/org/apache/hbase/archetypes/exemplars/shaded_client/HelloHBase.java b/hbase-archetypes/hbase-shaded-client-project/src/main/java/org/apache/hbase/archetypes/exemplars/shaded_client/HelloHBase.java
index 1c6b8bb..94a1e71 100644
--- a/hbase-archetypes/hbase-shaded-client-project/src/main/java/org/apache/hbase/archetypes/exemplars/shaded_client/HelloHBase.java
+++ b/hbase-archetypes/hbase-shaded-client-project/src/main/java/org/apache/hbase/archetypes/exemplars/shaded_client/HelloHBase.java
@@ -62,14 +62,14 @@ public final class HelloHBase {
/**
* ConnectionFactory#createConnection() automatically looks for
* hbase-site.xml (HBase configuration parameters) on the system's
- * CLASSPATH, to enable creation of Connection to HBase via Zookeeper.
+ * CLASSPATH, to enable creation of Connection to HBase via ZooKeeper.
*/
try (Connection connection = ConnectionFactory.createConnection();
Admin admin = connection.getAdmin()) {
admin.getClusterStatus(); // assure connection successfully established
System.out.println("\n*** Hello HBase! -- Connection has been "
- + "established via Zookeeper!!\n");
+ + "established via ZooKeeper!!\n");
createNamespaceAndTable(admin);
http://git-wip-us.apache.org/repos/asf/hbase/blob/0bf065a5/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ZooKeeperRegistry.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ZooKeeperRegistry.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ZooKeeperRegistry.java
index 5aabb0c..9208f6e 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ZooKeeperRegistry.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ZooKeeperRegistry.java
@@ -104,9 +104,9 @@ class ZooKeeperRegistry implements Registry {
LOG.info("ClusterId read in ZooKeeper is null");
}
} catch (KeeperException e) {
- LOG.warn("Can't retrieve clusterId from Zookeeper", e);
+ LOG.warn("Can't retrieve clusterId from ZooKeeper", e);
} catch (IOException e) {
- LOG.warn("Can't retrieve clusterId from Zookeeper", e);
+ LOG.warn("Can't retrieve clusterId from ZooKeeper", e);
} finally {
if (zkw != null) zkw.close();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/0bf065a5/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java
index 916eaf8..bd3930d 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java
@@ -48,7 +48,7 @@ import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.KeeperException.NoNodeException;
/**
- * This class provides an implementation of the ReplicationPeers interface using Zookeeper. The
+ * This class provides an implementation of the ReplicationPeers interface using ZooKeeper. The
* peers znode contains a list of all peer replication clusters and the current replication state of
* those clusters. It has one child peer znode for each peer cluster. The peer znode is named with
* the cluster id provided by the user in the HBase shell. The value of the peer znode contains the
http://git-wip-us.apache.org/repos/asf/hbase/blob/0bf065a5/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesZKImpl.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesZKImpl.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesZKImpl.java
index 764b5b0..2bb8ea8 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesZKImpl.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesZKImpl.java
@@ -41,7 +41,7 @@ import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
import org.apache.zookeeper.KeeperException;
/**
- * This class provides an implementation of the ReplicationQueues interface using Zookeeper. The
+ * This class provides an implementation of the ReplicationQueues interface using ZooKeeper. The
* base znode that this class works at is the myQueuesZnode. The myQueuesZnode contains a list of
* all outstanding WAL files on this region server that need to be replicated. The myQueuesZnode is
* the regionserver name (a concatenation of the region server\u2019s hostname, client port and start
http://git-wip-us.apache.org/repos/asf/hbase/blob/0bf065a5/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationTrackerZKImpl.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationTrackerZKImpl.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationTrackerZKImpl.java
index f9f2d43..6e4983e 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationTrackerZKImpl.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationTrackerZKImpl.java
@@ -34,7 +34,7 @@ import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
import org.apache.zookeeper.KeeperException;
/**
- * This class is a Zookeeper implementation of the ReplicationTracker interface. This class is
+ * This class is a ZooKeeper implementation of the ReplicationTracker interface. This class is
* responsible for handling replication events that are defined in the ReplicationListener
* interface.
*/
http://git-wip-us.apache.org/repos/asf/hbase/blob/0bf065a5/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/RecoverableZooKeeper.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/RecoverableZooKeeper.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/RecoverableZooKeeper.java
index c1eb214..e3c3b8c 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/RecoverableZooKeeper.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/RecoverableZooKeeper.java
@@ -130,9 +130,9 @@ public class RecoverableZooKeeper {
}
/**
- * Try to create a Zookeeper connection. Turns any exception encountered into a
+ * Try to create a ZooKeeper connection. Turns any exception encountered into a
* KeeperException.OperationTimeoutException so it can retried.
- * @return The created Zookeeper connection object
+ * @return The created ZooKeeper connection object
* @throws KeeperException
*/
protected synchronized ZooKeeper checkZk() throws KeeperException {
@@ -153,7 +153,7 @@ public class RecoverableZooKeeper {
LOG.info("Closing dead ZooKeeper connection, session" +
" was: 0x"+Long.toHexString(zk.getSessionId()));
zk.close();
- // reset the Zookeeper connection
+ // reset the ZooKeeper connection
zk = null;
}
checkZk();
http://git-wip-us.apache.org/repos/asf/hbase/blob/0bf065a5/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
index 9e01d09..ff2e21f 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
@@ -922,7 +922,7 @@ public class ZKUtil {
}
if (!groups.isEmpty()) {
LOG.warn("Znode ACL setting for group " + groups
- + " is skipped, Zookeeper doesn't support this feature presently.");
+ + " is skipped, ZooKeeper doesn't support this feature presently.");
}
}
// Certain znodes are accessed directly by the client,
@@ -1392,7 +1392,7 @@ public class ZKUtil {
}
List<ZKUtilOp> ops = new ArrayList<ZKUtil.ZKUtilOp>();
for (String eachRoot : pathRoots) {
- // Zookeeper Watches are one time triggers; When children of parent nodes are deleted
+ // ZooKeeper Watches are one time triggers; When children of parent nodes are deleted
// recursively, must set another watch, get notified of delete node
List<String> children = listChildrenBFSAndWatchThem(zkw, eachRoot);
// Delete the leaves first and eventually get rid of the root
http://git-wip-us.apache.org/repos/asf/hbase/blob/0bf065a5/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSource.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSource.java
index 290b8f5..6995198 100644
--- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSource.java
+++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSource.java
@@ -68,7 +68,7 @@ public interface MetricsMasterSource extends BaseSource {
String NUMBER_OF_REGION_SERVERS_DESC = "Number of RegionServers";
String DEAD_REGION_SERVERS_DESC = "Names of dead RegionServers";
String NUMBER_OF_DEAD_REGION_SERVERS_DESC = "Number of dead RegionServers";
- String ZOOKEEPER_QUORUM_DESC = "Zookeeper Quorum";
+ String ZOOKEEPER_QUORUM_DESC = "ZooKeeper Quorum";
String SERVER_NAME_DESC = "Server Name";
String CLUSTER_ID_DESC = "Cluster Id";
String IS_ACTIVE_MASTER_DESC = "Is Active Master";
http://git-wip-us.apache.org/repos/asf/hbase/blob/0bf065a5/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapper.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapper.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapper.java
index 5e67f83..64e0a8a 100644
--- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapper.java
+++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapper.java
@@ -44,9 +44,9 @@ public interface MetricsMasterWrapper {
String getClusterId();
/**
- * Get the Zookeeper Quorum Info
+ * Get the ZooKeeper Quorum Info
*
- * @return Zookeeper Quorum Info
+ * @return ZooKeeper Quorum Info
*/
String getZookeeperQuorum();
http://git-wip-us.apache.org/repos/asf/hbase/blob/0bf065a5/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
index 06f3dc3..a80745d 100644
--- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
+++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
@@ -326,7 +326,7 @@ public interface MetricsRegionServerSource extends BaseSource, JvmPauseMonitorSo
String SERVER_NAME_NAME = "serverName";
String CLUSTER_ID_NAME = "clusterId";
String RS_START_TIME_DESC = "RegionServer Start Time";
- String ZOOKEEPER_QUORUM_DESC = "Zookeeper Quorum";
+ String ZOOKEEPER_QUORUM_DESC = "ZooKeeper Quorum";
String SERVER_NAME_DESC = "Server Name";
String CLUSTER_ID_DESC = "Cluster Id";
String UPDATES_BLOCKED_TIME = "updatesBlockedTime";
http://git-wip-us.apache.org/repos/asf/hbase/blob/0bf065a5/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java
index cf0c7ac..0aa625c 100644
--- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java
+++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java
@@ -37,9 +37,9 @@ public interface MetricsRegionServerWrapper {
String getClusterId();
/**
- * Get the Zookeeper Quorum Info
+ * Get the ZooKeeper Quorum Info
*
- * @return Zookeeper Quorum Info
+ * @return ZooKeeper Quorum Info
*/
String getZookeeperQuorum();
http://git-wip-us.apache.org/repos/asf/hbase/blob/0bf065a5/hbase-it/src/test/java/org/apache/hadoop/hbase/DistributedHBaseCluster.java
----------------------------------------------------------------------
diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/DistributedHBaseCluster.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/DistributedHBaseCluster.java
index 31ca996..c83c5e4 100644
--- a/hbase-it/src/test/java/org/apache/hadoop/hbase/DistributedHBaseCluster.java
+++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/DistributedHBaseCluster.java
@@ -131,20 +131,20 @@ public class DistributedHBaseCluster extends HBaseCluster {
@Override
public void startZkNode(String hostname, int port) throws IOException {
- LOG.info("Starting Zookeeper node on: " + hostname);
+ LOG.info("Starting ZooKeeper node on: " + hostname);
clusterManager.start(ServiceType.ZOOKEEPER_SERVER, hostname, port);
}
@Override
public void killZkNode(ServerName serverName) throws IOException {
- LOG.info("Aborting Zookeeper node on: " + serverName.getServerName());
+ LOG.info("Aborting ZooKeeper node on: " + serverName.getServerName());
clusterManager.kill(ServiceType.ZOOKEEPER_SERVER,
serverName.getHostname(), serverName.getPort());
}
@Override
public void stopZkNode(ServerName serverName) throws IOException {
- LOG.info("Stopping Zookeeper node: " + serverName.getServerName());
+ LOG.info("Stopping ZooKeeper node: " + serverName.getServerName());
clusterManager.stop(ServiceType.ZOOKEEPER_SERVER,
serverName.getHostname(), serverName.getPort());
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/0bf065a5/hbase-it/src/test/java/org/apache/hadoop/hbase/HBaseClusterManager.java
----------------------------------------------------------------------
diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/HBaseClusterManager.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/HBaseClusterManager.java
index ba6a4a9..f13102d 100644
--- a/hbase-it/src/test/java/org/apache/hadoop/hbase/HBaseClusterManager.java
+++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/HBaseClusterManager.java
@@ -252,7 +252,7 @@ public class HBaseClusterManager extends Configured implements ClusterManager {
String tmp = conf.get("hbase.it.clustermanager.zookeeper.conf.dir",
System.getenv("ZOOCFGDIR"));
if (zookeeperHome == null) {
- throw new IOException("Zookeeper home configuration parameter i.e. " +
+ throw new IOException("ZooKeeper home configuration parameter i.e. " +
"'hbase.it.clustermanager.zookeeper.home' is not configured properly.");
}
if (tmp != null) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/0bf065a5/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestZKAndFSPermissions.java
----------------------------------------------------------------------
diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestZKAndFSPermissions.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestZKAndFSPermissions.java
index 3845846..018b406 100644
--- a/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestZKAndFSPermissions.java
+++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestZKAndFSPermissions.java
@@ -53,7 +53,7 @@ import org.junit.experimental.categories.Category;
/**
* An integration test which checks that the znodes in zookeeper and data in the FileSystem
* are protected for secure HBase deployments.
- * This test is intended to be run on clusters with kerberos authorization for HBase and Zookeeper.
+ * This test is intended to be run on clusters with kerberos authorization for HBase and ZooKeeper.
*
* If hbase.security.authentication is not set to kerberos, the test does not run unless -f is
* specified which bypasses the check. It is recommended to always run with -f on secure clusters
http://git-wip-us.apache.org/repos/asf/hbase/blob/0bf065a5/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon
index 4b1c192..38bcf22 100644
--- a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon
+++ b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon
@@ -273,13 +273,13 @@ AssignmentManager assignmentManager = master.getAssignmentManager();
<td>When ZooKeeper client version was compiled</td>
</tr>
<tr>
- <td>Zookeeper Quorum</td>
+ <td>ZooKeeper Quorum</td>
<%escape #n>
<td> <% formatZKString() %> </td>
<td>Addresses of all registered ZK servers. For more, see <a href="/zk.jsp">zk dump</a>.</td>
</tr>
<tr>
- <td>Zookeeper Base Path</td>
+ <td>ZooKeeper Base Path</td>
<td> <% master.getZooKeeper().getBaseZNode() %></td>
<td>Root node of this cluster in ZK.</td>
</tr>
http://git-wip-us.apache.org/repos/asf/hbase/blob/0bf065a5/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RSStatusTmpl.jamon
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RSStatusTmpl.jamon b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RSStatusTmpl.jamon
index b98f50d..21e9b9f 100644
--- a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RSStatusTmpl.jamon
+++ b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RSStatusTmpl.jamon
@@ -172,7 +172,7 @@ org.apache.hadoop.hbase.zookeeper.MasterAddressTracker;
<td>When ZooKeeper client version was compiled</td>
</tr>
<tr>
- <td>Zookeeper Quorum</td>
+ <td>ZooKeeper Quorum</td>
<td><% regionServer.getZooKeeper().getQuorum() %></td>
<td>Addresses of all registered ZK servers</td>
</tr>
http://git-wip-us.apache.org/repos/asf/hbase/blob/0bf065a5/hbase-server/src/main/java/org/apache/hadoop/hbase/ZKNamespaceManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/ZKNamespaceManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/ZKNamespaceManager.java
index b0aabb2..ee59c01 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/ZKNamespaceManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/ZKNamespaceManager.java
@@ -156,7 +156,7 @@ public class ZKNamespaceManager extends ZooKeeperListener {
refreshNodes(nodes);
} catch (KeeperException ke) {
LOG.error("Error reading data from zookeeper for path "+path, ke);
- watcher.abort("Zookeeper error get node children for path "+path, ke);
+ watcher.abort("ZooKeeper error get node children for path "+path, ke);
} catch (IOException e) {
LOG.error("Error deserializing namespace child from: "+path, e);
watcher.abort("Error deserializing namespace child from: " + path, e);
http://git-wip-us.apache.org/repos/asf/hbase/blob/0bf065a5/hbase-server/src/main/java/org/apache/hadoop/hbase/ZNodeClearer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/ZNodeClearer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/ZNodeClearer.java
index e68a068..652cbc0 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/ZNodeClearer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/ZNodeClearer.java
@@ -197,7 +197,7 @@ public class ZNodeClearer {
LOG.warn("Can't read the content of the znode file", e);
return false;
} catch (KeeperException e) {
- LOG.warn("Zookeeper exception deleting znode", e);
+ LOG.warn("ZooKeeper exception deleting znode", e);
return false;
} finally {
zkw.close();
http://git-wip-us.apache.org/repos/asf/hbase/blob/0bf065a5/hbase-server/src/main/java/org/apache/hadoop/hbase/coordination/ZkSplitLogWorkerCoordination.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/coordination/ZkSplitLogWorkerCoordination.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/coordination/ZkSplitLogWorkerCoordination.java
index 7e6708e..2687f2e 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/coordination/ZkSplitLogWorkerCoordination.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/coordination/ZkSplitLogWorkerCoordination.java
@@ -146,7 +146,7 @@ public class ZkSplitLogWorkerCoordination extends ZooKeeperListener implements
ZKSplitLogManagerCoordination.DEFAULT_TIMEOUT) / 3);
}
- /* Support functions for Zookeeper async callback */
+ /* Support functions for ZooKeeper async callback */
void getDataSetWatchFailure(String path) {
synchronized (grabTaskLock) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/0bf065a5/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateNamespaceProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateNamespaceProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateNamespaceProcedure.java
index d8daab7..a9a0968 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateNamespaceProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateNamespaceProcedure.java
@@ -298,7 +298,7 @@ public class CreateNamespaceProcedure
}
/**
- * Update Zookeeper.
+ * Update ZooKeeper.
* @param env MasterProcedureEnv
* @param nsDescriptor NamespaceDescriptor
* @throws IOException
@@ -310,7 +310,7 @@ public class CreateNamespaceProcedure
}
/**
- * rollback Zookeeper update.
+ * rollback ZooKeeper update.
* @param env MasterProcedureEnv
* @throws IOException
*/
@@ -373,4 +373,4 @@ public class CreateNamespaceProcedure
// the client does not know about this procedures.
return !isBootstrapNamespace();
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/0bf065a5/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteNamespaceProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteNamespaceProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteNamespaceProcedure.java
index 30844df..bd71aa3 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteNamespaceProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteNamespaceProcedure.java
@@ -292,7 +292,7 @@ public class DeleteNamespaceProcedure
}
/**
- * remove from Zookeeper.
+ * remove from ZooKeeper.
* @param env MasterProcedureEnv
* @param namespaceName name of the namespace in string format
* @throws IOException
@@ -304,7 +304,7 @@ public class DeleteNamespaceProcedure
}
/**
- * undo the remove from Zookeeper
+ * undo the remove from ZooKeeper
* @param env MasterProcedureEnv
* @throws IOException
*/
@@ -401,4 +401,4 @@ public class DeleteNamespaceProcedure
}
return traceEnabled;
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/0bf065a5/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyNamespaceProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyNamespaceProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyNamespaceProcedure.java
index 78334e8..1f885bd 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyNamespaceProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyNamespaceProcedure.java
@@ -251,7 +251,7 @@ public class ModifyNamespaceProcedure
}
/**
- * Update Zookeeper.
+ * Update ZooKeeper.
* @param env MasterProcedureEnv
* @throws IOException
*/
@@ -260,7 +260,7 @@ public class ModifyNamespaceProcedure
}
/**
- * Update Zookeeper during undo.
+ * Update ZooKeeper during undo.
* @param env MasterProcedureEnv
* @throws IOException
*/
@@ -285,4 +285,4 @@ public class ModifyNamespaceProcedure
}
return traceEnabled;
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/0bf065a5/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/mapreduce/SweepJob.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/mapreduce/SweepJob.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/mapreduce/SweepJob.java
index 6d06dfc..79cd39c 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/mapreduce/SweepJob.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/mapreduce/SweepJob.java
@@ -135,7 +135,7 @@ public class SweepJob {
* The running of the sweep tool on the same column family are mutually exclusive.
* The HBase major compaction and running of the sweep tool on the same column family
* are mutually exclusive.
- * The synchronization is done by the Zookeeper.
+ * The synchronization is done by the ZooKeeper.
* So in the beginning of the running, we need to make sure only this sweep tool is the only one
* that is currently running in this column family, and in this column family there're no major
* compaction in progress.
http://git-wip-us.apache.org/repos/asf/hbase/blob/0bf065a5/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/mapreduce/SweepJobNodeTracker.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/mapreduce/SweepJobNodeTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/mapreduce/SweepJobNodeTracker.java
index dea67a4..e7f6907 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/mapreduce/SweepJobNodeTracker.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/mapreduce/SweepJobNodeTracker.java
@@ -58,7 +58,7 @@ public class SweepJobNodeTracker extends ZooKeeperListener {
* Registers the watcher on the sweep job node.
* If there's no such a sweep job node, or it's not created by the sweep job that
* owns the current MR, the current process will be aborted.
- * This assumes the table lock uses the Zookeeper. It's a workaround and only used
+ * This assumes the table lock uses the ZooKeeper. It's a workaround and only used
* in the sweep tool, and the sweep tool will be removed after the mob file compaction
* is finished.
*/
http://git-wip-us.apache.org/repos/asf/hbase/blob/0bf065a5/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HMobStore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HMobStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HMobStore.java
index 54b942d..bfa1f80 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HMobStore.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HMobStore.java
@@ -462,7 +462,7 @@ public class HMobStore extends HStore {
// If it's major compaction, try to find whether there's a sweeper is running
// If yes, mark the major compaction as retainDeleteMarkers
if (compaction.getRequest().isAllFiles()) {
- // Use the Zookeeper to coordinate.
+ // Use the ZooKeeper to coordinate.
// 1. Acquire a operation lock.
// 1.1. If no, mark the major compaction as retainDeleteMarkers and continue the compaction.
// 1.2. If the lock is obtained, search the node of sweeping.
http://git-wip-us.apache.org/repos/asf/hbase/blob/0bf065a5/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSyncUp.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSyncUp.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSyncUp.java
index b86de12..bfc6421 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSyncUp.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSyncUp.java
@@ -40,7 +40,7 @@ import org.apache.hadoop.util.ToolRunner;
/**
* In a scenario of Replication based Disaster/Recovery, when hbase
* Master-Cluster crashes, this tool is used to sync-up the delta from Master to
- * Slave using the info from Zookeeper. The tool will run on Master-Cluser, and
+ * Slave using the info from ZooKeeper. The tool will run on Master-Cluser, and
* assume ZK, Filesystem and NetWork still available after hbase crashes
*
* hbase org.apache.hadoop.hbase.replication.regionserver.ReplicationSyncUp
http://git-wip-us.apache.org/repos/asf/hbase/blob/0bf065a5/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/ZKPermissionWatcher.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/ZKPermissionWatcher.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/ZKPermissionWatcher.java
index b3aa782..c4b3219 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/ZKPermissionWatcher.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/ZKPermissionWatcher.java
@@ -130,7 +130,7 @@ public class ZKPermissionWatcher extends ZooKeeperListener implements Closeable
} catch (KeeperException ke) {
LOG.error("Error reading data from zookeeper", ke);
// only option is to abort
- watcher.abort("Zookeeper error obtaining acl node children", ke);
+ watcher.abort("ZooKeeper error obtaining acl node children", ke);
}
}
});
@@ -170,7 +170,7 @@ public class ZKPermissionWatcher extends ZooKeeperListener implements Closeable
} catch (KeeperException ke) {
LOG.error("Error reading data from zookeeper for node " + entry, ke);
// only option is to abort
- watcher.abort("Zookeeper error getting data for node " + entry, ke);
+ watcher.abort("ZooKeeper error getting data for node " + entry, ke);
} catch (IOException ioe) {
LOG.error("Error reading permissions writables", ioe);
}
@@ -196,7 +196,7 @@ public class ZKPermissionWatcher extends ZooKeeperListener implements Closeable
}
} catch (KeeperException ke) {
LOG.error("Error reading data from zookeeper for path "+path, ke);
- watcher.abort("Zookeeper error get node children for path "+path, ke);
+ watcher.abort("ZooKeeper error get node children for path "+path, ke);
}
executor.submit(new Runnable() {
// allows subsequent nodeChildrenChanged event to preempt current processing of
http://git-wip-us.apache.org/repos/asf/hbase/blob/0bf065a5/hbase-server/src/main/java/org/apache/hadoop/hbase/security/token/AuthenticationTokenSecretManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/token/AuthenticationTokenSecretManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/token/AuthenticationTokenSecretManager.java
index de66827..5b488a9 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/token/AuthenticationTokenSecretManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/token/AuthenticationTokenSecretManager.java
@@ -107,7 +107,7 @@ public class AuthenticationTokenSecretManager
// try to become leader
this.leaderElector.start();
} catch (KeeperException ke) {
- LOG.error("Zookeeper initialization failed", ke);
+ LOG.error("ZooKeeper initialization failed", ke);
}
}
@@ -145,9 +145,9 @@ public class AuthenticationTokenSecretManager
AuthenticationKey masterKey = allKeys.get(identifier.getKeyId());
if(masterKey == null) {
if(zkWatcher.getWatcher().isAborted()) {
- LOG.error("ZookeeperWatcher is abort");
+ LOG.error("ZooKeeperWatcher is abort");
throw new InvalidToken("Token keys could not be sync from zookeeper"
- + " because of ZookeeperWatcher abort");
+ + " because of ZooKeeperWatcher abort");
}
synchronized (this) {
if (!leaderElector.isAlive() || leaderElector.isStopped()) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/0bf065a5/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/ZKVisibilityLabelWatcher.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/ZKVisibilityLabelWatcher.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/ZKVisibilityLabelWatcher.java
index 4941a54..f30136f 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/ZKVisibilityLabelWatcher.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/ZKVisibilityLabelWatcher.java
@@ -95,7 +95,7 @@ public class ZKVisibilityLabelWatcher extends ZooKeeperListener {
} catch (KeeperException ke) {
LOG.error("Error setting watcher on node " + path, ke);
// only option is to abort
- watcher.abort("Zookeeper error obtaining label node children", ke);
+ watcher.abort("ZooKeeper error obtaining label node children", ke);
}
}
}
@@ -119,7 +119,7 @@ public class ZKVisibilityLabelWatcher extends ZooKeeperListener {
} catch (KeeperException ke) {
LOG.error("Error reading data from zookeeper for node " + path, ke);
// only option is to abort
- watcher.abort("Zookeeper error getting data for node " + path, ke);
+ watcher.abort("ZooKeeper error getting data for node " + path, ke);
}
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/0bf065a5/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
index 31ae925..eec5ea4 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
@@ -1741,7 +1741,7 @@ public class HBaseFsck extends Configured implements Closeable {
HConstants.EMPTY_START_ROW, false, false);
if (rl == null) {
errors.reportError(ERROR_CODE.NULL_META_REGION,
- "META region was not found in Zookeeper");
+ "META region was not found in ZooKeeper");
return false;
}
for (HRegionLocation metaLocation : rl.getRegionLocations()) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/0bf065a5/hbase-server/src/main/java/org/apache/hadoop/hbase/zookeeper/lock/ZKInterProcessLockBase.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/zookeeper/lock/ZKInterProcessLockBase.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/zookeeper/lock/ZKInterProcessLockBase.java
index a6371cc..090acf7 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/zookeeper/lock/ZKInterProcessLockBase.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/zookeeper/lock/ZKInterProcessLockBase.java
@@ -111,7 +111,7 @@ public abstract class ZKInterProcessLockBase implements InterProcessLock {
private ZNodeComparator() {
}
- /** Parses sequenceId from the znode name. Zookeeper documentation
+ /** Parses sequenceId from the znode name. ZooKeeper documentation
* states: The sequence number is always fixed length of 10 digits, 0 padded
*/
public static long getChildSequenceId(String childZNode) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/0bf065a5/hbase-server/src/main/resources/hbase-webapps/master/zk.jsp
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/resources/hbase-webapps/master/zk.jsp b/hbase-server/src/main/resources/hbase-webapps/master/zk.jsp
index 0649554..687c950 100644
--- a/hbase-server/src/main/resources/hbase-webapps/master/zk.jsp
+++ b/hbase-server/src/main/resources/hbase-webapps/master/zk.jsp
@@ -75,7 +75,7 @@
<div class="container-fluid content">
<div class="row inner_header">
<div class="page-header">
- <h1>Zookeeper Dump</h1>
+ <h1>ZooKeeper Dump</h1>
</div>
</div>
<div class="row">
http://git-wip-us.apache.org/repos/asf/hbase/blob/0bf065a5/hbase-server/src/test/java/org/apache/hadoop/hbase/TestIPv6NIOServerSocketChannel.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestIPv6NIOServerSocketChannel.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestIPv6NIOServerSocketChannel.java
index ecbdb5d..a598bff 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestIPv6NIOServerSocketChannel.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestIPv6NIOServerSocketChannel.java
@@ -36,7 +36,7 @@ import org.junit.experimental.categories.Category;
import org.junit.rules.TestRule;
/**
- * This tests whether ServerSocketChannel works over ipv6, which Zookeeper
+ * This tests whether ServerSocketChannel works over ipv6, which ZooKeeper
* depends on. On Windows Oracle JDK 6, creating a ServerSocketChannel throws
* java.net.SocketException: Address family not supported by protocol family
* exception. It is a known JVM bug, seems to be only resolved for JDK7:
http://git-wip-us.apache.org/repos/asf/hbase/blob/0bf065a5/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterCoprocessorExceptionWithRemove.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterCoprocessorExceptionWithRemove.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterCoprocessorExceptionWithRemove.java
index 5048c73..bbb855c 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterCoprocessorExceptionWithRemove.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterCoprocessorExceptionWithRemove.java
@@ -169,7 +169,7 @@ public class TestMasterCoprocessorExceptionWithRemove {
new Abortable() {
@Override
public void abort(String why, Throwable e) {
- throw new RuntimeException("Fatal Zookeeper tracker error, why=", e);
+ throw new RuntimeException("Fatal ZooKeeper tracker error, why=", e);
}
@Override
public boolean isAborted() {
http://git-wip-us.apache.org/repos/asf/hbase/blob/0bf065a5/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/FilterTestingCluster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/FilterTestingCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/FilterTestingCluster.java
index bf61681..e587096 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/FilterTestingCluster.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/FilterTestingCluster.java
@@ -104,7 +104,7 @@ public class FilterTestingCluster {
} catch (MasterNotRunningException e) {
assertNull("Master is not running", e);
} catch (ZooKeeperConnectionException e) {
- assertNull("Cannot connect to Zookeeper", e);
+ assertNull("Cannot connect to ZooKeeper", e);
} catch (IOException e) {
assertNull("IOException", e);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/0bf065a5/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilterWrapper.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilterWrapper.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilterWrapper.java
index 9e1c011..4d7c808 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilterWrapper.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilterWrapper.java
@@ -180,7 +180,7 @@ public class TestFilterWrapper {
} catch (MasterNotRunningException e) {
assertNull("Master is not running", e);
} catch (ZooKeeperConnectionException e) {
- assertNull("Cannot connect to Zookeeper", e);
+ assertNull("Cannot connect to ZooKeeper", e);
} catch (IOException e) {
assertNull("Caught IOException", e);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/0bf065a5/hbase-server/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZooKeeperACL.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZooKeeperACL.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZooKeeperACL.java
index 50e886a..3f474b4 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZooKeeperACL.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZooKeeperACL.java
@@ -113,7 +113,7 @@ public class TestZooKeeperACL {
/**
* Create a node and check its ACL. When authentication is enabled on
- * Zookeeper, all nodes (except /hbase/root-region-server, /hbase/master
+ * ZooKeeper, all nodes (except /hbase/root-region-server, /hbase/master
* and /hbase/hbaseid) should be created so that only the hbase server user
* (master or region server user) that created them can access them, and
* this user should have all permissions on this node. For
@@ -138,7 +138,7 @@ public class TestZooKeeperACL {
}
/**
- * When authentication is enabled on Zookeeper, /hbase/root-region-server
+ * When authentication is enabled on ZooKeeper, /hbase/root-region-server
* should be created with 2 ACLs: one specifies that the hbase user has
* full access to the node; the other, that it is world-readable.
*/
@@ -175,7 +175,7 @@ public class TestZooKeeperACL {
}
/**
- * When authentication is enabled on Zookeeper, /hbase/master should be
+ * When authentication is enabled on ZooKeeper, /hbase/master should be
* created with 2 ACLs: one specifies that the hbase user has full access
* to the node; the other, that it is world-readable.
*/
@@ -211,7 +211,7 @@ public class TestZooKeeperACL {
}
/**
- * When authentication is enabled on Zookeeper, /hbase/hbaseid should be
+ * When authentication is enabled on ZooKeeper, /hbase/hbaseid should be
* created with 2 ACLs: one specifies that the hbase user has full access
* to the node; the other, that it is world-readable.
*/
http://git-wip-us.apache.org/repos/asf/hbase/blob/0bf065a5/src/main/asciidoc/_chapters/configuration.adoc
----------------------------------------------------------------------
diff --git a/src/main/asciidoc/_chapters/configuration.adoc b/src/main/asciidoc/_chapters/configuration.adoc
index 6aefd5a..d705db9 100644
--- a/src/main/asciidoc/_chapters/configuration.adoc
+++ b/src/main/asciidoc/_chapters/configuration.adoc
@@ -411,7 +411,7 @@ Set [var]+JAVA_HOME+ to point at the root of your +java+ install.
This is the default mode.
Standalone mode is what is described in the <<quickstart,quickstart>> section.
In standalone mode, HBase does not use HDFS -- it uses the local filesystem instead -- and it runs all HBase daemons and a local ZooKeeper all up in the same JVM.
-Zookeeper binds to a well known port so clients may talk to HBase.
+ZooKeeper binds to a well known port so clients may talk to HBase.
[[distributed]]
=== Distributed
@@ -453,7 +453,7 @@ In addition, the cluster is configured so that multiple cluster nodes enlist as
These configuration basics are all demonstrated in <<quickstart_fully_distributed,quickstart-fully-distributed>>.
.Distributed RegionServers
-Typically, your cluster will contain multiple RegionServers all running on different servers, as well as primary and backup Master and Zookeeper daemons.
+Typically, your cluster will contain multiple RegionServers all running on different servers, as well as primary and backup Master and ZooKeeper daemons.
The _conf/regionservers_ file on the master server contains a list of hosts whose RegionServers are associated with this cluster.
Each host is on a separate line.
All hosts listed in this file will have their RegionServer processes started and stopped when the master server starts or stops.
@@ -703,8 +703,8 @@ Below we show what the main configuration files -- _hbase-site.xml_, _regionserv
<name>hbase.cluster.distributed</name>
<value>true</value>
<description>The mode the cluster will be in. Possible values are
- false: standalone and pseudo-distributed setups with managed Zookeeper
- true: fully-distributed with unmanaged Zookeeper Quorum (see hbase-env.sh)
+ false: standalone and pseudo-distributed setups with managed ZooKeeper
+ true: fully-distributed with unmanaged ZooKeeper Quorum (see hbase-env.sh)
</description>
</property>
</configuration>
http://git-wip-us.apache.org/repos/asf/hbase/blob/0bf065a5/src/main/asciidoc/_chapters/getting_started.adoc
----------------------------------------------------------------------
diff --git a/src/main/asciidoc/_chapters/getting_started.adoc b/src/main/asciidoc/_chapters/getting_started.adoc
index 7ef91b0..276a908 100644
--- a/src/main/asciidoc/_chapters/getting_started.adoc
+++ b/src/main/asciidoc/_chapters/getting_started.adoc
@@ -288,7 +288,7 @@ $
=== Intermediate - Pseudo-Distributed Local Install
After working your way through <<quickstart,quickstart>>, you can re-configure HBase to run in pseudo-distributed mode.
-Pseudo-distributed mode means that HBase still runs completely on a single host, but each HBase daemon (HMaster, HRegionServer, and Zookeeper) runs as a separate process.
+Pseudo-distributed mode means that HBase still runs completely on a single host, but each HBase daemon (HMaster, HRegionServer, and ZooKeeper) runs as a separate process.
By default, unless you configure the `hbase.rootdir` property as described in <<quickstart,quickstart>>, your data is still stored in _/tmp/_.
In this walk-through, we store your data in HDFS instead, assuming you have HDFS available.
You can skip the HDFS configuration to continue storing your data in the local filesystem.
@@ -429,7 +429,7 @@ You can stop HBase the same way as in the <<quickstart,quickstart>> procedure, u
In reality, you need a fully-distributed configuration to fully test HBase and to use it in real-world scenarios.
In a distributed configuration, the cluster contains multiple nodes, each of which runs one or more HBase daemon.
-These include primary and backup Master instances, multiple Zookeeper nodes, and multiple RegionServer nodes.
+These include primary and backup Master instances, multiple ZooKeeper nodes, and multiple RegionServer nodes.
This advanced quickstart adds two more nodes to your cluster.
The architecture will be as follows:
http://git-wip-us.apache.org/repos/asf/hbase/blob/0bf065a5/src/main/asciidoc/_chapters/ops_mgt.adoc
----------------------------------------------------------------------
diff --git a/src/main/asciidoc/_chapters/ops_mgt.adoc b/src/main/asciidoc/_chapters/ops_mgt.adoc
index 6a10493..4c9c7c5 100644
--- a/src/main/asciidoc/_chapters/ops_mgt.adoc
+++ b/src/main/asciidoc/_chapters/ops_mgt.adoc
@@ -57,7 +57,7 @@ Some commands take arguments. Pass no args or -h for usage.
upgrade Upgrade hbase
master Run an HBase HMaster node
regionserver Run an HBase HRegionServer node
- zookeeper Run a Zookeeper server
+ zookeeper Run a ZooKeeper server
rest Run an HBase REST server
thrift Run the HBase Thrift server
thrift2 Run the HBase Thrift2 server
http://git-wip-us.apache.org/repos/asf/hbase/blob/0bf065a5/src/main/asciidoc/_chapters/zookeeper.adoc
----------------------------------------------------------------------
diff --git a/src/main/asciidoc/_chapters/zookeeper.adoc b/src/main/asciidoc/_chapters/zookeeper.adoc
index 565ef98..ee76d80 100644
--- a/src/main/asciidoc/_chapters/zookeeper.adoc
+++ b/src/main/asciidoc/_chapters/zookeeper.adoc
@@ -108,7 +108,7 @@ If running zookeeper 3.5+, you can ask hbase to make use of the new multi operat
.ZooKeeper Maintenance
[CAUTION]
====
-Be sure to set up the data dir cleaner described under link:http://zookeeper.apache.org/doc/r3.1.2/zookeeperAdmin.html#sc_maintenance[Zookeeper
+Be sure to set up the data dir cleaner described under link:http://zookeeper.apache.org/doc/r3.1.2/zookeeperAdmin.html#sc_maintenance[ZooKeeper
Maintenance] else you could have 'interesting' problems a couple of months in; i.e.
zookeeper could start dropping sessions if it has to run through a directory of hundreds of thousands of logs which is wont to do around leader reelection time -- a process rare but run on occasion whether because a machine is dropped or happens to hiccup.
====
@@ -120,7 +120,7 @@ To point HBase at an existing ZooKeeper cluster, one that is not managed by HBas
----
...
- # Tell HBase whether it should manage its own instance of Zookeeper or not.
+ # Tell HBase whether it should manage its own instance of ZooKeeper or not.
export HBASE_MANAGES_ZK=false
----
@@ -145,10 +145,10 @@ Additionally, see the link:http://wiki.apache.org/hadoop/ZooKeeper/FAQ#A7[ZooKee
[[zk.sasl.auth]]
== SASL Authentication with ZooKeeper
-Newer releases of Apache HBase (>= 0.92) will support connecting to a ZooKeeper Quorum that supports SASL authentication (which is available in Zookeeper versions 3.4.0 or later).
+Newer releases of Apache HBase (>= 0.92) will support connecting to a ZooKeeper Quorum that supports SASL authentication (which is available in ZooKeeper versions 3.4.0 or later).
This describes how to set up HBase to mutually authenticate with a ZooKeeper Quorum.
-ZooKeeper/HBase mutual authentication (link:https://issues.apache.org/jira/browse/HBASE-2418[HBASE-2418]) is required as part of a complete secure HBase configuration (link:https://issues.apache.org/jira/browse/HBASE-3025[HBASE-3025]). For simplicity of explication, this section ignores additional configuration required (Secure HDFS and Coprocessor configuration). It's recommended to begin with an HBase-managed Zookeeper configuration (as opposed to a standalone Zookeeper quorum) for ease of learning.
+ZooKeeper/HBase mutual authentication (link:https://issues.apache.org/jira/browse/HBASE-2418[HBASE-2418]) is required as part of a complete secure HBase configuration (link:https://issues.apache.org/jira/browse/HBASE-3025[HBASE-3025]). For simplicity of explication, this section ignores additional configuration required (Secure HDFS and Coprocessor configuration). It's recommended to begin with an HBase-managed ZooKeeper configuration (as opposed to a standalone ZooKeeper quorum) for ease of learning.
=== Operating System Prerequisites
@@ -165,7 +165,7 @@ Each user who will be an HBase client should also be given a Kerberos principal.
This principal should usually have a password assigned to it (as opposed to, as with the HBase servers, a keytab file) which only this user knows.
The client's principal's `maxrenewlife` should be set so that it can be renewed enough so that the user can complete their HBase client processes.
For example, if a user runs a long-running HBase client process that takes at most 3 days, we might create this user's principal within `kadmin` with: `addprinc -maxrenewlife 3days`.
-The Zookeeper client and server libraries manage their own ticket refreshment by running threads that wake up periodically to do the refreshment.
+The ZooKeeper client and server libraries manage their own ticket refreshment by running threads that wake up periodically to do the refreshment.
On each host that will run an HBase client (e.g. `hbase shell`), add the following file to the HBase home directory's _conf_ directory:
@@ -181,7 +181,7 @@ Client {
We'll refer to this JAAS configuration file as _$CLIENT_CONF_ below.
-=== HBase-managed Zookeeper Configuration
+=== HBase-managed ZooKeeper Configuration
On each node that will run a zookeeper, a master, or a regionserver, create a link:http://docs.oracle.com/javase/1.4.2/docs/guide/security/jgss/tutorials/LoginConfigFile.html[JAAS] configuration file in the conf directory of the node's _HBASE_HOME_ directory that looks like the following:
@@ -207,7 +207,7 @@ Client {
where the _$PATH_TO_HBASE_KEYTAB_ and _$PATH_TO_ZOOKEEPER_KEYTAB_ files are what you created above, and `$HOST` is the hostname for that node.
-The `Server` section will be used by the Zookeeper quorum server, while the `Client` section will be used by the HBase master and regionservers.
+The `Server` section will be used by the ZooKeeper quorum server, while the `Client` section will be used by the HBase master and regionservers.
The path to this file should be substituted for the text _$HBASE_SERVER_CONF_ in the _hbase-env.sh_ listing below.
The path to this file should be substituted for the text _$CLIENT_CONF_ in the _hbase-env.sh_ listing below.
@@ -255,7 +255,7 @@ Modify your _hbase-site.xml_ on each node that will run zookeeper, master or reg
</configuration>
----
-where `$ZK_NODES` is the comma-separated list of hostnames of the Zookeeper Quorum hosts.
+where `$ZK_NODES` is the comma-separated list of hostnames of the ZooKeeper Quorum hosts.
Start your hbase cluster by running one or more of the following set of commands on the appropriate hosts:
@@ -266,7 +266,7 @@ bin/hbase master start
bin/hbase regionserver start
----
-=== External Zookeeper Configuration
+=== External ZooKeeper Configuration
Add a JAAS configuration file that looks like:
@@ -326,7 +326,7 @@ Modify your _hbase-site.xml_ on each node that will run a master or regionserver
</configuration>
----
-where `$ZK_NODES` is the comma-separated list of hostnames of the Zookeeper Quorum hosts.
+where `$ZK_NODES` is the comma-separated list of hostnames of the ZooKeeper Quorum hosts.
Also on each of these hosts, create a JAAS configuration file containing:
@@ -346,7 +346,7 @@ Server {
where `$HOST` is the hostname of each Quorum host.
We will refer to the full pathname of this file as _$ZK_SERVER_CONF_ below.
-Start your Zookeepers on each Zookeeper Quorum host with:
+Start your ZooKeepers on each ZooKeeper Quorum host with:
[source,bourne]
----
@@ -362,9 +362,9 @@ bin/hbase master start
bin/hbase regionserver start
----
-=== Zookeeper Server Authentication Log Output
+=== ZooKeeper Server Authentication Log Output
-If the configuration above is successful, you should see something similar to the following in your Zookeeper server logs:
+If the configuration above is successful, you should see something similar to the following in your ZooKeeper server logs:
----
@@ -382,9 +382,9 @@ If the configuration above is successful, you should see something similar to th
11/12/05 22:43:59 INFO server.ZooKeeperServer: adding SASL authorization for authorizationID: hbase
----
-=== Zookeeper Client Authentication Log Output
+=== ZooKeeper Client Authentication Log Output
-On the Zookeeper client side (HBase master or regionserver), you should see something similar to the following:
+On the ZooKeeper client side (HBase master or regionserver), you should see something similar to the following:
----
[42/50] [abbrv] hbase git commit: HBASE-15740 Replication
source.shippedKBs metric is undercounting because it is in KB
Posted by sy...@apache.org.
HBASE-15740 Replication source.shippedKBs metric is undercounting because it is in KB
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/b75b2268
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/b75b2268
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/b75b2268
Branch: refs/heads/hbase-12439
Commit: b75b2268046d8b3e872060505260289d8ee43dd5
Parents: 541d1da
Author: Enis Soztutar <en...@apache.org>
Authored: Mon May 9 10:25:49 2016 -0700
Committer: Enis Soztutar <en...@apache.org>
Committed: Mon May 9 10:25:49 2016 -0700
----------------------------------------------------------------------
.../MetricsReplicationSourceSource.java | 5 +++-
.../MetricsReplicationGlobalSourceSource.java | 25 ++++++++++++++++++--
.../MetricsReplicationSourceSourceImpl.java | 13 ++++++++--
.../replication/regionserver/MetricsSource.java | 10 ++++----
.../regionserver/ReplicationSource.java | 2 +-
.../DummyRegionServerEndpointProtos.java | 21 ++++++++--------
6 files changed, 54 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/b75b2268/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSource.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSource.java
index 3aa01ab..271f0ac 100644
--- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSource.java
+++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSource.java
@@ -24,7 +24,10 @@ public interface MetricsReplicationSourceSource {
public static final String SOURCE_AGE_OF_LAST_SHIPPED_OP = "source.ageOfLastShippedOp";
public static final String SOURCE_SHIPPED_BATCHES = "source.shippedBatches";
+ @Deprecated
+ /** @deprecated Use SOURCE_SHIPPED_BYTES instead */
public static final String SOURCE_SHIPPED_KBS = "source.shippedKBs";
+ public static final String SOURCE_SHIPPED_BYTES = "source.shippedBytes";
public static final String SOURCE_SHIPPED_OPS = "source.shippedOps";
public static final String SOURCE_LOG_READ_IN_BYTES = "source.logReadInBytes";
@@ -41,7 +44,7 @@ public interface MetricsReplicationSourceSource {
void incrLogEditsFiltered(long size);
void incrBatchesShipped(int batches);
void incrOpsShipped(long ops);
- void incrShippedKBs(long size);
+ void incrShippedBytes(long size);
void incrLogReadInBytes(long size);
void incrLogReadInEdits(long size);
void clear();
http://git-wip-us.apache.org/repos/asf/hbase/blob/b75b2268/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationGlobalSourceSource.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationGlobalSourceSource.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationGlobalSourceSource.java
index 2526f32..476d2f7 100644
--- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationGlobalSourceSource.java
+++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationGlobalSourceSource.java
@@ -30,6 +30,8 @@ public class MetricsReplicationGlobalSourceSource implements MetricsReplicationS
private final MutableFastCounter logEditsFilteredCounter;
private final MutableFastCounter shippedBatchesCounter;
private final MutableFastCounter shippedOpsCounter;
+ private final MutableFastCounter shippedBytesCounter;
+ @Deprecated
private final MutableFastCounter shippedKBsCounter;
private final MutableFastCounter logReadInBytesCounter;
private final MutableFastCounter shippedHFilesCounter;
@@ -48,6 +50,8 @@ public class MetricsReplicationGlobalSourceSource implements MetricsReplicationS
shippedKBsCounter = rms.getMetricsRegistry().getCounter(SOURCE_SHIPPED_KBS, 0L);
+ shippedBytesCounter = rms.getMetricsRegistry().getCounter(SOURCE_SHIPPED_BYTES, 0L);
+
logReadInBytesCounter = rms.getMetricsRegistry().getCounter(SOURCE_LOG_READ_IN_BYTES, 0L);
logReadInEditsCounter = rms.getMetricsRegistry().getCounter(SOURCE_LOG_READ_IN_EDITS, 0L);
@@ -88,8 +92,25 @@ public class MetricsReplicationGlobalSourceSource implements MetricsReplicationS
shippedOpsCounter.incr(ops);
}
- @Override public void incrShippedKBs(long size) {
- shippedKBsCounter.incr(size);
+ @Override public void incrShippedBytes(long size) {
+ shippedBytesCounter.incr(size);
+ // obtained value maybe smaller than 1024. We should make sure that KB count
+ // eventually picks up even from multiple smaller updates.
+ incrementKBsCounter(shippedBytesCounter, shippedKBsCounter);
+ }
+
+ static void incrementKBsCounter(MutableFastCounter bytesCounter, MutableFastCounter kbsCounter) {
+ // Following code should be thread-safe.
+ long delta = 0;
+ while(true) {
+ long bytes = bytesCounter.value();
+ delta = (bytes / 1024) - kbsCounter.value();
+ if (delta > 0) {
+ kbsCounter.incr(delta);
+ } else {
+ break;
+ }
+ }
}
@Override public void incrLogReadInBytes(long size) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/b75b2268/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSourceImpl.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSourceImpl.java
index 03e3116..835e81c 100644
--- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSourceImpl.java
+++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSourceImpl.java
@@ -30,7 +30,9 @@ public class MetricsReplicationSourceSourceImpl implements MetricsReplicationSou
private final String logEditsFilteredKey;
private final String shippedBatchesKey;
private final String shippedOpsKey;
+ @Deprecated
private final String shippedKBsKey;
+ private final String shippedBytesKey;
private final String logReadInBytesKey;
private final String shippedHFilesKey;
private final String sizeOfHFileRefsQueueKey;
@@ -42,6 +44,7 @@ public class MetricsReplicationSourceSourceImpl implements MetricsReplicationSou
private final MutableFastCounter shippedBatchesCounter;
private final MutableFastCounter shippedOpsCounter;
private final MutableFastCounter shippedKBsCounter;
+ private final MutableFastCounter shippedBytesCounter;
private final MutableFastCounter logReadInBytesCounter;
private final MutableFastCounter shippedHFilesCounter;
private final MutableGaugeLong sizeOfHFileRefsQueueGauge;
@@ -65,6 +68,9 @@ public class MetricsReplicationSourceSourceImpl implements MetricsReplicationSou
shippedKBsKey = "source." + this.id + ".shippedKBs";
shippedKBsCounter = rms.getMetricsRegistry().getCounter(shippedKBsKey, 0L);
+ shippedBytesKey = "source." + this.id + ".shippedBytes";
+ shippedBytesCounter = rms.getMetricsRegistry().getCounter(shippedBytesKey, 0L);
+
logReadInBytesKey = "source." + this.id + ".logReadInBytes";
logReadInBytesCounter = rms.getMetricsRegistry().getCounter(logReadInBytesKey, 0L);
@@ -109,8 +115,10 @@ public class MetricsReplicationSourceSourceImpl implements MetricsReplicationSou
shippedOpsCounter.incr(ops);
}
- @Override public void incrShippedKBs(long size) {
- shippedKBsCounter.incr(size);
+ @Override public void incrShippedBytes(long size) {
+ shippedBytesCounter.incr(size);
+ MetricsReplicationGlobalSourceSource
+ .incrementKBsCounter(shippedBytesCounter, shippedKBsCounter);
}
@Override public void incrLogReadInBytes(long size) {
@@ -125,6 +133,7 @@ public class MetricsReplicationSourceSourceImpl implements MetricsReplicationSou
rms.removeMetric(shippedBatchesKey);
rms.removeMetric(shippedOpsKey);
rms.removeMetric(shippedKBsKey);
+ rms.removeMetric(shippedBytesKey);
rms.removeMetric(logReadInBytesKey);
rms.removeMetric(logReadInEditsKey);
http://git-wip-us.apache.org/repos/asf/hbase/blob/b75b2268/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsSource.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsSource.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsSource.java
index 4a044bf..b07f1d1 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsSource.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsSource.java
@@ -136,15 +136,15 @@ public class MetricsSource {
*
* @param batchSize the size of the batch that was shipped to sinks.
*/
- public void shipBatch(long batchSize, int sizeInKB) {
+ public void shipBatch(long batchSize, int sizeInBytes) {
singleSourceSource.incrBatchesShipped(1);
globalSourceSource.incrBatchesShipped(1);
singleSourceSource.incrOpsShipped(batchSize);
globalSourceSource.incrOpsShipped(batchSize);
- singleSourceSource.incrShippedKBs(sizeInKB);
- globalSourceSource.incrShippedKBs(sizeInKB);
+ singleSourceSource.incrShippedBytes(sizeInBytes);
+ globalSourceSource.incrShippedBytes(sizeInBytes);
}
/**
@@ -153,8 +153,8 @@ public class MetricsSource {
* @param batchSize the size of the batch that was shipped to sinks.
* @param hfiles total number of hfiles shipped to sinks.
*/
- public void shipBatch(long batchSize, int sizeInKB, long hfiles) {
- shipBatch(batchSize, sizeInKB);
+ public void shipBatch(long batchSize, int sizeInBytes, long hfiles) {
+ shipBatch(batchSize, sizeInBytes);
singleSourceSource.incrHFilesShipped(hfiles);
globalSourceSource.incrHFilesShipped(hfiles);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b75b2268/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
index 0f956c8..a25c80b 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
@@ -1060,7 +1060,7 @@ public class ReplicationSource extends Thread
totalReplicatedEdits.addAndGet(entries.size());
totalReplicatedOperations.addAndGet(currentNbOperations);
// FIXME check relationship between wal group and overall
- metrics.shipBatch(currentNbOperations, currentSize / 1024, currentNbHFiles);
+ metrics.shipBatch(currentNbOperations, currentSize, currentNbHFiles);
metrics.setAgeOfLastShippedOp(entries.get(entries.size() - 1).getKey().getWriteTime(),
walGroupId);
if (LOG.isTraceEnabled()) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/b75b2268/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/protobuf/generated/DummyRegionServerEndpointProtos.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/protobuf/generated/DummyRegionServerEndpointProtos.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/protobuf/generated/DummyRegionServerEndpointProtos.java
index 2ad3c59..a011b30 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/protobuf/generated/DummyRegionServerEndpointProtos.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/protobuf/generated/DummyRegionServerEndpointProtos.java
@@ -1,5 +1,5 @@
// Generated by the protocol buffer compiler. DO NOT EDIT!
-// source: hbase-server/src/test/protobuf/DummyRegionServerEndpoint.proto
+// source: DummyRegionServerEndpoint.proto
package org.apache.hadoop.hbase.coprocessor.protobuf.generated;
@@ -1185,16 +1185,15 @@ public final class DummyRegionServerEndpointProtos {
descriptor;
static {
java.lang.String[] descriptorData = {
- "\n>hbase-server/src/test/protobuf/DummyRe" +
- "gionServerEndpoint.proto\022\rhbase.test.pb\"" +
- "\016\n\014DummyRequest\"\036\n\rDummyResponse\022\r\n\005valu" +
- "e\030\001 \002(\t2\237\001\n\014DummyService\022F\n\tdummyCall\022\033." +
- "hbase.test.pb.DummyRequest\032\034.hbase.test." +
- "pb.DummyResponse\022G\n\ndummyThrow\022\033.hbase.t" +
- "est.pb.DummyRequest\032\034.hbase.test.pb.Dumm" +
- "yResponseB_\n6org.apache.hadoop.hbase.cop" +
- "rocessor.protobuf.generatedB\037DummyRegion" +
- "ServerEndpointProtos\210\001\001\240\001\001"
+ "\n\037DummyRegionServerEndpoint.proto\022\rhbase" +
+ ".test.pb\"\016\n\014DummyRequest\"\036\n\rDummyRespons" +
+ "e\022\r\n\005value\030\001 \002(\t2\237\001\n\014DummyService\022F\n\tdum" +
+ "myCall\022\033.hbase.test.pb.DummyRequest\032\034.hb" +
+ "ase.test.pb.DummyResponse\022G\n\ndummyThrow\022" +
+ "\033.hbase.test.pb.DummyRequest\032\034.hbase.tes" +
+ "t.pb.DummyResponseB_\n6org.apache.hadoop." +
+ "hbase.coprocessor.protobuf.generatedB\037Du" +
+ "mmyRegionServerEndpointProtos\210\001\001\240\001\001"
};
com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner =
new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() {
[24/50] [abbrv] hbase git commit: HBASE-15608 Remove PB references
from SnapShot related Exceptions (ram)
Posted by sy...@apache.org.
HBASE-15608 Remove PB references from SnapShot related Exceptions (ram)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/bec81b19
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/bec81b19
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/bec81b19
Branch: refs/heads/hbase-12439
Commit: bec81b1977ab3862c424cc0e0768c5d18bda7dee
Parents: 0d25291
Author: Ramkrishna <ra...@intel.com>
Authored: Fri May 6 17:06:25 2016 +0530
Committer: Ramkrishna <ra...@intel.com>
Committed: Fri May 6 17:06:25 2016 +0530
----------------------------------------------------------------------
.../apache/hadoop/hbase/client/HBaseAdmin.java | 2 +-
.../hbase/client/SnapshotDescription.java | 8 ++++
.../hadoop/hbase/protobuf/ProtobufUtil.java | 14 +++++++
.../snapshot/CorruptedSnapshotException.java | 2 +-
.../hbase/snapshot/HBaseSnapshotException.java | 2 +-
.../snapshot/RestoreSnapshotException.java | 2 +-
.../snapshot/SnapshotCreationException.java | 2 +-
.../snapshot/SnapshotDoesNotExistException.java | 2 +-
.../hbase/snapshot/SnapshotExistsException.java | 2 +-
.../procedure/CloneSnapshotProcedure.java | 3 +-
.../master/snapshot/MasterSnapshotVerifier.java | 16 +++++---
.../hbase/master/snapshot/SnapshotManager.java | 41 ++++++++++++--------
.../snapshot/SnapshotDescriptionUtils.java | 6 ++-
.../hadoop/hbase/snapshot/SnapshotManifest.java | 6 ++-
.../hbase/snapshot/SnapshotReferenceUtil.java | 15 ++++---
15 files changed, 84 insertions(+), 39 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/bec81b19/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
index 7371f03..4b0609d 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
@@ -2333,7 +2333,7 @@ public class HBaseAdmin implements Admin {
}
if (!done.getDone()) {
throw new SnapshotCreationException("Snapshot '" + snapshot.getName()
- + "' wasn't completed in expectedTime:" + max + " ms", snapshot);
+ + "' wasn't completed in expectedTime:" + max + " ms", snapshotDesc);
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/bec81b19/hbase-client/src/main/java/org/apache/hadoop/hbase/client/SnapshotDescription.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/SnapshotDescription.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/SnapshotDescription.java
index a455937..f737825 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/SnapshotDescription.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/SnapshotDescription.java
@@ -81,4 +81,12 @@ public class SnapshotDescription {
public int getVersion() {
return this.version;
}
+
+ @Override
+ public String toString() {
+ return "SnapshotDescription: name = " + ((name != null) ? name : null) + "/table = "
+ + ((table != null) ? table : null) + " /owner = " + ((owner != null) ? owner : null)
+ + (creationTime != -1 ? ("/creationtime = " + creationTime) : "")
+ + (version != -1 ? ("/version = " + version) : "");
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/bec81b19/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
index 9978c77..50a4920 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
@@ -3472,4 +3472,18 @@ public final class ProtobufUtil {
HBaseProtos.SnapshotDescription snapshot = builder.build();
return snapshot;
}
+
+ /**
+ * Convert from
+ * {@link org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription} to
+ * {@link SnapshotDescription}
+ * @param snapshotDesc the protobuf SnapshotDescription
+ * @return the POJO SnapshotDescription
+ */
+ public static SnapshotDescription
+ createSnapshotDesc(HBaseProtos.SnapshotDescription snapshotDesc) {
+ return new SnapshotDescription(snapshotDesc.getName(), snapshotDesc.getTable(),
+ createSnapshotType(snapshotDesc.getType()), snapshotDesc.getOwner(),
+ snapshotDesc.getCreationTime(), snapshotDesc.getVersion());
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/bec81b19/hbase-client/src/main/java/org/apache/hadoop/hbase/snapshot/CorruptedSnapshotException.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/snapshot/CorruptedSnapshotException.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/snapshot/CorruptedSnapshotException.java
index 2c6cc35..d0ef28d 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/snapshot/CorruptedSnapshotException.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/snapshot/CorruptedSnapshotException.java
@@ -19,7 +19,7 @@ package org.apache.hadoop.hbase.snapshot;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.classification.InterfaceStability;
-import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.client.SnapshotDescription;
/**
http://git-wip-us.apache.org/repos/asf/hbase/blob/bec81b19/hbase-client/src/main/java/org/apache/hadoop/hbase/snapshot/HBaseSnapshotException.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/snapshot/HBaseSnapshotException.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/snapshot/HBaseSnapshotException.java
index 5d03eab..2fe58ed 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/snapshot/HBaseSnapshotException.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/snapshot/HBaseSnapshotException.java
@@ -20,7 +20,7 @@ package org.apache.hadoop.hbase.snapshot;
import org.apache.hadoop.hbase.DoNotRetryIOException;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.classification.InterfaceStability;
-import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.client.SnapshotDescription;
/**
* General exception base class for when a snapshot fails
http://git-wip-us.apache.org/repos/asf/hbase/blob/bec81b19/hbase-client/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotException.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotException.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotException.java
index c36d482..70e8d3b 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotException.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotException.java
@@ -20,7 +20,7 @@ package org.apache.hadoop.hbase.snapshot;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.classification.InterfaceStability;
-import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.client.SnapshotDescription;
/**
* Thrown when a snapshot could not be restored due to a server-side error when restoring it.
http://git-wip-us.apache.org/repos/asf/hbase/blob/bec81b19/hbase-client/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotCreationException.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotCreationException.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotCreationException.java
index 2f7d2be..2738b3d 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotCreationException.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotCreationException.java
@@ -19,7 +19,7 @@ package org.apache.hadoop.hbase.snapshot;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.classification.InterfaceStability;
-import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.client.SnapshotDescription;
/**
* Thrown when a snapshot could not be created due to a server-side error when
http://git-wip-us.apache.org/repos/asf/hbase/blob/bec81b19/hbase-client/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDoesNotExistException.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDoesNotExistException.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDoesNotExistException.java
index d9ecd14..e088408 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDoesNotExistException.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDoesNotExistException.java
@@ -19,7 +19,7 @@ package org.apache.hadoop.hbase.snapshot;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.classification.InterfaceStability;
-import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.client.SnapshotDescription;
/**
http://git-wip-us.apache.org/repos/asf/hbase/blob/bec81b19/hbase-client/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotExistsException.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotExistsException.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotExistsException.java
index 172c89e..425f93a 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotExistsException.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotExistsException.java
@@ -19,7 +19,7 @@ package org.apache.hadoop.hbase.snapshot;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.classification.InterfaceStability;
-import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
+import org.apache.hadoop.hbase.client.SnapshotDescription;
/**
* Thrown when a snapshot exists but should not
http://git-wip-us.apache.org/repos/asf/hbase/blob/bec81b19/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CloneSnapshotProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CloneSnapshotProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CloneSnapshotProcedure.java
index c008b36..e91bd29 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CloneSnapshotProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CloneSnapshotProcedure.java
@@ -455,7 +455,8 @@ public class CloneSnapshotProcedure
String msg = "clone snapshot=" + ClientSnapshotDescriptionUtils.toString(snapshot) +
" failed because " + e.getMessage();
LOG.error(msg, e);
- IOException rse = new RestoreSnapshotException(msg, e, snapshot);
+ IOException rse = new RestoreSnapshotException(msg, e,
+ ProtobufUtil.createSnapshotDesc(snapshot));
// these handlers aren't futures so we need to register the error here.
monitorException.receive(new ForeignException("Master CloneSnapshotProcedure", rse));
http://git-wip-us.apache.org/repos/asf/hbase/blob/bec81b19/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/MasterSnapshotVerifier.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/MasterSnapshotVerifier.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/MasterSnapshotVerifier.java
index a28e111..40f5845 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/MasterSnapshotVerifier.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/MasterSnapshotVerifier.java
@@ -35,6 +35,7 @@ import org.apache.hadoop.hbase.client.RegionReplicaUtil;
import org.apache.hadoop.hbase.MetaTableAccessor;
import org.apache.hadoop.hbase.master.MasterServices;
import org.apache.hadoop.hbase.mob.MobUtils;
+import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
import org.apache.hadoop.hbase.snapshot.ClientSnapshotDescriptionUtils;
@@ -125,8 +126,9 @@ public final class MasterSnapshotVerifier {
private void verifySnapshotDescription(Path snapshotDir) throws CorruptedSnapshotException {
SnapshotDescription found = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
if (!this.snapshot.equals(found)) {
- throw new CorruptedSnapshotException("Snapshot read (" + found
- + ") doesn't equal snapshot we ran (" + snapshot + ").", snapshot);
+ throw new CorruptedSnapshotException(
+ "Snapshot read (" + found + ") doesn't equal snapshot we ran (" + snapshot + ").",
+ ProtobufUtil.createSnapshotDesc(snapshot));
}
}
@@ -137,12 +139,14 @@ public final class MasterSnapshotVerifier {
private void verifyTableInfo(final SnapshotManifest manifest) throws IOException {
HTableDescriptor htd = manifest.getTableDescriptor();
if (htd == null) {
- throw new CorruptedSnapshotException("Missing Table Descriptor", snapshot);
+ throw new CorruptedSnapshotException("Missing Table Descriptor",
+ ProtobufUtil.createSnapshotDesc(snapshot));
}
if (!htd.getNameAsString().equals(snapshot.getTable())) {
- throw new CorruptedSnapshotException("Invalid Table Descriptor. Expected "
- + snapshot.getTable() + " name, got " + htd.getNameAsString(), snapshot);
+ throw new CorruptedSnapshotException(
+ "Invalid Table Descriptor. Expected " + snapshot.getTable() + " name, got "
+ + htd.getNameAsString(), ProtobufUtil.createSnapshotDesc(snapshot));
}
}
@@ -216,7 +220,7 @@ public final class MasterSnapshotVerifier {
if (!region.equals(manifestRegionInfo)) {
String msg = "Manifest region info " + manifestRegionInfo +
"doesn't match expected region:" + region;
- throw new CorruptedSnapshotException(msg, snapshot);
+ throw new CorruptedSnapshotException(msg, ProtobufUtil.createSnapshotDesc(snapshot));
}
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/bec81b19/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java
index 50a1832..8ef63fb 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java
@@ -65,6 +65,7 @@ import org.apache.hadoop.hbase.procedure.ProcedureCoordinator;
import org.apache.hadoop.hbase.procedure.ProcedureCoordinatorRpcs;
import org.apache.hadoop.hbase.procedure.ZKProcedureCoordinatorRpcs;
import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
+import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameStringPair;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ProcedureDescription;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
@@ -275,7 +276,7 @@ public class SnapshotManager extends MasterProcedureManager implements Stoppable
public void deleteSnapshot(SnapshotDescription snapshot) throws SnapshotDoesNotExistException, IOException {
// check to see if it is completed
if (!isSnapshotCompleted(snapshot)) {
- throw new SnapshotDoesNotExistException(snapshot);
+ throw new SnapshotDoesNotExistException(ProtobufUtil.createSnapshotDesc(snapshot));
}
String snapshotName = snapshot.getName();
@@ -357,7 +358,7 @@ public class SnapshotManager extends MasterProcedureManager implements Stoppable
status = expected.getName() + " not found in proclist " + coordinator.getProcedureNames();
}
throw new HBaseSnapshotException("Snapshot " + ssString + " had an error. " + status, e,
- expected);
+ ProtobufUtil.createSnapshotDesc(expected));
}
// check to see if we are done
@@ -426,7 +427,7 @@ public class SnapshotManager extends MasterProcedureManager implements Stoppable
+ " because we are already running another snapshot "
+ (handler != null ? ("on the same table " +
ClientSnapshotDescriptionUtils.toString(handler.getSnapshot()))
- : "with the same name"), snapshot);
+ : "with the same name"), ProtobufUtil.createSnapshotDesc(snapshot));
}
// make sure we aren't running a restore on the same table
@@ -443,14 +444,16 @@ public class SnapshotManager extends MasterProcedureManager implements Stoppable
// recreate the working directory for the snapshot
if (!fs.mkdirs(workingDir)) {
- throw new SnapshotCreationException("Couldn't create working directory (" + workingDir
- + ") for snapshot" , snapshot);
+ throw new SnapshotCreationException(
+ "Couldn't create working directory (" + workingDir + ") for snapshot",
+ ProtobufUtil.createSnapshotDesc(snapshot));
}
} catch (HBaseSnapshotException e) {
throw e;
} catch (IOException e) {
throw new SnapshotCreationException(
- "Exception while checking to see if snapshot could be started.", e, snapshot);
+ "Exception while checking to see if snapshot could be started.", e,
+ ProtobufUtil.createSnapshotDesc(snapshot));
}
}
@@ -516,7 +519,8 @@ public class SnapshotManager extends MasterProcedureManager implements Stoppable
ClientSnapshotDescriptionUtils.toString(snapshot));
}
// fail the snapshot
- throw new SnapshotCreationException("Could not build snapshot handler", e, snapshot);
+ throw new SnapshotCreationException("Could not build snapshot handler", e,
+ ProtobufUtil.createSnapshotDesc(snapshot));
}
}
@@ -530,8 +534,9 @@ public class SnapshotManager extends MasterProcedureManager implements Stoppable
public void takeSnapshot(SnapshotDescription snapshot) throws IOException {
// check to see if we already completed the snapshot
if (isSnapshotCompleted(snapshot)) {
- throw new SnapshotExistsException("Snapshot '" + snapshot.getName()
- + "' already stored on the filesystem.", snapshot);
+ throw new SnapshotExistsException(
+ "Snapshot '" + snapshot.getName() + "' already stored on the filesystem.",
+ ProtobufUtil.createSnapshotDesc(snapshot));
}
LOG.debug("No existing snapshot, attempting snapshot...");
@@ -547,14 +552,16 @@ public class SnapshotManager extends MasterProcedureManager implements Stoppable
} catch (FileNotFoundException e) {
String msg = "Table:" + snapshot.getTable() + " info doesn't exist!";
LOG.error(msg);
- throw new SnapshotCreationException(msg, e, snapshot);
+ throw new SnapshotCreationException(msg, e, ProtobufUtil.createSnapshotDesc(snapshot));
} catch (IOException e) {
- throw new SnapshotCreationException("Error while geting table description for table "
- + snapshot.getTable(), e, snapshot);
+ throw new SnapshotCreationException(
+ "Error while geting table description for table " + snapshot.getTable(), e,
+ ProtobufUtil.createSnapshotDesc(snapshot));
}
if (desc == null) {
- throw new SnapshotCreationException("Table '" + snapshot.getTable()
- + "' doesn't exist, can't take snapshot.", snapshot);
+ throw new SnapshotCreationException(
+ "Table '" + snapshot.getTable() + "' doesn't exist, can't take snapshot.",
+ ProtobufUtil.createSnapshotDesc(snapshot));
}
SnapshotDescription.Builder builder = snapshot.toBuilder();
// if not specified, set the snapshot format
@@ -593,7 +600,8 @@ public class SnapshotManager extends MasterProcedureManager implements Stoppable
+ "', isn't open or closed, we don't know what to do!");
TablePartiallyOpenException tpoe = new TablePartiallyOpenException(snapshot.getTable()
+ " isn't fully open.");
- throw new SnapshotCreationException("Table is not entirely open or closed", tpoe, snapshot);
+ throw new SnapshotCreationException("Table is not entirely open or closed", tpoe,
+ ProtobufUtil.createSnapshotDesc(snapshot));
}
// call post coproc hook
@@ -750,7 +758,8 @@ public class SnapshotManager extends MasterProcedureManager implements Stoppable
// check if the snapshot exists
if (!fs.exists(snapshotDir)) {
LOG.error("A Snapshot named '" + reqSnapshot.getName() + "' does not exist.");
- throw new SnapshotDoesNotExistException(reqSnapshot);
+ throw new SnapshotDoesNotExistException(
+ ProtobufUtil.createSnapshotDesc(reqSnapshot));
}
// Get snapshot info from file system. The reqSnapshot is a "fake" snapshotInfo with
http://git-wip-us.apache.org/repos/asf/hbase/blob/bec81b19/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDescriptionUtils.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDescriptionUtils.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDescriptionUtils.java
index b6b608c..560a42b 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDescriptionUtils.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDescriptionUtils.java
@@ -29,6 +29,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
import org.apache.hadoop.hbase.security.User;
@@ -333,8 +334,9 @@ public final class SnapshotDescriptionUtils {
LOG.debug("Snapshot is done, just moving the snapshot from " + workingDir + " to "
+ finishedDir);
if (!fs.rename(workingDir, finishedDir)) {
- throw new SnapshotCreationException("Failed to move working directory(" + workingDir
- + ") to completed directory(" + finishedDir + ").", snapshot);
+ throw new SnapshotCreationException(
+ "Failed to move working directory(" + workingDir + ") to completed directory("
+ + finishedDir + ").", ProtobufUtil.createSnapshotDesc(snapshot));
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/bec81b19/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java
index df0c348..9d7af50 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java
@@ -158,7 +158,8 @@ public final class SnapshotManifest {
case SnapshotManifestV2.DESCRIPTOR_VERSION:
return new SnapshotManifestV2.ManifestBuilder(conf, fs, workingDir);
default:
- throw new CorruptedSnapshotException("Invalid Snapshot version: "+ desc.getVersion(), desc);
+ throw new CorruptedSnapshotException("Invalid Snapshot version: " + desc.getVersion(),
+ ProtobufUtil.createSnapshotDesc(desc));
}
}
@@ -391,7 +392,8 @@ public final class SnapshotManifest {
break;
}
default:
- throw new CorruptedSnapshotException("Invalid Snapshot version: "+ desc.getVersion(), desc);
+ throw new CorruptedSnapshotException("Invalid Snapshot version: " + desc.getVersion(),
+ ProtobufUtil.createSnapshotDesc(desc));
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/bec81b19/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotReferenceUtil.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotReferenceUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotReferenceUtil.java
index 0532ba7..ba8087b 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotReferenceUtil.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotReferenceUtil.java
@@ -40,6 +40,7 @@ import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.io.HFileLink;
import org.apache.hadoop.hbase.mob.MobUtils;
+import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
@@ -209,7 +210,8 @@ public final class SnapshotReferenceUtil {
throw new InterruptedIOException(e.getMessage());
} catch (ExecutionException e) {
if (e.getCause() instanceof CorruptedSnapshotException) {
- throw new CorruptedSnapshotException(e.getCause().getMessage(), snapshotDesc);
+ throw new CorruptedSnapshotException(e.getCause().getMessage(),
+ ProtobufUtil.createSnapshotDesc(snapshotDesc));
} else {
IOException ex = new IOException();
ex.initCause(e.getCause());
@@ -248,8 +250,9 @@ public final class SnapshotReferenceUtil {
String refRegion = refPath.getParent().getParent().getName();
refPath = HFileLink.createPath(table, refRegion, family, refPath.getName());
if (!HFileLink.buildFromHFileLinkPattern(conf, refPath).exists(fs)) {
- throw new CorruptedSnapshotException("Missing parent hfile for: " + fileName +
- " path=" + refPath, snapshot);
+ throw new CorruptedSnapshotException(
+ "Missing parent hfile for: " + fileName + " path=" + refPath,
+ ProtobufUtil.createSnapshotDesc(snapshot));
}
if (storeFile.hasReference()) {
@@ -285,14 +288,16 @@ public final class SnapshotReferenceUtil {
String msg = "hfile: " + fileName + " size does not match with the expected one. " +
" found=" + fstat.getLen() + " expected=" + storeFile.getFileSize();
LOG.error(msg);
- throw new CorruptedSnapshotException(msg, snapshot);
+ throw new CorruptedSnapshotException(msg,
+ ProtobufUtil.createSnapshotDesc(snapshot));
}
} catch (FileNotFoundException e) {
String msg = "Can't find hfile: " + fileName + " in the real (" +
link.getOriginPath() + ") or archive (" + link.getArchivePath()
+ ") directory for the primary table.";
LOG.error(msg);
- throw new CorruptedSnapshotException(msg, snapshot);
+ throw new CorruptedSnapshotException(msg,
+ ProtobufUtil.createSnapshotDesc(snapshot));
}
}
[33/50] [abbrv] hbase git commit: HBASE-15795 Cleanup all classes in
package org.apache.hadoop.hbase.ipc for code style
Posted by sy...@apache.org.
HBASE-15795 Cleanup all classes in package org.apache.hadoop.hbase.ipc
for code style
Signed-off-by: stack <st...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/e71ae601
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/e71ae601
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/e71ae601
Branch: refs/heads/hbase-12439
Commit: e71ae601f7ce40c7f917141ff7a4122f6205987b
Parents: 99e2deb
Author: Jurriaan Mous <ju...@jurmo.us>
Authored: Sat May 7 14:53:59 2016 +0200
Committer: stack <st...@apache.org>
Committed: Sat May 7 09:55:07 2016 -0700
----------------------------------------------------------------------
.../hadoop/hbase/ipc/AbstractRpcClient.java | 14 +-
.../hbase/ipc/AsyncServerResponseHandler.java | 14 +-
.../hadoop/hbase/ipc/BlockingRpcCallback.java | 4 +-
.../java/org/apache/hadoop/hbase/ipc/Call.java | 5 +-
.../hadoop/hbase/ipc/CallTimeoutException.java | 4 +-
.../apache/hadoop/hbase/ipc/ConnectionId.java | 14 +-
.../hadoop/hbase/ipc/CoprocessorRpcUtils.java | 2 +-
.../DelegatingPayloadCarryingRpcController.java | 2 +-
.../apache/hadoop/hbase/ipc/FailedServers.java | 13 +-
.../org/apache/hadoop/hbase/ipc/IPCUtil.java | 155 ++++++++-----------
.../hbase/ipc/MasterCoprocessorRpcChannel.java | 9 +-
.../hbase/ipc/RegionCoprocessorRpcChannel.java | 17 +-
.../ipc/RegionServerCoprocessorRpcChannel.java | 17 +-
.../org/apache/hadoop/hbase/ipc/RpcClient.java | 7 +-
.../hadoop/hbase/ipc/RpcClientFactory.java | 7 +-
.../apache/hadoop/hbase/ipc/RpcClientImpl.java | 135 +++++++++-------
.../hadoop/hbase/ipc/ServerRpcController.java | 8 +-
.../hbase/ipc/TimeLimitedRpcController.java | 10 +-
18 files changed, 224 insertions(+), 213 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/e71ae601/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AbstractRpcClient.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AbstractRpcClient.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AbstractRpcClient.java
index ec6332a..c091d1d 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AbstractRpcClient.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AbstractRpcClient.java
@@ -141,7 +141,9 @@ public abstract class AbstractRpcClient implements RpcClient {
// For NO CODEC, "hbase.client.rpc.codec" must be configured with empty string AND
// "hbase.client.default.rpc.codec" also -- because default is to do cell block encoding.
String className = conf.get(HConstants.RPC_CODEC_CONF_KEY, getDefaultCodec(this.conf));
- if (className == null || className.length() == 0) return null;
+ if (className == null || className.length() == 0) {
+ return null;
+ }
try {
return (Codec)Class.forName(className).newInstance();
} catch (Exception e) {
@@ -161,9 +163,11 @@ public abstract class AbstractRpcClient implements RpcClient {
*/
private static CompressionCodec getCompressor(final Configuration conf) {
String className = conf.get("hbase.client.rpc.compressor", null);
- if (className == null || className.isEmpty()) return null;
+ if (className == null || className.isEmpty()) {
+ return null;
+ }
try {
- return (CompressionCodec)Class.forName(className).newInstance();
+ return (CompressionCodec)Class.forName(className).newInstance();
} catch (Exception e) {
throw new RuntimeException("Failed getting compressor " + className, e);
}
@@ -252,8 +256,8 @@ public abstract class AbstractRpcClient implements RpcClient {
* will be a
* new Connection each time.
* @return A pair with the Message response and the Cell data (if any).
- * @throws InterruptedException
- * @throws java.io.IOException
+ * @throws InterruptedException if call is interrupted
+ * @throws java.io.IOException if transport failed
*/
protected abstract Pair<Message, CellScanner> call(PayloadCarryingRpcController pcrc,
Descriptors.MethodDescriptor md, Message param, Message returnType, User ticket,
http://git-wip-us.apache.org/repos/asf/hbase/blob/e71ae601/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncServerResponseHandler.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncServerResponseHandler.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncServerResponseHandler.java
index a0928b1..6fcca34 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncServerResponseHandler.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncServerResponseHandler.java
@@ -17,6 +17,13 @@
*/
package org.apache.hadoop.hbase.ipc;
+import com.google.protobuf.Message;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.ByteBufInputStream;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.channel.SimpleChannelInboundHandler;
+
import java.io.IOException;
import org.apache.hadoop.hbase.CellScanner;
@@ -25,13 +32,6 @@ import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.protobuf.generated.RPCProtos;
import org.apache.hadoop.ipc.RemoteException;
-import com.google.protobuf.Message;
-
-import io.netty.buffer.ByteBuf;
-import io.netty.buffer.ByteBufInputStream;
-import io.netty.channel.ChannelHandlerContext;
-import io.netty.channel.SimpleChannelInboundHandler;
-
/**
* Handles Hbase responses
*/
http://git-wip-us.apache.org/repos/asf/hbase/blob/e71ae601/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/BlockingRpcCallback.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/BlockingRpcCallback.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/BlockingRpcCallback.java
index 3aa59c7..0475e58 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/BlockingRpcCallback.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/BlockingRpcCallback.java
@@ -18,13 +18,13 @@
package org.apache.hadoop.hbase.ipc;
+import com.google.protobuf.RpcCallback;
+
import java.io.IOException;
import java.io.InterruptedIOException;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import com.google.protobuf.RpcCallback;
-
/**
* Simple {@link RpcCallback} implementation providing a
* {@link java.util.concurrent.Future}-like {@link BlockingRpcCallback#get()} method, which
http://git-wip-us.apache.org/repos/asf/hbase/blob/e71ae601/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/Call.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/Call.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/Call.java
index 5f90837..73bc0e2 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/Call.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/Call.java
@@ -19,14 +19,15 @@ package org.apache.hadoop.hbase.ipc;
import com.google.protobuf.Descriptors;
import com.google.protobuf.Message;
+
+import java.io.IOException;
+
import org.apache.hadoop.hbase.CellScanner;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.client.MetricsConnection;
import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
-import java.io.IOException;
-
/** A call waiting for a value. */
@InterfaceAudience.Private
public class Call {
http://git-wip-us.apache.org/repos/asf/hbase/blob/e71ae601/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/CallTimeoutException.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/CallTimeoutException.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/CallTimeoutException.java
index a81e5d1..1e31f72 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/CallTimeoutException.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/CallTimeoutException.java
@@ -17,11 +17,11 @@
*/
package org.apache.hadoop.hbase.ipc;
+import java.io.IOException;
+
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.classification.InterfaceStability;
-import java.io.IOException;
-
/**
* Client-side call timeout
*/
http://git-wip-us.apache.org/repos/asf/hbase/blob/e71ae601/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/ConnectionId.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/ConnectionId.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/ConnectionId.java
index 33fc880..08f8171 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/ConnectionId.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/ConnectionId.java
@@ -17,11 +17,11 @@
*/
package org.apache.hadoop.hbase.ipc;
+import java.net.InetSocketAddress;
+
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.security.User;
-import java.net.InetSocketAddress;
-
/**
* This class holds the address and the user ticket, etc. The client connections
* to servers are uniquely identified by <remoteAddress, ticket, serviceName>
@@ -58,14 +58,14 @@ public class ConnectionId {
@Override
public boolean equals(Object obj) {
- if (obj instanceof ConnectionId) {
- ConnectionId id = (ConnectionId) obj;
- return address.equals(id.address) &&
+ if (obj instanceof ConnectionId) {
+ ConnectionId id = (ConnectionId) obj;
+ return address.equals(id.address) &&
((ticket != null && ticket.equals(id.ticket)) ||
(ticket == id.ticket)) &&
this.serviceName == id.serviceName;
- }
- return false;
+ }
+ return false;
}
@Override // simply use the default Object#hashcode() ?
http://git-wip-us.apache.org/repos/asf/hbase/blob/e71ae601/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/CoprocessorRpcUtils.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/CoprocessorRpcUtils.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/CoprocessorRpcUtils.java
index 60044d4..63ff3e8 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/CoprocessorRpcUtils.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/CoprocessorRpcUtils.java
@@ -34,7 +34,7 @@ public final class CoprocessorRpcUtils {
* We assume that all HBase protobuf services share a common package name
* (defined in the .proto files).
*/
- private static String hbaseServicePackage;
+ private static final String hbaseServicePackage;
static {
Descriptors.ServiceDescriptor clientService = ClientProtos.ClientService.getDescriptor();
hbaseServicePackage = clientService.getFullName()
http://git-wip-us.apache.org/repos/asf/hbase/blob/e71ae601/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/DelegatingPayloadCarryingRpcController.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/DelegatingPayloadCarryingRpcController.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/DelegatingPayloadCarryingRpcController.java
index ad4224b..aafd492 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/DelegatingPayloadCarryingRpcController.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/DelegatingPayloadCarryingRpcController.java
@@ -27,7 +27,7 @@ import org.apache.hadoop.hbase.classification.InterfaceAudience;
*/
@InterfaceAudience.Private
public class DelegatingPayloadCarryingRpcController extends PayloadCarryingRpcController {
- private PayloadCarryingRpcController delegate;
+ private final PayloadCarryingRpcController delegate;
public DelegatingPayloadCarryingRpcController(PayloadCarryingRpcController delegate) {
this.delegate = delegate;
http://git-wip-us.apache.org/repos/asf/hbase/blob/e71ae601/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/FailedServers.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/FailedServers.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/FailedServers.java
index 16ec16c..868cdc6 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/FailedServers.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/FailedServers.java
@@ -17,22 +17,21 @@
*/
package org.apache.hadoop.hbase.ipc;
+import java.net.InetSocketAddress;
+import java.util.Iterator;
+import java.util.LinkedList;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.Pair;
-import java.net.InetSocketAddress;
-import java.util.Iterator;
-import java.util.LinkedList;
-
/**
* A class to manage a list of servers that failed recently.
*/
@InterfaceAudience.Private
public class FailedServers {
- private final LinkedList<Pair<Long, String>> failedServers = new
- LinkedList<Pair<Long, String>>();
+ private final LinkedList<Pair<Long, String>> failedServers = new LinkedList<>();
private final int recheckServersTimeout;
public FailedServers(Configuration conf) {
@@ -45,7 +44,7 @@ public class FailedServers {
*/
public synchronized void addToFailedServers(InetSocketAddress address) {
final long expiry = EnvironmentEdgeManager.currentTime() + recheckServersTimeout;
- failedServers.addFirst(new Pair<Long, String>(expiry, address.toString()));
+ failedServers.addFirst(new Pair<>(expiry, address.toString()));
}
/**
http://git-wip-us.apache.org/repos/asf/hbase/blob/e71ae601/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/IPCUtil.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/IPCUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/IPCUtil.java
index d98d81d..a87bc8d 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/IPCUtil.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/IPCUtil.java
@@ -17,21 +17,22 @@
*/
package org.apache.hadoop.hbase.ipc;
-import java.io.DataInput;
+import com.google.common.base.Preconditions;
+import com.google.protobuf.CodedOutputStream;
+import com.google.protobuf.Message;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.BufferOverflowException;
import java.nio.ByteBuffer;
-
import org.apache.commons.io.IOUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.CellScanner;
import org.apache.hadoop.hbase.DoNotRetryIOException;
import org.apache.hadoop.hbase.HBaseIOException;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.codec.Codec;
import org.apache.hadoop.hbase.io.BoundedByteBufferPool;
import org.apache.hadoop.hbase.io.ByteBufferInputStream;
@@ -45,10 +46,6 @@ import org.apache.hadoop.io.compress.CompressionInputStream;
import org.apache.hadoop.io.compress.Compressor;
import org.apache.hadoop.io.compress.Decompressor;
-import com.google.common.base.Preconditions;
-import com.google.protobuf.CodedOutputStream;
-import com.google.protobuf.Message;
-
/**
* Utility to help ipc'ing.
*/
@@ -83,13 +80,13 @@ public class IPCUtil {
/**
* Puts CellScanner Cells into a cell block using passed in <code>codec</code> and/or
* <code>compressor</code>.
- * @param codec
- * @param compressor
- * @param cellScanner
+ * @param codec to use for encoding
+ * @param compressor to use for encoding
+ * @param cellScanner to encode
* @return Null or byte buffer filled with a cellblock filled with passed-in Cells encoded using
- * passed in <code>codec</code> and/or <code>compressor</code>; the returned buffer has been
- * flipped and is ready for reading. Use limit to find total size.
- * @throws IOException
+ * passed in <code>codec</code> and/or <code>compressor</code>; the returned buffer has been
+ * flipped and is ready for reading. Use limit to find total size.
+ * @throws IOException if encoding the cells fail
*/
@SuppressWarnings("resource")
public ByteBuffer buildCellBlock(final Codec codec, final CompressionCodec compressor,
@@ -101,26 +98,30 @@ public class IPCUtil {
/**
* Puts CellScanner Cells into a cell block using passed in <code>codec</code> and/or
* <code>compressor</code>.
- * @param codec
- * @param compressor
- * @param cellScanner
+ * @param codec to use for encoding
+ * @param compressor to use for encoding
+ * @param cellScanner to encode
* @param pool Pool of ByteBuffers to make use of. Can be null and then we'll allocate
- * our own ByteBuffer.
+ * our own ByteBuffer.
* @return Null or byte buffer filled with a cellblock filled with passed-in Cells encoded using
- * passed in <code>codec</code> and/or <code>compressor</code>; the returned buffer has been
- * flipped and is ready for reading. Use limit to find total size. If <code>pool</code> was not
- * null, then this returned ByteBuffer came from there and should be returned to the pool when
- * done.
- * @throws IOException
+ * passed in <code>codec</code> and/or <code>compressor</code>; the returned buffer has been
+ * flipped and is ready for reading. Use limit to find total size. If <code>pool</code> was not
+ * null, then this returned ByteBuffer came from there and should be returned to the pool when
+ * done.
+ * @throws IOException if encoding the cells fail
*/
@SuppressWarnings("resource")
public ByteBuffer buildCellBlock(final Codec codec, final CompressionCodec compressor,
final CellScanner cellScanner, final BoundedByteBufferPool pool)
throws IOException {
- if (cellScanner == null) return null;
- if (codec == null) throw new CellScannerButNoCodecException();
+ if (cellScanner == null) {
+ return null;
+ }
+ if (codec == null) {
+ throw new CellScannerButNoCodecException();
+ }
int bufferSize = this.cellBlockBuildingInitialBufferSize;
- ByteBufferOutputStream baos = null;
+ ByteBufferOutputStream baos;
if (pool != null) {
ByteBuffer bb = pool.getBuffer();
bufferSize = bb.capacity();
@@ -137,15 +138,17 @@ public class IPCUtil {
}
baos = new ByteBufferOutputStream(bufferSize);
}
- OutputStream os = baos;
Compressor poolCompressor = null;
- try {
+ try (OutputStream os = baos) {
+ OutputStream os2Compress = os;
if (compressor != null) {
- if (compressor instanceof Configurable) ((Configurable)compressor).setConf(this.conf);
+ if (compressor instanceof Configurable) {
+ ((Configurable) compressor).setConf(this.conf);
+ }
poolCompressor = CodecPool.getCompressor(compressor);
- os = compressor.createOutputStream(os, poolCompressor);
+ os2Compress = compressor.createOutputStream(os, poolCompressor);
}
- Codec.Encoder encoder = codec.getEncoder(os);
+ Codec.Encoder encoder = codec.getEncoder(os2Compress);
int count = 0;
while (cellScanner.advance()) {
encoder.write(cellScanner.current());
@@ -154,12 +157,15 @@ public class IPCUtil {
encoder.flush();
// If no cells, don't mess around. Just return null (could be a bunch of existence checking
// gets or something -- stuff that does not return a cell).
- if (count == 0) return null;
+ if (count == 0) {
+ return null;
+ }
} catch (BufferOverflowException e) {
throw new DoNotRetryIOException(e);
} finally {
- os.close();
- if (poolCompressor != null) CodecPool.returnCompressor(poolCompressor);
+ if (poolCompressor != null) {
+ CodecPool.returnCompressor(poolCompressor);
+ }
}
if (LOG.isTraceEnabled()) {
if (bufferSize < baos.size()) {
@@ -171,10 +177,10 @@ public class IPCUtil {
}
/**
- * @param codec
- * @param cellBlock
+ * @param codec to use for cellblock
+ * @param cellBlock to encode
* @return CellScanner to work against the content of <code>cellBlock</code>
- * @throws IOException
+ * @throws IOException if encoding fails
*/
public CellScanner createCellScanner(final Codec codec, final CompressionCodec compressor,
final byte[] cellBlock) throws IOException {
@@ -191,12 +197,12 @@ public class IPCUtil {
}
/**
- * @param codec
+ * @param codec to use for cellblock
* @param cellBlock ByteBuffer containing the cells written by the Codec. The buffer should be
- * position()'ed at the start of the cell block and limit()'ed at the end.
+ * position()'ed at the start of the cell block and limit()'ed at the end.
* @return CellScanner to work against the content of <code>cellBlock</code>.
- * All cells created out of the CellScanner will share the same ByteBuffer being passed.
- * @throws IOException
+ * All cells created out of the CellScanner will share the same ByteBuffer being passed.
+ * @throws IOException if cell encoding fails
*/
public CellScanner createCellScannerReusingBuffers(final Codec codec,
final CompressionCodec compressor, ByteBuffer cellBlock) throws IOException {
@@ -212,11 +218,13 @@ public class IPCUtil {
private ByteBuffer decompress(CompressionCodec compressor, ByteBuffer cellBlock)
throws IOException {
// GZIPCodec fails w/ NPE if no configuration.
- if (compressor instanceof Configurable) ((Configurable) compressor).setConf(this.conf);
+ if (compressor instanceof Configurable) {
+ ((Configurable) compressor).setConf(this.conf);
+ }
Decompressor poolDecompressor = CodecPool.getDecompressor(compressor);
CompressionInputStream cis = compressor.createInputStream(new ByteBufferInputStream(cellBlock),
poolDecompressor);
- ByteBufferOutputStream bbos = null;
+ ByteBufferOutputStream bbos;
try {
// TODO: This is ugly. The buffer will be resized on us if we guess wrong.
// TODO: Reuse buffers.
@@ -232,33 +240,13 @@ public class IPCUtil {
}
/**
- * @param m Message to serialize delimited; i.e. w/ a vint of its size preceeding its
- * serialization.
- * @return The passed in Message serialized with delimiter. Return null if <code>m</code> is null
- * @throws IOException
- */
- public static ByteBuffer getDelimitedMessageAsByteBuffer(final Message m) throws IOException {
- if (m == null) return null;
- int serializedSize = m.getSerializedSize();
- int vintSize = CodedOutputStream.computeRawVarint32Size(serializedSize);
- byte [] buffer = new byte[serializedSize + vintSize];
- // Passing in a byte array saves COS creating a buffer which it does when using streams.
- CodedOutputStream cos = CodedOutputStream.newInstance(buffer);
- // This will write out the vint preamble and the message serialized.
- cos.writeMessageNoTag(m);
- cos.flush();
- cos.checkNoSpaceLeft();
- return ByteBuffer.wrap(buffer);
- }
-
- /**
* Write out header, param, and cell block if there is one.
- * @param dos
- * @param header
- * @param param
- * @param cellBlock
+ * @param dos Stream to write into
+ * @param header to write
+ * @param param to write
+ * @param cellBlock to write
* @return Total number of bytes written.
- * @throws IOException
+ * @throws IOException if write action fails
*/
public static int write(final OutputStream dos, final Message header, final Message param,
final ByteBuffer cellBlock)
@@ -267,7 +255,9 @@ public class IPCUtil {
// swoop. This is dictated by how the server is currently written. Server needs to change
// if we are to be able to write without the length prefixing.
int totalSize = IPCUtil.getTotalSizeWhenWrittenDelimited(header, param);
- if (cellBlock != null) totalSize += cellBlock.remaining();
+ if (cellBlock != null) {
+ totalSize += cellBlock.remaining();
+ }
return write(dos, header, param, cellBlock, totalSize);
}
@@ -278,36 +268,25 @@ public class IPCUtil {
dos.write(Bytes.toBytes(totalSize));
// This allocates a buffer that is the size of the message internally.
header.writeDelimitedTo(dos);
- if (param != null) param.writeDelimitedTo(dos);
- if (cellBlock != null) dos.write(cellBlock.array(), 0, cellBlock.remaining());
+ if (param != null) {
+ param.writeDelimitedTo(dos);
+ }
+ if (cellBlock != null) {
+ dos.write(cellBlock.array(), 0, cellBlock.remaining());
+ }
dos.flush();
return totalSize;
}
/**
- * Read in chunks of 8K (HBASE-7239)
- * @param in
- * @param dest
- * @param offset
- * @param len
- * @throws IOException
- */
- public static void readChunked(final DataInput in, byte[] dest, int offset, int len)
- throws IOException {
- int maxRead = 8192;
-
- for (; offset < len; offset += maxRead) {
- in.readFully(dest, offset, Math.min(len - offset, maxRead));
- }
- }
-
- /**
* @return Size on the wire when the two messages are written with writeDelimitedTo
*/
public static int getTotalSizeWhenWrittenDelimited(Message ... messages) {
int totalSize = 0;
for (Message m: messages) {
- if (m == null) continue;
+ if (m == null) {
+ continue;
+ }
totalSize += m.getSerializedSize();
totalSize += CodedOutputStream.computeRawVarint32Size(m.getSerializedSize());
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/e71ae601/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/MasterCoprocessorRpcChannel.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/MasterCoprocessorRpcChannel.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/MasterCoprocessorRpcChannel.java
index 68798ed..6fae5cb 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/MasterCoprocessorRpcChannel.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/MasterCoprocessorRpcChannel.java
@@ -15,9 +15,12 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.hadoop.hbase.ipc;
+import com.google.protobuf.Descriptors;
+import com.google.protobuf.Message;
+import com.google.protobuf.RpcController;
+
import java.io.IOException;
import org.apache.commons.logging.Log;
@@ -29,10 +32,6 @@ import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.CoprocessorServiceResponse;
-import com.google.protobuf.Descriptors;
-import com.google.protobuf.Message;
-import com.google.protobuf.RpcController;
-
/**
* Provides clients with an RPC connection to call coprocessor endpoint {@link com.google.protobuf.Service}s
* against the active master. An instance of this class may be obtained
http://git-wip-us.apache.org/repos/asf/hbase/blob/e71ae601/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RegionCoprocessorRpcChannel.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RegionCoprocessorRpcChannel.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RegionCoprocessorRpcChannel.java
index 4d3a453..55d6375 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RegionCoprocessorRpcChannel.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RegionCoprocessorRpcChannel.java
@@ -15,9 +15,12 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.hadoop.hbase.ipc;
+import com.google.protobuf.Descriptors;
+import com.google.protobuf.Message;
+import com.google.protobuf.RpcController;
+
import java.io.IOException;
import org.apache.commons.logging.Log;
@@ -32,10 +35,6 @@ import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.CoprocessorServiceResponse;
import org.apache.hadoop.hbase.util.Bytes;
-import com.google.protobuf.Descriptors;
-import com.google.protobuf.Message;
-import com.google.protobuf.RpcController;
-
/**
* Provides clients with an RPC connection to call coprocessor endpoint {@link com.google.protobuf.Service}s
* against a given table region. An instance of this class may be obtained
@@ -52,10 +51,10 @@ public class RegionCoprocessorRpcChannel extends SyncCoprocessorRpcChannel {
private final TableName table;
private final byte[] row;
private byte[] lastRegion;
- private int operationTimeout;
+ private final int operationTimeout;
- private RpcRetryingCallerFactory rpcCallerFactory;
- private RpcControllerFactory rpcControllerFactory;
+ private final RpcRetryingCallerFactory rpcCallerFactory;
+ private final RpcControllerFactory rpcControllerFactory;
/**
* Constructor
@@ -105,7 +104,7 @@ public class RegionCoprocessorRpcChannel extends SyncCoprocessorRpcChannel {
};
CoprocessorServiceResponse result = rpcCallerFactory.<CoprocessorServiceResponse> newCaller()
.callWithRetries(callable, operationTimeout);
- Message response = null;
+ Message response;
if (result.getValue().hasValue()) {
Message.Builder builder = responsePrototype.newBuilderForType();
ProtobufUtil.mergeFrom(builder, result.getValue().getValue());
http://git-wip-us.apache.org/repos/asf/hbase/blob/e71ae601/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RegionServerCoprocessorRpcChannel.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RegionServerCoprocessorRpcChannel.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RegionServerCoprocessorRpcChannel.java
index 6ffb579..c23d36c 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RegionServerCoprocessorRpcChannel.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RegionServerCoprocessorRpcChannel.java
@@ -11,28 +11,29 @@
package org.apache.hadoop.hbase.ipc;
+import com.google.protobuf.Descriptors;
+import com.google.protobuf.Message;
+import com.google.protobuf.RpcController;
+
import java.io.IOException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.client.ClusterConnection;
import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.CoprocessorServiceResponse;
-import com.google.protobuf.Descriptors;
-import com.google.protobuf.Message;
-import com.google.protobuf.RpcController;
/**
* Provides clients with an RPC connection to call coprocessor endpoint
* {@link com.google.protobuf.Service}s against a given region server. An instance of this class may
- * be obtained by calling {@link org.apache.hadoop.hbase.client.HBaseAdmin#coprocessorService(ServerName)},
- * but should normally only be used in creating a new {@link com.google.protobuf.Service} stub to
- * call the endpoint methods.
+ * be obtained by calling {@link org.apache.hadoop.hbase.client.HBaseAdmin#
+ * coprocessorService(ServerName)}, but should normally only be used in creating a new
+ * {@link com.google.protobuf.Service} stub to call the endpoint methods.
* @see org.apache.hadoop.hbase.client.HBaseAdmin#coprocessorService(ServerName)
*/
@InterfaceAudience.Private
@@ -59,7 +60,7 @@ public class RegionServerCoprocessorRpcChannel extends SyncCoprocessorRpcChannel
// TODO: Are we retrying here? Does not seem so. We should use RetryingRpcCaller
CoprocessorServiceResponse result =
ProtobufUtil.execRegionServerService(controller, connection.getClient(serverName), call);
- Message response = null;
+ Message response;
if (result.getValue().hasValue()) {
Message.Builder builder = responsePrototype.newBuilderForType();
ProtobufUtil.mergeFrom(builder, result.getValue().getValue());
http://git-wip-us.apache.org/repos/asf/hbase/blob/e71ae601/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcClient.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcClient.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcClient.java
index c3f4d02..f77f1ec 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcClient.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcClient.java
@@ -18,13 +18,14 @@
package org.apache.hadoop.hbase.ipc;
import com.google.protobuf.BlockingRpcChannel;
-import org.apache.hadoop.hbase.ServerName;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.security.User;
import java.io.Closeable;
import java.io.IOException;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.security.User;
+
/**
* Interface for RpcClient implementations so ConnectionManager can handle it.
*/
http://git-wip-us.apache.org/repos/asf/hbase/blob/e71ae601/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcClientFactory.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcClientFactory.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcClientFactory.java
index 822daca..07254e9 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcClientFactory.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcClientFactory.java
@@ -18,13 +18,14 @@
package org.apache.hadoop.hbase.ipc;
import com.google.common.annotations.VisibleForTesting;
+
+import java.net.SocketAddress;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.client.MetricsConnection;
import org.apache.hadoop.hbase.util.ReflectionUtils;
-import java.net.SocketAddress;
-
/**
* Factory to create a {@link org.apache.hadoop.hbase.ipc.RpcClient}
*/
@@ -74,7 +75,7 @@ public final class RpcClientFactory {
return ReflectionUtils.instantiateWithCustomCtor(
rpcClientClass,
new Class[] { Configuration.class, String.class, SocketAddress.class,
- MetricsConnection.class },
+ MetricsConnection.class },
new Object[] { conf, clusterId, localAddr, metrics }
);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/e71ae601/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcClientImpl.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcClientImpl.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcClientImpl.java
index 83d4adf..7b2500c 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcClientImpl.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcClientImpl.java
@@ -1,5 +1,4 @@
/**
- *
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -16,9 +15,14 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.hadoop.hbase.ipc;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.protobuf.Descriptors.MethodDescriptor;
+import com.google.protobuf.Message;
+import com.google.protobuf.Message.Builder;
+import com.google.protobuf.RpcCallback;
+
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.Closeable;
@@ -96,12 +100,6 @@ import org.apache.htrace.Span;
import org.apache.htrace.Trace;
import org.apache.htrace.TraceScope;
-import com.google.common.annotations.VisibleForTesting;
-import com.google.protobuf.Descriptors.MethodDescriptor;
-import com.google.protobuf.Message;
-import com.google.protobuf.Message.Builder;
-import com.google.protobuf.RpcCallback;
-
/**
* Does RPC against a cluster. Manages connections per regionserver in the cluster.
* <p>See HBaseServer
@@ -160,25 +158,25 @@ public class RpcClientImpl extends AbstractRpcClient {
* socket connected to a remote address. Calls are multiplexed through this
* socket: responses may be delivered out of order. */
protected class Connection extends Thread {
- private ConnectionHeader header; // connection header
+ private final ConnectionHeader header; // connection header
protected ConnectionId remoteId;
protected Socket socket = null; // connected socket
protected DataInputStream in;
protected DataOutputStream out;
- private Object outLock = new Object();
- private InetSocketAddress server; // server ip:port
+ private final Object outLock = new Object();
+ private final InetSocketAddress server; // server ip:port
private String serverPrincipal; // server's krb5 principal name
private AuthMethod authMethod; // authentication method
private boolean useSasl;
private Token<? extends TokenIdentifier> token;
private HBaseSaslRpcClient saslRpcClient;
- private int reloginMaxBackoff; // max pause before relogin on sasl failure
+ private final int reloginMaxBackoff; // max pause before relogin on sasl failure
private final Codec codec;
private final CompressionCodec compressor;
// currently active calls
protected final ConcurrentSkipListMap<Integer, Call> calls =
- new ConcurrentSkipListMap<Integer, Call>();
+ new ConcurrentSkipListMap<>();
protected final AtomicBoolean shouldCloseConnection = new AtomicBoolean();
protected final CallSender callSender;
@@ -228,7 +226,7 @@ public class RpcClientImpl extends AbstractRpcClient {
CallSender(String name, Configuration conf) {
int queueSize = conf.getInt("hbase.ipc.client.write.queueSize", 1000);
- callsToWrite = new ArrayBlockingQueue<CallFuture>(queueSize);
+ callsToWrite = new ArrayBlockingQueue<>(queueSize);
setDaemon(true);
setName(name + " - writer");
}
@@ -438,21 +436,27 @@ public class RpcClientImpl extends AbstractRpcClient {
socket.getOutputStream().close();
}
} catch (IOException ignored) { // Can happen if the socket is already closed
- if (LOG.isTraceEnabled()) LOG.trace("ignored", ignored);
+ if (LOG.isTraceEnabled()){
+ LOG.trace("ignored", ignored);
+ }
}
try {
if (socket.getInputStream() != null) {
socket.getInputStream().close();
}
} catch (IOException ignored) { // Can happen if the socket is already closed
- if (LOG.isTraceEnabled()) LOG.trace("ignored", ignored);
+ if (LOG.isTraceEnabled()){
+ LOG.trace("ignored", ignored);
+ }
}
try {
if (socket.getChannel() != null) {
socket.getChannel().close();
}
} catch (IOException ignored) { // Can happen if the socket is already closed
- if (LOG.isTraceEnabled()) LOG.trace("ignored", ignored);
+ if (LOG.isTraceEnabled()){
+ LOG.trace("ignored", ignored);
+ }
}
try {
socket.close();
@@ -665,8 +669,8 @@ public class RpcClientImpl extends AbstractRpcClient {
return null;
} else {
String msg = "Couldn't setup connection for " +
- UserGroupInformation.getLoginUser().getUserName() +
- " to " + serverPrincipal;
+ UserGroupInformation.getLoginUser().getUserName() +
+ " to " + serverPrincipal;
LOG.warn(msg);
throw (IOException) new IOException(msg).initCause(ex);
}
@@ -735,7 +739,9 @@ public class RpcClientImpl extends AbstractRpcClient {
}
}
boolean continueSasl;
- if (ticket == null) throw new FatalConnectionException("ticket/user is null");
+ if (ticket == null){
+ throw new FatalConnectionException("ticket/user is null");
+ }
try {
continueSasl = ticket.doAs(new PrivilegedExceptionAction<Boolean>() {
@Override
@@ -868,11 +874,8 @@ public class RpcClientImpl extends AbstractRpcClient {
}
protected void tracedWriteRequest(Call call, int priority, Span span) throws IOException {
- TraceScope ts = Trace.continueSpan(span);
- try {
+ try (TraceScope ignored = Trace.continueSpan(span)) {
writeRequest(call, priority, span);
- } finally {
- ts.close();
}
}
@@ -903,7 +906,7 @@ public class RpcClientImpl extends AbstractRpcClient {
if (priority != PayloadCarryingRpcController.PRIORITY_UNSET) {
builder.setPriority(priority);
}
- RequestHeader header = builder.build();
+ RequestHeader requestHeader = builder.build();
setupIOstreams();
@@ -913,13 +916,15 @@ public class RpcClientImpl extends AbstractRpcClient {
checkIsOpen();
IOException writeException = null;
synchronized (this.outLock) {
- if (Thread.interrupted()) throw new InterruptedIOException();
+ if (Thread.interrupted()){
+ throw new InterruptedIOException();
+ }
calls.put(call.id, call); // We put first as we don't want the connection to become idle.
checkIsOpen(); // Now we're checking that it didn't became idle in between.
try {
- call.callStats.setRequestSizeBytes(IPCUtil.write(this.out, header, call.param,
+ call.callStats.setRequestSizeBytes(IPCUtil.write(this.out, requestHeader, call.param,
cellBlock));
} catch (IOException e) {
// We set the value inside the synchronized block, this way the next in line
@@ -941,7 +946,9 @@ public class RpcClientImpl extends AbstractRpcClient {
doNotify();
// Now that we notified, we can rethrow the exception if any. Otherwise we're good.
- if (writeException != null) throw writeException;
+ if (writeException != null){
+ throw writeException;
+ }
}
@edu.umd.cs.findbugs.annotations.SuppressWarnings(value="NN_NAKED_NOTIFY",
@@ -956,7 +963,9 @@ public class RpcClientImpl extends AbstractRpcClient {
* Because only one receiver, so no synchronization on in.
*/
protected void readResponse() {
- if (shouldCloseConnection.get()) return;
+ if (shouldCloseConnection.get()){
+ return;
+ }
Call call = null;
boolean expectedCall = false;
try {
@@ -1015,12 +1024,16 @@ public class RpcClientImpl extends AbstractRpcClient {
EnvironmentEdgeManager.currentTime() - call.callStats.getStartTime());
}
} catch (IOException e) {
- if (expectedCall) call.setException(e);
+ if (expectedCall){
+ call.setException(e);
+ }
if (e instanceof SocketTimeoutException) {
// Clean up open calls but don't treat this as a fatal condition,
// since we expect certain responses to not make it by the specified
// {@link ConnectionId#rpcTimeout}.
- if (LOG.isTraceEnabled()) LOG.trace("ignored", e);
+ if (LOG.isTraceEnabled()){
+ LOG.trace("ignored", e);
+ }
} else {
// Treat this as a fatal condition and close this connection
markClosed(e);
@@ -1054,7 +1067,9 @@ public class RpcClientImpl extends AbstractRpcClient {
}
protected synchronized boolean markClosed(IOException e) {
- if (e == null) throw new NullPointerException();
+ if (e == null){
+ throw new NullPointerException();
+ }
boolean ret = shouldCloseConnection.compareAndSet(false, true);
if (ret) {
@@ -1124,7 +1139,7 @@ public class RpcClientImpl extends AbstractRpcClient {
super(conf, clusterId, localAddr, metrics);
this.socketFactory = factory;
- this.connections = new PoolMap<ConnectionId, Connection>(getPoolType(conf), getPoolSize(conf));
+ this.connections = new PoolMap<>(getPoolType(conf), getPoolSize(conf));
this.failedServers = new FailedServers(conf);
}
@@ -1156,8 +1171,12 @@ public class RpcClientImpl extends AbstractRpcClient {
* using this client. */
@Override
public void close() {
- if (LOG.isDebugEnabled()) LOG.debug("Stopping rpc client");
- if (!running.compareAndSet(true, false)) return;
+ if (LOG.isDebugEnabled()){
+ LOG.debug("Stopping rpc client");
+ }
+ if (!running.compareAndSet(true, false)){
+ return;
+ }
Set<Connection> connsToClose = null;
// wake up all connections
@@ -1172,7 +1191,7 @@ public class RpcClientImpl extends AbstractRpcClient {
// at all (if CallSender has a cancelled Call it can happen). See HBASE-13851
if (!conn.isAlive()) {
if (connsToClose == null) {
- connsToClose = new HashSet<Connection>();
+ connsToClose = new HashSet<>();
}
connsToClose.add(conn);
}
@@ -1207,8 +1226,8 @@ public class RpcClientImpl extends AbstractRpcClient {
* {@link UserProvider#getCurrent()} makes a new instance of User each time so will be a
* new Connection each time.
* @return A pair with the Message response and the Cell data (if any).
- * @throws InterruptedException
- * @throws IOException
+ * @throws InterruptedException if the call is interupted
+ * @throws IOException if something fails on the connection
*/
@Override
protected Pair<Message, CellScanner> call(PayloadCarryingRpcController pcrc, MethodDescriptor md,
@@ -1228,17 +1247,17 @@ public class RpcClientImpl extends AbstractRpcClient {
final CallFuture cts;
if (connection.callSender != null) {
cts = connection.callSender.sendCall(call, pcrc.getPriority(), Trace.currentSpan());
- pcrc.notifyOnCancel(new RpcCallback<Object>() {
- @Override
- public void run(Object parameter) {
- connection.callSender.remove(cts);
- }
- });
- if (pcrc.isCanceled()) {
- // To finish if the call was cancelled before we set the notification (race condition)
- call.callComplete();
- return new Pair<Message, CellScanner>(call.response, call.cells);
+ pcrc.notifyOnCancel(new RpcCallback<Object>() {
+ @Override
+ public void run(Object parameter) {
+ connection.callSender.remove(cts);
}
+ });
+ if (pcrc.isCanceled()) {
+ // To finish if the call was cancelled before we set the notification (race condition)
+ call.callComplete();
+ return new Pair<>(call.response, call.cells);
+ }
} else {
cts = null;
connection.tracedWriteRequest(call, pcrc.getPriority(), Trace.currentSpan());
@@ -1246,7 +1265,9 @@ public class RpcClientImpl extends AbstractRpcClient {
while (!call.done) {
if (call.checkAndSetTimeout()) {
- if (cts != null) connection.callSender.remove(cts);
+ if (cts != null){
+ connection.callSender.remove(cts);
+ }
break;
}
if (connection.shouldCloseConnection.get()) {
@@ -1255,12 +1276,16 @@ public class RpcClientImpl extends AbstractRpcClient {
}
try {
synchronized (call) {
- if (call.done) break;
+ if (call.done){
+ break;
+ }
call.wait(Math.min(call.remainingTime(), 1000) + 1);
}
} catch (InterruptedException e) {
call.setException(new InterruptedIOException());
- if (cts != null) connection.callSender.remove(cts);
+ if (cts != null) {
+ connection.callSender.remove(cts);
+ }
throw e;
}
}
@@ -1274,7 +1299,7 @@ public class RpcClientImpl extends AbstractRpcClient {
throw wrapException(addr, call.error);
}
- return new Pair<Message, CellScanner>(call.response, call.cells);
+ return new Pair<>(call.response, call.cells);
}
@@ -1303,12 +1328,14 @@ public class RpcClientImpl extends AbstractRpcClient {
}
/**
- * Get a connection from the pool, or create a new one and add it to the
+ * Get a connection from the pool, or create a new one and add it to the
* pool. Connections to a given host/port are reused.
*/
protected Connection getConnection(User ticket, Call call, InetSocketAddress addr)
throws IOException {
- if (!running.get()) throw new StoppedRpcClientException();
+ if (!running.get()){
+ throw new StoppedRpcClientException();
+ }
Connection connection;
ConnectionId remoteId =
new ConnectionId(ticket, call.md.getService().getName(), addr);
http://git-wip-us.apache.org/repos/asf/hbase/blob/e71ae601/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/ServerRpcController.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/ServerRpcController.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/ServerRpcController.java
index aa407f7..b899eb8 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/ServerRpcController.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/ServerRpcController.java
@@ -18,14 +18,14 @@
package org.apache.hadoop.hbase.ipc;
+import com.google.protobuf.RpcCallback;
+import com.google.protobuf.RpcController;
+
import java.io.IOException;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.util.StringUtils;
-import com.google.protobuf.RpcCallback;
-import com.google.protobuf.RpcController;
-
/**
* Used for server-side protobuf RPC service invocations. This handler allows
* invocation exceptions to easily be passed through to the RPC server from coprocessor
@@ -55,7 +55,7 @@ public class ServerRpcController implements RpcController {
/**
* The exception thrown within
* {@link com.google.protobuf.Service#callMethod(
- * Descriptors.MethodDescriptor, RpcController, Message, RpcCallback)},
+ * Descriptors.MethodDescriptor, RpcController, Message, RpcCallback)}
* if any.
*/
// TODO: it would be good widen this to just Throwable, but IOException is what we allow now
http://git-wip-us.apache.org/repos/asf/hbase/blob/e71ae601/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/TimeLimitedRpcController.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/TimeLimitedRpcController.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/TimeLimitedRpcController.java
index de502cb..cf08ea9 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/TimeLimitedRpcController.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/TimeLimitedRpcController.java
@@ -18,14 +18,14 @@
package org.apache.hadoop.hbase.ipc;
+import com.google.protobuf.RpcCallback;
+import com.google.protobuf.RpcController;
+
import java.io.IOException;
import java.util.concurrent.atomic.AtomicReference;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import com.google.protobuf.RpcCallback;
-import com.google.protobuf.RpcController;
-
@InterfaceAudience.Private
public class TimeLimitedRpcController implements RpcController {
@@ -35,10 +35,10 @@ public class TimeLimitedRpcController implements RpcController {
protected volatile Integer callTimeout;
protected volatile boolean cancelled = false;
protected final AtomicReference<RpcCallback<Object>> cancellationCb =
- new AtomicReference<RpcCallback<Object>>(null);
+ new AtomicReference<>(null);
protected final AtomicReference<RpcCallback<IOException>> failureCb =
- new AtomicReference<RpcCallback<IOException>>(null);
+ new AtomicReference<>(null);
private IOException exception;
[18/50] [abbrv] hbase git commit: HBASE-15575 Rename table DDL
*Handler methods in MasterObserver to more meaningful names (Stephen Yuan
Jiang)
Posted by sy...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase/blob/b7ce55f0/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterObserver.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterObserver.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterObserver.java
index 4b43cfe..9e04c50 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterObserver.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterObserver.java
@@ -59,7 +59,7 @@ public interface MasterObserver extends Coprocessor {
* @param ctx the environment to interact with the framework and master
* @param desc the HTableDescriptor for the table
* @param regions the initial regions created for the table
- * @throws IOException
+ * @throws IOException if something went wrong
*/
void preCreateTable(final ObserverContext<MasterCoprocessorEnvironment> ctx,
HTableDescriptor desc, HRegionInfo[] regions) throws IOException;
@@ -70,10 +70,11 @@ public interface MasterObserver extends Coprocessor {
* @param ctx the environment to interact with the framework and master
* @param desc the HTableDescriptor for the table
* @param regions the initial regions created for the table
- * @throws IOException
+ * @throws IOException if something went wrong
*/
void postCreateTable(final ObserverContext<MasterCoprocessorEnvironment> ctx,
HTableDescriptor desc, HRegionInfo[] regions) throws IOException;
+
/**
* Called before a new table is created by
* {@link org.apache.hadoop.hbase.master.HMaster}. Called as part of create
@@ -82,8 +83,12 @@ public interface MasterObserver extends Coprocessor {
* @param ctx the environment to interact with the framework and master
* @param desc the HTableDescriptor for the table
* @param regions the initial regions created for the table
- * @throws IOException
+ * @throws IOException if something went wrong
+ * @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0
+ * (<a href="https://issues.apache.org/jira/browse/HBASE-15575">HBASE-15575</a>).
+ * Use {@link #preCreateTableAction(ObserverContext, HTableDescriptor, HRegionInfo[])}.
*/
+ @Deprecated
void preCreateTableHandler(final ObserverContext<MasterCoprocessorEnvironment>
ctx, HTableDescriptor desc, HRegionInfo[] regions) throws IOException;
@@ -94,17 +99,61 @@ public interface MasterObserver extends Coprocessor {
* @param ctx the environment to interact with the framework and master
* @param desc the HTableDescriptor for the table
* @param regions the initial regions created for the table
- * @throws IOException
+ * @throws IOException if something went wrong
+ * @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0
+ * (<a href="https://issues.apache.org/jira/browse/HBASE-15575">HBASE-15575</a>).
+ * Use {@link #postCompletedCreateTableAction(ObserverContext, HTableDescriptor, HRegionInfo[])}
*/
+ @Deprecated
void postCreateTableHandler(final ObserverContext<MasterCoprocessorEnvironment>
ctx, HTableDescriptor desc, HRegionInfo[] regions) throws IOException;
/**
+ * Called before a new table is created by
+ * {@link org.apache.hadoop.hbase.master.HMaster}. Called as part of create
+ * table procedure and it is async to the create RPC call.
+ * It can't bypass the default action, e.g., ctx.bypass() won't have effect.
+ *
+ * Implementation note: This replaces the deprecated
+ * {@link #preCreateTableHandler(ObserverContext, HTableDescriptor, HRegionInfo[])} method.
+ * Make sure to implement only one of the two as both are called.
+ *
+ * @param ctx the environment to interact with the framework and master
+ * @param desc the HTableDescriptor for the table
+ * @param regions the initial regions created for the table
+ * @throws IOException if something went wrong
+ */
+ void preCreateTableAction(
+ final ObserverContext<MasterCoprocessorEnvironment> ctx,
+ final HTableDescriptor desc,
+ final HRegionInfo[] regions) throws IOException;
+
+ /**
+ * Called after the createTable operation has been requested. Called as part
+ * of create table RPC call. Called as part of create table procedure and
+ * it is async to the create RPC call.
+ *
+ * Implementation note: This replaces the deprecated
+ * {@link #postCreateTableHandler(ObserverContext, HTableDescriptor, HRegionInfo[])} method.
+ * Make sure to implement only one of the two as both are called.
+ *
+ * @param ctx the environment to interact with the framework and master
+ * @param desc the HTableDescriptor for the table
+ * @param regions the initial regions created for the table
+ * @throws IOException if something went wrong
+ */
+ void postCompletedCreateTableAction(
+ final ObserverContext<MasterCoprocessorEnvironment> ctx,
+ final HTableDescriptor desc,
+ final HRegionInfo[] regions) throws IOException;
+
+ /**
* Called before {@link org.apache.hadoop.hbase.master.HMaster} deletes a
* table. Called as part of delete table RPC call.
* It can't bypass the default action, e.g., ctx.bypass() won't have effect.
* @param ctx the environment to interact with the framework and master
* @param tableName the name of the table
+ * @throws IOException if something went wrong
*/
void preDeleteTable(final ObserverContext<MasterCoprocessorEnvironment> ctx,
TableName tableName) throws IOException;
@@ -114,6 +163,7 @@ public interface MasterObserver extends Coprocessor {
* of delete table RPC call.
* @param ctx the environment to interact with the framework and master
* @param tableName the name of the table
+ * @throws IOException if something went wrong
*/
void postDeleteTable(final ObserverContext<MasterCoprocessorEnvironment> ctx,
TableName tableName) throws IOException;
@@ -125,7 +175,12 @@ public interface MasterObserver extends Coprocessor {
* It can't bypass the default action, e.g., ctx.bypass() won't have effect.
* @param ctx the environment to interact with the framework and master
* @param tableName the name of the table
+ * @throws IOException if something went wrong
+ * @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0
+ * (<a href="https://issues.apache.org/jira/browse/HBASE-15575">HBASE-15575</a>).
+ * Use {@link #preDeleteTableAction(ObserverContext, TableName)}.
*/
+ @Deprecated
void preDeleteTableHandler(
final ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName)
throws IOException;
@@ -137,11 +192,51 @@ public interface MasterObserver extends Coprocessor {
* It can't bypass the default action, e.g., ctx.bypass() won't have effect.
* @param ctx the environment to interact with the framework and master
* @param tableName the name of the table
+ * @throws IOException if something went wrong
+ * @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0
+ * (<a href="https://issues.apache.org/jira/browse/HBASE-15575">HBASE-15575</a>).
+ * Use {@link #postCompletedDeleteTableAction(ObserverContext, TableName)}.
*/
+ @Deprecated
void postDeleteTableHandler(
final ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName)
throws IOException;
+ /**
+ * Called before {@link org.apache.hadoop.hbase.master.HMaster} deletes a
+ * table. Called as part of delete table procedure and
+ * it is async to the delete RPC call.
+ * It can't bypass the default action, e.g., ctx.bypass() won't have effect.
+ *
+ * Implementation note: This replaces the deprecated
+ * {@link #preDeleteTableHandler(ObserverContext, TableName)} method.
+ * Make sure to implement only one of the two as both are called.
+ *
+ * @param ctx the environment to interact with the framework and master
+ * @param tableName the name of the table
+ * @throws IOException if something went wrong
+ */
+ void preDeleteTableAction(
+ final ObserverContext<MasterCoprocessorEnvironment> ctx, final TableName tableName)
+ throws IOException;
+
+ /**
+ * Called after {@link org.apache.hadoop.hbase.master.HMaster} deletes a
+ * table. Called as part of delete table procedure and it is async to the
+ * delete RPC call.
+ * It can't bypass the default action, e.g., ctx.bypass() won't have effect.
+ *
+ * Implementation note: This replaces the deprecated
+ * {@link #postDeleteTableHandler(ObserverContext, TableName)} method.
+ * Make sure to implement only one of the two as both are called.
+ *
+ * @param ctx the environment to interact with the framework and master
+ * @param tableName the name of the table
+ * @throws IOException if something went wrong
+ */
+ void postCompletedDeleteTableAction(
+ final ObserverContext<MasterCoprocessorEnvironment> ctx, final TableName tableName)
+ throws IOException;
/**
* Called before {@link org.apache.hadoop.hbase.master.HMaster} truncates a
@@ -149,6 +244,7 @@ public interface MasterObserver extends Coprocessor {
* It can't bypass the default action, e.g., ctx.bypass() won't have effect.
* @param ctx the environment to interact with the framework and master
* @param tableName the name of the table
+ * @throws IOException if something went wrong
*/
void preTruncateTable(final ObserverContext<MasterCoprocessorEnvironment> ctx,
TableName tableName) throws IOException;
@@ -160,6 +256,7 @@ public interface MasterObserver extends Coprocessor {
* truncate operation is terminated.
* @param ctx the environment to interact with the framework and master
* @param tableName the name of the table
+ * @throws IOException if something went wrong
*/
void postTruncateTable(final ObserverContext<MasterCoprocessorEnvironment> ctx,
TableName tableName) throws IOException;
@@ -171,7 +268,12 @@ public interface MasterObserver extends Coprocessor {
* It can't bypass the default action, e.g., ctx.bypass() won't have effect.
* @param ctx the environment to interact with the framework and master
* @param tableName the name of the table
+ * @throws IOException if something went wrong
+ * @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0
+ * (<a href="https://issues.apache.org/jira/browse/HBASE-15575">HBASE-15575</a>).
+ * Use {@link #preTruncateTableAction(ObserverContext, TableName)}.
*/
+ @Deprecated
void preTruncateTableHandler(
final ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName)
throws IOException;
@@ -183,18 +285,60 @@ public interface MasterObserver extends Coprocessor {
* It can't bypass the default action, e.g., ctx.bypass() won't have effect.
* @param ctx the environment to interact with the framework and master
* @param tableName the name of the table
+ * @throws IOException if something went wrong
+ * @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0
+ * (<a href="https://issues.apache.org/jira/browse/HBASE-15575">HBASE-15575</a>).
+ * Use {@link #postCompletedTruncateTableAction(ObserverContext, TableName)}.
*/
+ @Deprecated
void postTruncateTableHandler(
final ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName)
throws IOException;
/**
+ * Called before {@link org.apache.hadoop.hbase.master.HMaster} truncates a
+ * table. Called as part of truncate table procedure and it is async
+ * to the truncate RPC call.
+ * It can't bypass the default action, e.g., ctx.bypass() won't have effect.
+ *
+ * Implementation note: This replaces the deprecated
+ * {@link #preTruncateTableHandler(ObserverContext, TableName)} method.
+ * Make sure to implement only one of the two as both are called.
+ *
+ * @param ctx the environment to interact with the framework and master
+ * @param tableName the name of the table
+ * @throws IOException if something went wrong
+ */
+ void preTruncateTableAction(
+ final ObserverContext<MasterCoprocessorEnvironment> ctx, final TableName tableName)
+ throws IOException;
+
+ /**
+ * Called after {@link org.apache.hadoop.hbase.master.HMaster} truncates a
+ * table. Called as part of truncate table procedure and it is async to the
+ * truncate RPC call.
+ * It can't bypass the default action, e.g., ctx.bypass() won't have effect.
+ *
+ * Implementation note: This replaces the deprecated
+ * {@link #postTruncateTableHandler(ObserverContext, TableName)} method.
+ * Make sure to implement only one of the two as both are called.
+ *
+ * @param ctx the environment to interact with the framework and master
+ * @param tableName the name of the table
+ * @throws IOException if something went wrong
+ */
+ void postCompletedTruncateTableAction(
+ final ObserverContext<MasterCoprocessorEnvironment> ctx, final TableName tableName)
+ throws IOException;
+
+ /**
* Called prior to modifying a table's properties. Called as part of modify
* table RPC call.
* It can't bypass the default action, e.g., ctx.bypass() won't have effect.
* @param ctx the environment to interact with the framework and master
* @param tableName the name of the table
* @param htd the HTableDescriptor
+ * @throws IOException if something went wrong
*/
void preModifyTable(final ObserverContext<MasterCoprocessorEnvironment> ctx,
final TableName tableName, HTableDescriptor htd) throws IOException;
@@ -205,6 +349,7 @@ public interface MasterObserver extends Coprocessor {
* @param ctx the environment to interact with the framework and master
* @param tableName the name of the table
* @param htd the HTableDescriptor
+ * @throws IOException if something went wrong
*/
void postModifyTable(final ObserverContext<MasterCoprocessorEnvironment> ctx,
final TableName tableName, HTableDescriptor htd) throws IOException;
@@ -216,7 +361,12 @@ public interface MasterObserver extends Coprocessor {
* @param ctx the environment to interact with the framework and master
* @param tableName the name of the table
* @param htd the HTableDescriptor
+ * @throws IOException if something went wrong
+ * @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0
+ * (<a href="https://issues.apache.org/jira/browse/HBASE-15575">HBASE-15575</a>).
+ * Use {@link #preModifyTableAction(ObserverContext, TableName, HTableDescriptor)}.
*/
+ @Deprecated
void preModifyTableHandler(
final ObserverContext<MasterCoprocessorEnvironment> ctx,
final TableName tableName, HTableDescriptor htd) throws IOException;
@@ -228,17 +378,61 @@ public interface MasterObserver extends Coprocessor {
* @param ctx the environment to interact with the framework and master
* @param tableName the name of the table
* @param htd the HTableDescriptor
+ * @throws IOException if something went wrong
+ * @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0
+ * (<a href="https://issues.apache.org/jira/browse/HBASE-13645">HBASE-13645</a>).
+ * Use {@link #postCompletedModifyTableAction(ObserverContext, TableName, HTableDescriptor)}.
*/
+ @Deprecated
void postModifyTableHandler(
final ObserverContext<MasterCoprocessorEnvironment> ctx,
final TableName tableName, HTableDescriptor htd) throws IOException;
/**
+ * Called prior to modifying a table's properties. Called as part of modify
+ * table procedure and it is async to the modify table RPC call.
+ * It can't bypass the default action, e.g., ctx.bypass() won't have effect.
+ *
+ * Implementation note: This replaces the deprecated
+ * {@link #preModifyTableHandler(ObserverContext, TableName, HTableDescriptor)} method.
+ * Make sure to implement only one of the two as both are called.
+ *
+ * @param ctx the environment to interact with the framework and master
+ * @param tableName the name of the table
+ * @param htd the HTableDescriptor
+ * @throws IOException if something went wrong
+ */
+ void preModifyTableAction(
+ final ObserverContext<MasterCoprocessorEnvironment> ctx,
+ final TableName tableName,
+ final HTableDescriptor htd) throws IOException;
+
+ /**
+ * Called after to modifying a table's properties. Called as part of modify
+ * table procedure and it is async to the modify table RPC call.
+ * It can't bypass the default action, e.g., ctx.bypass() won't have effect.
+ *
+ * Implementation note: This replaces the deprecated
+ * {@link #postModifyTableHandler(ObserverContext, TableName, HTableDescriptor)} method.
+ * Make sure to implement only one of the two as both are called.
+ *
+ * @param ctx the environment to interact with the framework and master
+ * @param tableName the name of the table
+ * @param htd the HTableDescriptor
+ * @throws IOException if something went wrong
+ */
+ void postCompletedModifyTableAction(
+ final ObserverContext<MasterCoprocessorEnvironment> ctx,
+ final TableName tableName,
+ final HTableDescriptor htd) throws IOException;
+
+ /**
* Called prior to adding a new column family to the table. Called as part of
* add column RPC call.
* @param ctx the environment to interact with the framework and master
* @param tableName the name of the table
* @param columnFamily the HColumnDescriptor
+ * @throws IOException if something went wrong
* @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0
* (<a href="https://issues.apache.org/jira/browse/HBASE-13645">HBASE-13645</a>).
* Use {@link #preAddColumnFamily(ObserverContext, TableName, HColumnDescriptor)}.
@@ -258,6 +452,7 @@ public interface MasterObserver extends Coprocessor {
* @param ctx the environment to interact with the framework and master
* @param tableName the name of the table
* @param columnFamily the HColumnDescriptor
+ * @throws IOException if something went wrong
*/
void preAddColumnFamily(final ObserverContext<MasterCoprocessorEnvironment> ctx,
TableName tableName, HColumnDescriptor columnFamily) throws IOException;
@@ -268,6 +463,7 @@ public interface MasterObserver extends Coprocessor {
* @param ctx the environment to interact with the framework and master
* @param tableName the name of the table
* @param columnFamily the HColumnDescriptor
+ * @throws IOException if something went wrong
* @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0
* (<a href="https://issues.apache.org/jira/browse/HBASE-13645">HBASE-13645</a>).
* Use {@link #postAddColumnFamily(ObserverContext, TableName, HColumnDescriptor)}.
@@ -287,6 +483,7 @@ public interface MasterObserver extends Coprocessor {
* @param ctx the environment to interact with the framework and master
* @param tableName the name of the table
* @param columnFamily the HColumnDescriptor
+ * @throws IOException if something went wrong
*/
void postAddColumnFamily(final ObserverContext<MasterCoprocessorEnvironment> ctx,
TableName tableName, HColumnDescriptor columnFamily) throws IOException;
@@ -297,9 +494,10 @@ public interface MasterObserver extends Coprocessor {
* @param ctx the environment to interact with the framework and master
* @param tableName the name of the table
* @param columnFamily the HColumnDescriptor
+ * @throws IOException if something went wrong
* @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0
- * (<a href="https://issues.apache.org/jira/browse/HBASE-13645">HBASE-13645</a>). Use
- * {@link #preAddColumnFamilyHandler(ObserverContext, TableName, HColumnDescriptor)}.
+ * (<a href="https://issues.apache.org/jira/browse/HBASE-13645">HBASE-13645</a>). Use
+ * {@link #preAddColumnFamilyAction(ObserverContext, TableName, HColumnDescriptor)}.
*/
@Deprecated
void preAddColumnHandler(
@@ -308,7 +506,7 @@ public interface MasterObserver extends Coprocessor {
/**
* Called prior to adding a new column family to the table. Called as part of
- * add column handler.
+ * add column procedure.
*
* Implementation note: This replaces the deprecated
* {@link #preAddColumnHandler(ObserverContext, TableName, HColumnDescriptor)} method.
@@ -317,11 +515,12 @@ public interface MasterObserver extends Coprocessor {
* @param ctx the environment to interact with the framework and master
* @param tableName the name of the table
* @param columnFamily the HColumnDescriptor
+ * @throws IOException if something went wrong
*/
- void preAddColumnFamilyHandler(
+ void preAddColumnFamilyAction(
final ObserverContext<MasterCoprocessorEnvironment> ctx,
- TableName tableName, HColumnDescriptor columnFamily) throws IOException;
-
+ final TableName tableName,
+ final HColumnDescriptor columnFamily) throws IOException;
/**
* Called after the new column family has been created. Called as part of
@@ -329,9 +528,10 @@ public interface MasterObserver extends Coprocessor {
* @param ctx the environment to interact with the framework and master
* @param tableName the name of the table
* @param columnFamily the HColumnDescriptor
+ * @throws IOException if something went wrong
* @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0
- * (<a href="https://issues.apache.org/jira/browse/HBASE-13645">HBASE-13645</a>). Use
- * {@link #postAddColumnFamilyHandler(ObserverContext, TableName, HColumnDescriptor)}.
+ * (<a href="https://issues.apache.org/jira/browse/HBASE-13645">HBASE-13645</a>). Use
+ * {@link #postCompletedAddColumnFamilyAction(ObserverContext, TableName, HColumnDescriptor)}.
*/
@Deprecated
void postAddColumnHandler(
@@ -340,7 +540,7 @@ public interface MasterObserver extends Coprocessor {
/**
* Called after the new column family has been created. Called as part of
- * add column handler.
+ * add column procedure.
*
* Implementation note: This replaces the deprecated
* {@link #postAddColumnHandler(ObserverContext, TableName, HColumnDescriptor)} method.
@@ -349,10 +549,12 @@ public interface MasterObserver extends Coprocessor {
* @param ctx the environment to interact with the framework and master
* @param tableName the name of the table
* @param columnFamily the HColumnDescriptor
+ * @throws IOException if something went wrong
*/
- void postAddColumnFamilyHandler(
+ void postCompletedAddColumnFamilyAction(
final ObserverContext<MasterCoprocessorEnvironment> ctx,
- TableName tableName, HColumnDescriptor columnFamily) throws IOException;
+ final TableName tableName,
+ final HColumnDescriptor columnFamily) throws IOException;
/**
* Called prior to modifying a column family's attributes. Called as part of
@@ -360,6 +562,7 @@ public interface MasterObserver extends Coprocessor {
* @param ctx the environment to interact with the framework and master
* @param tableName the name of the table
* @param columnFamily the HColumnDescriptor
+ * @throws IOException if something went wrong
* @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0
* (<a href="https://issues.apache.org/jira/browse/HBASE-13645">HBASE-13645</a>).
* Use {@link #preModifyColumnFamily(ObserverContext, TableName, HColumnDescriptor)}.
@@ -379,6 +582,7 @@ public interface MasterObserver extends Coprocessor {
* @param ctx the environment to interact with the framework and master
* @param tableName the name of the table
* @param columnFamily the HColumnDescriptor
+ * @throws IOException if something went wrong
*/
void preModifyColumnFamily(final ObserverContext<MasterCoprocessorEnvironment> ctx,
TableName tableName, HColumnDescriptor columnFamily) throws IOException;
@@ -389,6 +593,7 @@ public interface MasterObserver extends Coprocessor {
* @param ctx the environment to interact with the framework and master
* @param tableName the name of the table
* @param columnFamily the HColumnDescriptor
+ * @throws IOException if something went wrong
* @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0
* (<a href="https://issues.apache.org/jira/browse/HBASE-13645">HBASE-13645</a>).
* Use {@link #postModifyColumnFamily(ObserverContext, TableName, HColumnDescriptor)}.
@@ -408,6 +613,7 @@ public interface MasterObserver extends Coprocessor {
* @param ctx the environment to interact with the framework and master
* @param tableName the name of the table
* @param columnFamily the HColumnDescriptor
+ * @throws IOException if something went wrong
*/
void postModifyColumnFamily(final ObserverContext<MasterCoprocessorEnvironment> ctx,
TableName tableName, HColumnDescriptor columnFamily) throws IOException;
@@ -418,9 +624,10 @@ public interface MasterObserver extends Coprocessor {
* @param ctx the environment to interact with the framework and master
* @param tableName the name of the table
* @param columnFamily the HColumnDescriptor
+ * @throws IOException if something went wrong
* @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0
- * (<a href="https://issues.apache.org/jira/browse/HBASE-13645">HBASE-13645</a>).
- * Use {@link #preModifyColumnFamilyHandler(ObserverContext, TableName, HColumnDescriptor)}.
+ * (<a href="https://issues.apache.org/jira/browse/HBASE-13645">HBASE-13645</a>).
+ * Use {@link #preModifyColumnFamilyAction(ObserverContext, TableName, HColumnDescriptor)}.
*/
@Deprecated
void preModifyColumnHandler(
@@ -429,7 +636,7 @@ public interface MasterObserver extends Coprocessor {
/**
* Called prior to modifying a column family's attributes. Called as part of
- * modify column handler.
+ * modify column procedure.
*
* Implementation note: This replaces the deprecated
* {@link #preModifyColumnHandler(ObserverContext, TableName, HColumnDescriptor)} method.
@@ -438,10 +645,12 @@ public interface MasterObserver extends Coprocessor {
* @param ctx the environment to interact with the framework and master
* @param tableName the name of the table
* @param columnFamily the HColumnDescriptor
+ * @throws IOException if something went wrong
*/
- void preModifyColumnFamilyHandler(
+ void preModifyColumnFamilyAction(
final ObserverContext<MasterCoprocessorEnvironment> ctx,
- TableName tableName, HColumnDescriptor columnFamily) throws IOException;
+ final TableName tableName,
+ final HColumnDescriptor columnFamily) throws IOException;
/**
* Called after the column family has been updated. Called as part of modify
@@ -449,9 +658,10 @@ public interface MasterObserver extends Coprocessor {
* @param ctx the environment to interact with the framework and master
* @param tableName the name of the table
* @param columnFamily the HColumnDescriptor
+ * @throws IOException if something went wrong
* @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0
- * (<a href="https://issues.apache.org/jira/browse/HBASE-13645">HBASE-13645</a>).
- * Use {@link #postModifyColumnFamilyHandler(ObserverContext, TableName, HColumnDescriptor)}.
+ * (<a href="https://issues.apache.org/jira/browse/HBASE-13645">HBASE-13645</a>). Use
+ * {@link #postCompletedModifyColumnFamilyAction(ObserverContext,TableName,HColumnDescriptor)}.
*/
@Deprecated
void postModifyColumnHandler(
@@ -460,7 +670,7 @@ public interface MasterObserver extends Coprocessor {
/**
* Called after the column family has been updated. Called as part of modify
- * column handler.
+ * column procedure.
*
* Implementation note: This replaces the deprecated
* {@link #postModifyColumnHandler(ObserverContext, TableName, HColumnDescriptor)} method.
@@ -469,10 +679,12 @@ public interface MasterObserver extends Coprocessor {
* @param ctx the environment to interact with the framework and master
* @param tableName the name of the table
* @param columnFamily the HColumnDescriptor
+ * @throws IOException if something went wrong
*/
- void postModifyColumnFamilyHandler(
+ void postCompletedModifyColumnFamilyAction(
final ObserverContext<MasterCoprocessorEnvironment> ctx,
- TableName tableName, HColumnDescriptor columnFamily) throws IOException;
+ final TableName tableName,
+ final HColumnDescriptor columnFamily) throws IOException;
/**
* Called prior to deleting the entire column family. Called as part of
@@ -480,6 +692,7 @@ public interface MasterObserver extends Coprocessor {
* @param ctx the environment to interact with the framework and master
* @param tableName the name of the table
* @param columnFamily the column family
+ * @throws IOException if something went wrong
* @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0
* (<a href="https://issues.apache.org/jira/browse/HBASE-13645">HBASE-13645</a>).
* Use {@link #preDeleteColumnFamily(ObserverContext, TableName, byte[])}.
@@ -499,6 +712,7 @@ public interface MasterObserver extends Coprocessor {
* @param ctx the environment to interact with the framework and master
* @param tableName the name of the table
* @param columnFamily the column
+ * @throws IOException if something went wrong
*/
void preDeleteColumnFamily(final ObserverContext<MasterCoprocessorEnvironment> ctx,
final TableName tableName, final byte[] columnFamily) throws IOException;
@@ -509,6 +723,7 @@ public interface MasterObserver extends Coprocessor {
* @param ctx the environment to interact with the framework and master
* @param tableName the name of the table
* @param columnFamily the column family
+ * @throws IOException if something went wrong
* @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0
* (<a href="https://issues.apache.org/jira/browse/HBASE-13645">HBASE-13645</a>).
* Use {@link #postDeleteColumnFamily(ObserverContext, TableName, byte[])}.
@@ -528,6 +743,7 @@ public interface MasterObserver extends Coprocessor {
* @param ctx the environment to interact with the framework and master
* @param tableName the name of the table
* @param columnFamily the column family
+ * @throws IOException if something went wrong
*/
void postDeleteColumnFamily(final ObserverContext<MasterCoprocessorEnvironment> ctx,
final TableName tableName, final byte[] columnFamily) throws IOException;
@@ -538,9 +754,10 @@ public interface MasterObserver extends Coprocessor {
* @param ctx the environment to interact with the framework and master
* @param tableName the name of the table
* @param columnFamily the column family
+ * @throws IOException if something went wrong
* @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0
* (<a href="https://issues.apache.org/jira/browse/HBASE-13645">HBASE-13645</a>).
- * Use {@link #preDeleteColumnFamilyHandler(ObserverContext, TableName, byte[])}.
+ * Use {@link #preDeleteColumnFamilyAction(ObserverContext, TableName, byte[])}.
*/
@Deprecated
void preDeleteColumnHandler(
@@ -549,7 +766,7 @@ public interface MasterObserver extends Coprocessor {
/**
* Called prior to deleting the entire column family. Called as part of
- * delete column handler.
+ * delete column procedure.
*
* Implementation note: This replaces the deprecated
* {@link #preDeleteColumnHandler(ObserverContext, TableName, byte[])} method.
@@ -558,8 +775,9 @@ public interface MasterObserver extends Coprocessor {
* @param ctx the environment to interact with the framework and master
* @param tableName the name of the table
* @param columnFamily the column family
+ * @throws IOException if something went wrong
*/
- void preDeleteColumnFamilyHandler(
+ void preDeleteColumnFamilyAction(
final ObserverContext<MasterCoprocessorEnvironment> ctx,
final TableName tableName, final byte[] columnFamily) throws IOException;
@@ -569,9 +787,10 @@ public interface MasterObserver extends Coprocessor {
* @param ctx the environment to interact with the framework and master
* @param tableName the name of the table
* @param columnFamily the column family
+ * @throws IOException if something went wrong
* @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0
- * (<a href="https://issues.apache.org/jira/browse/HBASE-13645">HBASE-13645</a>).
- * Use {@link #postDeleteColumnFamilyHandler(ObserverContext, TableName, byte[])}.
+ * (<a href="https://issues.apache.org/jira/browse/HBASE-13645">HBASE-13645</a>).
+ * Use {@link #postCompletedDeleteColumnFamilyAction(ObserverContext, TableName, byte[])}.
*/
@Deprecated
void postDeleteColumnHandler(
@@ -580,7 +799,7 @@ public interface MasterObserver extends Coprocessor {
/**
* Called after the column family has been deleted. Called as part of
- * delete column handler.
+ * delete column procedure.
*
* Implementation note: This replaces the deprecated
* {@link #postDeleteColumnHandler(ObserverContext, TableName, byte[])} method.
@@ -589,8 +808,9 @@ public interface MasterObserver extends Coprocessor {
* @param ctx the environment to interact with the framework and master
* @param tableName the name of the table
* @param columnFamily the column family
+ * @throws IOException if something went wrong
*/
- void postDeleteColumnFamilyHandler(
+ void postCompletedDeleteColumnFamilyAction(
final ObserverContext<MasterCoprocessorEnvironment> ctx,
final TableName tableName, final byte[] columnFamily) throws IOException;
@@ -599,6 +819,7 @@ public interface MasterObserver extends Coprocessor {
* It can't bypass the default action, e.g., ctx.bypass() won't have effect.
* @param ctx the environment to interact with the framework and master
* @param tableName the name of the table
+ * @throws IOException if something went wrong
*/
void preEnableTable(final ObserverContext<MasterCoprocessorEnvironment> ctx,
final TableName tableName) throws IOException;
@@ -608,6 +829,7 @@ public interface MasterObserver extends Coprocessor {
* of enable table RPC call.
* @param ctx the environment to interact with the framework and master
* @param tableName the name of the table
+ * @throws IOException if something went wrong
*/
void postEnableTable(final ObserverContext<MasterCoprocessorEnvironment> ctx,
final TableName tableName) throws IOException;
@@ -618,7 +840,12 @@ public interface MasterObserver extends Coprocessor {
* It can't bypass the default action, e.g., ctx.bypass() won't have effect.
* @param ctx the environment to interact with the framework and master
* @param tableName the name of the table
+ * @throws IOException if something went wrong
+ * @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0
+ * (<a href="https://issues.apache.org/jira/browse/HBASE-15575">HBASE-15575</a>).
+ * Use {@link #preEnableTableAction(ObserverContext, TableName)}.
*/
+ @Deprecated
void preEnableTableHandler(
final ObserverContext<MasterCoprocessorEnvironment> ctx,
final TableName tableName) throws IOException;
@@ -628,17 +855,56 @@ public interface MasterObserver extends Coprocessor {
* of enable table handler and it is async to the enable table RPC call.
* @param ctx the environment to interact with the framework and master
* @param tableName the name of the table
+ * @throws IOException if something went wrong
+ * @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0
+ * (<a href="https://issues.apache.org/jira/browse/HBASE-15575">HBASE-15575</a>).
+ * Use {@link #postCompletedEnableTableAction(ObserverContext, TableName)}.
*/
+ @Deprecated
void postEnableTableHandler(
final ObserverContext<MasterCoprocessorEnvironment> ctx,
final TableName tableName) throws IOException;
/**
+ * Called prior to enabling a table. Called as part of enable table procedure
+ * and it is async to the enable table RPC call.
+ * It can't bypass the default action, e.g., ctx.bypass() won't have effect.
+ *
+ * Implementation note: This replaces the deprecated
+ * {@link #preEnableTableHandler(ObserverContext, TableName)} method.
+ * Make sure to implement only one of the two as both are called.
+ *
+ * @param ctx the environment to interact with the framework and master
+ * @param tableName the name of the table
+ * @throws IOException if something went wrong
+ */
+ void preEnableTableAction(
+ final ObserverContext<MasterCoprocessorEnvironment> ctx,
+ final TableName tableName) throws IOException;
+
+ /**
+ * Called after the enableTable operation has been requested. Called as part
+ * of enable table procedure and it is async to the enable table RPC call.
+ *
+ * Implementation note: This replaces the deprecated
+ * {@link #postEnableTableHandler(ObserverContext, TableName)} method.
+ * Make sure to implement only one of the two as both are called.
+ *
+ * @param ctx the environment to interact with the framework and master
+ * @param tableName the name of the table
+ * @throws IOException if something went wrong
+ */
+ void postCompletedEnableTableAction(
+ final ObserverContext<MasterCoprocessorEnvironment> ctx,
+ final TableName tableName) throws IOException;
+
+ /**
* Called prior to disabling a table. Called as part of disable table RPC
* call.
* It can't bypass the default action, e.g., ctx.bypass() won't have effect.
* @param ctx the environment to interact with the framework and master
* @param tableName the name of the table
+ * @throws IOException if something went wrong
*/
void preDisableTable(final ObserverContext<MasterCoprocessorEnvironment> ctx,
final TableName tableName) throws IOException;
@@ -648,6 +914,7 @@ public interface MasterObserver extends Coprocessor {
* of disable table RPC call.
* @param ctx the environment to interact with the framework and master
* @param tableName the name of the table
+ * @throws IOException if something went wrong
*/
void postDisableTable(final ObserverContext<MasterCoprocessorEnvironment> ctx,
final TableName tableName) throws IOException;
@@ -658,7 +925,12 @@ public interface MasterObserver extends Coprocessor {
* It can't bypass the default action, e.g., ctx.bypass() won't have effect.
* @param ctx the environment to interact with the framework and master
* @param tableName the name of the table
+ * @throws IOException if something went wrong
+ * @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0
+ * (<a href="https://issues.apache.org/jira/browse/HBASE-15575">HBASE-15575</a>).
+ * Use {@link #preDisableTableAction(ObserverContext, TableName)}.
*/
+ @Deprecated
void preDisableTableHandler(
final ObserverContext<MasterCoprocessorEnvironment> ctx,
final TableName tableName) throws IOException;
@@ -668,15 +940,55 @@ public interface MasterObserver extends Coprocessor {
* of disable table handler and it is asyn to the disable table RPC call.
* @param ctx the environment to interact with the framework and master
* @param tableName the name of the table
+ * @throws IOException if something went wrong
+ * @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0
+ * (<a href="https://issues.apache.org/jira/browse/HBASE-15575">HBASE-15575</a>).
+ * Use {@link #postCompletedDisableTableAction(ObserverContext, TableName)}.
*/
+ @Deprecated
void postDisableTableHandler(
final ObserverContext<MasterCoprocessorEnvironment> ctx,
final TableName tableName) throws IOException;
/**
+ * Called prior to disabling a table. Called as part of disable table procedure
+ * and it is asyn to the disable table RPC call.
+ * It can't bypass the default action, e.g., ctx.bypass() won't have effect.
+ *
+ * Implementation note: This replaces the deprecated
+ * {@link #preDisableTableHandler(ObserverContext, TableName)} method.
+ * Make sure to implement only one of the two as both are called.
+ *
+ * @param ctx the environment to interact with the framework and master
+ * @param tableName the name of the table
+ * @throws IOException if something went wrong
+ */
+ void preDisableTableAction(
+ final ObserverContext<MasterCoprocessorEnvironment> ctx,
+ final TableName tableName) throws IOException;
+
+ /**
+ * Called after the disableTable operation has been requested. Called as part
+ * of disable table procedure and it is asyn to the disable table RPC call.
+ *
+ * Implementation note: This replaces the deprecated
+ * {@link #postDisableTableHandler(ObserverContext, TableName)} method.
+ * Make sure to implement only one of the two as both are called.
+ *
+ * @param ctx the environment to interact with the framework and master
+ * @param tableName the name of the table
+ * @throws IOException if something went wrong
+ */
+ void postCompletedDisableTableAction(
+ final ObserverContext<MasterCoprocessorEnvironment> ctx,
+ final TableName tableName) throws IOException;
+
+ /**
* Called before a abortProcedure request has been processed.
* @param ctx the environment to interact with the framework and master
- * @throws IOException
+ * @param procEnv procedure executor
+ * @param procId the Id of the procedure
+ * @throws IOException if something went wrong
*/
public void preAbortProcedure(
ObserverContext<MasterCoprocessorEnvironment> ctx,
@@ -686,6 +998,7 @@ public interface MasterObserver extends Coprocessor {
/**
* Called after a abortProcedure request has been processed.
* @param ctx the environment to interact with the framework and master
+ * @throws IOException if something went wrong
*/
public void postAbortProcedure(ObserverContext<MasterCoprocessorEnvironment> ctx)
throws IOException;
@@ -693,7 +1006,7 @@ public interface MasterObserver extends Coprocessor {
/**
* Called before a listProcedures request has been processed.
* @param ctx the environment to interact with the framework and master
- * @throws IOException
+ * @throws IOException if something went wrong
*/
void preListProcedures(ObserverContext<MasterCoprocessorEnvironment> ctx)
throws IOException;
@@ -702,6 +1015,7 @@ public interface MasterObserver extends Coprocessor {
* Called after a listProcedures request has been processed.
* @param ctx the environment to interact with the framework and master
* @param procInfoList the list of procedures about to be returned
+ * @throws IOException if something went wrong
*/
void postListProcedures(
ObserverContext<MasterCoprocessorEnvironment> ctx,
@@ -872,7 +1186,7 @@ public interface MasterObserver extends Coprocessor {
* @param ctx the environment to interact with the framework and master
* @param snapshot the SnapshotDescriptor for the snapshot
* @param hTableDescriptor the hTableDescriptor of the table to snapshot
- * @throws IOException
+ * @throws IOException if something went wrong
*/
void preSnapshot(final ObserverContext<MasterCoprocessorEnvironment> ctx,
final SnapshotDescription snapshot, final HTableDescriptor hTableDescriptor)
@@ -884,7 +1198,7 @@ public interface MasterObserver extends Coprocessor {
* @param ctx the environment to interact with the framework and master
* @param snapshot the SnapshotDescriptor for the snapshot
* @param hTableDescriptor the hTableDescriptor of the table to snapshot
- * @throws IOException
+ * @throws IOException if something went wrong
*/
void postSnapshot(final ObserverContext<MasterCoprocessorEnvironment> ctx,
final SnapshotDescription snapshot, final HTableDescriptor hTableDescriptor)
@@ -895,7 +1209,7 @@ public interface MasterObserver extends Coprocessor {
* It can't bypass the default action, e.g., ctx.bypass() won't have effect.
* @param ctx the environment to interact with the framework and master
* @param snapshot the SnapshotDescriptor of the snapshot to list
- * @throws IOException
+ * @throws IOException if something went wrong
*/
void preListSnapshot(final ObserverContext<MasterCoprocessorEnvironment> ctx,
final SnapshotDescription snapshot) throws IOException;
@@ -905,7 +1219,7 @@ public interface MasterObserver extends Coprocessor {
* It can't bypass the default action, e.g., ctx.bypass() won't have effect.
* @param ctx the environment to interact with the framework and master
* @param snapshot the SnapshotDescriptor of the snapshot to list
- * @throws IOException
+ * @throws IOException if something went wrong
*/
void postListSnapshot(final ObserverContext<MasterCoprocessorEnvironment> ctx,
final SnapshotDescription snapshot) throws IOException;
@@ -917,7 +1231,7 @@ public interface MasterObserver extends Coprocessor {
* @param ctx the environment to interact with the framework and master
* @param snapshot the SnapshotDescriptor for the snapshot
* @param hTableDescriptor the hTableDescriptor of the table to create
- * @throws IOException
+ * @throws IOException if something went wrong
*/
void preCloneSnapshot(final ObserverContext<MasterCoprocessorEnvironment> ctx,
final SnapshotDescription snapshot, final HTableDescriptor hTableDescriptor)
@@ -929,7 +1243,7 @@ public interface MasterObserver extends Coprocessor {
* @param ctx the environment to interact with the framework and master
* @param snapshot the SnapshotDescriptor for the snapshot
* @param hTableDescriptor the hTableDescriptor of the table to create
- * @throws IOException
+ * @throws IOException if something went wrong
*/
void postCloneSnapshot(final ObserverContext<MasterCoprocessorEnvironment> ctx,
final SnapshotDescription snapshot, final HTableDescriptor hTableDescriptor)
@@ -942,7 +1256,7 @@ public interface MasterObserver extends Coprocessor {
* @param ctx the environment to interact with the framework and master
* @param snapshot the SnapshotDescriptor for the snapshot
* @param hTableDescriptor the hTableDescriptor of the table to restore
- * @throws IOException
+ * @throws IOException if something went wrong
*/
void preRestoreSnapshot(final ObserverContext<MasterCoprocessorEnvironment> ctx,
final SnapshotDescription snapshot, final HTableDescriptor hTableDescriptor)
@@ -954,7 +1268,7 @@ public interface MasterObserver extends Coprocessor {
* @param ctx the environment to interact with the framework and master
* @param snapshot the SnapshotDescriptor for the snapshot
* @param hTableDescriptor the hTableDescriptor of the table to restore
- * @throws IOException
+ * @throws IOException if something went wrong
*/
void postRestoreSnapshot(final ObserverContext<MasterCoprocessorEnvironment> ctx,
final SnapshotDescription snapshot, final HTableDescriptor hTableDescriptor)
@@ -966,7 +1280,7 @@ public interface MasterObserver extends Coprocessor {
* It can't bypass the default action, e.g., ctx.bypass() won't have effect.
* @param ctx the environment to interact with the framework and master
* @param snapshot the SnapshotDescriptor of the snapshot to delete
- * @throws IOException
+ * @throws IOException if something went wrong
*/
void preDeleteSnapshot(final ObserverContext<MasterCoprocessorEnvironment> ctx,
final SnapshotDescription snapshot) throws IOException;
@@ -976,7 +1290,7 @@ public interface MasterObserver extends Coprocessor {
* Called as part of deleteSnapshot RPC call.
* @param ctx the environment to interact with the framework and master
* @param snapshot the SnapshotDescriptor of the snapshot to delete
- * @throws IOException
+ * @throws IOException if something went wrong
*/
void postDeleteSnapshot(final ObserverContext<MasterCoprocessorEnvironment> ctx,
final SnapshotDescription snapshot) throws IOException;
@@ -987,7 +1301,7 @@ public interface MasterObserver extends Coprocessor {
* @param tableNamesList the list of table names, or null if querying for all
* @param descriptors an empty list, can be filled with what to return if bypassing
* @param regex regular expression used for filtering the table names
- * @throws IOException
+ * @throws IOException if something went wrong
*/
void preGetTableDescriptors(ObserverContext<MasterCoprocessorEnvironment> ctx,
List<TableName> tableNamesList, List<HTableDescriptor> descriptors,
@@ -999,7 +1313,7 @@ public interface MasterObserver extends Coprocessor {
* @param tableNamesList the list of table names, or null if querying for all
* @param descriptors the list of descriptors about to be returned
* @param regex regular expression used for filtering the table names
- * @throws IOException
+ * @throws IOException if something went wrong
*/
void postGetTableDescriptors(ObserverContext<MasterCoprocessorEnvironment> ctx,
List<TableName> tableNamesList, List<HTableDescriptor> descriptors,
@@ -1010,7 +1324,7 @@ public interface MasterObserver extends Coprocessor {
* @param ctx the environment to interact with the framework and master
* @param descriptors an empty list, can be filled with what to return if bypassing
* @param regex regular expression used for filtering the table names
- * @throws IOException
+ * @throws IOException if something went wrong
*/
void preGetTableNames(ObserverContext<MasterCoprocessorEnvironment> ctx,
List<HTableDescriptor> descriptors, String regex) throws IOException;
@@ -1020,7 +1334,7 @@ public interface MasterObserver extends Coprocessor {
* @param ctx the environment to interact with the framework and master
* @param descriptors the list of descriptors about to be returned
* @param regex regular expression used for filtering the table names
- * @throws IOException
+ * @throws IOException if something went wrong
*/
void postGetTableNames(ObserverContext<MasterCoprocessorEnvironment> ctx,
List<HTableDescriptor> descriptors, String regex) throws IOException;
@@ -1033,7 +1347,7 @@ public interface MasterObserver extends Coprocessor {
* It can't bypass the default action, e.g., ctx.bypass() won't have effect.
* @param ctx the environment to interact with the framework and master
* @param ns the NamespaceDescriptor for the table
- * @throws IOException
+ * @throws IOException if something went wrong
*/
void preCreateNamespace(final ObserverContext<MasterCoprocessorEnvironment> ctx,
NamespaceDescriptor ns) throws IOException;
@@ -1041,7 +1355,7 @@ public interface MasterObserver extends Coprocessor {
* Called after the createNamespace operation has been requested.
* @param ctx the environment to interact with the framework and master
* @param ns the NamespaceDescriptor for the table
- * @throws IOException
+ * @throws IOException if something went wrong
*/
void postCreateNamespace(final ObserverContext<MasterCoprocessorEnvironment> ctx,
NamespaceDescriptor ns) throws IOException;
@@ -1052,6 +1366,7 @@ public interface MasterObserver extends Coprocessor {
* It can't bypass the default action, e.g., ctx.bypass() won't have effect.
* @param ctx the environment to interact with the framework and master
* @param namespace the name of the namespace
+ * @throws IOException if something went wrong
*/
void preDeleteNamespace(final ObserverContext<MasterCoprocessorEnvironment> ctx,
String namespace) throws IOException;
@@ -1060,6 +1375,7 @@ public interface MasterObserver extends Coprocessor {
* Called after the deleteNamespace operation has been requested.
* @param ctx the environment to interact with the framework and master
* @param namespace the name of the namespace
+ * @throws IOException if something went wrong
*/
void postDeleteNamespace(final ObserverContext<MasterCoprocessorEnvironment> ctx,
String namespace) throws IOException;
@@ -1069,6 +1385,7 @@ public interface MasterObserver extends Coprocessor {
* It can't bypass the default action, e.g., ctx.bypass() won't have effect.
* @param ctx the environment to interact with the framework and master
* @param ns the NamespaceDescriptor
+ * @throws IOException if something went wrong
*/
void preModifyNamespace(final ObserverContext<MasterCoprocessorEnvironment> ctx,
NamespaceDescriptor ns) throws IOException;
@@ -1077,6 +1394,7 @@ public interface MasterObserver extends Coprocessor {
* Called after the modifyNamespace operation has been requested.
* @param ctx the environment to interact with the framework and master
* @param ns the NamespaceDescriptor
+ * @throws IOException if something went wrong
*/
void postModifyNamespace(final ObserverContext<MasterCoprocessorEnvironment> ctx,
NamespaceDescriptor ns) throws IOException;
@@ -1085,7 +1403,7 @@ public interface MasterObserver extends Coprocessor {
* Called before a getNamespaceDescriptor request has been processed.
* @param ctx the environment to interact with the framework and master
* @param namespace the name of the namespace
- * @throws IOException
+ * @throws IOException if something went wrong
*/
void preGetNamespaceDescriptor(ObserverContext<MasterCoprocessorEnvironment> ctx,
String namespace) throws IOException;
@@ -1094,7 +1412,7 @@ public interface MasterObserver extends Coprocessor {
* Called after a getNamespaceDescriptor request has been processed.
* @param ctx the environment to interact with the framework and master
* @param ns the NamespaceDescriptor
- * @throws IOException
+ * @throws IOException if something went wrong
*/
void postGetNamespaceDescriptor(ObserverContext<MasterCoprocessorEnvironment> ctx,
NamespaceDescriptor ns) throws IOException;
@@ -1103,7 +1421,7 @@ public interface MasterObserver extends Coprocessor {
* Called before a listNamespaceDescriptors request has been processed.
* @param ctx the environment to interact with the framework and master
* @param descriptors an empty list, can be filled with what to return if bypassing
- * @throws IOException
+ * @throws IOException if something went wrong
*/
void preListNamespaceDescriptors(ObserverContext<MasterCoprocessorEnvironment> ctx,
List<NamespaceDescriptor> descriptors) throws IOException;
@@ -1112,7 +1430,7 @@ public interface MasterObserver extends Coprocessor {
* Called after a listNamespaceDescriptors request has been processed.
* @param ctx the environment to interact with the framework and master
* @param descriptors the list of descriptors about to be returned
- * @throws IOException
+ * @throws IOException if something went wrong
*/
void postListNamespaceDescriptors(ObserverContext<MasterCoprocessorEnvironment> ctx,
List<NamespaceDescriptor> descriptors) throws IOException;
@@ -1122,7 +1440,7 @@ public interface MasterObserver extends Coprocessor {
* Called before the table memstore is flushed to disk.
* @param ctx the environment to interact with the framework and master
* @param tableName the name of the table
- * @throws IOException
+ * @throws IOException if something went wrong
*/
void preTableFlush(final ObserverContext<MasterCoprocessorEnvironment> ctx,
final TableName tableName) throws IOException;
@@ -1131,7 +1449,7 @@ public interface MasterObserver extends Coprocessor {
* Called after the table memstore is flushed to disk.
* @param ctx the environment to interact with the framework and master
* @param tableName the name of the table
- * @throws IOException
+ * @throws IOException if something went wrong
*/
void postTableFlush(final ObserverContext<MasterCoprocessorEnvironment> ctx,
final TableName tableName) throws IOException;
@@ -1141,7 +1459,7 @@ public interface MasterObserver extends Coprocessor {
* @param ctx the environment to interact with the framework and master
* @param userName the name of user
* @param quotas the quota settings
- * @throws IOException
+ * @throws IOException if something went wrong
*/
void preSetUserQuota(final ObserverContext<MasterCoprocessorEnvironment> ctx,
final String userName, final Quotas quotas) throws IOException;
@@ -1151,7 +1469,7 @@ public interface MasterObserver extends Coprocessor {
* @param ctx the environment to interact with the framework and master
* @param userName the name of user
* @param quotas the quota settings
- * @throws IOException
+ * @throws IOException if something went wrong
*/
void postSetUserQuota(final ObserverContext<MasterCoprocessorEnvironment> ctx,
final String userName, final Quotas quotas) throws IOException;
@@ -1162,7 +1480,7 @@ public interface MasterObserver extends Coprocessor {
* @param userName the name of user
* @param tableName the name of the table
* @param quotas the quota settings
- * @throws IOException
+ * @throws IOException if something went wrong
*/
void preSetUserQuota(final ObserverContext<MasterCoprocessorEnvironment> ctx,
final String userName, final TableName tableName, final Quotas quotas) throws IOException;
@@ -1173,7 +1491,7 @@ public interface MasterObserver extends Coprocessor {
* @param userName the name of user
* @param tableName the name of the table
* @param quotas the quota settings
- * @throws IOException
+ * @throws IOException if something went wrong
*/
void postSetUserQuota(final ObserverContext<MasterCoprocessorEnvironment> ctx,
final String userName, final TableName tableName, final Quotas quotas) throws IOException;
@@ -1184,7 +1502,7 @@ public interface MasterObserver extends Coprocessor {
* @param userName the name of user
* @param namespace the name of the namespace
* @param quotas the quota settings
- * @throws IOException
+ * @throws IOException if something went wrong
*/
void preSetUserQuota(final ObserverContext<MasterCoprocessorEnvironment> ctx,
final String userName, final String namespace, final Quotas quotas) throws IOException;
@@ -1195,7 +1513,7 @@ public interface MasterObserver extends Coprocessor {
* @param userName the name of user
* @param namespace the name of the namespace
* @param quotas the quota settings
- * @throws IOException
+ * @throws IOException if something went wrong
*/
void postSetUserQuota(final ObserverContext<MasterCoprocessorEnvironment> ctx,
final String userName, final String namespace, final Quotas quotas) throws IOException;
@@ -1205,7 +1523,7 @@ public interface MasterObserver extends Coprocessor {
* @param ctx the environment to interact with the framework and master
* @param tableName the name of the table
* @param quotas the quota settings
- * @throws IOException
+ * @throws IOException if something went wrong
*/
void preSetTableQuota(final ObserverContext<MasterCoprocessorEnvironment> ctx,
final TableName tableName, final Quotas quotas) throws IOException;
@@ -1215,7 +1533,7 @@ public interface MasterObserver extends Coprocessor {
* @param ctx the environment to interact with the framework and master
* @param tableName the name of the table
* @param quotas the quota settings
- * @throws IOException
+ * @throws IOException if something went wrong
*/
void postSetTableQuota(final ObserverContext<MasterCoprocessorEnvironment> ctx,
final TableName tableName, final Quotas quotas) throws IOException;
@@ -1225,7 +1543,7 @@ public interface MasterObserver extends Coprocessor {
* @param ctx the environment to interact with the framework and master
* @param namespace the name of the namespace
* @param quotas the quota settings
- * @throws IOException
+ * @throws IOException if something went wrong
*/
void preSetNamespaceQuota(final ObserverContext<MasterCoprocessorEnvironment> ctx,
final String namespace, final Quotas quotas) throws IOException;
@@ -1235,7 +1553,7 @@ public interface MasterObserver extends Coprocessor {
* @param ctx the environment to interact with the framework and master
* @param namespace the name of the namespace
* @param quotas the quota settings
- * @throws IOException
+ * @throws IOException if something went wrong
*/
void postSetNamespaceQuota(final ObserverContext<MasterCoprocessorEnvironment> ctx,
final String namespace, final Quotas quotas) throws IOException;
http://git-wip-us.apache.org/repos/asf/hbase/blob/b7ce55f0/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index ff3d792..c2cab33 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -1082,7 +1082,7 @@ public class HMaster extends HRegionServer implements MasterServices {
// at a time. To do concurrency, would need fencing of enable/disable of
// tables.
// Any time changing this maxThreads to > 1, pls see the comment at
- // AccessController#postCreateTableHandler
+ // AccessController#postCompletedCreateTableAction
this.service.startExecutorService(ExecutorType.MASTER_TABLE_OPERATIONS, 1);
startProcedureExecutor();
http://git-wip-us.apache.org/repos/asf/hbase/blob/b7ce55f0/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java
index 14e8c20..459fd01 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java
@@ -240,24 +240,26 @@ public class MasterCoprocessorHost
});
}
- public void preCreateTableHandler(final HTableDescriptor htd, final HRegionInfo[] regions)
+ public void preCreateTableAction(final HTableDescriptor htd, final HRegionInfo[] regions)
throws IOException {
execOperation(coprocessors.isEmpty() ? null : new CoprocessorOperation() {
@Override
public void call(MasterObserver oserver, ObserverContext<MasterCoprocessorEnvironment> ctx)
throws IOException {
oserver.preCreateTableHandler(ctx, htd, regions);
+ oserver.preCreateTableAction(ctx, htd, regions);
}
});
}
- public void postCreateTableHandler(final HTableDescriptor htd, final HRegionInfo[] regions)
- throws IOException {
+ public void postCompletedCreateTableAction(
+ final HTableDescriptor htd, final HRegionInfo[] regions) throws IOException {
execOperation(coprocessors.isEmpty() ? null : new CoprocessorOperation() {
@Override
public void call(MasterObserver oserver, ObserverContext<MasterCoprocessorEnvironment> ctx)
throws IOException {
oserver.postCreateTableHandler(ctx, htd, regions);
+ oserver.postCompletedCreateTableAction(ctx, htd, regions);
}
});
}
@@ -282,22 +284,24 @@ public class MasterCoprocessorHost
});
}
- public void preDeleteTableHandler(final TableName tableName) throws IOException {
+ public void preDeleteTableAction(final TableName tableName) throws IOException {
execOperation(coprocessors.isEmpty() ? null : new CoprocessorOperation() {
@Override
public void call(MasterObserver oserver, ObserverContext<MasterCoprocessorEnvironment> ctx)
throws IOException {
oserver.preDeleteTableHandler(ctx, tableName);
+ oserver.preDeleteTableAction(ctx, tableName);
}
});
}
- public void postDeleteTableHandler(final TableName tableName) throws IOException {
+ public void postCompletedDeleteTableAction(final TableName tableName) throws IOException {
execOperation(coprocessors.isEmpty() ? null : new CoprocessorOperation() {
@Override
public void call(MasterObserver oserver, ObserverContext<MasterCoprocessorEnvironment> ctx)
throws IOException {
oserver.postDeleteTableHandler(ctx, tableName);
+ oserver.postCompletedDeleteTableAction(ctx, tableName);
}
});
}
@@ -322,22 +326,24 @@ public class MasterCoprocessorHost
});
}
- public void preTruncateTableHandler(final TableName tableName) throws IOException {
+ public void preTruncateTableAction(final TableName tableName) throws IOException {
execOperation(coprocessors.isEmpty() ? null : new CoprocessorOperation() {
@Override
public void call(MasterObserver oserver, ObserverContext<MasterCoprocessorEnvironment> ctx)
throws IOException {
oserver.preTruncateTableHandler(ctx, tableName);
+ oserver.preTruncateTableAction(ctx, tableName);
}
});
}
- public void postTruncateTableHandler(final TableName tableName) throws IOException {
+ public void postCompletedTruncateTableAction(final TableName tableName) throws IOException {
execOperation(coprocessors.isEmpty() ? null : new CoprocessorOperation() {
@Override
public void call(MasterObserver oserver, ObserverContext<MasterCoprocessorEnvironment> ctx)
throws IOException {
oserver.postTruncateTableHandler(ctx, tableName);
+ oserver.postCompletedTruncateTableAction(ctx, tableName);
}
});
}
@@ -364,24 +370,26 @@ public class MasterCoprocessorHost
});
}
- public void preModifyTableHandler(final TableName tableName, final HTableDescriptor htd)
+ public void preModifyTableAction(final TableName tableName, final HTableDescriptor htd)
throws IOException {
execOperation(coprocessors.isEmpty() ? null : new CoprocessorOperation() {
@Override
public void call(MasterObserver oserver, ObserverContext<MasterCoprocessorEnvironment> ctx)
throws IOException {
oserver.preModifyTableHandler(ctx, tableName, htd);
+ oserver.preModifyTableAction(ctx, tableName, htd);
}
});
}
- public void postModifyTableHandler(final TableName tableName, final HTableDescriptor htd)
+ public void postCompletedModifyTableAction(final TableName tableName, final HTableDescriptor htd)
throws IOException {
execOperation(coprocessors.isEmpty() ? null : new CoprocessorOperation() {
@Override
public void call(MasterObserver oserver, ObserverContext<MasterCoprocessorEnvironment> ctx)
throws IOException {
oserver.postModifyTableHandler(ctx, tableName, htd);
+ oserver.postCompletedModifyTableAction(ctx, tableName, htd);
}
});
}
@@ -410,27 +418,30 @@ public class MasterCoprocessorHost
});
}
- public boolean preAddColumnHandler(final TableName tableName,
- final HColumnDescriptor columnFamily)
+ public boolean preAddColumnFamilyAction(
+ final TableName tableName,
+ final HColumnDescriptor columnFamily)
throws IOException {
return execOperation(coprocessors.isEmpty() ? null : new CoprocessorOperation() {
@Override
public void call(MasterObserver oserver, ObserverContext<MasterCoprocessorEnvironment> ctx)
throws IOException {
oserver.preAddColumnHandler(ctx, tableName, columnFamily);
- oserver.preAddColumnFamilyHandler(ctx, tableName, columnFamily);
+ oserver.preAddColumnFamilyAction(ctx, tableName, columnFamily);
}
});
}
- public void postAddColumnHandler(final TableName tableName, final HColumnDescriptor columnFamily)
+ public void postCompletedAddColumnFamilyAction(
+ final TableName tableName,
+ final HColumnDescriptor columnFamily)
throws IOException {
execOperation(coprocessors.isEmpty() ? null : new CoprocessorOperation() {
@Override
public void call(MasterObserver oserver, ObserverContext<MasterCoprocessorEnvironment> ctx)
throws IOException {
oserver.postAddColumnHandler(ctx, tableName, columnFamily);
- oserver.postAddColumnFamilyHandler(ctx, tableName, columnFamily);
+ oserver.postCompletedAddColumnFamilyAction(ctx, tableName, columnFamily);
}
});
}
@@ -459,26 +470,28 @@ public class MasterCoprocessorHost
});
}
- public boolean preModifyColumnHandler(final TableName tableName,
+ public boolean preModifyColumnFamilyAction(
+ final TableName tableName,
final HColumnDescriptor columnFamily) throws IOException {
return execOperation(coprocessors.isEmpty() ? null : new CoprocessorOperation() {
@Override
public void call(MasterObserver oserver, ObserverContext<MasterCoprocessorEnvironment> ctx)
throws IOException {
oserver.preModifyColumnHandler(ctx, tableName, columnFamily);
- oserver.preModifyColumnFamilyHandler(ctx, tableName, columnFamily);
+ oserver.preModifyColumnFamilyAction(ctx, tableName, columnFamily);
}
});
}
- public void postModifyColumnHandler(final TableName tableName,
+ public void postCompletedModifyColumnFamilyAction(
+ final TableName tableName,
final HColumnDescriptor columnFamily) throws IOException {
execOperation(coprocessors.isEmpty() ? null : new CoprocessorOperation() {
@Override
public void call(MasterObserver oserver, ObserverContext<MasterCoprocessorEnvironment> ctx)
throws IOException {
oserver.postModifyColumnHandler(ctx, tableName, columnFamily);
- oserver.postModifyColumnFamilyHandler(ctx, tableName, columnFamily);
+ oserver.postCompletedModifyColumnFamilyAction(ctx, tableName, columnFamily);
}
});
}
@@ -507,26 +520,28 @@ public class MasterCoprocessorHost
});
}
- public boolean preDeleteColumnHandler(final TableName tableName, final byte[] columnFamily)
+ public boolean preDeleteColumnFamilyAction(
+ final TableName tableName,
+ final byte[] columnFamily)
throws IOException {
return execOperation(coprocessors.isEmpty() ? null : new CoprocessorOperation() {
@Override
public void call(MasterObserver oserver, ObserverContext<MasterCoprocessorEnvironment> ctx)
throws IOException {
oserver.preDeleteColumnHandler(ctx, tableName, columnFamily);
- oserver.preDeleteColumnFamilyHandler(ctx, tableName, columnFamily);
+ oserver.preDeleteColumnFamilyAction(ctx, tableName, columnFamily);
}
});
}
- public void postDeleteColumnHandler(final TableName tableName, final byte[] columnFamily)
- throws IOException {
+ public void postCompletedDeleteColumnFamilyAction(
+ final TableName tableName, final byte[] columnFamily) throws IOException {
execOperation(coprocessors.isEmpty() ? null : new CoprocessorOperation() {
@Override
public void call(MasterObserver oserver, ObserverContext<MasterCoprocessorEnvironment> ctx)
throws IOException {
oserver.postDeleteColumnHandler(ctx, tableName, columnFamily);
- oserver.postDeleteColumnFamilyHandler(ctx, tableName, columnFamily);
+ oserver.postCompletedDeleteColumnFamilyAction(ctx, tableName, columnFamily);
}
});
}
@@ -551,22 +566,24 @@ public class MasterCoprocessorHost
});
}
- public void preEnableTableHandler(final TableName tableName) throws IOException {
+ public void preEnableTableAction(final TableName tableName) throws IOException {
execOperation(coprocessors.isEmpty() ? null : new CoprocessorOperation() {
@Override
public void call(MasterObserver oserver, ObserverContext<MasterCoprocessorEnvironment> ctx)
throws IOException {
oserver.preEnableTableHandler(ctx, tableName);
+ oserver.preEnableTableAction(ctx, tableName);
}
});
}
- public void postEnableTableHandler(final TableName tableName) throws IOException {
+ public void postCompletedEnableTableAction(final TableName tableName) throws IOException {
execOperation(coprocessors.isEmpty() ? null : new CoprocessorOperation() {
@Override
public void call(MasterObserver oserver, ObserverContext<MasterCoprocessorEnvironment> ctx)
throws IOException {
oserver.postEnableTableHandler(ctx, tableName);
+ oserver.postCompletedEnableTableAction(ctx, tableName);
}
});
}
@@ -591,22 +608,24 @@ public class MasterCoprocessorHost
});
}
- public void preDisableTableHandler(final TableName tableName) throws IOException {
+ public void preDisableTableAction(final TableName tableName) throws IOException {
execOperation(coprocessors.isEmpty() ? null : new CoprocessorOperation() {
@Override
public void call(MasterObserver oserver, ObserverContext<MasterCoprocessorEnvironment> ctx)
throws IOException {
oserver.preDisableTableHandler(ctx, tableName);
+ oserver.preDisableTableAction(ctx, tableName);
}
});
}
- public void postDisableTableHandler(final TableName tableName) throws IOException {
+ public void postCompletedDisableTableAction(final TableName tableName) throws IOException {
execOperation(coprocessors.isEmpty() ? null : new CoprocessorOperation() {
@Override
public void call(MasterObserver oserver, ObserverContext<MasterCoprocessorEnvironment> ctx)
throws IOException {
oserver.postDisableTableHandler(ctx, tableName);
+ oserver.postCompletedDisableTableAction(ctx, tableName);
}
});
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b7ce55f0/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/DisableTableHandler.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/DisableTableHandler.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/DisableTableHandler.java
index d34f25e..bfa9b5f 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/DisableTableHandler.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/DisableTableHandler.java
@@ -126,11 +126,11 @@ public class DisableTableHandler extends EventHandler {
MasterCoprocessorHost cpHost = ((HMaster) this.server)
.getMasterCoprocessorHost();
if (cpHost != null) {
- cpHost.preDisableTableHandler(this.tableName);
+ cpHost.preDisableTableAction(this.tableName);
}
handleDisableTable();
if (cpHost != null) {
- cpHost.postDisableTableHandler(this.tableName);
+ cpHost.postCompletedDisableTableAction(this.tableName);
}
} catch (IOException e) {
LOG.error("Error trying to disable table " + this.tableName, e);
http://git-wip-us.apache.org/repos/asf/hbase/blob/b7ce55f0/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/EnableTableHandler.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/EnableTableHandler.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/EnableTableHandler.java
index fd3d4c7..7edf3f1 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/EnableTableHandler.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/EnableTableHandler.java
@@ -135,11 +135,11 @@ public class EnableTableHandler extends EventHandler {
MasterCoprocessorHost cpHost = ((HMaster) this.server)
.getMasterCoprocessorHost();
if (cpHost != null) {
- cpHost.preEnableTableHandler(this.tableName);
+ cpHost.preEnableTableAction(this.tableName);
}
handleEnableTable();
if (cpHost != null) {
- cpHost.postEnableTableHandler(this.tableName);
+ cpHost.postCompletedEnableTableAction(this.tableName);
}
} catch (IOException | InterruptedException e) {
LOG.error("Error trying to enable the table " + this.tableName, e);
http://git-wip-us.apache.org/repos/asf/hbase/blob/b7ce55f0/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/AddColumnFamilyProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/AddColumnFamilyProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/AddColumnFamilyProcedure.java
index 2571829..4a6c67d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/AddColumnFamilyProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/AddColumnFamilyProcedure.java
@@ -383,10 +383,10 @@ public class AddColumnFamilyProcedure
public Void run() throws Exception {
switch (state) {
case ADD_COLUMN_FAMILY_PRE_OPERATION:
- cpHost.preAddColumnHandler(tableName, cfDescriptor);
+ cpHost.preAddColumnFamilyAction(tableName, cfDescriptor);
break;
case ADD_COLUMN_FAMILY_POST_OPERATION:
- cpHost.postAddColumnHandler(tableName, cfDescriptor);
+ cpHost.postCompletedAddColumnFamilyAction(tableName, cfDescriptor);
break;
default:
throw new UnsupportedOperationException(this + " unhandled state=" + state);
http://git-wip-us.apache.org/repos/asf/hbase/blob/b7ce55f0/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CloneSnapshotProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CloneSnapshotProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CloneSnapshotProcedure.java
index 2a84a15..c008b36 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CloneSnapshotProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CloneSnapshotProcedure.java
@@ -376,7 +376,7 @@ public class CloneSnapshotProcedure
user.doAs(new PrivilegedExceptionAction<Void>() {
@Override
public Void run() throws Exception {
- cpHost.preCreateTableHandler(hTableDescriptor, null);
+ cpHost.preCreateTableAction(hTableDescriptor, null);
return null;
}
});
@@ -398,7 +398,7 @@ public class CloneSnapshotProcedure
user.doAs(new PrivilegedExceptionAction<Void>() {
@Override
public Void run() throws Exception {
- cpHost.postCreateTableHandler(hTableDescriptor, regions);
+ cpHost.postCompletedCreateTableAction(hTableDescriptor, regions);
return null;
}
});
http://git-wip-us.apache.org/repos/asf/hbase/blob/b7ce55f0/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java
index 51fc74c..110bccd 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java
@@ -311,7 +311,7 @@ public class CreateTableProcedure
user.doAs(new PrivilegedExceptionAction<Void>() {
@Override
public Void run() throws Exception {
- cpHost.preCreateTableHandler(hTableDescriptor, regions);
+ cpHost.preCreateTableAction(hTableDescriptor, regions);
return null;
}
});
@@ -327,7 +327,7 @@ public class CreateTableProcedure
user.doAs(new PrivilegedExceptionAction<Void>() {
@Override
public Void run() throws Exception {
- cpHost.postCreateTableHandler(hTableDescriptor, regions);
+ cpHost.postCompletedCreateTableAction(hTableDescriptor, regions);
return null;
}
});
http://git-wip-us.apache.org/repos/asf/hbase/blob/b7ce55f0/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteColumnFamilyProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteColumnFamilyProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteColumnFamilyProcedure.java
index 442ed72..1e510b3 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteColumnFamilyProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteColumnFamilyProcedure.java
@@ -408,10 +408,10 @@ public class DeleteColumnFamilyProcedure
public Void run() throws Exception {
switch (state) {
case DELETE_COLUMN_FAMILY_PRE_OPERATION:
- cpHost.preDeleteColumnHandler(tableName, familyName);
+ cpHost.preDeleteColumnFamilyAction(tableName, familyName);
break;
case DELETE_COLUMN_FAMILY_POST_OPERATION:
- cpHost.postDeleteColumnHandler(tableName, familyName);
+ cpHost.postCompletedDeleteColumnFamilyAction(tableName, familyName);
break;
default:
throw new UnsupportedOperationException(this + " unhandled state=" + state);
http://git-wip-us.apache.org/repos/asf/hbase/blob/b7ce55f0/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteTableProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteTableProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteTableProcedure.java
index ff22b88..4c92fb5 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteTableProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteTableProcedure.java
@@ -269,7 +269,7 @@ public class DeleteTableProcedure
user.doAs(new PrivilegedExceptionAction<Void>() {
@Override
public Void run() throws Exception {
- cpHost.preDeleteTableHandler(tableName);
+ cpHost.preDeleteTableAction(tableName);
return null;
}
});
@@ -287,7 +287,7 @@ public class DeleteTableProcedure
user.doAs(new PrivilegedExceptionAction<Void>() {
@Override
public Void run() throws Exception {
- cpHost.postDeleteTableHandler(tableName);
+ cpHost.postCompletedDeleteTableAction(tableName);
return null;
}
});
http://git-wip-us.apache.org/repos/asf/hbase/blob/b7ce55f0/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DisableTableProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DisableTableProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DisableTableProcedure.java
index 005069d..6a65690 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DisableTableProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DisableTableProcedure.java
@@ -464,10 +464,10 @@ public class DisableTableProcedure
public Void run() throws Exception {
switch (state) {
case DISABLE_TABLE_PRE_OPERATION:
- cpHost.preDisableTableHandler(tableName);
+ cpHost.preDisableTableAction(tableName);
break;
case DISABLE_TABLE_POST_OPERATION:
- cpHost.postDisableTableHandler(tableName);
+ cpHost.postCompletedDisableTableAction(tableName);
break;
default:
throw new UnsupportedOperationException(this + " unhandled state=" + state);
http://git-wip-us.apache.org/repos/asf/hbase/blob/b7ce55f0/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/EnableTableProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/EnableTableProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/EnableTableProcedure.java
index d8af419..862231e 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/EnableTableProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/EnableTableProcedure.java
@@ -566,10 +566,10 @@ public class EnableTableProcedure
public Void run() throws Exception {
switch (state) {
case ENABLE_TABLE_PRE_OPERATION:
- cpHost.preEnableTableHandler(getTableName());
+ cpHost.preEnableTableAction(getTableName());
break;
case ENABLE_TABLE_POST_OPERATION:
- cpHost.postEnableTableHandler(getTableName());
+ cpHost.postCompletedEnableTableAction(getTableName());
break;
default:
throw new UnsupportedOperationException(this + " unhandled state=" + state);
http://git-wip-us.apache.org/repos/asf/hbase/blob/b7ce55f0/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyColumnFamilyProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyColumnFamilyProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyColumnFamilyProcedure.java
index 0479c36..0b7d8a2 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyColumnFamilyProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyColumnFamilyProcedure.java
@@ -364,10 +364,10 @@ public class ModifyColumnFamilyProcedure
public Void run() throws Exception {
switch (state) {
case MODIFY_COLUMN_FAMILY_PRE_OPERATION:
- cpHost.preModifyColumnHandler(tableName, cfDescriptor);
+ cpHost.preModifyColumnFamilyAction(tableName, cfDescriptor);
break;
case MODIFY_COLUMN_FAMILY_POST_OPERATION:
- cpHost.postModifyColumnHandler(tableName, cfDescriptor);
+ cpHost.postCompletedModifyColumnFamilyAction(tableName, cfDescriptor);
break;
default:
throw new UnsupportedOperationException(this + " unhandled state=" + state);
http://git-wip-us.apache.org/repos/asf/hbase/blob/b7ce55f0/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyTableProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyTableProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyTableProcedure.java
index 8e907ec..8f088da 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyTableProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyTableProcedure.java
@@ -477,10 +477,10 @@ public class ModifyTableProcedure
public Void run() throws Exception {
switch (state) {
case MODIFY_TABLE_PRE_OPERATION:
- cpHost.preModifyTableHandler(getTableName(), modifiedHTableDescriptor);
+ cpHost.preModifyTableAction(getTableName(), modifiedHTableDescriptor);
break;
case MODIFY_TABLE_POST_OPERATION:
- cpHost.postModifyTableHandler(getTableName(), modifiedHTableDescriptor);
+ cpHost.postCompletedModifyTableAction(getTableName(), modifiedHTableDescriptor);
break;
default:
throw new UnsupportedOperationException(this + " unhandled state=" + state);
[47/50] [abbrv] hbase git commit: HBASE-15801 Upgrade checkstyle for
all branches
Posted by sy...@apache.org.
HBASE-15801 Upgrade checkstyle for all branches
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/224b03b2
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/224b03b2
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/224b03b2
Branch: refs/heads/hbase-12439
Commit: 224b03b2a5258c819988cd02363bba7b4e0121f2
Parents: 9d17e3d
Author: zhangduo <zh...@apache.org>
Authored: Mon May 9 14:38:51 2016 +0800
Committer: zhangduo <zh...@apache.org>
Committed: Tue May 10 09:33:46 2016 +0800
----------------------------------------------------------------------
pom.xml | 11 +++++++++++
1 file changed, 11 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/224b03b2/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 0b3c61a..6652b78 100644
--- a/pom.xml
+++ b/pom.xml
@@ -883,6 +883,11 @@
<artifactId>hbase-checkstyle</artifactId>
<version>${project.version}</version>
</dependency>
+ <dependency>
+ <groupId>com.puppycrawl.tools</groupId>
+ <artifactId>checkstyle</artifactId>
+ <version>${checkstyle.version}</version>
+ </dependency>
</dependencies>
<configuration>
<configLocation>hbase/checkstyle.xml</configLocation>
@@ -998,6 +1003,11 @@
<artifactId>hbase-checkstyle</artifactId>
<version>${project.version}</version>
</dependency>
+ <dependency>
+ <groupId>com.puppycrawl.tools</groupId>
+ <artifactId>checkstyle</artifactId>
+ <version>${checkstyle.version}</version>
+ </dependency>
</dependencies>
<configuration>
<configLocation>hbase/checkstyle.xml</configLocation>
@@ -1248,6 +1258,7 @@
<maven.antrun.version>1.8</maven.antrun.version>
<jamon.plugin.version>2.4.1</jamon.plugin.version>
<findbugs-annotations>1.3.9-1</findbugs-annotations>
+ <checkstyle.version>6.18</checkstyle.version>
<javadoc.version>2.10.3</javadoc.version>
<asciidoctor.plugin.version>1.5.2.1</asciidoctor.plugin.version>
<!-- General Packaging -->
[15/50] [abbrv] hbase git commit: HBASE-15563 'counter' may overflow
in BoundedGroupingStrategy (Matt Warhaftig)
Posted by sy...@apache.org.
HBASE-15563 'counter' may overflow in BoundedGroupingStrategy (Matt Warhaftig)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/efcf94de
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/efcf94de
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/efcf94de
Branch: refs/heads/hbase-12439
Commit: efcf94def7d1b0a93bc9f0693719a0648e0c8f7a
Parents: c0ce47d
Author: tedyu <yu...@gmail.com>
Authored: Thu May 5 10:28:33 2016 -0700
Committer: tedyu <yu...@gmail.com>
Committed: Thu May 5 10:28:33 2016 -0700
----------------------------------------------------------------------
.../hadoop/hbase/wal/BoundedGroupingStrategy.java | 14 +++++++++++++-
1 file changed, 13 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/efcf94de/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/BoundedGroupingStrategy.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/BoundedGroupingStrategy.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/BoundedGroupingStrategy.java
index 65c774e..06f8792 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/BoundedGroupingStrategy.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/BoundedGroupingStrategy.java
@@ -46,7 +46,7 @@ public class BoundedGroupingStrategy implements RegionGroupingStrategy{
String idStr = Bytes.toString(identifier);
String groupName = groupNameCache.get(idStr);
if (null == groupName) {
- groupName = groupNames[counter.getAndIncrement() % groupNames.length];
+ groupName = groupNames[getAndIncrAtomicInteger(counter, groupNames.length)];
String extantName = groupNameCache.putIfAbsent(idStr, groupName);
if (extantName != null) {
return extantName;
@@ -55,6 +55,18 @@ public class BoundedGroupingStrategy implements RegionGroupingStrategy{
return groupName;
}
+ // Non-blocking incrementing & resetting of AtomicInteger.
+ private int getAndIncrAtomicInteger(AtomicInteger atomicInt, int reset) {
+ for (;;) {
+ int current = atomicInt.get();
+ int next = (current + 1);
+ if (next == reset) {
+ next = 0;
+ }
+ if (atomicInt.compareAndSet(current, next)) return current;
+ }
+ }
+
@Override
public void init(Configuration config, String providerId) {
int regionGroupNumber = config.getInt(NUM_REGION_GROUPS, DEFAULT_NUM_REGION_GROUPS);
[19/50] [abbrv] hbase git commit: HBASE-15575 Rename table DDL
*Handler methods in MasterObserver to more meaningful names (Stephen Yuan
Jiang)
Posted by sy...@apache.org.
HBASE-15575 Rename table DDL *Handler methods in MasterObserver to more meaningful names (Stephen Yuan Jiang)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/b7ce55f0
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/b7ce55f0
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/b7ce55f0
Branch: refs/heads/hbase-12439
Commit: b7ce55f001015613273a4d7eed1d9e1afa92184a
Parents: 8b0ce77
Author: Stephen Yuan Jiang <sy...@gmail.com>
Authored: Thu May 5 11:44:22 2016 -0700
Committer: Stephen Yuan Jiang <sy...@gmail.com>
Committed: Thu May 5 11:44:22 2016 -0700
----------------------------------------------------------------------
.../hbase/rsgroup/RSGroupAdminEndpoint.java | 136 +++++-
.../BaseMasterAndRegionObserver.java | 129 +++++-
.../hbase/coprocessor/BaseMasterObserver.java | 429 ++++++++++++++++--
.../hbase/coprocessor/MasterObserver.java | 450 ++++++++++++++++---
.../org/apache/hadoop/hbase/master/HMaster.java | 2 +-
.../hbase/master/MasterCoprocessorHost.java | 73 +--
.../master/handler/DisableTableHandler.java | 4 +-
.../master/handler/EnableTableHandler.java | 4 +-
.../procedure/AddColumnFamilyProcedure.java | 4 +-
.../procedure/CloneSnapshotProcedure.java | 4 +-
.../master/procedure/CreateTableProcedure.java | 4 +-
.../procedure/DeleteColumnFamilyProcedure.java | 4 +-
.../master/procedure/DeleteTableProcedure.java | 4 +-
.../master/procedure/DisableTableProcedure.java | 4 +-
.../master/procedure/EnableTableProcedure.java | 4 +-
.../procedure/ModifyColumnFamilyProcedure.java | 4 +-
.../master/procedure/ModifyTableProcedure.java | 4 +-
.../procedure/TruncateTableProcedure.java | 4 +-
.../hbase/security/access/AccessController.java | 6 +-
.../hbase/coprocessor/TestMasterObserver.java | 367 +++++++++------
.../hbase/master/TestTableLockManager.java | 12 +-
.../master/handler/TestEnableTableHandler.java | 14 +-
.../hbase/namespace/TestNamespaceAuditor.java | 18 +-
.../hbase/security/access/SecureTestUtil.java | 14 +-
.../hadoop/hbase/util/BaseTestHBaseFsck.java | 18 +-
25 files changed, 1362 insertions(+), 354 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/b7ce55f0/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminEndpoint.java
----------------------------------------------------------------------
diff --git a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminEndpoint.java b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminEndpoint.java
index 6bc7508..7172f06 100644
--- a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminEndpoint.java
+++ b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminEndpoint.java
@@ -310,6 +310,7 @@ public class RSGroupAdminEndpoint extends RSGroupAdminService
HRegionInfo[] regions) throws IOException {
}
+ @Deprecated
@Override
public void preCreateTableHandler(ObserverContext<MasterCoprocessorEnvironment> ctx,
HTableDescriptor desc,
@@ -317,27 +318,56 @@ public class RSGroupAdminEndpoint extends RSGroupAdminService
}
@Override
+ public void preCreateTableAction(
+ final ObserverContext<MasterCoprocessorEnvironment> ctx,
+ final HTableDescriptor desc,
+ final HRegionInfo[] regions) throws IOException {
+ }
+
+ @Deprecated
+ @Override
public void postCreateTableHandler(ObserverContext<MasterCoprocessorEnvironment> ctx,
HTableDescriptor desc,
HRegionInfo[] regions) throws IOException {
}
@Override
+ public void postCompletedCreateTableAction(
+ final ObserverContext<MasterCoprocessorEnvironment> ctx,
+ final HTableDescriptor desc,
+ final HRegionInfo[] regions) throws IOException {
+ }
+
+ @Override
public void preDeleteTable(ObserverContext<MasterCoprocessorEnvironment> ctx,
TableName tableName) throws IOException {
}
+ @Deprecated
@Override
public void preDeleteTableHandler(ObserverContext<MasterCoprocessorEnvironment> ctx,
TableName tableName) throws IOException {
}
@Override
+ public void preDeleteTableAction(
+ final ObserverContext<MasterCoprocessorEnvironment> ctx,
+ final TableName tableName) throws IOException {
+ }
+
+ @Deprecated
+ @Override
public void postDeleteTableHandler(ObserverContext<MasterCoprocessorEnvironment> ctx,
TableName tableName) throws IOException {
}
@Override
+ public void postCompletedDeleteTableAction(
+ final ObserverContext<MasterCoprocessorEnvironment> ctx,
+ final TableName tableName) throws IOException {
+ }
+
+ @Override
public void preTruncateTable(ObserverContext<MasterCoprocessorEnvironment> ctx,
TableName tableName) throws IOException {
}
@@ -347,17 +377,31 @@ public class RSGroupAdminEndpoint extends RSGroupAdminService
TableName tableName) throws IOException {
}
+ @Deprecated
@Override
public void preTruncateTableHandler(ObserverContext<MasterCoprocessorEnvironment> ctx,
TableName tableName) throws IOException {
}
@Override
+ public void preTruncateTableAction(
+ final ObserverContext<MasterCoprocessorEnvironment> ctx,
+ final TableName tableName) throws IOException {
+ }
+
+ @Deprecated
+ @Override
public void postTruncateTableHandler(ObserverContext<MasterCoprocessorEnvironment> ctx,
TableName tableName) throws IOException {
}
@Override
+ public void postCompletedTruncateTableAction(
+ final ObserverContext<MasterCoprocessorEnvironment> ctx,
+ final TableName tableName) throws IOException {
+ }
+
+ @Override
public void preModifyTable(ObserverContext<MasterCoprocessorEnvironment> ctx,
TableName tableName,
HTableDescriptor htd) throws IOException {
@@ -369,6 +413,7 @@ public class RSGroupAdminEndpoint extends RSGroupAdminService
HTableDescriptor htd) throws IOException {
}
+ @Deprecated
@Override
public void preModifyTableHandler(ObserverContext<MasterCoprocessorEnvironment> ctx,
TableName tableName,
@@ -376,12 +421,27 @@ public class RSGroupAdminEndpoint extends RSGroupAdminService
}
@Override
+ public void preModifyTableAction(
+ final ObserverContext<MasterCoprocessorEnvironment> ctx,
+ final TableName tableName,
+ final HTableDescriptor htd) throws IOException {
+ }
+
+ @Deprecated
+ @Override
public void postModifyTableHandler(ObserverContext<MasterCoprocessorEnvironment> ctx,
TableName tableName,
HTableDescriptor htd) throws IOException {
}
@Override
+ public void postCompletedModifyTableAction(
+ final ObserverContext<MasterCoprocessorEnvironment> ctx,
+ final TableName tableName,
+ final HTableDescriptor htd) throws IOException {
+ }
+
+ @Override
public void preAddColumn(ObserverContext<MasterCoprocessorEnvironment> ctx,
TableName tableName,
HColumnDescriptor columnFamily) throws IOException {
@@ -405,6 +465,7 @@ public class RSGroupAdminEndpoint extends RSGroupAdminService
HColumnDescriptor columnFamily) throws IOException {
}
+ @Deprecated
@Override
public void preAddColumnHandler(ObserverContext<MasterCoprocessorEnvironment> ctx,
TableName tableName,
@@ -412,11 +473,13 @@ public class RSGroupAdminEndpoint extends RSGroupAdminService
}
@Override
- public void preAddColumnFamilyHandler(ObserverContext<MasterCoprocessorEnvironment> ctx,
- TableName tableName,
- HColumnDescriptor columnFamily) throws IOException {
+ public void preAddColumnFamilyAction(
+ final ObserverContext<MasterCoprocessorEnvironment> ctx,
+ final TableName tableName,
+ final HColumnDescriptor columnFamily) throws IOException {
}
+ @Deprecated
@Override
public void postAddColumnHandler(ObserverContext<MasterCoprocessorEnvironment> ctx,
TableName tableName,
@@ -424,9 +487,10 @@ public class RSGroupAdminEndpoint extends RSGroupAdminService
}
@Override
- public void postAddColumnFamilyHandler(ObserverContext<MasterCoprocessorEnvironment> ctx,
- TableName tableName,
- HColumnDescriptor columnFamily) throws IOException {
+ public void postCompletedAddColumnFamilyAction(
+ final ObserverContext<MasterCoprocessorEnvironment> ctx,
+ final TableName tableName,
+ final HColumnDescriptor columnFamily) throws IOException {
}
@Override
@@ -453,6 +517,7 @@ public class RSGroupAdminEndpoint extends RSGroupAdminService
}
+ @Deprecated
@Override
public void preModifyColumnHandler(ObserverContext<MasterCoprocessorEnvironment> ctx, TableName
tableName, HColumnDescriptor columnFamily) throws IOException {
@@ -460,12 +525,15 @@ public class RSGroupAdminEndpoint extends RSGroupAdminService
}
@Override
- public void preModifyColumnFamilyHandler(ObserverContext<MasterCoprocessorEnvironment> ctx,
- TableName tableName, HColumnDescriptor columnFamily)
+ public void preModifyColumnFamilyAction(
+ final ObserverContext<MasterCoprocessorEnvironment> ctx,
+ final TableName tableName,
+ final HColumnDescriptor columnFamily)
throws IOException {
}
+ @Deprecated
@Override
public void postModifyColumnHandler(ObserverContext<MasterCoprocessorEnvironment> ctx,
TableName tableName, HColumnDescriptor columnFamily) throws
@@ -474,8 +542,10 @@ public class RSGroupAdminEndpoint extends RSGroupAdminService
}
@Override
- public void postModifyColumnFamilyHandler(ObserverContext<MasterCoprocessorEnvironment> ctx,
- TableName tableName, HColumnDescriptor columnFamily)
+ public void postCompletedModifyColumnFamilyAction(
+ final ObserverContext<MasterCoprocessorEnvironment> ctx,
+ final TableName tableName,
+ final HColumnDescriptor columnFamily)
throws IOException {
}
@@ -504,6 +574,7 @@ public class RSGroupAdminEndpoint extends RSGroupAdminService
}
+ @Deprecated
@Override
public void preDeleteColumnHandler(ObserverContext<MasterCoprocessorEnvironment> ctx, TableName
tableName, byte[] columnFamily) throws IOException {
@@ -511,12 +582,15 @@ public class RSGroupAdminEndpoint extends RSGroupAdminService
}
@Override
- public void preDeleteColumnFamilyHandler(ObserverContext<MasterCoprocessorEnvironment> ctx,
- TableName tableName, byte[] columnFamily) throws
+ public void preDeleteColumnFamilyAction(
+ final ObserverContext<MasterCoprocessorEnvironment> ctx,
+ final TableName tableName,
+ final byte[] columnFamily) throws
IOException {
}
+ @Deprecated
@Override
public void postDeleteColumnHandler(ObserverContext<MasterCoprocessorEnvironment> ctx,
TableName tableName, byte[] columnFamily) throws IOException {
@@ -524,8 +598,10 @@ public class RSGroupAdminEndpoint extends RSGroupAdminService
}
@Override
- public void postDeleteColumnFamilyHandler(ObserverContext<MasterCoprocessorEnvironment> ctx,
- TableName tableName, byte[] columnFamily) throws
+ public void postCompletedDeleteColumnFamilyAction(
+ final ObserverContext<MasterCoprocessorEnvironment> ctx,
+ final TableName tableName,
+ final byte[] columnFamily) throws
IOException {
}
@@ -542,6 +618,7 @@ public class RSGroupAdminEndpoint extends RSGroupAdminService
}
+ @Deprecated
@Override
public void preEnableTableHandler(ObserverContext<MasterCoprocessorEnvironment> ctx, TableName
tableName) throws IOException {
@@ -549,12 +626,27 @@ public class RSGroupAdminEndpoint extends RSGroupAdminService
}
@Override
+ public void preEnableTableAction(
+ final ObserverContext<MasterCoprocessorEnvironment> ctx,
+ final TableName tableName) throws IOException {
+
+ }
+
+ @Deprecated
+ @Override
public void postEnableTableHandler(ObserverContext<MasterCoprocessorEnvironment> ctx, TableName
tableName) throws IOException {
}
@Override
+ public void postCompletedEnableTableAction(
+ final ObserverContext<MasterCoprocessorEnvironment> ctx,
+ final TableName tableName) throws IOException {
+
+ }
+
+ @Override
public void preDisableTable(ObserverContext<MasterCoprocessorEnvironment> ctx, TableName
tableName) throws IOException {
@@ -566,6 +658,7 @@ public class RSGroupAdminEndpoint extends RSGroupAdminService
}
+ @Deprecated
@Override
public void preDisableTableHandler(ObserverContext<MasterCoprocessorEnvironment> ctx, TableName
tableName) throws IOException {
@@ -573,12 +666,27 @@ public class RSGroupAdminEndpoint extends RSGroupAdminService
}
@Override
+ public void preDisableTableAction(
+ final ObserverContext<MasterCoprocessorEnvironment> ctx,
+ final TableName tableName) throws IOException {
+
+ }
+
+ @Deprecated
+ @Override
public void postDisableTableHandler(ObserverContext<MasterCoprocessorEnvironment> ctx,
TableName tableName) throws IOException {
}
@Override
+ public void postCompletedDisableTableAction(
+ final ObserverContext<MasterCoprocessorEnvironment> ctx,
+ final TableName tableName) throws IOException {
+
+ }
+
+ @Override
public void preMove(ObserverContext<MasterCoprocessorEnvironment> ctx, HRegionInfo region,
ServerName srcServer, ServerName destServer) throws IOException {
http://git-wip-us.apache.org/repos/asf/hbase/blob/b7ce55f0/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseMasterAndRegionObserver.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseMasterAndRegionObserver.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseMasterAndRegionObserver.java
index 74d9fe1..5a69708 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseMasterAndRegionObserver.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseMasterAndRegionObserver.java
@@ -67,6 +67,7 @@ public class BaseMasterAndRegionObserver extends BaseRegionObserver
HRegionInfo regionA, HRegionInfo regionB) throws IOException {
}
+ @Deprecated
@Override
public void preCreateTableHandler(
final ObserverContext<MasterCoprocessorEnvironment> ctx,
@@ -74,12 +75,27 @@ public class BaseMasterAndRegionObserver extends BaseRegionObserver
}
@Override
+ public void preCreateTableAction(
+ final ObserverContext<MasterCoprocessorEnvironment> ctx,
+ final HTableDescriptor desc,
+ final HRegionInfo[] regions) throws IOException {
+ }
+
+ @Deprecated
+ @Override
public void postCreateTableHandler(
final ObserverContext<MasterCoprocessorEnvironment> ctx,
HTableDescriptor desc, HRegionInfo[] regions) throws IOException {
}
@Override
+ public void postCompletedCreateTableAction(
+ final ObserverContext<MasterCoprocessorEnvironment> ctx,
+ final HTableDescriptor desc,
+ final HRegionInfo[] regions) throws IOException {
+ }
+
+ @Override
public void preDeleteTable(ObserverContext<MasterCoprocessorEnvironment> ctx,
TableName tableName) throws IOException {
}
@@ -89,6 +105,7 @@ public class BaseMasterAndRegionObserver extends BaseRegionObserver
TableName tableName) throws IOException {
}
+ @Deprecated
@Override
public void preDeleteTableHandler(
final ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName)
@@ -96,12 +113,25 @@ public class BaseMasterAndRegionObserver extends BaseRegionObserver
}
@Override
+ public void preDeleteTableAction(
+ final ObserverContext<MasterCoprocessorEnvironment> ctx, final TableName tableName)
+ throws IOException{
+ }
+
+ @Deprecated
+ @Override
public void postDeleteTableHandler(
final ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName)
throws IOException {
}
@Override
+ public void postCompletedDeleteTableAction(
+ final ObserverContext<MasterCoprocessorEnvironment> ctx, final TableName tableName)
+ throws IOException {
+ }
+
+ @Override
public void preTruncateTable(ObserverContext<MasterCoprocessorEnvironment> ctx,
TableName tableName) throws IOException {
}
@@ -111,6 +141,7 @@ public class BaseMasterAndRegionObserver extends BaseRegionObserver
TableName tableName) throws IOException {
}
+ @Deprecated
@Override
public void preTruncateTableHandler(
final ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName)
@@ -118,22 +149,37 @@ public class BaseMasterAndRegionObserver extends BaseRegionObserver
}
@Override
+ public void preTruncateTableAction(
+ final ObserverContext<MasterCoprocessorEnvironment> ctx, final TableName tableName)
+ throws IOException {
+ }
+
+ @Deprecated
+ @Override
public void postTruncateTableHandler(
final ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName)
throws IOException {
}
@Override
+ public void postCompletedTruncateTableAction(
+ final ObserverContext<MasterCoprocessorEnvironment> ctx, final TableName tableName)
+ throws IOException {
+ }
+
+ @Override
public void preModifyTable(ObserverContext<MasterCoprocessorEnvironment> ctx,
TableName tableName, HTableDescriptor htd) throws IOException {
}
+ @Deprecated
@Override
public void postModifyTableHandler(
ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName,
HTableDescriptor htd) throws IOException {
}
+ @Deprecated
@Override
public void preModifyTableHandler(
ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName,
@@ -141,6 +187,20 @@ public class BaseMasterAndRegionObserver extends BaseRegionObserver
}
@Override
+ public void preModifyTableAction(
+ final ObserverContext<MasterCoprocessorEnvironment> ctx,
+ final TableName tableName,
+ final HTableDescriptor htd) throws IOException {
+ }
+
+ @Override
+ public void postCompletedModifyTableAction(
+ final ObserverContext<MasterCoprocessorEnvironment> ctx,
+ final TableName tableName,
+ final HTableDescriptor htd) throws IOException {
+ }
+
+ @Override
public void postModifyTable(ObserverContext<MasterCoprocessorEnvironment> ctx,
TableName tableName, HTableDescriptor htd) throws IOException {
}
@@ -225,9 +285,10 @@ public class BaseMasterAndRegionObserver extends BaseRegionObserver
}
@Override
- public void preAddColumnFamilyHandler(
- ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName,
- HColumnDescriptor columnFamily) throws IOException {
+ public void preAddColumnFamilyAction(
+ final ObserverContext<MasterCoprocessorEnvironment> ctx,
+ final TableName tableName,
+ final HColumnDescriptor columnFamily) throws IOException {
}
@Deprecated
@@ -238,9 +299,10 @@ public class BaseMasterAndRegionObserver extends BaseRegionObserver
}
@Override
- public void postAddColumnFamilyHandler(
- ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName,
- HColumnDescriptor columnFamily) throws IOException {
+ public void postCompletedAddColumnFamilyAction(
+ final ObserverContext<MasterCoprocessorEnvironment> ctx,
+ final TableName tableName,
+ final HColumnDescriptor columnFamily) throws IOException {
}
@Deprecated
@@ -273,9 +335,10 @@ public class BaseMasterAndRegionObserver extends BaseRegionObserver
}
@Override
- public void preModifyColumnFamilyHandler(
- ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName,
- HColumnDescriptor columnFamily) throws IOException {
+ public void preModifyColumnFamilyAction(
+ final ObserverContext<MasterCoprocessorEnvironment> ctx,
+ final TableName tableName,
+ final HColumnDescriptor columnFamily) throws IOException {
}
@Deprecated
@@ -286,9 +349,10 @@ public class BaseMasterAndRegionObserver extends BaseRegionObserver
}
@Override
- public void postModifyColumnFamilyHandler(
- ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName,
- HColumnDescriptor columnFamily) throws IOException {
+ public void postCompletedModifyColumnFamilyAction(
+ final ObserverContext<MasterCoprocessorEnvironment> ctx,
+ final TableName tableName,
+ final HColumnDescriptor columnFamily) throws IOException {
}
@Deprecated
@@ -321,9 +385,10 @@ public class BaseMasterAndRegionObserver extends BaseRegionObserver
}
@Override
- public void preDeleteColumnFamilyHandler(
- ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName,
- byte[] columnFamily) throws IOException {
+ public void preDeleteColumnFamilyAction(
+ final ObserverContext<MasterCoprocessorEnvironment> ctx,
+ final TableName tableName,
+ final byte[] columnFamily) throws IOException {
}
@Deprecated
@@ -334,9 +399,10 @@ public class BaseMasterAndRegionObserver extends BaseRegionObserver
}
@Override
- public void postDeleteColumnFamilyHandler(
- ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName,
- byte[] columnFamily) throws IOException {
+ public void postCompletedDeleteColumnFamilyAction(
+ final ObserverContext<MasterCoprocessorEnvironment> ctx,
+ final TableName tableName,
+ final byte[] columnFamily) throws IOException {
}
@@ -350,6 +416,7 @@ public class BaseMasterAndRegionObserver extends BaseRegionObserver
TableName tableName) throws IOException {
}
+ @Deprecated
@Override
public void preEnableTableHandler(
ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName)
@@ -357,12 +424,24 @@ public class BaseMasterAndRegionObserver extends BaseRegionObserver
}
@Override
+ public void preEnableTableAction(
+ final ObserverContext<MasterCoprocessorEnvironment> ctx, final TableName tableName)
+ throws IOException {
+ }
+ @Deprecated
+ @Override
public void postEnableTableHandler(
ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName)
throws IOException {
}
@Override
+ public void postCompletedEnableTableAction(
+ final ObserverContext<MasterCoprocessorEnvironment> ctx, final TableName tableName)
+ throws IOException {
+ }
+
+ @Override
public void preDisableTable(ObserverContext<MasterCoprocessorEnvironment> ctx,
TableName tableName) throws IOException {
}
@@ -372,6 +451,7 @@ public class BaseMasterAndRegionObserver extends BaseRegionObserver
TableName tableName) throws IOException {
}
+ @Deprecated
@Override
public void preDisableTableHandler(
ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName)
@@ -379,12 +459,25 @@ public class BaseMasterAndRegionObserver extends BaseRegionObserver
}
@Override
+ public void preDisableTableAction(
+ final ObserverContext<MasterCoprocessorEnvironment> ctx, final TableName tableName)
+ throws IOException {
+ }
+
+ @Deprecated
+ @Override
public void postDisableTableHandler(
ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName)
throws IOException {
}
@Override
+ public void postCompletedDisableTableAction(
+ final ObserverContext<MasterCoprocessorEnvironment> ctx, final TableName tableName)
+ throws IOException {
+ }
+
+ @Override
public void preAbortProcedure(
ObserverContext<MasterCoprocessorEnvironment> ctx,
final ProcedureExecutor<MasterProcedureEnv> procEnv,
http://git-wip-us.apache.org/repos/asf/hbase/blob/b7ce55f0/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseMasterObserver.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseMasterObserver.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseMasterObserver.java
index 3574bbd..71dab06 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseMasterObserver.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseMasterObserver.java
@@ -56,6 +56,20 @@ public class BaseMasterObserver implements MasterObserver {
HTableDescriptor desc, HRegionInfo[] regions) throws IOException {
}
+ /**
+ * Called before a new table is created by
+ * {@link org.apache.hadoop.hbase.master.HMaster}. Called as part of create
+ * table handler and it is async to the create RPC call.
+ * It can't bypass the default action, e.g., ctx.bypass() won't have effect.
+ * @param ctx the environment to interact with the framework and master
+ * @param desc the HTableDescriptor for the table
+ * @param regions the initial regions created for the table
+ * @throws IOException if something went wrong
+ * @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0
+ * (<a href="https://issues.apache.org/jira/browse/HBASE-15575">HBASE-15575</a>).
+ * Use {@link #preCreateTableAction(ObserverContext, HTableDescriptor, HRegionInfo[])}.
+ */
+ @Deprecated
@Override
public void preCreateTableHandler(
final ObserverContext<MasterCoprocessorEnvironment> ctx,
@@ -63,12 +77,39 @@ public class BaseMasterObserver implements MasterObserver {
}
@Override
+ public void preCreateTableAction(
+ final ObserverContext<MasterCoprocessorEnvironment> ctx,
+ final HTableDescriptor desc,
+ final HRegionInfo[] regions) throws IOException {
+ }
+
+ /**
+ * Called after the createTable operation has been requested. Called as part
+ * of create table RPC call. Called as part of create table handler and
+ * it is async to the create RPC call.
+ * @param ctx the environment to interact with the framework and master
+ * @param desc the HTableDescriptor for the table
+ * @param regions the initial regions created for the table
+ * @throws IOException if something went wrong
+ * @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0
+ * (<a href="https://issues.apache.org/jira/browse/HBASE-15575">HBASE-15575</a>).
+ * Use {@link #postCompletedCreateTableAction(ObserverContext, HTableDescriptor, HRegionInfo[])}
+ */
+ @Deprecated
+ @Override
public void postCreateTableHandler(
final ObserverContext<MasterCoprocessorEnvironment> ctx,
HTableDescriptor desc, HRegionInfo[] regions) throws IOException {
}
@Override
+ public void postCompletedCreateTableAction(
+ final ObserverContext<MasterCoprocessorEnvironment> ctx,
+ final HTableDescriptor desc,
+ final HRegionInfo[] regions) throws IOException {
+ }
+
+ @Override
public void preDispatchMerge(final ObserverContext<MasterCoprocessorEnvironment> ctx,
HRegionInfo regionA, HRegionInfo regionB) throws IOException {
}
@@ -88,6 +129,19 @@ public class BaseMasterObserver implements MasterObserver {
TableName tableName) throws IOException {
}
+ /**
+ * Called before {@link org.apache.hadoop.hbase.master.HMaster} deletes a
+ * table. Called as part of delete table handler and
+ * it is async to the delete RPC call.
+ * It can't bypass the default action, e.g., ctx.bypass() won't have effect.
+ * @param ctx the environment to interact with the framework and master
+ * @param tableName the name of the table
+ * @throws IOException if something went wrong
+ * @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0
+ * (<a href="https://issues.apache.org/jira/browse/HBASE-15575">HBASE-15575</a>).
+ * Use {@link #preDeleteTableAction(ObserverContext, TableName)}.
+ */
+ @Deprecated
@Override
public void preDeleteTableHandler(
final ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName)
@@ -95,12 +149,37 @@ public class BaseMasterObserver implements MasterObserver {
}
@Override
+ public void preDeleteTableAction(
+ final ObserverContext<MasterCoprocessorEnvironment> ctx, final TableName tableName)
+ throws IOException{
+ }
+
+ /**
+ * Called after {@link org.apache.hadoop.hbase.master.HMaster} deletes a
+ * table. Called as part of delete table handler and it is async to the
+ * delete RPC call.
+ * It can't bypass the default action, e.g., ctx.bypass() won't have effect.
+ * @param ctx the environment to interact with the framework and master
+ * @param tableName the name of the table
+ * @throws IOException if something went wrong
+ * @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0
+ * (<a href="https://issues.apache.org/jira/browse/HBASE-15575">HBASE-15575</a>).
+ * Use {@link #postCompletedDeleteTableAction(ObserverContext, TableName)}.
+ */
+ @Deprecated
+ @Override
public void postDeleteTableHandler(
final ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName)
throws IOException {
}
@Override
+ public void postCompletedDeleteTableAction(
+ final ObserverContext<MasterCoprocessorEnvironment> ctx, final TableName tableName)
+ throws IOException {
+ }
+
+ @Override
public void preTruncateTable(ObserverContext<MasterCoprocessorEnvironment> ctx,
TableName tableName) throws IOException {
}
@@ -110,6 +189,19 @@ public class BaseMasterObserver implements MasterObserver {
TableName tableName) throws IOException {
}
+ /**
+ * Called before {@link org.apache.hadoop.hbase.master.HMaster} truncates a
+ * table. Called as part of truncate table handler and it is sync
+ * to the truncate RPC call.
+ * It can't bypass the default action, e.g., ctx.bypass() won't have effect.
+ * @param ctx the environment to interact with the framework and master
+ * @param tableName the name of the table
+ * @throws IOException if something went wrong
+ * @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0
+ * (<a href="https://issues.apache.org/jira/browse/HBASE-15575">HBASE-15575</a>).
+ * Use {@link #preTruncateTableAction(ObserverContext, TableName)}.
+ */
+ @Deprecated
@Override
public void preTruncateTableHandler(
final ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName)
@@ -117,31 +209,96 @@ public class BaseMasterObserver implements MasterObserver {
}
@Override
+ public void preTruncateTableAction(
+ final ObserverContext<MasterCoprocessorEnvironment> ctx, final TableName tableName)
+ throws IOException {
+ }
+
+ /**
+ * Called after {@link org.apache.hadoop.hbase.master.HMaster} truncates a
+ * table. Called as part of truncate table handler and it is sync to the
+ * truncate RPC call.
+ * It can't bypass the default action, e.g., ctx.bypass() won't have effect.
+ * @param ctx the environment to interact with the framework and master
+ * @param tableName the name of the table
+ * @throws IOException if something went wrong
+ * @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0
+ * (<a href="https://issues.apache.org/jira/browse/HBASE-15575">HBASE-15575</a>).
+ * Use {@link #postCompletedTruncateTableAction(ObserverContext, TableName)}.
+ */
+ @Deprecated
+ @Override
public void postTruncateTableHandler(
final ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName)
throws IOException {
}
@Override
+ public void postCompletedTruncateTableAction(
+ final ObserverContext<MasterCoprocessorEnvironment> ctx, final TableName tableName)
+ throws IOException {
+ }
+
+ @Override
public void preModifyTable(ObserverContext<MasterCoprocessorEnvironment> ctx,
TableName tableName, HTableDescriptor htd) throws IOException {
}
@Override
- public void postModifyTableHandler(
+ public void postModifyTable(ObserverContext<MasterCoprocessorEnvironment> ctx,
+ TableName tableName, HTableDescriptor htd) throws IOException {
+ }
+
+ /**
+ * Called prior to modifying a table's properties. Called as part of modify
+ * table handler and it is async to the modify table RPC call.
+ * It can't bypass the default action, e.g., ctx.bypass() won't have effect.
+ * @param ctx the environment to interact with the framework and master
+ * @param tableName the name of the table
+ * @param htd the HTableDescriptor
+ * @throws IOException if something went wrong
+ * @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0
+ * (<a href="https://issues.apache.org/jira/browse/HBASE-15575">HBASE-15575</a>).
+ * Use {@link #preModifyTableAction(ObserverContext, TableName, HTableDescriptor)}.
+ */
+ @Deprecated
+ @Override
+ public void preModifyTableHandler(
ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName,
HTableDescriptor htd) throws IOException {
}
@Override
- public void preModifyTableHandler(
+ public void preModifyTableAction(
+ final ObserverContext<MasterCoprocessorEnvironment> ctx,
+ final TableName tableName,
+ final HTableDescriptor htd) throws IOException {
+ }
+
+ /**
+ * Called after to modifying a table's properties. Called as part of modify
+ * table handler and it is async to the modify table RPC call.
+ * It can't bypass the default action, e.g., ctx.bypass() won't have effect.
+ * @param ctx the environment to interact with the framework and master
+ * @param tableName the name of the table
+ * @param htd the HTableDescriptor
+ * @throws IOException if something went wrong
+ * @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0
+ * (<a href="https://issues.apache.org/jira/browse/HBASE-13645">HBASE-13645</a>).
+ * Use {@link #postCompletedModifyTableAction(ObserverContext, TableName, HTableDescriptor)}.
+ */
+ @Deprecated
+ @Override
+ public void postModifyTableHandler(
ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName,
HTableDescriptor htd) throws IOException {
}
@Override
- public void postModifyTable(ObserverContext<MasterCoprocessorEnvironment> ctx,
- TableName tableName, HTableDescriptor htd) throws IOException {
+ public void postCompletedModifyTableAction(
+ final ObserverContext<MasterCoprocessorEnvironment> ctx,
+ final TableName tableName,
+ final HTableDescriptor htd) throws IOException {
}
@Override
@@ -198,6 +355,17 @@ public class BaseMasterObserver implements MasterObserver {
List<NamespaceDescriptor> descriptors) throws IOException {
}
+ /**
+ * Called prior to adding a new column family to the table. Called as part of
+ * add column RPC call.
+ * @param ctx the environment to interact with the framework and master
+ * @param tableName the name of the table
+ * @param columnFamily the HColumnDescriptor
+ * @throws IOException if something went wrong
+ * @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0
+ * (<a href="https://issues.apache.org/jira/browse/HBASE-13645">HBASE-13645</a>).
+ * Use {@link #preAddColumnFamily(ObserverContext, TableName, HColumnDescriptor)}.
+ */
@Deprecated
@Override
public void preAddColumn(ObserverContext<MasterCoprocessorEnvironment> ctx,
@@ -209,6 +377,17 @@ public class BaseMasterObserver implements MasterObserver {
TableName tableName, HColumnDescriptor columnFamily) throws IOException {
}
+ /**
+ * Called after the new column family has been created. Called as part of
+ * add column RPC call.
+ * @param ctx the environment to interact with the framework and master
+ * @param tableName the name of the table
+ * @param columnFamily the HColumnDescriptor
+ * @throws IOException if something went wrong
+ * @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0
+ * (<a href="https://issues.apache.org/jira/browse/HBASE-13645">HBASE-13645</a>).
+ * Use {@link #postAddColumnFamily(ObserverContext, TableName, HColumnDescriptor)}.
+ */
@Deprecated
@Override
public void postAddColumn(ObserverContext<MasterCoprocessorEnvironment> ctx,
@@ -220,6 +399,17 @@ public class BaseMasterObserver implements MasterObserver {
TableName tableName, HColumnDescriptor columnFamily) throws IOException {
}
+ /**
+ * Called prior to adding a new column family to the table. Called as part of
+ * add column handler.
+ * @param ctx the environment to interact with the framework and master
+ * @param tableName the name of the table
+ * @param columnFamily the HColumnDescriptor
+ * @throws IOException if something went wrong
+ * @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0
+ * (<a href="https://issues.apache.org/jira/browse/HBASE-13645">HBASE-13645</a>). Use
+ * {@link #preAddColumnFamilyAction(ObserverContext, TableName, HColumnDescriptor)}.
+ */
@Deprecated
@Override
public void preAddColumnHandler(
@@ -228,11 +418,23 @@ public class BaseMasterObserver implements MasterObserver {
}
@Override
- public void preAddColumnFamilyHandler(
- ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName,
- HColumnDescriptor columnFamily) throws IOException {
- }
-
+ public void preAddColumnFamilyAction(
+ final ObserverContext<MasterCoprocessorEnvironment> ctx,
+ final TableName tableName,
+ final HColumnDescriptor columnFamily) throws IOException {
+ }
+
+ /**
+ * Called after the new column family has been created. Called as part of
+ * add column handler.
+ * @param ctx the environment to interact with the framework and master
+ * @param tableName the name of the table
+ * @param columnFamily the HColumnDescriptor
+ * @throws IOException if something went wrong
+ * @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0
+ * (<a href="https://issues.apache.org/jira/browse/HBASE-13645">HBASE-13645</a>). Use
+ * {@link #postCompletedAddColumnFamilyAction(ObserverContext, TableName, HColumnDescriptor)}.
+ */
@Deprecated
@Override
public void postAddColumnHandler(
@@ -241,11 +443,23 @@ public class BaseMasterObserver implements MasterObserver {
}
@Override
- public void postAddColumnFamilyHandler(
- ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName,
- HColumnDescriptor columnFamily) throws IOException {
- }
-
+ public void postCompletedAddColumnFamilyAction(
+ final ObserverContext<MasterCoprocessorEnvironment> ctx,
+ final TableName tableName,
+ final HColumnDescriptor columnFamily) throws IOException {
+ }
+
+ /**
+ * Called prior to modifying a column family's attributes. Called as part of
+ * modify column RPC call.
+ * @param ctx the environment to interact with the framework and master
+ * @param tableName the name of the table
+ * @param columnFamily the HColumnDescriptor
+ * @throws IOException if something went wrong
+ * @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0
+ * (<a href="https://issues.apache.org/jira/browse/HBASE-13645">HBASE-13645</a>).
+ * Use {@link #preModifyColumnFamily(ObserverContext, TableName, HColumnDescriptor)}.
+ */
@Deprecated
@Override
public void preModifyColumn(ObserverContext<MasterCoprocessorEnvironment> ctx,
@@ -257,6 +471,17 @@ public class BaseMasterObserver implements MasterObserver {
TableName tableName, HColumnDescriptor columnFamily) throws IOException {
}
+ /**
+ * Called after the column family has been updated. Called as part of modify
+ * column RPC call.
+ * @param ctx the environment to interact with the framework and master
+ * @param tableName the name of the table
+ * @param columnFamily the HColumnDescriptor
+ * @throws IOException if something went wrong
+ * @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0
+ * (<a href="https://issues.apache.org/jira/browse/HBASE-13645">HBASE-13645</a>).
+ * Use {@link #postModifyColumnFamily(ObserverContext, TableName, HColumnDescriptor)}.
+ */
@Deprecated
@Override
public void postModifyColumn(ObserverContext<MasterCoprocessorEnvironment> ctx,
@@ -268,6 +493,17 @@ public class BaseMasterObserver implements MasterObserver {
TableName tableName, HColumnDescriptor columnFamily) throws IOException {
}
+ /**
+ * Called prior to modifying a column family's attributes. Called as part of
+ * modify column handler.
+ * @param ctx the environment to interact with the framework and master
+ * @param tableName the name of the table
+ * @param columnFamily the HColumnDescriptor
+ * @throws IOException if something went wrong
+ * @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0
+ * (<a href="https://issues.apache.org/jira/browse/HBASE-13645">HBASE-13645</a>).
+ * Use {@link #preModifyColumnFamilyAction(ObserverContext, TableName, HColumnDescriptor)}.
+ */
@Deprecated
@Override
public void preModifyColumnHandler(
@@ -276,11 +512,23 @@ public class BaseMasterObserver implements MasterObserver {
}
@Override
- public void preModifyColumnFamilyHandler(
- ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName,
- HColumnDescriptor columnFamily) throws IOException {
- }
-
+ public void preModifyColumnFamilyAction(
+ final ObserverContext<MasterCoprocessorEnvironment> ctx,
+ final TableName tableName,
+ final HColumnDescriptor columnFamily) throws IOException {
+ }
+
+ /**
+ * Called after the column family has been updated. Called as part of modify
+ * column handler.
+ * @param ctx the environment to interact with the framework and master
+ * @param tableName the name of the table
+ * @param columnFamily the HColumnDescriptor
+ * @throws IOException if something went wrong
+ * @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0
+ * (<a href="https://issues.apache.org/jira/browse/HBASE-13645">HBASE-13645</a>). Use
+ * {@link #postCompletedModifyColumnFamilyAction(ObserverContext,TableName,HColumnDescriptor)}.
+ */
@Deprecated
@Override
public void postModifyColumnHandler(
@@ -289,11 +537,23 @@ public class BaseMasterObserver implements MasterObserver {
}
@Override
- public void postModifyColumnFamilyHandler(
- ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName,
- HColumnDescriptor columnFamily) throws IOException {
- }
-
+ public void postCompletedModifyColumnFamilyAction(
+ final ObserverContext<MasterCoprocessorEnvironment> ctx,
+ final TableName tableName,
+ final HColumnDescriptor columnFamily) throws IOException {
+ }
+
+ /**
+ * Called prior to deleting the entire column family. Called as part of
+ * delete column RPC call.
+ * @param ctx the environment to interact with the framework and master
+ * @param tableName the name of the table
+ * @param columnFamily the column family
+ * @throws IOException if something went wrong
+ * @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0
+ * (<a href="https://issues.apache.org/jira/browse/HBASE-13645">HBASE-13645</a>).
+ * Use {@link #preDeleteColumnFamily(ObserverContext, TableName, byte[])}.
+ */
@Deprecated
@Override
public void preDeleteColumn(ObserverContext<MasterCoprocessorEnvironment> ctx,
@@ -305,6 +565,17 @@ public class BaseMasterObserver implements MasterObserver {
TableName tableName, byte[] columnFamily) throws IOException {
}
+ /**
+ * Called after the column family has been deleted. Called as part of delete
+ * column RPC call.
+ * @param ctx the environment to interact with the framework and master
+ * @param tableName the name of the table
+ * @param columnFamily the column family
+ * @throws IOException if something went wrong
+ * @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0
+ * (<a href="https://issues.apache.org/jira/browse/HBASE-13645">HBASE-13645</a>).
+ * Use {@link #postDeleteColumnFamily(ObserverContext, TableName, byte[])}.
+ */
@Deprecated
@Override
public void postDeleteColumn(ObserverContext<MasterCoprocessorEnvironment> ctx,
@@ -316,6 +587,17 @@ public class BaseMasterObserver implements MasterObserver {
TableName tableName, byte[] columnFamily) throws IOException {
}
+ /**
+ * Called prior to deleting the entire column family. Called as part of
+ * delete column handler.
+ * @param ctx the environment to interact with the framework and master
+ * @param tableName the name of the table
+ * @param columnFamily the column family
+ * @throws IOException if something went wrong
+ * @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0
+ * (<a href="https://issues.apache.org/jira/browse/HBASE-13645">HBASE-13645</a>).
+ * Use {@link #preDeleteColumnFamilyAction(ObserverContext, TableName, byte[])}.
+ */
@Deprecated
@Override
public void preDeleteColumnHandler(
@@ -324,11 +606,23 @@ public class BaseMasterObserver implements MasterObserver {
}
@Override
- public void preDeleteColumnFamilyHandler(
- ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName,
- byte[] columnFamily) throws IOException {
- }
-
+ public void preDeleteColumnFamilyAction(
+ final ObserverContext<MasterCoprocessorEnvironment> ctx,
+ final TableName tableName,
+ final byte[] columnFamily) throws IOException {
+ }
+
+ /**
+ * Called after the column family has been deleted. Called as part of
+ * delete column handler.
+ * @param ctx the environment to interact with the framework and master
+ * @param tableName the name of the table
+ * @param columnFamily the column family
+ * @throws IOException if something went wrong
+ * @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0
+ * (<a href="https://issues.apache.org/jira/browse/HBASE-13645">HBASE-13645</a>).
+ * Use {@link #postCompletedDeleteColumnFamilyAction(ObserverContext, TableName, byte[])}.
+ */
@Deprecated
@Override
public void postDeleteColumnHandler(
@@ -337,9 +631,10 @@ public class BaseMasterObserver implements MasterObserver {
}
@Override
- public void postDeleteColumnFamilyHandler(
- ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName,
- byte[] columnFamily) throws IOException {
+ public void postCompletedDeleteColumnFamilyAction(
+ final ObserverContext<MasterCoprocessorEnvironment> ctx,
+ final TableName tableName,
+ final byte[] columnFamily) throws IOException {
}
@@ -353,6 +648,18 @@ public class BaseMasterObserver implements MasterObserver {
TableName tableName) throws IOException {
}
+ /**
+ * Called prior to enabling a table. Called as part of enable table handler
+ * and it is async to the enable table RPC call.
+ * It can't bypass the default action, e.g., ctx.bypass() won't have effect.
+ * @param ctx the environment to interact with the framework and master
+ * @param tableName the name of the table
+ * @throws IOException if something went wrong
+ * @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0
+ * (<a href="https://issues.apache.org/jira/browse/HBASE-15575">HBASE-15575</a>).
+ * Use {@link #preEnableTableAction(ObserverContext, TableName)}.
+ */
+ @Deprecated
@Override
public void preEnableTableHandler(
ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName)
@@ -360,12 +667,35 @@ public class BaseMasterObserver implements MasterObserver {
}
@Override
+ public void preEnableTableAction(
+ ObserverContext<MasterCoprocessorEnvironment> ctx, final TableName tableName)
+ throws IOException {
+ }
+
+ /**
+ * Called after the enableTable operation has been requested. Called as part
+ * of enable table handler and it is async to the enable table RPC call.
+ * @param ctx the environment to interact with the framework and master
+ * @param tableName the name of the table
+ * @throws IOException if something went wrong
+ * @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0
+ * (<a href="https://issues.apache.org/jira/browse/HBASE-15575">HBASE-15575</a>).
+ * Use {@link #postCompletedEnableTableAction(ObserverContext, TableName)}.
+ */
+ @Deprecated
+ @Override
public void postEnableTableHandler(
ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName)
throws IOException {
}
@Override
+ public void postCompletedEnableTableAction(
+ ObserverContext<MasterCoprocessorEnvironment> ctx, final TableName tableName)
+ throws IOException {
+ }
+
+ @Override
public void preDisableTable(ObserverContext<MasterCoprocessorEnvironment> ctx,
TableName tableName) throws IOException {
}
@@ -375,6 +705,18 @@ public class BaseMasterObserver implements MasterObserver {
TableName tableName) throws IOException {
}
+ /**
+ * Called prior to disabling a table. Called as part of disable table handler
+ * and it is asyn to the disable table RPC call.
+ * It can't bypass the default action, e.g., ctx.bypass() won't have effect.
+ * @param ctx the environment to interact with the framework and master
+ * @param tableName the name of the table
+ * @throws IOException if something went wrong
+ * @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0
+ * (<a href="https://issues.apache.org/jira/browse/HBASE-15575">HBASE-15575</a>).
+ * Use {@link #preDisableTableAction(ObserverContext, TableName)}.
+ */
+ @Deprecated
@Override
public void preDisableTableHandler(
ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName)
@@ -382,12 +724,35 @@ public class BaseMasterObserver implements MasterObserver {
}
@Override
+ public void preDisableTableAction(
+ ObserverContext<MasterCoprocessorEnvironment> ctx, final TableName tableName)
+ throws IOException {
+ }
+
+ /**
+ * Called after the disableTable operation has been requested. Called as part
+ * of disable table handler and it is asyn to the disable table RPC call.
+ * @param ctx the environment to interact with the framework and master
+ * @param tableName the name of the table
+ * @throws IOException if something went wrong
+ * @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0
+ * (<a href="https://issues.apache.org/jira/browse/HBASE-15575">HBASE-15575</a>).
+ * Use {@link #postCompletedDisableTableAction(ObserverContext, TableName)}.
+ */
+ @Deprecated
+ @Override
public void postDisableTableHandler(
ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName)
throws IOException {
}
@Override
+ public void postCompletedDisableTableAction(
+ ObserverContext<MasterCoprocessorEnvironment> ctx, final TableName tableName)
+ throws IOException {
+ }
+
+ @Override
public void preAbortProcedure(
ObserverContext<MasterCoprocessorEnvironment> ctx,
final ProcedureExecutor<MasterProcedureEnv> procEnv,
[32/50] [abbrv] hbase git commit: HBASE-15778 replace master.am and
master.sm direct access with getter calls
Posted by sy...@apache.org.
HBASE-15778 replace master.am and master.sm direct access with getter calls
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/99e2deb8
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/99e2deb8
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/99e2deb8
Branch: refs/heads/hbase-12439
Commit: 99e2deb86ff2285e22800e55259033a791d4612d
Parents: 513ca34
Author: Matteo Bertozzi <ma...@cloudera.com>
Authored: Sat May 7 05:19:15 2016 -0700
Committer: Matteo Bertozzi <ma...@cloudera.com>
Committed: Sat May 7 05:19:15 2016 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hbase/master/HMaster.java | 8 +++---
.../hadoop/hbase/master/MasterRpcServices.java | 29 ++++++++++----------
.../master/TestAssignmentManagerOnCluster.java | 8 +++---
.../apache/hadoop/hbase/master/TestMaster.java | 2 +-
.../hbase/master/TestMasterNoCluster.java | 4 +--
.../hbase/master/TestRegionPlacement.java | 14 +++++-----
6 files changed, 33 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/99e2deb8/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 3829b35..109097b 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -260,7 +260,7 @@ public class HMaster extends HRegionServer implements MasterServices {
LoadBalancerTracker loadBalancerTracker;
// Tracker for split and merge state
- SplitOrMergeTracker splitOrMergeTracker;
+ private SplitOrMergeTracker splitOrMergeTracker;
// Tracker for region normalizer state
private RegionNormalizerTracker regionNormalizerTracker;
@@ -274,10 +274,10 @@ public class HMaster extends HRegionServer implements MasterServices {
private MasterWalManager walManager;
// server manager to deal with region server info
- volatile ServerManager serverManager;
+ private volatile ServerManager serverManager;
// manager of assignment nodes in zookeeper
- AssignmentManager assignmentManager;
+ private AssignmentManager assignmentManager;
// buffer for "fatal error" notices from region servers
// in the cluster. This is only used for assisting
@@ -299,7 +299,7 @@ public class HMaster extends HRegionServer implements MasterServices {
private final ProcedureEvent serverCrashProcessingEnabled =
new ProcedureEvent("server crash processing");
- LoadBalancer balancer;
+ private LoadBalancer balancer;
private RegionNormalizer normalizer;
private BalancerChore balancerChore;
private RegionNormalizerChore normalizerChore;
http://git-wip-us.apache.org/repos/asf/hbase/blob/99e2deb8/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
index 18ee4fe..8a040fa 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
@@ -165,7 +165,7 @@ public class MasterRpcServices extends RSRpcServices
}
try {
if (mode == BalanceSwitchMode.SYNC) {
- synchronized (master.balancer) {
+ synchronized (master.getLoadBalancer()) {
master.loadBalancerTracker.setBalancerOn(newValue);
}
} else {
@@ -232,7 +232,8 @@ public class MasterRpcServices extends RSRpcServices
throw new ServiceException(ioe);
}
byte[] encodedRegionName = request.getRegionName().toByteArray();
- RegionStoreSequenceIds ids = master.serverManager.getLastFlushedSequenceId(encodedRegionName);
+ RegionStoreSequenceIds ids = master.getServerManager()
+ .getLastFlushedSequenceId(encodedRegionName);
return ResponseConverter.buildGetLastFlushedSequenceIdResponse(ids);
}
@@ -243,8 +244,8 @@ public class MasterRpcServices extends RSRpcServices
master.checkServiceStarted();
ClusterStatusProtos.ServerLoad sl = request.getLoad();
ServerName serverName = ProtobufUtil.toServerName(request.getServer());
- ServerLoad oldLoad = master.serverManager.getLoad(serverName);
- master.serverManager.regionServerReport(serverName, new ServerLoad(sl));
+ ServerLoad oldLoad = master.getServerManager().getLoad(serverName);
+ master.getServerManager().regionServerReport(serverName, new ServerLoad(sl));
if (sl != null && master.metricsMaster != null) {
// Up our metrics.
master.metricsMaster.incrementRequests(sl.getTotalNumberOfRequests()
@@ -266,7 +267,7 @@ public class MasterRpcServices extends RSRpcServices
request.getPort(), request.getServerStartCode());
// if regionserver passed hostname to use,
// then use it instead of doing a reverse DNS lookup
- ServerName rs = master.serverManager.regionServerStartup(request, ia);
+ ServerName rs = master.getServerManager().regionServerStartup(request, ia);
// Send back some config info
RegionServerStartupResponse.Builder resp = createConfigurationSubset();
@@ -326,7 +327,7 @@ public class MasterRpcServices extends RSRpcServices
LOG.warn("assignRegion specifier type: expected: " + RegionSpecifierType.REGION_NAME
+ " actual: " + type);
}
- RegionStates regionStates = master.assignmentManager.getRegionStates();
+ RegionStates regionStates = master.getAssignmentManager().getRegionStates();
HRegionInfo regionInfo = regionStates.getRegionInfo(regionName);
if (regionInfo == null) throw new UnknownRegionException(Bytes.toString(regionName));
if (master.cpHost != null) {
@@ -336,7 +337,7 @@ public class MasterRpcServices extends RSRpcServices
}
LOG.info(master.getClientIdAuditPrefix()
+ " assign " + regionInfo.getRegionNameAsString());
- master.assignmentManager.assign(regionInfo, true);
+ master.getAssignmentManager().assign(regionInfo, true);
if (master.cpHost != null) {
master.cpHost.postAssign(regionInfo);
}
@@ -503,7 +504,7 @@ public class MasterRpcServices extends RSRpcServices
+ request.getRegionA().getType() + ", region_b="
+ request.getRegionB().getType());
}
- RegionStates regionStates = master.assignmentManager.getRegionStates();
+ RegionStates regionStates = master.getAssignmentManager().getRegionStates();
RegionState regionStateA = regionStates.getRegionState(Bytes.toString(encodedNameOfRegionA));
RegionState regionStateB = regionStates.getRegionState(Bytes.toString(encodedNameOfRegionB));
if (regionStateA == null || regionStateB == null) {
@@ -767,7 +768,7 @@ public class MasterRpcServices extends RSRpcServices
try {
master.checkInitialized();
- Pair<Integer,Integer> pair = master.assignmentManager.getReopenStatus(tableName);
+ Pair<Integer,Integer> pair = master.getAssignmentManager().getReopenStatus(tableName);
GetSchemaAlterStatusResponse.Builder ret = GetSchemaAlterStatusResponse.newBuilder();
ret.setYetToUpdateRegions(pair.getFirst());
ret.setTotalRegions(pair.getSecond());
@@ -1155,7 +1156,7 @@ public class MasterRpcServices extends RSRpcServices
master.cpHost.preRegionOffline(hri);
}
LOG.info(master.getClientIdAuditPrefix() + " offline " + hri.getRegionNameAsString());
- master.assignmentManager.regionOffline(hri);
+ master.getAssignmentManager().regionOffline(hri);
if (master.cpHost != null) {
master.cpHost.postRegionOffline(hri);
}
@@ -1301,7 +1302,7 @@ public class MasterRpcServices extends RSRpcServices
}
LOG.debug(master.getClientIdAuditPrefix() + " unassign " + hri.getRegionNameAsString()
+ " in current location if it is online and reassign.force=" + force);
- master.assignmentManager.unassign(hri);
+ master.getAssignmentManager().unassign(hri);
if (master.cpHost != null) {
master.cpHost.postUnassign(hri, force);
}
@@ -1320,16 +1321,16 @@ public class MasterRpcServices extends RSRpcServices
RegionStateTransition rt = req.getTransition(0);
TableName tableName = ProtobufUtil.toTableName(
rt.getRegionInfo(0).getTableName());
- RegionStates regionStates = master.assignmentManager.getRegionStates();
+ RegionStates regionStates = master.getAssignmentManager().getRegionStates();
if (!(TableName.META_TABLE_NAME.equals(tableName)
&& regionStates.getRegionState(HRegionInfo.FIRST_META_REGIONINFO) != null)
- && !master.assignmentManager.isFailoverCleanupDone()) {
+ && !master.getAssignmentManager().isFailoverCleanupDone()) {
// Meta region is assigned before master finishes the
// failover cleanup. So no need this check for it
throw new PleaseHoldException("Master is rebuilding user regions");
}
ServerName sn = ProtobufUtil.toServerName(req.getServer());
- String error = master.assignmentManager.onRegionTransition(sn, rt);
+ String error = master.getAssignmentManager().onRegionTransition(sn, rt);
ReportRegionStateTransitionResponse.Builder rrtr =
ReportRegionStateTransitionResponse.newBuilder();
if (error != null) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/99e2deb8/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManagerOnCluster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManagerOnCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManagerOnCluster.java
index fdfc1b3..95ef6d5 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManagerOnCluster.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManagerOnCluster.java
@@ -234,7 +234,7 @@ public class TestAssignmentManagerOnCluster {
MetaTableAccessor.addRegionToMeta(meta, hri);
master = TEST_UTIL.getHBaseCluster().getMaster();
- Set<ServerName> onlineServers = master.serverManager.getOnlineServers().keySet();
+ Set<ServerName> onlineServers = master.getServerManager().getOnlineServers().keySet();
assertFalse("There should be some servers online", onlineServers.isEmpty());
// Use the first server as the destination server
@@ -243,7 +243,7 @@ public class TestAssignmentManagerOnCluster {
// Created faked dead server that is still online in master
deadServer = ServerName.valueOf(destServer.getHostname(),
destServer.getPort(), destServer.getStartcode() - 100L);
- master.serverManager.recordNewServerWithLock(deadServer, ServerLoad.EMPTY_SERVERLOAD);
+ master.getServerManager().recordNewServerWithLock(deadServer, ServerLoad.EMPTY_SERVERLOAD);
final AssignmentManager am = master.getAssignmentManager();
RegionPlan plan = new RegionPlan(hri, null, deadServer);
@@ -260,7 +260,7 @@ public class TestAssignmentManagerOnCluster {
assertFalse("Region should be assigned", am.getRegionStates().isRegionInTransition(hri));
} finally {
if (deadServer != null) {
- master.serverManager.expireServer(deadServer);
+ master.getServerManager().expireServer(deadServer);
}
TEST_UTIL.deleteTable(table);
@@ -1337,7 +1337,7 @@ public class TestAssignmentManagerOnCluster {
public void enableSSH(boolean enabled) {
this.enabled.set(enabled);
if (enabled) {
- serverManager.processQueuedDeadServers();
+ getServerManager().processQueuedDeadServers();
}
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/99e2deb8/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMaster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMaster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMaster.java
index c03afbc..19a5d8a 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMaster.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMaster.java
@@ -101,7 +101,7 @@ public class TestMaster {
LOG.info("Splitting table");
TEST_UTIL.getHBaseAdmin().split(TABLENAME);
LOG.info("Waiting for split result to be about to open");
- RegionStates regionStates = m.assignmentManager.getRegionStates();
+ RegionStates regionStates = m.getAssignmentManager().getRegionStates();
while (regionStates.getRegionsOfTable(TABLENAME).size() <= 1) {
Thread.sleep(100);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/99e2deb8/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterNoCluster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterNoCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterNoCluster.java
index cafee7a..374c773 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterNoCluster.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterNoCluster.java
@@ -277,7 +277,7 @@ public class TestMasterNoCluster {
InterruptedException, KeeperException, CoordinatedStateException {
super.initializeZKBasedSystemTrackers();
// Record a newer server in server manager at first
- serverManager.recordNewServerWithLock(newServer, ServerLoad.EMPTY_SERVERLOAD);
+ getServerManager().recordNewServerWithLock(newServer, ServerLoad.EMPTY_SERVERLOAD);
List<ServerName> onlineServers = new ArrayList<ServerName>();
onlineServers.add(deadServer);
@@ -310,7 +310,7 @@ public class TestMasterNoCluster {
LOG.info("Master is initialized");
assertFalse("The dead server should not be pulled in",
- master.serverManager.isServerOnline(deadServer));
+ master.getServerManager().isServerOnline(deadServer));
} finally {
master.stopMaster();
master.join();
http://git-wip-us.apache.org/repos/asf/hbase/blob/99e2deb8/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionPlacement.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionPlacement.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionPlacement.java
index bae38c6..d7f0a32 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionPlacement.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionPlacement.java
@@ -160,7 +160,7 @@ public class TestRegionPlacement {
// Check when a RS stops, the regions get assigned to their secondary/tertiary
killRandomServerAndVerifyAssignment();
-
+
// also verify that the AssignmentVerificationReport has the correct information
reports = rp.verifyRegionPlacement(false);
report = reports.get(0);
@@ -179,7 +179,7 @@ public class TestRegionPlacement {
RegionPlacementMaintainer.printAssignmentPlan(currentPlan);
}
- private void killRandomServerAndVerifyAssignment()
+ private void killRandomServerAndVerifyAssignment()
throws IOException, InterruptedException, KeeperException {
ServerName serverToKill = null;
int killIndex = 0;
@@ -216,15 +216,15 @@ public class TestRegionPlacement {
entry.getValue()[1] + " " + entry.getValue()[2]);
}
}
- int orig = TEST_UTIL.getHBaseCluster().getMaster().assignmentManager.getNumRegionsOpened();
+ int orig = TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager().getNumRegionsOpened();
TEST_UTIL.getHBaseCluster().stopRegionServer(serverToKill);
TEST_UTIL.getHBaseCluster().waitForRegionServerToStop(serverToKill, 60000);
- int curr = TEST_UTIL.getHBaseCluster().getMaster().assignmentManager.getNumRegionsOpened();
+ int curr = TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager().getNumRegionsOpened();
while (curr - orig < regionsToVerify.size()) {
LOG.debug("Waiting for " + regionsToVerify.size() + " to come online " +
" Current #regions " + curr + " Original #regions " + orig);
Thread.sleep(200);
- curr = TEST_UTIL.getHBaseCluster().getMaster().assignmentManager.getNumRegionsOpened();
+ curr = TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager().getNumRegionsOpened();
}
// now verify
for (Map.Entry<HRegionInfo, Pair<ServerName, ServerName>> entry : regionsToVerify.entrySet()) {
@@ -369,7 +369,7 @@ public class TestRegionPlacement {
throws InterruptedException, IOException {
MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
HMaster m = cluster.getMaster();
- int lastRegionOpenedCount = m.assignmentManager.getNumRegionsOpened();
+ int lastRegionOpenedCount = m.getAssignmentManager().getNumRegionsOpened();
// get the assignments start to execute
m.balance();
@@ -378,7 +378,7 @@ public class TestRegionPlacement {
int attempt = 0;
int currentRegionOpened, regionMovement;
do {
- currentRegionOpened = m.assignmentManager.getNumRegionsOpened();
+ currentRegionOpened = m.getAssignmentManager().getNumRegionsOpened();
regionMovement= currentRegionOpened - lastRegionOpenedCount;
LOG.debug("There are " + regionMovement + "/" + expected +
" regions moved after " + attempt + " attempts");
[39/50] [abbrv] hbase git commit: HBASE-15609 Remove PB references
from Result,
DoubleColumnInterpreter and any such public facing class for 2.0 (Ram)
Posted by sy...@apache.org.
HBASE-15609 Remove PB references from Result, DoubleColumnInterpreter and
any such public facing class for 2.0 (Ram)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/97ad33c6
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/97ad33c6
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/97ad33c6
Branch: refs/heads/hbase-12439
Commit: 97ad33c6918dc39811fc86d82b38e43e60361bf5
Parents: 58888e0
Author: Ramkrishna <ra...@intel.com>
Authored: Mon May 9 14:56:00 2016 +0530
Committer: Ramkrishna <ra...@intel.com>
Committed: Mon May 9 14:56:00 2016 +0530
----------------------------------------------------------------------
.../java/org/apache/hadoop/hbase/ClusterId.java | 4 +-
.../org/apache/hadoop/hbase/ClusterStatus.java | 113 ----------------
.../hadoop/hbase/client/AsyncProcess.java | 6 +-
.../hbase/client/ClusterStatusListener.java | 3 +-
.../apache/hadoop/hbase/client/HBaseAdmin.java | 5 +-
.../hadoop/hbase/client/MetricsConnection.java | 7 +-
.../hadoop/hbase/client/RegionLoadStats.java | 50 ++++++++
.../org/apache/hadoop/hbase/client/Result.java | 20 +--
.../hadoop/hbase/client/ResultStatsUtil.java | 5 +-
.../hbase/client/ServerStatisticTracker.java | 3 +-
.../hadoop/hbase/client/StatisticTrackable.java | 4 +-
.../hbase/client/backoff/ServerStatistics.java | 6 +-
.../BigDecimalColumnInterpreter.java | 5 +-
.../coprocessor/DoubleColumnInterpreter.java | 3 +-
.../coprocessor/LongColumnInterpreter.java | 5 +-
.../hbase/coprocessor/ColumnInterpreter.java | 5 +-
.../hadoop/hbase/protobuf/ProtobufUtil.java | 128 +++++++++++++++++++
.../client/TestClientExponentialBackoff.java | 5 +-
.../org/apache/hadoop/hbase/ProcedureInfo.java | 76 +----------
.../org/apache/hadoop/hbase/ProcedureState.java | 30 +++++
.../org/apache/hadoop/hbase/ProcedureUtil.java | 103 +++++++++++++++
.../hadoop/hbase/procedure2/Procedure.java | 21 ++-
.../store/wal/ProcedureWALFormatReader.java | 3 +-
.../procedure2/ProcedureTestingUtility.java | 12 +-
.../hadoop/hbase/master/MasterRpcServices.java | 7 +-
.../master/procedure/ProcedureSyncWait.java | 2 +-
.../apache/hadoop/hbase/client/TestResult.java | 3 +-
.../master/procedure/TestProcedureAdmin.java | 2 +-
28 files changed, 385 insertions(+), 251 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/97ad33c6/hbase-client/src/main/java/org/apache/hadoop/hbase/ClusterId.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ClusterId.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ClusterId.java
index 6a3b14f..c127627 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ClusterId.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ClusterId.java
@@ -81,7 +81,7 @@ public class ClusterId {
/**
* @return A pb instance to represent this instance.
*/
- ClusterIdProtos.ClusterId convert() {
+ public ClusterIdProtos.ClusterId convert() {
ClusterIdProtos.ClusterId.Builder builder = ClusterIdProtos.ClusterId.newBuilder();
return builder.setClusterId(this.id).build();
}
@@ -90,7 +90,7 @@ public class ClusterId {
* @param cid
* @return A {@link ClusterId} made from the passed in <code>cid</code>
*/
- static ClusterId convert(final ClusterIdProtos.ClusterId cid) {
+ public static ClusterId convert(final ClusterIdProtos.ClusterId cid) {
return new ClusterId(cid.getClusterId());
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/97ad33c6/hbase-client/src/main/java/org/apache/hadoop/hbase/ClusterStatus.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ClusterStatus.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ClusterStatus.java
index 4b73dda..bc97a95 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ClusterStatus.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ClusterStatus.java
@@ -346,117 +346,4 @@ public class ClusterStatus extends VersionedWritable {
}
return sb.toString();
}
-
- /**
- * Convert a ClusterStatus to a protobuf ClusterStatus
- *
- * @return the protobuf ClusterStatus
- */
- public ClusterStatusProtos.ClusterStatus convert() {
- ClusterStatusProtos.ClusterStatus.Builder builder =
- ClusterStatusProtos.ClusterStatus.newBuilder();
- builder.setHbaseVersion(HBaseVersionFileContent.newBuilder().setVersion(getHBaseVersion()));
-
- if (liveServers != null){
- for (Map.Entry<ServerName, ServerLoad> entry : liveServers.entrySet()) {
- LiveServerInfo.Builder lsi =
- LiveServerInfo.newBuilder().setServer(ProtobufUtil.toServerName(entry.getKey()));
- lsi.setServerLoad(entry.getValue().obtainServerLoadPB());
- builder.addLiveServers(lsi.build());
- }
- }
-
- if (deadServers != null){
- for (ServerName deadServer : deadServers) {
- builder.addDeadServers(ProtobufUtil.toServerName(deadServer));
- }
- }
-
- if (intransition != null) {
- for (Map.Entry<String, RegionState> rit : getRegionsInTransition().entrySet()) {
- ClusterStatusProtos.RegionState rs = rit.getValue().convert();
- RegionSpecifier.Builder spec =
- RegionSpecifier.newBuilder().setType(RegionSpecifierType.REGION_NAME);
- spec.setValue(ByteStringer.wrap(Bytes.toBytes(rit.getKey())));
-
- RegionInTransition pbRIT =
- RegionInTransition.newBuilder().setSpec(spec.build()).setRegionState(rs).build();
- builder.addRegionsInTransition(pbRIT);
- }
- }
-
- if (clusterId != null) {
- builder.setClusterId(new ClusterId(clusterId).convert());
- }
-
- if (masterCoprocessors != null) {
- for (String coprocessor : masterCoprocessors) {
- builder.addMasterCoprocessors(HBaseProtos.Coprocessor.newBuilder().setName(coprocessor));
- }
- }
-
- if (master != null){
- builder.setMaster(ProtobufUtil.toServerName(getMaster()));
- }
-
- if (backupMasters != null) {
- for (ServerName backup : backupMasters) {
- builder.addBackupMasters(ProtobufUtil.toServerName(backup));
- }
- }
-
- if (balancerOn != null){
- builder.setBalancerOn(balancerOn);
- }
-
- return builder.build();
- }
-
- /**
- * Convert a protobuf ClusterStatus to a ClusterStatus
- *
- * @param proto the protobuf ClusterStatus
- * @return the converted ClusterStatus
- */
- public static ClusterStatus convert(ClusterStatusProtos.ClusterStatus proto) {
-
- Map<ServerName, ServerLoad> servers = null;
- servers = new HashMap<ServerName, ServerLoad>(proto.getLiveServersList().size());
- for (LiveServerInfo lsi : proto.getLiveServersList()) {
- servers.put(ProtobufUtil.toServerName(
- lsi.getServer()), new ServerLoad(lsi.getServerLoad()));
- }
-
- Collection<ServerName> deadServers = null;
- deadServers = new ArrayList<ServerName>(proto.getDeadServersList().size());
- for (HBaseProtos.ServerName sn : proto.getDeadServersList()) {
- deadServers.add(ProtobufUtil.toServerName(sn));
- }
-
- Collection<ServerName> backupMasters = null;
- backupMasters = new ArrayList<ServerName>(proto.getBackupMastersList().size());
- for (HBaseProtos.ServerName sn : proto.getBackupMastersList()) {
- backupMasters.add(ProtobufUtil.toServerName(sn));
- }
-
- Map<String, RegionState> rit = null;
- rit = new HashMap<String, RegionState>(proto.getRegionsInTransitionList().size());
- for (RegionInTransition region : proto.getRegionsInTransitionList()) {
- String key = new String(region.getSpec().getValue().toByteArray());
- RegionState value = RegionState.convert(region.getRegionState());
- rit.put(key, value);
- }
-
- String[] masterCoprocessors = null;
- final int numMasterCoprocessors = proto.getMasterCoprocessorsCount();
- masterCoprocessors = new String[numMasterCoprocessors];
- for (int i = 0; i < numMasterCoprocessors; i++) {
- masterCoprocessors[i] = proto.getMasterCoprocessors(i).getName();
- }
-
- return new ClusterStatus(proto.getHbaseVersion().getVersion(),
- ClusterId.convert(proto.getClusterId()).toString(),servers,deadServers,
- ProtobufUtil.toServerName(proto.getMaster()),backupMasters,rit,masterCoprocessors,
- proto.getBalancerOn());
- }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/97ad33c6/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncProcess.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncProcess.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncProcess.java
index b2c758d..6de8c82 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncProcess.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncProcess.java
@@ -55,6 +55,7 @@ import org.apache.hadoop.hbase.client.backoff.ServerStatistics;
import org.apache.hadoop.hbase.client.coprocessor.Batch;
import org.apache.hadoop.hbase.exceptions.ClientExceptionsUtil;
import org.apache.hadoop.hbase.ipc.RpcControllerFactory;
+import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
@@ -1718,10 +1719,11 @@ class AsyncProcess {
for (Map.Entry<byte[], MultiResponse.RegionResult> regionStats : results.entrySet()) {
byte[] regionName = regionStats.getKey();
ClientProtos.RegionLoadStats stat = regionStats.getValue().getStat();
+ RegionLoadStats regionLoadstats = ProtobufUtil.createRegionLoadStats(stat);
ResultStatsUtil.updateStats(AsyncProcess.this.connection.getStatisticsTracker(), server,
- regionName, stat);
+ regionName, regionLoadstats);
ResultStatsUtil.updateStats(AsyncProcess.this.connection.getConnectionMetrics(),
- server, regionName, stat);
+ server, regionName, regionLoadstats);
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/97ad33c6/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClusterStatusListener.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClusterStatusListener.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClusterStatusListener.java
index 5756232..9efb33d 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClusterStatusListener.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClusterStatusListener.java
@@ -49,6 +49,7 @@ import org.apache.hadoop.hbase.HBaseInterfaceAudience;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos;
import org.apache.hadoop.hbase.util.Addressing;
import org.apache.hadoop.hbase.util.ExceptionUtil;
@@ -257,7 +258,7 @@ class ClusterStatusListener implements Closeable {
ByteBufInputStream bis = new ByteBufInputStream(dp.content());
try {
ClusterStatusProtos.ClusterStatus csp = ClusterStatusProtos.ClusterStatus.parseFrom(bis);
- ClusterStatus ncs = ClusterStatus.convert(csp);
+ ClusterStatus ncs = ProtobufUtil.convert(csp);
receive(ncs);
} finally {
bis.close();
http://git-wip-us.apache.org/repos/asf/hbase/blob/97ad33c6/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
index 4b0609d..a2ef34b 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
@@ -53,6 +53,7 @@ import org.apache.hadoop.hbase.MetaTableAccessor;
import org.apache.hadoop.hbase.NamespaceDescriptor;
import org.apache.hadoop.hbase.NotServingRegionException;
import org.apache.hadoop.hbase.ProcedureInfo;
+import org.apache.hadoop.hbase.ProcedureUtil;
import org.apache.hadoop.hbase.RegionLocations;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableExistsException;
@@ -1847,7 +1848,7 @@ public class HBaseAdmin implements Admin {
PayloadCarryingRpcController controller = rpcControllerFactory.newController();
controller.setCallTimeout(callTimeout);
GetClusterStatusRequest req = RequestConverter.buildGetClusterStatusRequest();
- return ClusterStatus.convert(master.getClusterStatus(controller, req).getClusterStatus());
+ return ProtobufUtil.convert(master.getClusterStatus(controller, req).getClusterStatus());
}
});
}
@@ -2013,7 +2014,7 @@ public class HBaseAdmin implements Admin {
controller, ListProceduresRequest.newBuilder().build()).getProcedureList();
ProcedureInfo[] procInfoList = new ProcedureInfo[procList.size()];
for (int i = 0; i < procList.size(); i++) {
- procInfoList[i] = ProcedureInfo.convert(procList.get(i));
+ procInfoList[i] = ProcedureUtil.convert(procList.get(i));
}
return procInfoList;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/97ad33c6/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MetricsConnection.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MetricsConnection.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MetricsConnection.java
index 2839fb0..a5dc7fb 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MetricsConnection.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MetricsConnection.java
@@ -28,7 +28,6 @@ import com.codahale.metrics.JmxReporter;
import com.codahale.metrics.RatioGauge;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ClientService;
import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutateRequest;
import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto.MutationType;
@@ -156,7 +155,7 @@ public class MetricsConnection implements StatisticTrackable {
HEAP_BASE + this.name));
}
- public void update(ClientProtos.RegionLoadStats regionStatistics) {
+ public void update(RegionLoadStats regionStatistics) {
this.memstoreLoadHist.update(regionStatistics.getMemstoreLoad());
this.heapOccupancyHist.update(regionStatistics.getHeapOccupancy());
}
@@ -200,7 +199,7 @@ public class MetricsConnection implements StatisticTrackable {
return;
}
Result result = (Result) r;
- ClientProtos.RegionLoadStats stats = result.getStats();
+ RegionLoadStats stats = result.getStats();
if (stats == null) {
return;
}
@@ -209,7 +208,7 @@ public class MetricsConnection implements StatisticTrackable {
@Override
public void updateRegionStats(ServerName serverName, byte[] regionName,
- ClientProtos.RegionLoadStats stats) {
+ RegionLoadStats stats) {
String name = serverName.getServerName() + "," + Bytes.toStringBinary(regionName);
ConcurrentMap<byte[], RegionStats> rsStats = null;
if (serverStats.containsKey(serverName)) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/97ad33c6/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionLoadStats.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionLoadStats.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionLoadStats.java
new file mode 100644
index 0000000..443026f
--- /dev/null
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionLoadStats.java
@@ -0,0 +1,50 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.client;
+
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+/**
+ * POJO representing region server load
+ */
+public class RegionLoadStats {
+ int memstoreLoad;
+ int heapOccupancy;
+ int compactionPressure;
+
+ public RegionLoadStats(int memstoreLoad, int heapOccupancy, int compactionPressure) {
+ this.memstoreLoad = memstoreLoad;
+ this.heapOccupancy = heapOccupancy;
+ this.compactionPressure = compactionPressure;
+ }
+
+ public int getMemstoreLoad() {
+ return this.memstoreLoad;
+ }
+
+ public int getHeapOccupancy() {
+ return this.heapOccupancy;
+ }
+
+ public int getCompactionPressure() {
+ return this.compactionPressure;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/97ad33c6/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Result.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Result.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Result.java
index d2a49c2..c7393f6 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Result.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Result.java
@@ -111,7 +111,7 @@ public class Result implements CellScannable, CellScanner {
* Index for where we are when Result is acting as a {@link CellScanner}.
*/
private int cellScannerIndex = INITIAL_CELLSCANNER_INDEX;
- private ClientProtos.RegionLoadStats stats;
+ private RegionLoadStats stats;
private final boolean readonly;
@@ -909,25 +909,11 @@ public class Result implements CellScannable, CellScanner {
}
/**
- * Add load information about the region to the information about the result
- * @param loadStats statistics about the current region from which this was returned
- * @deprecated use {@link #setStatistics(ClientProtos.RegionLoadStats)} instead
- * @throws UnsupportedOperationException if invoked on instance of EMPTY_RESULT
- * (which is supposed to be immutable).
- */
- @InterfaceAudience.Private
- @Deprecated
- public void addResults(ClientProtos.RegionLoadStats loadStats) {
- checkReadonly();
- this.stats = loadStats;
- }
-
- /**
* Set load information about the region to the information about the result
* @param loadStats statistics about the current region from which this was returned
*/
@InterfaceAudience.Private
- public void setStatistics(ClientProtos.RegionLoadStats loadStats) {
+ public void setStatistics(RegionLoadStats loadStats) {
this.stats = loadStats;
}
@@ -935,7 +921,7 @@ public class Result implements CellScannable, CellScanner {
* @return the associated statistics about the region from which this was returned. Can be
* <tt>null</tt> if stats are disabled.
*/
- public ClientProtos.RegionLoadStats getStats() {
+ public RegionLoadStats getStats() {
return stats;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/97ad33c6/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ResultStatsUtil.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ResultStatsUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ResultStatsUtil.java
index 6537d79..b5bf6c1 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ResultStatsUtil.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ResultStatsUtil.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.hbase.client;
import org.apache.hadoop.hbase.HRegionLocation;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
/**
* A {@link Result} with some statistics about the server/region status
@@ -50,7 +49,7 @@ public final class ResultStatsUtil {
}
Result result = (Result) r;
// early exit if there are no stats to collect
- ClientProtos.RegionLoadStats stats = result.getStats();
+ RegionLoadStats stats = result.getStats();
if(stats == null){
return r;
}
@@ -60,7 +59,7 @@ public final class ResultStatsUtil {
}
public static void updateStats(StatisticTrackable tracker, ServerName server, byte[] regionName,
- ClientProtos.RegionLoadStats stats) {
+ RegionLoadStats stats) {
if (regionName != null && stats != null && tracker != null) {
tracker.updateRegionStats(server, regionName, stats);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/97ad33c6/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ServerStatisticTracker.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ServerStatisticTracker.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ServerStatisticTracker.java
index b8e7923..cb21e8b 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ServerStatisticTracker.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ServerStatisticTracker.java
@@ -23,7 +23,6 @@ import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.client.backoff.ServerStatistics;
-import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
import java.util.concurrent.ConcurrentHashMap;
@@ -37,7 +36,7 @@ public class ServerStatisticTracker implements StatisticTrackable {
new ConcurrentHashMap<ServerName, ServerStatistics>();
@Override
- public void updateRegionStats(ServerName server, byte[] region, ClientProtos.RegionLoadStats
+ public void updateRegionStats(ServerName server, byte[] region, RegionLoadStats
currentStats) {
ServerStatistics stat = stats.get(server);
http://git-wip-us.apache.org/repos/asf/hbase/blob/97ad33c6/hbase-client/src/main/java/org/apache/hadoop/hbase/client/StatisticTrackable.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/StatisticTrackable.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/StatisticTrackable.java
index 7bb49e7..69f06ff 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/StatisticTrackable.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/StatisticTrackable.java
@@ -18,7 +18,6 @@ package org.apache.hadoop.hbase.client;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
/**
* Parent interface for an object to get updates about per-region statistics.
@@ -28,6 +27,5 @@ public interface StatisticTrackable {
/**
* Update stats per region.
* */
- void updateRegionStats(ServerName server, byte[] region, ClientProtos.RegionLoadStats
- stats);
+ void updateRegionStats(ServerName server, byte[] region, RegionLoadStats stats);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/97ad33c6/hbase-client/src/main/java/org/apache/hadoop/hbase/client/backoff/ServerStatistics.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/backoff/ServerStatistics.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/backoff/ServerStatistics.java
index 2072573..e33e2bc 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/backoff/ServerStatistics.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/backoff/ServerStatistics.java
@@ -18,7 +18,7 @@
package org.apache.hadoop.hbase.client.backoff;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
+import org.apache.hadoop.hbase.client.RegionLoadStats;
import org.apache.hadoop.hbase.util.Bytes;
import java.util.Map;
@@ -39,7 +39,7 @@ public class ServerStatistics {
* @param region
* @param currentStats
*/
- public void update(byte[] region, ClientProtos.RegionLoadStats currentStats) {
+ public void update(byte[] region, RegionLoadStats currentStats) {
RegionStatistics regionStat = this.stats.get(region);
if(regionStat == null){
regionStat = new RegionStatistics();
@@ -59,7 +59,7 @@ public class ServerStatistics {
private int heapOccupancy = 0;
private int compactionPressure = 0;
- public void update(ClientProtos.RegionLoadStats currentStats) {
+ public void update(RegionLoadStats currentStats) {
this.memstoreLoad = currentStats.getMemstoreLoad();
this.heapOccupancy = currentStats.getHeapOccupancy();
this.compactionPressure = currentStats.getCompactionPressure();
http://git-wip-us.apache.org/repos/asf/hbase/blob/97ad33c6/hbase-client/src/main/java/org/apache/hadoop/hbase/client/coprocessor/BigDecimalColumnInterpreter.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/coprocessor/BigDecimalColumnInterpreter.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/coprocessor/BigDecimalColumnInterpreter.java
index 7d0715b..5d1cc91 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/coprocessor/BigDecimalColumnInterpreter.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/coprocessor/BigDecimalColumnInterpreter.java
@@ -24,7 +24,9 @@ import java.math.RoundingMode;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.HBaseInterfaceAudience;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
import org.apache.hadoop.hbase.coprocessor.ColumnInterpreter;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.BigDecimalMsg;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.EmptyMsg;
@@ -36,7 +38,8 @@ import org.apache.hadoop.hbase.util.Bytes;
* is required at the RegionServer also.
*
*/
-@InterfaceAudience.Private
+@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.COPROC)
+@InterfaceStability.Evolving
public class BigDecimalColumnInterpreter extends ColumnInterpreter<BigDecimal, BigDecimal,
EmptyMsg, BigDecimalMsg, BigDecimalMsg> {
http://git-wip-us.apache.org/repos/asf/hbase/blob/97ad33c6/hbase-client/src/main/java/org/apache/hadoop/hbase/client/coprocessor/DoubleColumnInterpreter.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/coprocessor/DoubleColumnInterpreter.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/coprocessor/DoubleColumnInterpreter.java
index 5b8571f..c33bb00 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/coprocessor/DoubleColumnInterpreter.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/coprocessor/DoubleColumnInterpreter.java
@@ -22,6 +22,7 @@ import java.io.IOException;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.HBaseInterfaceAudience;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.classification.InterfaceStability;
import org.apache.hadoop.hbase.coprocessor.ColumnInterpreter;
@@ -37,7 +38,7 @@ import org.apache.hadoop.hbase.util.Bytes;
* TestDoubleColumnInterpreter methods for its sample usage.
* Its methods handle null arguments gracefully.
*/
-@InterfaceAudience.Public
+@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.COPROC)
@InterfaceStability.Evolving
public class DoubleColumnInterpreter extends ColumnInterpreter<Double, Double,
EmptyMsg, DoubleMsg, DoubleMsg>{
http://git-wip-us.apache.org/repos/asf/hbase/blob/97ad33c6/hbase-client/src/main/java/org/apache/hadoop/hbase/client/coprocessor/LongColumnInterpreter.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/coprocessor/LongColumnInterpreter.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/coprocessor/LongColumnInterpreter.java
index 586575a..f738b10 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/coprocessor/LongColumnInterpreter.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/coprocessor/LongColumnInterpreter.java
@@ -22,7 +22,9 @@ import java.io.IOException;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.HBaseInterfaceAudience;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
import org.apache.hadoop.hbase.coprocessor.ColumnInterpreter;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.EmptyMsg;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.LongMsg;
@@ -36,7 +38,8 @@ import org.apache.hadoop.hbase.util.Bytes;
* TestAggregateProtocol methods for its sample usage.
* Its methods handle null arguments gracefully.
*/
-@InterfaceAudience.Private
+@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.COPROC)
+@InterfaceStability.Evolving
public class LongColumnInterpreter extends ColumnInterpreter<Long, Long,
EmptyMsg, LongMsg, LongMsg> {
http://git-wip-us.apache.org/repos/asf/hbase/blob/97ad33c6/hbase-client/src/main/java/org/apache/hadoop/hbase/coprocessor/ColumnInterpreter.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/coprocessor/ColumnInterpreter.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/coprocessor/ColumnInterpreter.java
index e247c08..e22bd24 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/coprocessor/ColumnInterpreter.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/coprocessor/ColumnInterpreter.java
@@ -22,7 +22,9 @@ package org.apache.hadoop.hbase.coprocessor;
import java.io.IOException;
import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.HBaseInterfaceAudience;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
import com.google.protobuf.Message;
@@ -53,7 +55,8 @@ import com.google.protobuf.Message;
* @param Q PB message that is used to transport Cell (<T>) instance
* @param R PB message that is used to transport Promoted (<S>) instance
*/
-@InterfaceAudience.Private
+@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.COPROC)
+@InterfaceStability.Evolving
public abstract class ColumnInterpreter<T, S, P extends Message,
Q extends Message, R extends Message> {
http://git-wip-us.apache.org/repos/asf/hbase/blob/97ad33c6/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
index 62dfd45..9c0dd6a 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
@@ -44,6 +44,8 @@ import static org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpeci
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellScanner;
import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.ClusterId;
+import org.apache.hadoop.hbase.ClusterStatus;
import org.apache.hadoop.hbase.DoNotRetryIOException;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HColumnDescriptor;
@@ -52,6 +54,7 @@ import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.NamespaceDescriptor;
+import org.apache.hadoop.hbase.ServerLoad;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.Tag;
@@ -66,6 +69,7 @@ import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Increment;
import org.apache.hadoop.hbase.client.Mutation;
import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.RegionLoadStats;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.SnapshotDescription;
@@ -77,6 +81,7 @@ import org.apache.hadoop.hbase.filter.ByteArrayComparable;
import org.apache.hadoop.hbase.filter.Filter;
import org.apache.hadoop.hbase.io.LimitInputStream;
import org.apache.hadoop.hbase.io.TimeRange;
+import org.apache.hadoop.hbase.master.RegionState;
import org.apache.hadoop.hbase.protobuf.generated.AccessControlProtos;
import org.apache.hadoop.hbase.protobuf.generated.AccessControlProtos.AccessControlService;
import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService;
@@ -113,7 +118,10 @@ import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto.Del
import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto.MutationType;
import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ScanRequest;
import org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos;
+import org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos.LiveServerInfo;
+import org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos.RegionInTransition;
import org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos.RegionLoad;
+import org.apache.hadoop.hbase.protobuf.generated.FSProtos.HBaseVersionFileContent;
import org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos;
import org.apache.hadoop.hbase.protobuf.generated.FilterProtos;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos;
@@ -3492,4 +3500,124 @@ public final class ProtobufUtil {
createSnapshotType(snapshotDesc.getType()), snapshotDesc.getOwner(),
snapshotDesc.getCreationTime(), snapshotDesc.getVersion());
}
+
+ /**
+ * Convert a protobuf ClusterStatus to a ClusterStatus
+ *
+ * @param proto the protobuf ClusterStatus
+ * @return the converted ClusterStatus
+ */
+ public static ClusterStatus convert(ClusterStatusProtos.ClusterStatus proto) {
+
+ Map<ServerName, ServerLoad> servers = null;
+ servers = new HashMap<ServerName, ServerLoad>(proto.getLiveServersList().size());
+ for (LiveServerInfo lsi : proto.getLiveServersList()) {
+ servers.put(ProtobufUtil.toServerName(
+ lsi.getServer()), new ServerLoad(lsi.getServerLoad()));
+ }
+
+ Collection<ServerName> deadServers = null;
+ deadServers = new ArrayList<ServerName>(proto.getDeadServersList().size());
+ for (HBaseProtos.ServerName sn : proto.getDeadServersList()) {
+ deadServers.add(ProtobufUtil.toServerName(sn));
+ }
+
+ Collection<ServerName> backupMasters = null;
+ backupMasters = new ArrayList<ServerName>(proto.getBackupMastersList().size());
+ for (HBaseProtos.ServerName sn : proto.getBackupMastersList()) {
+ backupMasters.add(ProtobufUtil.toServerName(sn));
+ }
+
+ Map<String, RegionState> rit = null;
+ rit = new HashMap<String, RegionState>(proto.getRegionsInTransitionList().size());
+ for (RegionInTransition region : proto.getRegionsInTransitionList()) {
+ String key = new String(region.getSpec().getValue().toByteArray());
+ RegionState value = RegionState.convert(region.getRegionState());
+ rit.put(key, value);
+ }
+
+ String[] masterCoprocessors = null;
+ final int numMasterCoprocessors = proto.getMasterCoprocessorsCount();
+ masterCoprocessors = new String[numMasterCoprocessors];
+ for (int i = 0; i < numMasterCoprocessors; i++) {
+ masterCoprocessors[i] = proto.getMasterCoprocessors(i).getName();
+ }
+
+ return new ClusterStatus(proto.getHbaseVersion().getVersion(),
+ ClusterId.convert(proto.getClusterId()).toString(),servers,deadServers,
+ ProtobufUtil.toServerName(proto.getMaster()),backupMasters,rit,masterCoprocessors,
+ proto.getBalancerOn());
+ }
+
+ /**
+ * Convert a ClusterStatus to a protobuf ClusterStatus
+ *
+ * @return the protobuf ClusterStatus
+ */
+ public static ClusterStatusProtos.ClusterStatus convert(ClusterStatus status) {
+ ClusterStatusProtos.ClusterStatus.Builder builder =
+ ClusterStatusProtos.ClusterStatus.newBuilder();
+ builder
+ .setHbaseVersion(HBaseVersionFileContent.newBuilder().setVersion(status.getHBaseVersion()));
+
+ if (status.getServers() != null) {
+ for (ServerName serverName : status.getServers()) {
+ LiveServerInfo.Builder lsi =
+ LiveServerInfo.newBuilder().setServer(ProtobufUtil.toServerName(serverName));
+ status.getLoad(serverName);
+ lsi.setServerLoad(status.getLoad(serverName).obtainServerLoadPB());
+ builder.addLiveServers(lsi.build());
+ }
+ }
+
+ if (status.getDeadServerNames() != null) {
+ for (ServerName deadServer : status.getDeadServerNames()) {
+ builder.addDeadServers(ProtobufUtil.toServerName(deadServer));
+ }
+ }
+
+ if (status.getRegionsInTransition() != null) {
+ for (Map.Entry<String, RegionState> rit : status.getRegionsInTransition().entrySet()) {
+ ClusterStatusProtos.RegionState rs = rit.getValue().convert();
+ RegionSpecifier.Builder spec =
+ RegionSpecifier.newBuilder().setType(RegionSpecifierType.REGION_NAME);
+ spec.setValue(ByteStringer.wrap(Bytes.toBytes(rit.getKey())));
+
+ RegionInTransition pbRIT =
+ RegionInTransition.newBuilder().setSpec(spec.build()).setRegionState(rs).build();
+ builder.addRegionsInTransition(pbRIT);
+ }
+ }
+
+ if (status.getClusterId() != null) {
+ builder.setClusterId(new ClusterId(status.getClusterId()).convert());
+ }
+
+ if (status.getMasterCoprocessors() != null) {
+ for (String coprocessor : status.getMasterCoprocessors()) {
+ builder.addMasterCoprocessors(HBaseProtos.Coprocessor.newBuilder().setName(coprocessor));
+ }
+ }
+
+ if (status.getMaster() != null) {
+ builder.setMaster(ProtobufUtil.toServerName(status.getMaster()));
+ }
+
+ if (status.getBackupMasters() != null) {
+ for (ServerName backup : status.getBackupMasters()) {
+ builder.addBackupMasters(ProtobufUtil.toServerName(backup));
+ }
+ }
+
+ if (status.getBalancerOn() != null) {
+ builder.setBalancerOn(status.getBalancerOn());
+ }
+
+ return builder.build();
+ }
+
+ public static RegionLoadStats createRegionLoadStats(ClientProtos.RegionLoadStats stats) {
+ return new RegionLoadStats(stats.getMemstoreLoad(), stats.getHeapOccupancy(),
+ stats.getCompactionPressure());
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/97ad33c6/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestClientExponentialBackoff.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestClientExponentialBackoff.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestClientExponentialBackoff.java
index dcd4fca..838c40e 100644
--- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestClientExponentialBackoff.java
+++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestClientExponentialBackoff.java
@@ -21,6 +21,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.client.backoff.ExponentialClientBackoffPolicy;
import org.apache.hadoop.hbase.client.backoff.ServerStatistics;
+import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
import org.apache.hadoop.hbase.testclassification.ClientTests;
import org.apache.hadoop.hbase.testclassification.SmallTests;
@@ -157,7 +158,7 @@ public class TestClientExponentialBackoff {
ClientProtos.RegionLoadStats stat = ClientProtos.RegionLoadStats.newBuilder()
.setMemstoreLoad
(load).build();
- stats.update(regionname, stat);
+ stats.update(regionname, ProtobufUtil.createRegionLoadStats(stat));
}
private void update(ServerStatistics stats, int memstoreLoad, int heapOccupancy,
@@ -167,6 +168,6 @@ public class TestClientExponentialBackoff {
.setHeapOccupancy(heapOccupancy)
.setCompactionPressure(compactionPressure)
.build();
- stats.update(regionname, stat);
+ stats.update(regionname, ProtobufUtil.createRegionLoadStats(stat));
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/97ad33c6/hbase-common/src/main/java/org/apache/hadoop/hbase/ProcedureInfo.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/ProcedureInfo.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/ProcedureInfo.java
index 562ad7d..5bbff87 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/ProcedureInfo.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/ProcedureInfo.java
@@ -22,11 +22,7 @@ import java.io.IOException;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.classification.InterfaceStability;
-import org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage;
-import org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos;
-import org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.ProcedureState;
import org.apache.hadoop.hbase.security.User;
-import org.apache.hadoop.hbase.util.ByteStringer;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.ForeignExceptionUtil;
import org.apache.hadoop.hbase.util.NonceKey;
@@ -44,7 +40,7 @@ public class ProcedureInfo implements Cloneable {
private final ProcedureState procState;
private final long parentId;
private final NonceKey nonceKey;
- private final ForeignExceptionMessage exception;
+ private final ProcedureUtil.ForeignExceptionMsg exception;
private final long lastUpdate;
private final long startTime;
private final byte[] result;
@@ -59,7 +55,7 @@ public class ProcedureInfo implements Cloneable {
final ProcedureState procState,
final long parentId,
final NonceKey nonceKey,
- final ForeignExceptionMessage exception,
+ final ProcedureUtil.ForeignExceptionMsg exception,
final long lastUpdate,
final long startTime,
final byte[] result) {
@@ -156,24 +152,24 @@ public class ProcedureInfo implements Cloneable {
public IOException getException() {
if (isFailed()) {
- return ForeignExceptionUtil.toIOException(exception);
+ return ForeignExceptionUtil.toIOException(exception.getForeignExchangeMessage());
}
return null;
}
@InterfaceAudience.Private
- public ForeignExceptionMessage getForeignExceptionMessage() {
+ public ProcedureUtil.ForeignExceptionMsg getForeignExceptionMessage() {
return exception;
}
public String getExceptionCause() {
assert isFailed();
- return exception.getGenericException().getClassName();
+ return exception.getForeignExchangeMessage().getGenericException().getClassName();
}
public String getExceptionMessage() {
assert isFailed();
- return exception.getGenericException().getMessage();
+ return exception.getForeignExchangeMessage().getGenericException().getMessage();
}
public String getExceptionFullMessage() {
@@ -217,65 +213,6 @@ public class ProcedureInfo implements Cloneable {
}
/**
- * @return Convert the current {@link ProcedureInfo} into a Protocol Buffers Procedure
- * instance.
- */
- @InterfaceAudience.Private
- public static ProcedureProtos.Procedure convertToProcedureProto(
- final ProcedureInfo procInfo) {
- ProcedureProtos.Procedure.Builder builder = ProcedureProtos.Procedure.newBuilder();
-
- builder.setClassName(procInfo.getProcName());
- builder.setProcId(procInfo.getProcId());
- builder.setStartTime(procInfo.getStartTime());
- builder.setState(procInfo.getProcState());
- builder.setLastUpdate(procInfo.getLastUpdate());
-
- if (procInfo.hasParentId()) {
- builder.setParentId(procInfo.getParentId());
- }
-
- if (procInfo.getProcOwner() != null) {
- builder.setOwner(procInfo.getProcOwner());
- }
-
- if (procInfo.isFailed()) {
- builder.setException(procInfo.getForeignExceptionMessage());
- }
-
- if (procInfo.hasResultData()) {
- builder.setResult(ByteStringer.wrap(procInfo.getResult()));
- }
-
- return builder.build();
- }
-
- /**
- * Helper to convert the protobuf object.
- * @return Convert the current Protocol Buffers Procedure to {@link ProcedureInfo}
- * instance.
- */
- @InterfaceAudience.Private
- public static ProcedureInfo convert(final ProcedureProtos.Procedure procProto) {
- NonceKey nonceKey = null;
- if (procProto.getNonce() != HConstants.NO_NONCE) {
- nonceKey = new NonceKey(procProto.getNonceGroup(), procProto.getNonce());
- }
-
- return new ProcedureInfo(
- procProto.getProcId(),
- procProto.getClassName(),
- procProto.getOwner(),
- procProto.getState(),
- procProto.hasParentId() ? procProto.getParentId() : -1,
- nonceKey,
- procProto.hasException() ? procProto.getException() : null,
- procProto.getLastUpdate(),
- procProto.getStartTime(),
- procProto.hasResult() ? procProto.getResult().toByteArray() : null);
- }
-
- /**
* Check if the user is this procedure's owner
* @param procInfo the procedure to check
* @param user the user
@@ -293,4 +230,5 @@ public class ProcedureInfo implements Cloneable {
}
return procOwner.equals(user.getShortName());
}
+
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/97ad33c6/hbase-common/src/main/java/org/apache/hadoop/hbase/ProcedureState.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/ProcedureState.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/ProcedureState.java
new file mode 100644
index 0000000..306d285
--- /dev/null
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/ProcedureState.java
@@ -0,0 +1,30 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase;
+
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+
+/**
+ * POJO representing Procedure State
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public enum ProcedureState {
+ INITIALIZING, RUNNABLE, WAITING, WAITING_TIMEOUT, ROLLEDBACK, FINISHED;
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/97ad33c6/hbase-common/src/main/java/org/apache/hadoop/hbase/ProcedureUtil.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/ProcedureUtil.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/ProcedureUtil.java
new file mode 100644
index 0000000..2c935f3
--- /dev/null
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/ProcedureUtil.java
@@ -0,0 +1,103 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase;
+
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos;
+import org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage;
+import org.apache.hadoop.hbase.util.ByteStringer;
+import org.apache.hadoop.hbase.util.NonceKey;
+
+@InterfaceAudience.Private
+
+/**
+ * Helper to convert to/from ProcedureProtos
+ */
+public class ProcedureUtil {
+
+ private ProcedureUtil() {
+ }
+ /**
+ * @return Convert the current {@link ProcedureInfo} into a Protocol Buffers Procedure
+ * instance.
+ */
+ @InterfaceAudience.Private
+ public static ProcedureProtos.Procedure convertToProcedureProto(final ProcedureInfo procInfo) {
+ ProcedureProtos.Procedure.Builder builder = ProcedureProtos.Procedure.newBuilder();
+
+ builder.setClassName(procInfo.getProcName());
+ builder.setProcId(procInfo.getProcId());
+ builder.setStartTime(procInfo.getStartTime());
+ builder.setState(ProcedureProtos.ProcedureState.valueOf(procInfo.getProcState().name()));
+ builder.setLastUpdate(procInfo.getLastUpdate());
+
+ if (procInfo.hasParentId()) {
+ builder.setParentId(procInfo.getParentId());
+ }
+
+ if (procInfo.getProcOwner() != null) {
+ builder.setOwner(procInfo.getProcOwner());
+ }
+
+ if (procInfo.isFailed()) {
+ builder.setException(procInfo.getForeignExceptionMessage().getForeignExchangeMessage());
+ }
+
+ if (procInfo.hasResultData()) {
+ builder.setResult(ByteStringer.wrap(procInfo.getResult()));
+ }
+
+ return builder.build();
+ }
+
+ /**
+ * Helper to convert the protobuf object.
+ * @return Convert the current Protocol Buffers Procedure to {@link ProcedureInfo}
+ * instance.
+ */
+ @InterfaceAudience.Private
+ public static ProcedureInfo convert(final ProcedureProtos.Procedure procProto) {
+ NonceKey nonceKey = null;
+ if (procProto.getNonce() != HConstants.NO_NONCE) {
+ nonceKey = new NonceKey(procProto.getNonceGroup(), procProto.getNonce());
+ }
+
+ return new ProcedureInfo(procProto.getProcId(), procProto.getClassName(), procProto.getOwner(),
+ convertToProcedureState(procProto.getState()),
+ procProto.hasParentId() ? procProto.getParentId() : -1, nonceKey,
+ procProto.hasException() ? new ForeignExceptionMsg(procProto.getException()) : null,
+ procProto.getLastUpdate(), procProto.getStartTime(),
+ procProto.hasResult() ? procProto.getResult().toByteArray() : null);
+ }
+
+ public static ProcedureState convertToProcedureState(ProcedureProtos.ProcedureState state) {
+ return ProcedureState.valueOf(state.name());
+ }
+
+ public static class ForeignExceptionMsg {
+ private ForeignExceptionMessage exception;
+
+ public ForeignExceptionMsg(ForeignExceptionMessage exception) {
+ this.exception = exception;
+ }
+
+ public ForeignExceptionMessage getForeignExchangeMessage() {
+ return this.exception;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/97ad33c6/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/Procedure.java
----------------------------------------------------------------------
diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/Procedure.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/Procedure.java
index 2cc67be..7e58420 100644
--- a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/Procedure.java
+++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/Procedure.java
@@ -29,6 +29,7 @@ import java.util.Map;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.ProcedureInfo;
+import org.apache.hadoop.hbase.ProcedureUtil;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.classification.InterfaceStability;
import org.apache.hadoop.hbase.exceptions.TimeoutIOException;
@@ -688,18 +689,14 @@ public abstract class Procedure<TEnvironment> implements Comparable<Procedure> {
@InterfaceAudience.Private
public static ProcedureInfo createProcedureInfo(final Procedure proc, final NonceKey nonceKey) {
RemoteProcedureException exception = proc.hasException() ? proc.getException() : null;
- return new ProcedureInfo(
- proc.getProcId(),
- proc.toStringClass(),
- proc.getOwner(),
- proc.getState(),
- proc.hasParent() ? proc.getParentProcId() : -1,
- nonceKey,
- exception != null ?
- RemoteProcedureException.toProto(exception.getSource(), exception.getCause()) : null,
- proc.getLastUpdate(),
- proc.getStartTime(),
- proc.getResult());
+ return new ProcedureInfo(proc.getProcId(), proc.toStringClass(), proc.getOwner(),
+ ProcedureUtil.convertToProcedureState(proc.getState()),
+ proc.hasParent() ? proc.getParentProcId() : -1, nonceKey,
+ exception != null
+ ? new ProcedureUtil.ForeignExceptionMsg(
+ RemoteProcedureException.toProto(exception.getSource(), exception.getCause()))
+ : null,
+ proc.getLastUpdate(), proc.getStartTime(), proc.getResult());
}
/**
http://git-wip-us.apache.org/repos/asf/hbase/blob/97ad33c6/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormatReader.java
----------------------------------------------------------------------
diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormatReader.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormatReader.java
index 312eedb..b1b201b 100644
--- a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormatReader.java
+++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormatReader.java
@@ -24,6 +24,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.hbase.ProcedureInfo;
+import org.apache.hadoop.hbase.ProcedureUtil;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.classification.InterfaceStability;
import org.apache.hadoop.hbase.procedure2.Procedure;
@@ -288,7 +289,7 @@ public class ProcedureWALFormatReader {
}
public ProcedureInfo convertToInfo() {
- return ProcedureInfo.convert(proto);
+ return ProcedureUtil.convert(proto);
}
@Override
http://git-wip-us.apache.org/repos/asf/hbase/blob/97ad33c6/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.java
----------------------------------------------------------------------
diff --git a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.java b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.java
index 9edaec9..4cf48d2 100644
--- a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.java
+++ b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.java
@@ -175,9 +175,12 @@ public class ProcedureTestingUtility {
}
public static void assertProcNotFailed(final ProcedureInfo result) {
- ForeignExceptionMessage exception = result.getForeignExceptionMessage();
- String msg = exception != null ? result.getExceptionFullMessage() : "no exception found";
- assertFalse(msg, result.isFailed());
+ if (result.getForeignExceptionMessage() != null) {
+ ForeignExceptionMessage exception =
+ result.getForeignExceptionMessage().getForeignExchangeMessage();
+ String msg = exception != null ? result.getExceptionFullMessage() : "no exception found";
+ assertFalse(msg, result.isFailed());
+ }
}
public static void assertIsAbortException(final ProcedureInfo result) {
@@ -204,7 +207,8 @@ public class ProcedureTestingUtility {
public static Throwable getExceptionCause(final ProcedureInfo procInfo) {
assert procInfo.getForeignExceptionMessage() != null;
- return RemoteProcedureException.fromProto(procInfo.getForeignExceptionMessage()).getCause();
+ return RemoteProcedureException
+ .fromProto(procInfo.getForeignExceptionMessage().getForeignExchangeMessage()).getCause();
}
public static class TestProcedure extends Procedure<Void> {
http://git-wip-us.apache.org/repos/asf/hbase/blob/97ad33c6/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
index 8a040fa..a281900 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
@@ -36,6 +36,7 @@ import org.apache.hadoop.hbase.MetaTableAccessor;
import org.apache.hadoop.hbase.NamespaceDescriptor;
import org.apache.hadoop.hbase.PleaseHoldException;
import org.apache.hadoop.hbase.ProcedureInfo;
+import org.apache.hadoop.hbase.ProcedureUtil;
import org.apache.hadoop.hbase.ServerLoad;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableName;
@@ -707,7 +708,7 @@ public class MasterRpcServices extends RSRpcServices
GetClusterStatusResponse.Builder response = GetClusterStatusResponse.newBuilder();
try {
master.checkInitialized();
- response.setClusterStatus(master.getClusterStatus().convert());
+ response.setClusterStatus(ProtobufUtil.convert(master.getClusterStatus()));
} catch (IOException e) {
throw new ServiceException(e);
}
@@ -958,7 +959,7 @@ public class MasterRpcServices extends RSRpcServices
builder.setStartTime(result.getStartTime());
builder.setLastUpdate(result.getLastUpdate());
if (result.isFailed()) {
- builder.setException(result.getForeignExceptionMessage());
+ builder.setException(result.getForeignExceptionMessage().getForeignExchangeMessage());
}
if (result.hasResultData()) {
builder.setResult(ByteStringer.wrap(result.getResult()));
@@ -1018,7 +1019,7 @@ public class MasterRpcServices extends RSRpcServices
ListProceduresResponse.Builder response =
ListProceduresResponse.newBuilder();
for(ProcedureInfo p: master.listProcedures()) {
- response.addProcedure(ProcedureInfo.convertToProcedureProto(p));
+ response.addProcedure(ProcedureUtil.convertToProcedureProto(p));
}
return response.build();
} catch (IOException e) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/97ad33c6/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ProcedureSyncWait.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ProcedureSyncWait.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ProcedureSyncWait.java
index 6a7e267..8a5eb35 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ProcedureSyncWait.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ProcedureSyncWait.java
@@ -83,7 +83,7 @@ public final class ProcedureSyncWait {
if (result.isFailed()) {
// If the procedure fails, we should always have an exception captured. Throw it.
throw RemoteProcedureException.fromProto(
- result.getForeignExceptionMessage()).unwrapRemoteException();
+ result.getForeignExceptionMessage().getForeignExchangeMessage()).unwrapRemoteException();
}
return result.getResult();
} else {
http://git-wip-us.apache.org/repos/asf/hbase/blob/97ad33c6/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestResult.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestResult.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestResult.java
index 6c76ce4..6baf9e0 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestResult.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestResult.java
@@ -35,7 +35,6 @@ import org.apache.hadoop.hbase.CellComparator;
import org.apache.hadoop.hbase.CellScanner;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
import org.apache.hadoop.hbase.testclassification.ClientTests;
import org.apache.hadoop.hbase.testclassification.SmallTests;
import org.apache.hadoop.hbase.util.Bytes;
@@ -245,7 +244,7 @@ public class TestResult extends TestCase {
LOG.debug("As expected: " + ex.getMessage());
}
try {
- emptyResult.addResults(ClientProtos.RegionLoadStats.getDefaultInstance());
+ emptyResult.addResults(new RegionLoadStats(0, 0, 0));
fail("UnsupportedOperationException should have been thrown!");
} catch (UnsupportedOperationException ex) {
LOG.debug("As expected: " + ex.getMessage());
http://git-wip-us.apache.org/repos/asf/hbase/blob/97ad33c6/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestProcedureAdmin.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestProcedureAdmin.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestProcedureAdmin.java
index 17b6e09..31190c1 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestProcedureAdmin.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestProcedureAdmin.java
@@ -29,12 +29,12 @@ import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.ProcedureInfo;
+import org.apache.hadoop.hbase.ProcedureState;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
import org.apache.hadoop.hbase.testclassification.MasterTests;
import org.apache.hadoop.hbase.testclassification.MediumTests;
-import org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.ProcedureState;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
[21/50] [abbrv] hbase git commit: HBASE-15767 upgrade httpclient to
4.3.6
Posted by sy...@apache.org.
HBASE-15767 upgrade httpclient to 4.3.6
Signed-off-by: Sean Busbey <bu...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/c716a754
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/c716a754
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/c716a754
Branch: refs/heads/hbase-12439
Commit: c716a75416b8ef01f2c4e233f1340f3dbf33198e
Parents: 0bf065a
Author: tedyu <yu...@gmail.com>
Authored: Thu May 5 15:41:26 2016 -0500
Committer: Sean Busbey <bu...@apache.org>
Committed: Thu May 5 15:41:26 2016 -0500
----------------------------------------------------------------------
hbase-rest/pom.xml | 4 ++--
hbase-server/pom.xml | 4 ++--
pom.xml | 6 +++---
3 files changed, 7 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/c716a754/hbase-rest/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-rest/pom.xml b/hbase-rest/pom.xml
index 7bacf95..a66be01 100644
--- a/hbase-rest/pom.xml
+++ b/hbase-rest/pom.xml
@@ -270,8 +270,8 @@
<artifactId>commons-cli</artifactId>
</dependency>
<dependency>
- <groupId>commons-httpclient</groupId>
- <artifactId>commons-httpclient</artifactId>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
</dependency>
<dependency>
<groupId>commons-lang</groupId>
http://git-wip-us.apache.org/repos/asf/hbase/blob/c716a754/hbase-server/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-server/pom.xml b/hbase-server/pom.xml
index 2631417..9532d1e 100644
--- a/hbase-server/pom.xml
+++ b/hbase-server/pom.xml
@@ -395,8 +395,8 @@
<optional>true</optional>
</dependency>
<dependency>
- <groupId>commons-httpclient</groupId>
- <artifactId>commons-httpclient</artifactId>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
</dependency>
<dependency>
<groupId>commons-codec</groupId>
http://git-wip-us.apache.org/repos/asf/hbase/blob/c716a754/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index cf6501c..0b3c61a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1213,7 +1213,7 @@
<disruptor.version>3.3.0</disruptor.version>
<!-- Do not use versions earlier than 3.2.2 due to a security vulnerability -->
<collections.version>3.2.2</collections.version>
- <httpclient.version>3.1</httpclient.version>
+ <httpclient.version>4.3.6</httpclient.version>
<metrics-core.version>3.1.2</metrics-core.version>
<guava.version>12.0.1</guava.version>
<jsr305.version>1.3.9</jsr305.version>
@@ -1502,8 +1502,8 @@
<version>${collections.version}</version>
</dependency>
<dependency>
- <groupId>commons-httpclient</groupId>
- <artifactId>commons-httpclient</artifactId>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
<version>${httpclient.version}</version>
</dependency>
<dependency>
[26/50] [abbrv] hbase git commit: HBASE-15782 TestShell fails due to
some moved types
Posted by sy...@apache.org.
HBASE-15782 TestShell fails due to some moved types
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/387c7e6b
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/387c7e6b
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/387c7e6b
Branch: refs/heads/hbase-12439
Commit: 387c7e6b083fddeae2a7ebe1fef3546f38ef9fb5
Parents: 34e9a6f
Author: tedyu <yu...@gmail.com>
Authored: Fri May 6 06:11:12 2016 -0700
Committer: tedyu <yu...@gmail.com>
Committed: Fri May 6 06:11:12 2016 -0700
----------------------------------------------------------------------
hbase-shell/src/main/ruby/hbase/admin.rb | 11 +++++------
1 file changed, 5 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/387c7e6b/hbase-shell/src/main/ruby/hbase/admin.rb
----------------------------------------------------------------------
diff --git a/hbase-shell/src/main/ruby/hbase/admin.rb b/hbase-shell/src/main/ruby/hbase/admin.rb
index 88486c0..7ea315f 100644
--- a/hbase-shell/src/main/ruby/hbase/admin.rb
+++ b/hbase-shell/src/main/ruby/hbase/admin.rb
@@ -24,7 +24,6 @@ java_import org.apache.hadoop.hbase.util.RegionSplitter
java_import org.apache.hadoop.hbase.util.Bytes
java_import org.apache.hadoop.hbase.ServerName
java_import org.apache.hadoop.hbase.TableName
-java_import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos::SnapshotDescription
# Wrapper for org.apache.hadoop.hbase.client.HBaseAdmin
@@ -68,9 +67,9 @@ module Hbase
end
compact_type = nil
if type == "NORMAL"
- compact_type = org.apache.hadoop.hbase.client.Admin::CompactType::NORMAL
+ compact_type = org.apache.hadoop.hbase.client.CompactType::NORMAL
elsif type == "MOB"
- compact_type = org.apache.hadoop.hbase.client.Admin::CompactType::MOB
+ compact_type = org.apache.hadoop.hbase.client.CompactType::MOB
else
raise ArgumentError, "only NORMAL or MOB accepted for type!"
end
@@ -96,9 +95,9 @@ module Hbase
end
compact_type = nil
if type == "NORMAL"
- compact_type = org.apache.hadoop.hbase.client.Admin::CompactType::NORMAL
+ compact_type = org.apache.hadoop.hbase.client.CompactType::NORMAL
elsif type == "MOB"
- compact_type = org.apache.hadoop.hbase.client.Admin::CompactType::MOB
+ compact_type = org.apache.hadoop.hbase.client.CompactType::MOB
else
raise ArgumentError, "only NORMAL or MOB accepted for type!"
end
@@ -955,7 +954,7 @@ module Hbase
args.each do |arg|
if arg[SKIP_FLUSH] == true
@admin.snapshot(snapshot_name, table_name,
- SnapshotDescription::Type::SKIPFLUSH)
+ org.apache.hadoop.hbase.client.SnapshotType::SKIPFLUSH)
else
@admin.snapshot(snapshot_name, table_name)
end
[43/50] [abbrv] hbase git commit: HBASE-15807 - Update
report-flakies.py to look for "FAILED" status in test report. - Remove
duplicate testcase names in the result
Posted by sy...@apache.org.
HBASE-15807 - Update report-flakies.py to look for "FAILED" status in test report. - Remove duplicate testcase names in the result
Tested:
Ran it manually.
(Apekshit)
Change-Id: I2a7751eefe729b2a69c0f78596f72b6a0eb39b66
Signed-off-by: stack <st...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/2eced6f0
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/2eced6f0
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/2eced6f0
Branch: refs/heads/hbase-12439
Commit: 2eced6f0393abbc6ebbe6d37fffe0184e7255457
Parents: b75b226
Author: Apekshit <ap...@gmail.com>
Authored: Mon May 9 11:02:06 2016 -0700
Committer: stack <st...@apache.org>
Committed: Mon May 9 11:51:16 2016 -0700
----------------------------------------------------------------------
dev-support/report-flakies.py | 18 ++++++++++++------
1 file changed, 12 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/2eced6f0/dev-support/report-flakies.py
----------------------------------------------------------------------
diff --git a/dev-support/report-flakies.py b/dev-support/report-flakies.py
index 65faa7c..e5e66cc 100755
--- a/dev-support/report-flakies.py
+++ b/dev-support/report-flakies.py
@@ -99,7 +99,8 @@ for job_url in jobs_list:
bad_tests = set()
for build in build_id_to_results:
for test in build_id_to_results[build]:
- if build_id_to_results[build][test] == "REGRESSION":
+ if (build_id_to_results[build][test] == "REGRESSION"
+ or build_id_to_results[build][test] == "FAILED"):
bad_tests.add(test)
global_bad_tests.add(test)
@@ -123,7 +124,7 @@ for job_url in jobs_list:
print "{:>100} {:6} {:10} {:2.0f}%".format(bad_test, fail, total, fail*100.0/total)
else:
print "No flaky tests founds."
- if len(builds_ids) == len(build_ids_without_result):
+ if len(build_ids) == len(build_ids_without_result):
print "None of the analyzed builds have test result."
print "Builds analyzed: " + str(build_ids)
@@ -131,16 +132,21 @@ for job_url in jobs_list:
print ""
if args.mvn:
- includes = ""
- excludes = ""
+ # There might be multiple tests failing within each TestCase, avoid duplication of TestCase names.
+ test_cases = set()
for test in global_bad_tests:
test = re.sub(".*\.", "", test) # Remove package name prefix.
test = re.sub("#.*", "", test) # Remove individual unittest's name
- includes += test + ","
- excludes += "**/" + test + ".java,"
+ test_cases.add(test)
+
+ includes = ",".join(test_cases)
with open("./includes", "w") as inc_file:
inc_file.write(includes)
inc_file.close()
+
+ excludes = ""
+ for test_case in test_cases:
+ excludes += "**/" + test_case + ".java,"
with open("./excludes", "w") as exc_file:
exc_file.write(excludes)
exc_file.close()
[38/50] [abbrv] hbase git commit: HBASE-15738 Ensure artifacts in
project dist area include required md5 file
Posted by sy...@apache.org.
HBASE-15738 Ensure artifacts in project dist area include required md5 file
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/58888e09
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/58888e09
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/58888e09
Branch: refs/heads/hbase-12439
Commit: 58888e0911fabe9556c59b92217881cd56a0ec58
Parents: 48b77db
Author: Nick Dimiduk <nd...@apache.org>
Authored: Sun May 1 12:29:29 2016 -0700
Committer: Nick Dimiduk <nd...@apache.org>
Committed: Sun May 8 18:06:31 2016 -0700
----------------------------------------------------------------------
dev-support/make_rc.sh | 2 ++
src/main/asciidoc/_chapters/developer.adoc | 2 ++
2 files changed, 4 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/58888e09/dev-support/make_rc.sh
----------------------------------------------------------------------
diff --git a/dev-support/make_rc.sh b/dev-support/make_rc.sh
index 50e8efd..b88a984 100755
--- a/dev-support/make_rc.sh
+++ b/dev-support/make_rc.sh
@@ -77,6 +77,8 @@ echo "DONE"
echo "Check the content of ${archivedir}. If good, sign and push to dist.apache.org"
echo " cd ${archivedir}"
echo ' for i in *.tar.gz; do echo $i; gpg --print-mds $i > $i.mds ; done'
+echo ' for i in *.tar.gz; do echo $i; gpg --print-md MD5 $i > $i.md5 ; done'
+echo ' for i in *.tar.gz; do echo $i; gpg --print-md SHA512 $i > $i.sha ; done'
echo ' for i in *.tar.gz; do echo $i; gpg --armor --output $i.asc --detach-sig $i ; done'
echo ' rsync -av ${archivedir}/*.gz ${archivedir}/*.mds ${archivedir}/*.asc ~/repos/dist-dev/hbase-VERSION/'
echo "Check the content deployed to maven. If good, close the repo and record links of temporary staging repo"
http://git-wip-us.apache.org/repos/asf/hbase/blob/58888e09/src/main/asciidoc/_chapters/developer.adoc
----------------------------------------------------------------------
diff --git a/src/main/asciidoc/_chapters/developer.adoc b/src/main/asciidoc/_chapters/developer.adoc
index 09adb4e..0b284bb 100644
--- a/src/main/asciidoc/_chapters/developer.adoc
+++ b/src/main/asciidoc/_chapters/developer.adoc
@@ -706,6 +706,8 @@ In the _version directory_ run the following commands:
----
$ for i in *.tar.gz; do echo $i; gpg --print-mds $i > $i.mds ; done
+$ for i in *.tar.gz; do echo $i; gpg --print-md MD5 $i > $i.md5 ; done
+$ for i in *.tar.gz; do echo $i; gpg --print-md SHA512 $i > $i.sha ; done
$ for i in *.tar.gz; do echo $i; gpg --armor --output $i.asc --detach-sig $i ; done
$ cd ..
# Presuming our 'version directory' is named 0.96.0RC0, now copy it up to people.apache.org.
[40/50] [abbrv] hbase git commit: HBASE-15609 Addendum fix
compilation error
Posted by sy...@apache.org.
HBASE-15609 Addendum fix compilation error
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/05378cbf
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/05378cbf
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/05378cbf
Branch: refs/heads/hbase-12439
Commit: 05378cbf69957b76e8559185ea20235242a8b2e6
Parents: 97ad33c
Author: zhangduo <zh...@apache.org>
Authored: Mon May 9 20:45:04 2016 +0800
Committer: zhangduo <zh...@apache.org>
Committed: Mon May 9 20:45:04 2016 +0800
----------------------------------------------------------------------
.../org/apache/hadoop/hbase/master/ClusterStatusPublisher.java | 3 ++-
.../test/java/org/apache/hadoop/hbase/client/TestResult.java | 6 ------
2 files changed, 2 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/05378cbf/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterStatusPublisher.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterStatusPublisher.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterStatusPublisher.java
index e90aae6..194e023 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterStatusPublisher.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterStatusPublisher.java
@@ -59,6 +59,7 @@ import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.ScheduledChore;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos;
import org.apache.hadoop.hbase.util.Addressing;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
@@ -332,7 +333,7 @@ public class ClusterStatusPublisher extends ScheduledChore {
@Override
protected void encode(ChannelHandlerContext channelHandlerContext,
ClusterStatus clusterStatus, List<Object> objects) {
- ClusterStatusProtos.ClusterStatus csp = clusterStatus.convert();
+ ClusterStatusProtos.ClusterStatus csp = ProtobufUtil.convert(clusterStatus);
objects.add(new DatagramPacket(Unpooled.wrappedBuffer(csp.toByteArray()), isa));
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/05378cbf/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestResult.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestResult.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestResult.java
index 6baf9e0..0e93ee0 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestResult.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestResult.java
@@ -244,12 +244,6 @@ public class TestResult extends TestCase {
LOG.debug("As expected: " + ex.getMessage());
}
try {
- emptyResult.addResults(new RegionLoadStats(0, 0, 0));
- fail("UnsupportedOperationException should have been thrown!");
- } catch (UnsupportedOperationException ex) {
- LOG.debug("As expected: " + ex.getMessage());
- }
- try {
emptyResult.setExists(true);
fail("UnsupportedOperationException should have been thrown!");
} catch (UnsupportedOperationException ex) {
[27/50] [abbrv] hbase git commit: HBASE-15781 Remove unused
TableEventHandler and TotesHRegionInfo
Posted by sy...@apache.org.
HBASE-15781 Remove unused TableEventHandler and TotesHRegionInfo
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/d90f0571
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/d90f0571
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/d90f0571
Branch: refs/heads/hbase-12439
Commit: d90f0571e66211ef6ce3b97dd14f35ce8c6bdc9d
Parents: 387c7e6
Author: Matteo Bertozzi <ma...@cloudera.com>
Authored: Fri May 6 10:05:42 2016 -0700
Committer: Matteo Bertozzi <ma...@cloudera.com>
Committed: Fri May 6 10:05:42 2016 -0700
----------------------------------------------------------------------
.../hbase/master/handler/TableEventHandler.java | 259 -------------------
.../hbase/master/handler/TotesHRegionInfo.java | 36 ---
2 files changed, 295 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/d90f0571/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/TableEventHandler.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/TableEventHandler.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/TableEventHandler.java
deleted file mode 100644
index e93ad57..0000000
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/TableEventHandler.java
+++ /dev/null
@@ -1,259 +0,0 @@
-/**
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.master.handler;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.NavigableMap;
-import java.util.TreeMap;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hbase.CoordinatedStateException;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.HRegionLocation;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.InvalidFamilyOperationException;
-import org.apache.hadoop.hbase.MetaTableAccessor;
-import org.apache.hadoop.hbase.Server;
-import org.apache.hadoop.hbase.ServerName;
-import org.apache.hadoop.hbase.TableDescriptor;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.TableNotDisabledException;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.client.RegionLocator;
-import org.apache.hadoop.hbase.client.TableState;
-import org.apache.hadoop.hbase.executor.EventHandler;
-import org.apache.hadoop.hbase.executor.EventType;
-import org.apache.hadoop.hbase.master.BulkReOpen;
-import org.apache.hadoop.hbase.master.MasterServices;
-import org.apache.hadoop.hbase.master.TableLockManager.TableLock;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
-
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-
-/**
- * Base class for performing operations against tables.
- * Checks on whether the process can go forward are done in constructor rather
- * than later on in {@link #process()}. The idea is to fail fast rather than
- * later down in an async invocation of {@link #process()} (which currently has
- * no means of reporting back issues once started).
- */
-@InterfaceAudience.Private
-public abstract class TableEventHandler extends EventHandler {
- private static final Log LOG = LogFactory.getLog(TableEventHandler.class);
- protected final MasterServices masterServices;
- protected final TableName tableName;
- protected TableLock tableLock;
- private boolean isPrepareCalled = false;
-
- public TableEventHandler(EventType eventType, TableName tableName, Server server,
- MasterServices masterServices) {
- super(server, eventType);
- this.masterServices = masterServices;
- this.tableName = tableName;
- }
-
- public TableEventHandler prepare() throws IOException {
- //acquire the table write lock, blocking
- this.tableLock = masterServices.getTableLockManager()
- .writeLock(tableName, eventType.toString());
- this.tableLock.acquire();
- boolean success = false;
- try {
- try {
- this.masterServices.checkTableModifiable(tableName);
- } catch (TableNotDisabledException ex) {
- if (isOnlineSchemaChangeAllowed()
- && eventType.isOnlineSchemaChangeSupported()) {
- LOG.debug("Ignoring table not disabled exception " +
- "for supporting online schema changes.");
- } else {
- throw ex;
- }
- }
- prepareWithTableLock();
- success = true;
- } finally {
- if (!success ) {
- releaseTableLock();
- }
- }
- this.isPrepareCalled = true;
- return this;
- }
-
- /** Called from prepare() while holding the table lock. Subclasses
- * can do extra initialization, and not worry about the releasing
- * the table lock. */
- protected void prepareWithTableLock() throws IOException {
- }
-
- private boolean isOnlineSchemaChangeAllowed() {
- return this.server.getConfiguration().getBoolean(
- "hbase.online.schema.update.enable", false);
- }
-
- @Override
- public void process() {
- if (!isPrepareCalled) {
- //For proper table locking semantics, the implementor should ensure to call
- //TableEventHandler.prepare() before calling process()
- throw new RuntimeException("Implementation should have called prepare() first");
- }
- try {
- LOG.info("Handling table operation " + eventType + " on table " +
- tableName);
-
- List<HRegionInfo> hris;
- if (TableName.META_TABLE_NAME.equals(tableName)) {
- hris = new MetaTableLocator().getMetaRegions(server.getZooKeeper());
- } else {
- hris = MetaTableAccessor.getTableRegions(server.getConnection(), tableName);
- }
- handleTableOperation(hris);
- if (eventType.isOnlineSchemaChangeSupported() && this.masterServices.
- getAssignmentManager().getTableStateManager().isTableState(
- tableName, TableState.State.ENABLED)) {
- if (reOpenAllRegions(hris)) {
- LOG.info("Completed table operation " + eventType + " on table " +
- tableName);
- } else {
- LOG.warn("Error on reopening the regions");
- }
- }
- completed(null);
- } catch (IOException e) {
- LOG.error("Error manipulating table " + tableName, e);
- completed(e);
- } catch (CoordinatedStateException e) {
- LOG.error("Error manipulating table " + tableName, e);
- completed(e);
- } finally {
- releaseTableLock();
- }
- }
-
- protected void releaseTableLock() {
- if (this.tableLock != null) {
- try {
- this.tableLock.release();
- } catch (IOException ex) {
- LOG.warn("Could not release the table lock", ex);
- }
- }
- }
-
- /**
- * Called after that process() is completed.
- * @param exception null if process() is successful or not null if something has failed.
- */
- protected void completed(final Throwable exception) {
- }
-
- public boolean reOpenAllRegions(List<HRegionInfo> regions) throws IOException {
- boolean done = false;
- LOG.info("Bucketing regions by region server...");
- List<HRegionLocation> regionLocations = null;
- Connection connection = this.masterServices.getConnection();
- try (RegionLocator locator = connection.getRegionLocator(tableName)) {
- regionLocations = locator.getAllRegionLocations();
- }
- // Convert List<HRegionLocation> to Map<HRegionInfo, ServerName>.
- NavigableMap<HRegionInfo, ServerName> hri2Sn = new TreeMap<HRegionInfo, ServerName>();
- for (HRegionLocation location: regionLocations) {
- hri2Sn.put(location.getRegionInfo(), location.getServerName());
- }
- TreeMap<ServerName, List<HRegionInfo>> serverToRegions = Maps.newTreeMap();
- List<HRegionInfo> reRegions = new ArrayList<HRegionInfo>();
- for (HRegionInfo hri : regions) {
- ServerName sn = hri2Sn.get(hri);
- // Skip the offlined split parent region
- // See HBASE-4578 for more information.
- if (null == sn) {
- LOG.info("Skip " + hri);
- continue;
- }
- if (!serverToRegions.containsKey(sn)) {
- LinkedList<HRegionInfo> hriList = Lists.newLinkedList();
- serverToRegions.put(sn, hriList);
- }
- reRegions.add(hri);
- serverToRegions.get(sn).add(hri);
- }
-
- LOG.info("Reopening " + reRegions.size() + " regions on "
- + serverToRegions.size() + " region servers.");
- this.masterServices.getAssignmentManager().setRegionsToReopen(reRegions);
- BulkReOpen bulkReopen = new BulkReOpen(this.server, serverToRegions,
- this.masterServices.getAssignmentManager());
- while (true) {
- try {
- if (bulkReopen.bulkReOpen()) {
- done = true;
- break;
- } else {
- LOG.warn("Timeout before reopening all regions");
- }
- } catch (InterruptedException e) {
- LOG.warn("Reopen was interrupted");
- // Preserve the interrupt.
- Thread.currentThread().interrupt();
- break;
- }
- }
- return done;
- }
-
-
- /**
- * Gets a TableDescriptor from the masterServices. Can Throw exceptions.
- *
- * @return Table descriptor for this table
- * @throws FileNotFoundException
- * @throws IOException
- */
- public TableDescriptor getTableDescriptor()
- throws FileNotFoundException, IOException {
- TableDescriptor htd =
- this.masterServices.getTableDescriptors().getDescriptor(tableName);
- if (htd == null) {
- throw new IOException("HTableDescriptor missing for " + tableName);
- }
- return htd;
- }
-
- byte [] hasColumnFamily(final HTableDescriptor htd, final byte [] cf)
- throws InvalidFamilyOperationException {
- if (!htd.hasFamily(cf)) {
- throw new InvalidFamilyOperationException("Column family '" +
- Bytes.toString(cf) + "' does not exist");
- }
- return cf;
- }
-
- protected abstract void handleTableOperation(List<HRegionInfo> regions)
- throws IOException, CoordinatedStateException;
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/d90f0571/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/TotesHRegionInfo.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/TotesHRegionInfo.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/TotesHRegionInfo.java
deleted file mode 100644
index 1a30de4..0000000
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/TotesHRegionInfo.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/**
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.master.handler;
-
-
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.HRegionInfo;
-
-/**
- * Implementors tote an HRegionInfo instance.
- * This is a marker interface that can be put on {@link java.beans.EventHandler}s that
- * have an {@link HRegionInfo}.
- */
-@InterfaceAudience.Private
-public interface TotesHRegionInfo {
- /**
- * @return HRegionInfo instance.
- */
- HRegionInfo getHRegionInfo();
-}