You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by te...@apache.org on 2015/05/09 20:07:06 UTC
hbase git commit: HBASE-13593 Quota support for namespace should take
snapshot restore and clone into account (Ashish Singhi)
Repository: hbase
Updated Branches:
refs/heads/branch-1 fdaeb8601 -> 1c7389cbe
HBASE-13593 Quota support for namespace should take snapshot restore and clone into account (Ashish Singhi)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/1c7389cb
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/1c7389cb
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/1c7389cb
Branch: refs/heads/branch-1
Commit: 1c7389cbe20f6ca2e49f715020c9deea64521c09
Parents: fdaeb86
Author: tedyu <yu...@gmail.com>
Authored: Sat May 9 11:07:46 2015 -0700
Committer: tedyu <yu...@gmail.com>
Committed: Sat May 9 11:07:46 2015 -0700
----------------------------------------------------------------------
.../hbase/master/snapshot/SnapshotManager.java | 37 +++++-
.../hbase/namespace/NamespaceAuditor.java | 14 ++
.../hbase/namespace/NamespaceStateManager.java | 27 ++++
.../hadoop/hbase/quotas/MasterQuotaManager.java | 6 +
.../hbase/namespace/TestNamespaceAuditor.java | 129 +++++++++++++++++++
5 files changed, 211 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/1c7389cb/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java
index 59446cb..853e3cd 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java
@@ -734,7 +734,16 @@ public class SnapshotManager extends MasterProcedureManager implements Stoppable
if (cpHost != null) {
cpHost.preRestoreSnapshot(reqSnapshot, snapshotTableDesc);
}
- restoreSnapshot(snapshot, snapshotTableDesc);
+ try {
+ // Table already exist. Check and update the region quota for this table namespace
+ checkAndUpdateNamespaceRegionQuota(manifest, tableName);
+ restoreSnapshot(snapshot, snapshotTableDesc);
+ } catch (IOException e) {
+ this.master.getMasterQuotaManager().removeTableFromNamespaceQuota(tableName);
+ LOG.error("Exception occurred while restoring the snapshot " + snapshot.getName()
+ + " as table " + tableName.getNameAsString(), e);
+ throw e;
+ }
LOG.info("Restore snapshot=" + snapshot.getName() + " as table=" + tableName);
if (cpHost != null) {
@@ -745,7 +754,15 @@ public class SnapshotManager extends MasterProcedureManager implements Stoppable
if (cpHost != null) {
cpHost.preCloneSnapshot(reqSnapshot, htd);
}
- cloneSnapshot(snapshot, htd);
+ try {
+ checkAndUpdateNamespaceQuota(manifest, tableName);
+ cloneSnapshot(snapshot, htd);
+ } catch (IOException e) {
+ this.master.getMasterQuotaManager().removeTableFromNamespaceQuota(tableName);
+ LOG.error("Exception occurred while cloning the snapshot " + snapshot.getName()
+ + " as table " + tableName.getNameAsString(), e);
+ throw e;
+ }
LOG.info("Clone snapshot=" + snapshot.getName() + " as table=" + tableName);
if (cpHost != null) {
@@ -753,6 +770,22 @@ public class SnapshotManager extends MasterProcedureManager implements Stoppable
}
}
}
+
+ private void checkAndUpdateNamespaceQuota(SnapshotManifest manifest, TableName tableName)
+ throws IOException {
+ if (this.master.getMasterQuotaManager().isQuotaEnabled()) {
+ this.master.getMasterQuotaManager().checkNamespaceTableAndRegionQuota(tableName,
+ manifest.getRegionManifestsMap().size());
+ }
+ }
+
+ private void checkAndUpdateNamespaceRegionQuota(SnapshotManifest manifest, TableName tableName)
+ throws IOException {
+ if (this.master.getMasterQuotaManager().isQuotaEnabled()) {
+ this.master.getMasterQuotaManager().checkAndUpdateNamespaceRegionQuota(tableName,
+ manifest.getRegionManifestsMap().size());
+ }
+ }
/**
* Restore the specified snapshot.
http://git-wip-us.apache.org/repos/asf/hbase/blob/1c7389cb/hbase-server/src/main/java/org/apache/hadoop/hbase/namespace/NamespaceAuditor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/namespace/NamespaceAuditor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/namespace/NamespaceAuditor.java
index 99ad2fc..f493c95 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/namespace/NamespaceAuditor.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/namespace/NamespaceAuditor.java
@@ -68,6 +68,20 @@ public class NamespaceAuditor {
checkTableTypeAndThrowException(tName);
}
}
+
+ /**
+ * Check and update region count quota for an existing table.
+ * @param tName - table name for which region count to be updated.
+ * @param regions - Number of regions that will be added.
+ * @throws IOException Signals that an I/O exception has occurred.
+ */
+ public void checkQuotaToUpdateRegion(TableName tName, int regions) throws IOException {
+ if (stateManager.isInitialized()) {
+ stateManager.checkAndUpdateNamespaceRegionCount(tName, regions);
+ } else {
+ checkTableTypeAndThrowException(tName);
+ }
+ }
private void checkTableTypeAndThrowException(TableName name) throws IOException {
if (name.isSystemTable()) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/1c7389cb/hbase-server/src/main/java/org/apache/hadoop/hbase/namespace/NamespaceStateManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/namespace/NamespaceStateManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/namespace/NamespaceStateManager.java
index 6ee7d00..f61e2bd 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/namespace/NamespaceStateManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/namespace/NamespaceStateManager.java
@@ -108,6 +108,33 @@ class NamespaceStateManager extends ZooKeeperListener {
}
return true;
}
+
+ /**
+ * Check and update region count for an existing table. To handle scenarios like restore snapshot
+ * @param TableName name of the table for region count needs to be checked and updated
+ * @param incr count of regions
+ * @throws QuotaExceededException if quota exceeds for the number of regions allowed in a
+ * namespace
+ * @throws IOException Signals that an I/O exception has occurred.
+ */
+ synchronized void checkAndUpdateNamespaceRegionCount(TableName name, int incr)
+ throws IOException {
+ String namespace = name.getNamespaceAsString();
+ NamespaceDescriptor nspdesc = getNamespaceDescriptor(namespace);
+ if (nspdesc != null) {
+ NamespaceTableAndRegionInfo currentStatus = getState(namespace);
+ int regionCountOfTable = currentStatus.getRegionCountOfTable(name);
+ if ((currentStatus.getRegionCount() - regionCountOfTable + incr) > TableNamespaceManager
+ .getMaxRegions(nspdesc)) {
+ throw new QuotaExceededException("The table " + name.getNameAsString()
+ + " region count cannot be updated as it would exceed maximum number "
+ + "of regions allowed in the namespace. The total number of regions permitted is "
+ + TableNamespaceManager.getMaxRegions(nspdesc));
+ }
+ currentStatus.removeTable(name);
+ currentStatus.addTable(name, incr);
+ }
+ }
private NamespaceDescriptor getNamespaceDescriptor(String namespaceAsString) {
try {
http://git-wip-us.apache.org/repos/asf/hbase/blob/1c7389cb/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java
index be1d4c5..65759e8 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java
@@ -325,6 +325,12 @@ public class MasterQuotaManager implements RegionStateListener {
namespaceQuotaManager.checkQuotaToCreateTable(tName, regions);
}
}
+
+ public void checkAndUpdateNamespaceRegionQuota(TableName tName, int regions) throws IOException {
+ if (enabled) {
+ namespaceQuotaManager.checkQuotaToUpdateRegion(tName, regions);
+ }
+ }
public void onRegionMerged(HRegionInfo hri) throws IOException {
if (enabled) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/1c7389cb/hbase-server/src/test/java/org/apache/hadoop/hbase/namespace/TestNamespaceAuditor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/namespace/TestNamespaceAuditor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/namespace/TestNamespaceAuditor.java
index 8540dcf..f9d3765 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/namespace/TestNamespaceAuditor.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/namespace/TestNamespaceAuditor.java
@@ -46,6 +46,7 @@ import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Mutation;
+import org.apache.hadoop.hbase.client.RegionLocator;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.coprocessor.BaseMasterObserver;
import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
@@ -68,6 +69,7 @@ import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.HRegionServer;
import org.apache.hadoop.hbase.regionserver.Region;
import org.apache.hadoop.hbase.regionserver.RegionServerCoprocessorHost;
+import org.apache.hadoop.hbase.snapshot.RestoreSnapshotException;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.FSUtils;
@@ -679,4 +681,131 @@ public class TestNamespaceAuditor {
ADMIN.createTable(tableDescOne);
ADMIN.createTable(tableDescTwo, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 4);
}
+
+ @Test(expected = QuotaExceededException.class, timeout = 30000)
+ public void testCloneSnapshotQuotaExceed() throws Exception {
+ String nsp = prefix + "_testTableQuotaExceedWithCloneSnapshot";
+ NamespaceDescriptor nspDesc =
+ NamespaceDescriptor.create(nsp).addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "1")
+ .build();
+ ADMIN.createNamespace(nspDesc);
+ assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(nsp));
+ TableName tableName = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1");
+ TableName cloneTableName = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table2");
+ HTableDescriptor tableDescOne = new HTableDescriptor(tableName);
+ ADMIN.createTable(tableDescOne);
+ String snapshot = "snapshot_testTableQuotaExceedWithCloneSnapshot";
+ ADMIN.snapshot(snapshot, tableName);
+ ADMIN.cloneSnapshot(snapshot, cloneTableName);
+ ADMIN.deleteSnapshot(snapshot);
+ }
+
+ @Test(timeout = 180000)
+ public void testCloneSnapshot() throws Exception {
+ String nsp = prefix + "_testCloneSnapshot";
+ NamespaceDescriptor nspDesc =
+ NamespaceDescriptor.create(nsp).addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2")
+ .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "20").build();
+ ADMIN.createNamespace(nspDesc);
+ assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(nsp));
+ TableName tableName = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1");
+ TableName cloneTableName = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table2");
+ HTableDescriptor tableDescOne = new HTableDescriptor(tableName);
+
+ ADMIN.createTable(tableDescOne, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 4);
+ String snapshot = "snapshot_testCloneSnapshot";
+ ADMIN.snapshot(snapshot, tableName);
+ ADMIN.cloneSnapshot(snapshot, cloneTableName);
+
+ int tableLength;
+ try (RegionLocator locator = ADMIN.getConnection().getRegionLocator(tableName)) {
+ tableLength = locator.getStartKeys().length;
+ }
+ assertEquals(tableName.getNameAsString() + " should have four regions.", 4, tableLength);
+
+ try (RegionLocator locator = ADMIN.getConnection().getRegionLocator(cloneTableName)) {
+ tableLength = locator.getStartKeys().length;
+ }
+ assertEquals(cloneTableName.getNameAsString() + " should have four regions.", 4, tableLength);
+
+ NamespaceTableAndRegionInfo nstate = getNamespaceState(nsp);
+ assertEquals("Total tables count should be 2.", 2, nstate.getTables().size());
+ assertEquals("Total regions count should be.", 8, nstate.getRegionCount());
+
+ ADMIN.deleteSnapshot(snapshot);
+ }
+
+ @Test(timeout = 180000)
+ public void testRestoreSnapshot() throws Exception {
+ String nsp = prefix + "_testRestoreSnapshot";
+ NamespaceDescriptor nspDesc =
+ NamespaceDescriptor.create(nsp)
+ .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "10").build();
+ ADMIN.createNamespace(nspDesc);
+ assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(nsp));
+ TableName tableName1 = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1");
+ HTableDescriptor tableDescOne = new HTableDescriptor(tableName1);
+ ADMIN.createTable(tableDescOne, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 4);
+
+ NamespaceTableAndRegionInfo nstate = getNamespaceState(nsp);
+ assertEquals("Intial region count should be 4.", 4, nstate.getRegionCount());
+
+ String snapshot = "snapshot_testRestoreSnapshot";
+ ADMIN.snapshot(snapshot, tableName1);
+
+ List<HRegionInfo> regions = ADMIN.getTableRegions(tableName1);
+ Collections.sort(regions);
+
+ ADMIN.split(tableName1, Bytes.toBytes("JJJ"));
+ Thread.sleep(2000);
+ assertEquals("Total regions count should be 5.", 5, nstate.getRegionCount());
+
+ ADMIN.disableTable(tableName1);
+ ADMIN.restoreSnapshot(snapshot);
+
+ assertEquals("Total regions count should be 4 after restore.", 4, nstate.getRegionCount());
+
+ ADMIN.enableTable(tableName1);
+ ADMIN.deleteSnapshot(snapshot);
+ }
+
+ @Test(timeout = 180000)
+ public void testRestoreSnapshotQuotaExceed() throws Exception {
+ String nsp = prefix + "_testRestoreSnapshotQuotaExceed";
+ NamespaceDescriptor nspDesc =
+ NamespaceDescriptor.create(nsp)
+ .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "10").build();
+ ADMIN.createNamespace(nspDesc);
+ NamespaceDescriptor ndesc = ADMIN.getNamespaceDescriptor(nsp);
+ assertNotNull("Namespace descriptor found null.", ndesc);
+ TableName tableName1 = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1");
+ HTableDescriptor tableDescOne = new HTableDescriptor(tableName1);
+ ADMIN.createTable(tableDescOne, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 4);
+
+ NamespaceTableAndRegionInfo nstate = getNamespaceState(nsp);
+ assertEquals("Intial region count should be 4.", 4, nstate.getRegionCount());
+
+ String snapshot = "snapshot_testRestoreSnapshotQuotaExceed";
+ ADMIN.snapshot(snapshot, tableName1);
+
+ List<HRegionInfo> regions = ADMIN.getTableRegions(tableName1);
+ Collections.sort(regions);
+
+ ADMIN.split(tableName1, Bytes.toBytes("JJJ"));
+ Thread.sleep(2000);
+ assertEquals("Total regions count should be 5.", 5, nstate.getRegionCount());
+
+ ndesc.setConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "2");
+ ADMIN.modifyNamespace(ndesc);
+
+ ADMIN.disableTable(tableName1);
+ try {
+ ADMIN.restoreSnapshot(snapshot);
+ fail("Region quota is exceeded so QuotaExceededException should be thrown but HBaseAdmin"
+ + " wraps IOException into RestoreSnapshotException");
+ } catch (RestoreSnapshotException ignore) {
+ }
+ ADMIN.enableTable(tableName1);
+ ADMIN.deleteSnapshot(snapshot);
+ }
}