You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by el...@apache.org on 2017/05/19 17:13:36 UTC
[36/50] [abbrv] hbase git commit: HBASE-17447 Implement a
MasterObserver for automatically deleting space quotas
HBASE-17447 Implement a MasterObserver for automatically deleting space quotas
When a table or namespace is deleted, it would be nice to automatically
delete the quota on said table/NS. It's possible that not all people
would want this functionality so we can leave it up to the user to
configure this Observer.
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/a69b68e9
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/a69b68e9
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/a69b68e9
Branch: refs/heads/HBASE-16961
Commit: a69b68e94d714f1c36837ad22d7088ad9829d813
Parents: 6007442
Author: Josh Elser <el...@apache.org>
Authored: Thu Mar 16 18:54:01 2017 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Fri May 19 12:28:49 2017 -0400
----------------------------------------------------------------------
.../hbase/quotas/MasterSpaceQuotaObserver.java | 85 ++++++++++
.../quotas/TestMasterSpaceQuotaObserver.java | 169 +++++++++++++++++++
src/main/asciidoc/_chapters/ops_mgt.adoc | 17 ++
3 files changed, 271 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/a69b68e9/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterSpaceQuotaObserver.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterSpaceQuotaObserver.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterSpaceQuotaObserver.java
new file mode 100644
index 0000000..a3abf32
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterSpaceQuotaObserver.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.quotas;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.CoprocessorEnvironment;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment;
+import org.apache.hadoop.hbase.coprocessor.MasterObserver;
+import org.apache.hadoop.hbase.coprocessor.ObserverContext;
+import org.apache.hadoop.hbase.master.MasterServices;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.Quotas;
+
+/**
+ * An observer to automatically delete space quotas when a table/namespace
+ * are deleted.
+ */
+@InterfaceAudience.Private
+public class MasterSpaceQuotaObserver implements MasterObserver {
+ private CoprocessorEnvironment cpEnv;
+ private Configuration conf;
+ private boolean quotasEnabled = false;
+
+ @Override
+ public void start(CoprocessorEnvironment ctx) throws IOException {
+ this.cpEnv = ctx;
+ this.conf = cpEnv.getConfiguration();
+ this.quotasEnabled = QuotaUtil.isQuotaEnabled(conf);
+ }
+
+ @Override
+ public void postDeleteTable(
+ ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName) throws IOException {
+ // Do nothing if quotas aren't enabled
+ if (!quotasEnabled) {
+ return;
+ }
+ final MasterServices master = ctx.getEnvironment().getMasterServices();
+ final Connection conn = master.getConnection();
+ Quotas quotas = QuotaUtil.getTableQuota(master.getConnection(), tableName);
+ if (null != quotas && quotas.hasSpace()) {
+ QuotaSettings settings = QuotaSettingsFactory.removeTableSpaceLimit(tableName);
+ try (Admin admin = conn.getAdmin()) {
+ admin.setQuota(settings);
+ }
+ }
+ }
+
+ @Override
+ public void postDeleteNamespace(
+ ObserverContext<MasterCoprocessorEnvironment> ctx, String namespace) throws IOException {
+ // Do nothing if quotas aren't enabled
+ if (!quotasEnabled) {
+ return;
+ }
+ final MasterServices master = ctx.getEnvironment().getMasterServices();
+ final Connection conn = master.getConnection();
+ Quotas quotas = QuotaUtil.getNamespaceQuota(master.getConnection(), namespace);
+ if (null != quotas && quotas.hasSpace()) {
+ QuotaSettings settings = QuotaSettingsFactory.removeNamespaceSpaceLimit(namespace);
+ try (Admin admin = conn.getAdmin()) {
+ admin.setQuota(settings);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/a69b68e9/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestMasterSpaceQuotaObserver.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestMasterSpaceQuotaObserver.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestMasterSpaceQuotaObserver.java
new file mode 100644
index 0000000..a1eee4f
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestMasterSpaceQuotaObserver.java
@@ -0,0 +1,169 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.quotas;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.NamespaceDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.rules.TestName;
+
+/**
+ * Test class for {@link MasterSpaceQuotaObserver}.
+ */
+@Category(MediumTests.class)
+public class TestMasterSpaceQuotaObserver {
+ private static final Log LOG = LogFactory.getLog(TestSpaceQuotas.class);
+ private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+
+ @Rule
+ public TestName testName = new TestName();
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ Configuration conf = TEST_UTIL.getConfiguration();
+ conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, MasterSpaceQuotaObserver.class.getName());
+ conf.setBoolean(QuotaUtil.QUOTA_CONF_KEY, true);
+ TEST_UTIL.startMiniCluster(1);
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ TEST_UTIL.shutdownMiniCluster();
+ }
+
+ @Before
+ public void removeAllQuotas() throws Exception {
+ final Connection conn = TEST_UTIL.getConnection();
+ // Wait for the quota table to be created
+ if (!conn.getAdmin().tableExists(QuotaUtil.QUOTA_TABLE_NAME)) {
+ do {
+ LOG.debug("Quota table does not yet exist");
+ Thread.sleep(1000);
+ } while (!conn.getAdmin().tableExists(QuotaUtil.QUOTA_TABLE_NAME));
+ } else {
+ // Or, clean up any quotas from previous test runs.
+ QuotaRetriever scanner = QuotaRetriever.open(TEST_UTIL.getConfiguration());
+ for (QuotaSettings quotaSettings : scanner) {
+ final String namespace = quotaSettings.getNamespace();
+ final TableName tableName = quotaSettings.getTableName();
+ if (null != namespace) {
+ LOG.debug("Deleting quota for namespace: " + namespace);
+ QuotaUtil.deleteNamespaceQuota(conn, namespace);
+ } else {
+ assert null != tableName;
+ LOG.debug("Deleting quota for table: "+ tableName);
+ QuotaUtil.deleteTableQuota(conn, tableName);
+ }
+ }
+ }
+ }
+
+ @Test
+ public void testTableQuotaRemoved() throws Exception {
+ final Connection conn = TEST_UTIL.getConnection();
+ final Admin admin = conn.getAdmin();
+ final TableName tn = TableName.valueOf(testName.getMethodName());
+ // Drop the table if it somehow exists
+ if (admin.tableExists(tn)) {
+ admin.disableTable(tn);
+ admin.deleteTable(tn);
+ }
+
+ // Create a table
+ HTableDescriptor tableDesc = new HTableDescriptor(tn);
+ tableDesc.addFamily(new HColumnDescriptor("F1"));
+ admin.createTable(tableDesc);
+ assertEquals(0, getNumSpaceQuotas());
+
+ // Set a quota
+ QuotaSettings settings = QuotaSettingsFactory.limitTableSpace(
+ tn, 1024L, SpaceViolationPolicy.NO_INSERTS);
+ admin.setQuota(settings);
+ assertEquals(1, getNumSpaceQuotas());
+
+ // Delete the table and observe the quota being automatically deleted as well
+ admin.disableTable(tn);
+ admin.deleteTable(tn);
+ assertEquals(0, getNumSpaceQuotas());
+ }
+
+ @Test
+ public void testNamespaceQuotaRemoved() throws Exception {
+ final Connection conn = TEST_UTIL.getConnection();
+ final Admin admin = conn.getAdmin();
+ final String ns = testName.getMethodName();
+ // Drop the ns if it somehow exists
+ if (namespaceExists(ns)) {
+ admin.deleteNamespace(ns);
+ }
+
+ // Create the ns
+ NamespaceDescriptor desc = NamespaceDescriptor.create(ns).build();
+ admin.createNamespace(desc);
+ assertEquals(0, getNumSpaceQuotas());
+
+ // Set a quota
+ QuotaSettings settings = QuotaSettingsFactory.limitNamespaceSpace(
+ ns, 1024L, SpaceViolationPolicy.NO_INSERTS);
+ admin.setQuota(settings);
+ assertEquals(1, getNumSpaceQuotas());
+
+ // Delete the table and observe the quota being automatically deleted as well
+ admin.deleteNamespace(ns);
+ assertEquals(0, getNumSpaceQuotas());
+ }
+
+ public boolean namespaceExists(String ns) throws IOException {
+ NamespaceDescriptor[] descs = TEST_UTIL.getAdmin().listNamespaceDescriptors();
+ for (NamespaceDescriptor desc : descs) {
+ if (ns.equals(desc.getName())) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ public int getNumSpaceQuotas() throws Exception {
+ QuotaRetriever scanner = QuotaRetriever.open(TEST_UTIL.getConfiguration());
+ int numSpaceQuotas = 0;
+ for (QuotaSettings quotaSettings : scanner) {
+ if (quotaSettings.getQuotaType() == QuotaType.SPACE) {
+ numSpaceQuotas++;
+ }
+ }
+ return numSpaceQuotas;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/a69b68e9/src/main/asciidoc/_chapters/ops_mgt.adoc
----------------------------------------------------------------------
diff --git a/src/main/asciidoc/_chapters/ops_mgt.adoc b/src/main/asciidoc/_chapters/ops_mgt.adoc
index f9009f3..f60395b 100644
--- a/src/main/asciidoc/_chapters/ops_mgt.adoc
+++ b/src/main/asciidoc/_chapters/ops_mgt.adoc
@@ -1948,6 +1948,23 @@ and also will disallow all writes then the usage exceeds this limit. Because the
on 'ns1:t1', this table can grow up to 100TB, but only if 'ns1:t2' and 'ns1:t3' have a usage of zero bytes.
Practically, it's limit is 100TB less the current usage of 'ns1:t2' and 'ns1:t3'.
+[[ops.space.quota.deletion]]
+=== Automatic Space Quota Deletion
+
+By default, if a table or namespace is deleted that has a space quota, the quota itself is
+not also deleted. In some cases, it may be desirable for the space quota to be automatically deleted.
+In these cases, the user may configure the MasterSpaceQuotaObserver to delete any space quota
+automatically in hbase-site.xml.
+
+[source,java]
+----
+
+ <property>
+ <name>hbase.coprocessor.master.classes</name>
+ <value>...,org.apache.hadoop.hbase.quotas.MasterSpaceQuotaObserver</value>
+ </property>
+----
+
[[ops.backup]]
== HBase Backup