You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by sa...@apache.org on 2019/08/23 20:20:41 UTC
[hbase] branch HBASE-22895 updated (f504679 -> 00cfdca)
This is an automated email from the ASF dual-hosted git repository.
sakthi pushed a change to branch HBASE-22895
in repository https://gitbox.apache.org/repos/asf/hbase.git.
discard f504679 HBASE-22895 Fix the flakey TestSpaceQuotas
add 74fb204 HBASE-22464 Improvements to hbase-vote script
add 808f051 HBASE-22755 Removed deprecated methods from Mutation
add 2d45801 HBASE-21400 correct spelling error of 'initilize' in comment
new 00cfdca HBASE-22895 Fix the flakey TestSpaceQuotas
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (f504679)
\
N -- N -- N refs/heads/HBASE-22895 (00cfdca)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
dev-support/hbase-vote.sh | 22 +++++++++++++-------
.../org/apache/hadoop/hbase/client/Mutation.java | 24 ----------------------
.../java/org/apache/hadoop/hbase/client/Put.java | 11 ----------
.../hadoop/hbase/ipc/BlockingRpcConnection.java | 2 +-
.../org/apache/hadoop/hbase/wal/WALSplitUtil.java | 3 ++-
.../regionserver/TestSerialReplicationChecker.java | 2 +-
.../TestHBaseFsckCleanReplicationBarriers.java | 2 +-
7 files changed, 20 insertions(+), 46 deletions(-)
[hbase] 01/01: HBASE-22895 Fix the flakey TestSpaceQuotas
Posted by sa...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sakthi pushed a commit to branch HBASE-22895
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 00cfdca2203f1f6c5ae8b2612dae885a86bbb6d8
Author: Sakthi <sa...@apache.org>
AuthorDate: Thu Aug 22 17:18:05 2019 -0700
HBASE-22895 Fix the flakey TestSpaceQuotas
---
.../hbase/quotas/SpaceQuotaHelperForTests.java | 177 ++++-
.../quotas/TestSpaceQuotaBasicFunctioning.java | 224 ++++++
.../hbase/quotas/TestSpaceQuotaDropTable.java | 109 +++
.../hbase/quotas/TestSpaceQuotaIncrease.java | 101 +++
.../hbase/quotas/TestSpaceQuotaOnBulkLoad.java | 190 ++++++
.../quotas/TestSpaceQuotaOnNonExistingTables.java | 86 +++
.../hadoop/hbase/quotas/TestSpaceQuotaRemoval.java | 190 ++++++
.../hbase/quotas/TestSpaceQuotaSwitchPolicies.java | 117 ++++
.../hadoop/hbase/quotas/TestSpaceQuotas.java | 748 ---------------------
9 files changed, 1191 insertions(+), 751 deletions(-)
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/SpaceQuotaHelperForTests.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/SpaceQuotaHelperForTests.java
index 84463b3..2940700 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/SpaceQuotaHelperForTests.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/SpaceQuotaHelperForTests.java
@@ -16,6 +16,8 @@
*/
package org.apache.hadoop.hbase.quotas;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import java.io.IOException;
@@ -36,11 +38,19 @@ import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.MiniHBaseCluster;
import org.apache.hadoop.hbase.NamespaceDescriptor;
import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.TableNotEnabledException;
import org.apache.hadoop.hbase.Waiter.Predicate;
import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Append;
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Increment;
+import org.apache.hadoop.hbase.client.Mutation;
import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.TableDescriptor;
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
@@ -49,6 +59,7 @@ import org.apache.hadoop.hbase.regionserver.HStore;
import org.apache.hadoop.hbase.regionserver.HStoreFile;
import org.apache.hadoop.hbase.regionserver.TestHRegionServerBulkLoad;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.util.StringUtils;
import org.apache.yetus.audience.InterfaceAudience;
import org.junit.rules.TestName;
import org.slf4j.Logger;
@@ -71,6 +82,7 @@ public class SpaceQuotaHelperForTests {
private final HBaseTestingUtility testUtil;
private final TestName testName;
private final AtomicLong counter;
+ private static final int NUM_RETRIES = 10;
public SpaceQuotaHelperForTests(
HBaseTestingUtility testUtil, TestName testName, AtomicLong counter) {
@@ -119,9 +131,168 @@ public class SpaceQuotaHelperForTests {
}
/**
+ * Writes the given mutation into a table until it violates the given policy.
+ * Verifies that the policy has been violated & then returns the name of
+ * the table created & written into.
+ */
+ TableName writeUntilViolationAndVerifyViolation(
+ SpaceViolationPolicy policyToViolate, Mutation m) throws Exception {
+ final TableName tn = writeUntilViolation(policyToViolate);
+ verifyViolation(policyToViolate, tn, m);
+ return tn;
+ }
+
+ /**
+ * Writes the given mutation into a table until it violates the given policy.
+ * Returns the name of the table created & written into.
+ */
+ TableName writeUntilViolation(SpaceViolationPolicy policyToViolate) throws Exception {
+ TableName tn = createTableWithRegions(10);
+ setQuotaLimit(tn, policyToViolate, 2L);
+ // Write more data than should be allowed and flush it to disk
+ writeData(tn, 3L * SpaceQuotaHelperForTests.ONE_MEGABYTE);
+
+ // This should be sufficient time for the chores to run and see the change.
+ Thread.sleep(5000);
+
+ return tn;
+ }
+
+ /**
+ * Verifies that the given policy on the given table has been violated
+ */
+ void verifyViolation(SpaceViolationPolicy policyToViolate, TableName tn, Mutation m)
+ throws Exception {
+ // But let's try a few times to get the exception before failing
+ boolean sawError = false;
+ String msg = "";
+ for (int i = 0; i < NUM_RETRIES && !sawError; i++) {
+ try (Table table = testUtil.getConnection().getTable(tn)) {
+ if (m instanceof Put) {
+ table.put((Put) m);
+ } else if (m instanceof Delete) {
+ table.delete((Delete) m);
+ } else if (m instanceof Append) {
+ table.append((Append) m);
+ } else if (m instanceof Increment) {
+ table.increment((Increment) m);
+ } else {
+ fail(
+ "Failed to apply " + m.getClass().getSimpleName() +
+ " to the table. Programming error");
+ }
+ LOG.info("Did not reject the " + m.getClass().getSimpleName() + ", will sleep and retry");
+ Thread.sleep(2000);
+ } catch (Exception e) {
+ msg = StringUtils.stringifyException(e);
+ if ((policyToViolate.equals(SpaceViolationPolicy.DISABLE)
+ && e instanceof TableNotEnabledException) || msg.contains(policyToViolate.name())) {
+ LOG.info("Got the expected exception={}", msg);
+ sawError = true;
+ break;
+ } else {
+ LOG.warn("Did not get the expected exception, will sleep and retry", e);
+ Thread.sleep(2000);
+ }
+ }
+ }
+ if (!sawError) {
+ try (Table quotaTable = testUtil.getConnection().getTable(QuotaUtil.QUOTA_TABLE_NAME)) {
+ ResultScanner scanner = quotaTable.getScanner(new Scan());
+ Result result = null;
+ LOG.info("Dumping contents of hbase:quota table");
+ while ((result = scanner.next()) != null) {
+ LOG.info(Bytes.toString(result.getRow()) + " => " + result.toString());
+ }
+ scanner.close();
+ }
+ } else {
+ if (policyToViolate.equals(SpaceViolationPolicy.DISABLE)) {
+ assertTrue(
+ msg.contains("TableNotEnabledException") || msg.contains(policyToViolate.name()));
+ } else {
+ assertTrue("Expected exception message to contain the word '" + policyToViolate.name()
+ + "', but was " + msg,
+ msg.contains(policyToViolate.name()));
+ }
+ }
+ assertTrue(
+ "Expected to see an exception writing data to a table exceeding its quota", sawError);
+ }
+
+ /**
+ * Verifies that no policy has been violated on the given table
+ */
+ void verifyNoViolation(TableName tn, Mutation m) throws Exception {
+ // But let's try a few times to write data before failing
+ boolean sawSuccess = false;
+ for (int i = 0; i < NUM_RETRIES && !sawSuccess; i++) {
+ try (Table table = testUtil.getConnection().getTable(tn)) {
+ if (m instanceof Put) {
+ table.put((Put) m);
+ } else if (m instanceof Delete) {
+ table.delete((Delete) m);
+ } else if (m instanceof Append) {
+ table.append((Append) m);
+ } else if (m instanceof Increment) {
+ table.increment((Increment) m);
+ } else {
+ fail("Failed to apply " + m.getClass().getSimpleName() + " to the table."
+ + " Programming error");
+ }
+ sawSuccess = true;
+ } catch (Exception e) {
+ LOG.info("Rejected the " + m.getClass().getSimpleName() + ", will sleep and retry");
+ Thread.sleep(2000);
+ }
+ }
+ if (!sawSuccess) {
+ try (Table quotaTable = testUtil.getConnection().getTable(QuotaUtil.QUOTA_TABLE_NAME)) {
+ ResultScanner scanner = quotaTable.getScanner(new Scan());
+ Result result = null;
+ LOG.info("Dumping contents of hbase:quota table");
+ while ((result = scanner.next()) != null) {
+ LOG.info(Bytes.toString(result.getRow()) + " => " + result.toString());
+ }
+ scanner.close();
+ }
+ }
+ assertTrue("Expected to succeed in writing data to a table not having quota ", sawSuccess);
+ }
+
+ /**
+ * Sets the given quota (policy & limit) on the passed table.
+ */
+ void setQuotaLimit(final TableName tn, SpaceViolationPolicy policy, long sizeInMBs)
+ throws Exception {
+ final long sizeLimit = sizeInMBs * SpaceQuotaHelperForTests.ONE_MEGABYTE;
+ QuotaSettings settings = QuotaSettingsFactory.limitTableSpace(tn, sizeLimit, policy);
+ testUtil.getAdmin().setQuota(settings);
+ LOG.debug("Quota limit set for table = {}, limit = {}", tn, sizeLimit);
+ }
+
+ /**
+ * Removes the space quota from the given table
+ */
+ void removeQuotaFromtable(final TableName tn) throws Exception {
+ QuotaSettings removeQuota = QuotaSettingsFactory.removeTableSpaceLimit(tn);
+ testUtil.getAdmin().setQuota(removeQuota);
+ LOG.debug("Space quota settings removed from the table ", tn);
+ }
+
+ /**
+ * Removes all quotas defined in the HBase quota table.
+ */
+ void removeAllQuotas() throws Exception {
+ final Connection conn = testUtil.getConnection();
+ removeAllQuotas(conn);
+ assertEquals(0, listNumDefinedQuotas(conn));
+ }
+
+ /**
* Removes all quotas defined in the HBase quota table.
*/
- void removeAllQuotas(Connection conn) throws IOException, InterruptedException {
+ void removeAllQuotas(Connection conn) throws IOException {
// Wait for the quota table to be created
if (!conn.getAdmin().tableExists(QuotaUtil.QUOTA_TABLE_NAME)) {
waitForQuotaTable(conn);
@@ -167,14 +338,14 @@ public class SpaceQuotaHelperForTests {
/**
* Waits 30seconds for the HBase quota table to exist.
*/
- public void waitForQuotaTable(Connection conn) throws IOException {
+ void waitForQuotaTable(Connection conn) throws IOException {
waitForQuotaTable(conn, 30_000);
}
/**
* Waits {@code timeout} milliseconds for the HBase quota table to exist.
*/
- public void waitForQuotaTable(Connection conn, long timeout) throws IOException {
+ void waitForQuotaTable(Connection conn, long timeout) throws IOException {
testUtil.waitFor(timeout, 1000, new Predicate<IOException>() {
@Override
public boolean evaluate() throws IOException {
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestSpaceQuotaBasicFunctioning.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestSpaceQuotaBasicFunctioning.java
new file mode 100644
index 0000000..754c96e
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestSpaceQuotaBasicFunctioning.java
@@ -0,0 +1,224 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.quotas;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.DoNotRetryIOException;
+import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Append;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Increment;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.security.AccessDeniedException;
+import org.apache.hadoop.hbase.testclassification.LargeTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.util.StringUtils;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.rules.TestName;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+@Category(LargeTests.class)
+public class TestSpaceQuotaBasicFunctioning {
+
+ @ClassRule
+ public static final HBaseClassTestRule CLASS_RULE =
+ HBaseClassTestRule.forClass(TestSpaceQuotaBasicFunctioning.class);
+
+ private static final Logger LOG = LoggerFactory.getLogger(TestSpaceQuotaBasicFunctioning.class);
+ private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+ private static final int NUM_RETRIES = 10;
+
+ @Rule
+ public TestName testName = new TestName();
+ private SpaceQuotaHelperForTests helper;
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ Configuration conf = TEST_UTIL.getConfiguration();
+ SpaceQuotaHelperForTests.updateConfigForQuotas(conf);
+ TEST_UTIL.startMiniCluster(1);
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ TEST_UTIL.shutdownMiniCluster();
+ }
+
+ @Before
+ public void removeAllQuotas() throws Exception {
+ helper = new SpaceQuotaHelperForTests(TEST_UTIL, testName, new AtomicLong(0));
+ helper.removeAllQuotas();
+ }
+
+ @Test
+ public void testNoInsertsWithPut() throws Exception {
+ Put p = new Put(Bytes.toBytes("to_reject"));
+ p.addColumn(Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"),
+ Bytes.toBytes("reject"));
+ helper.writeUntilViolationAndVerifyViolation(SpaceViolationPolicy.NO_INSERTS, p);
+ }
+
+ @Test
+ public void testNoInsertsWithAppend() throws Exception {
+ Append a = new Append(Bytes.toBytes("to_reject"));
+ a.addColumn(Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"),
+ Bytes.toBytes("reject"));
+ helper.writeUntilViolationAndVerifyViolation(SpaceViolationPolicy.NO_INSERTS, a);
+ }
+
+ @Test
+ public void testNoInsertsWithIncrement() throws Exception {
+ Increment i = new Increment(Bytes.toBytes("to_reject"));
+ i.addColumn(Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("count"), 0);
+ helper.writeUntilViolationAndVerifyViolation(SpaceViolationPolicy.NO_INSERTS, i);
+ }
+
+ @Test
+ public void testDeletesAfterNoInserts() throws Exception {
+ final TableName tn = helper.writeUntilViolation(SpaceViolationPolicy.NO_INSERTS);
+ // Try a couple of times to verify that the quota never gets enforced, same as we
+ // do when we're trying to catch the failure.
+ Delete d = new Delete(Bytes.toBytes("should_not_be_rejected"));
+ for (int i = 0; i < NUM_RETRIES; i++) {
+ try (Table t = TEST_UTIL.getConnection().getTable(tn)) {
+ t.delete(d);
+ }
+ }
+ }
+
+ @Test
+ public void testNoWritesWithPut() throws Exception {
+ Put p = new Put(Bytes.toBytes("to_reject"));
+ p.addColumn(Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"),
+ Bytes.toBytes("reject"));
+ helper.writeUntilViolationAndVerifyViolation(SpaceViolationPolicy.NO_WRITES, p);
+ }
+
+ @Test
+ public void testNoWritesWithAppend() throws Exception {
+ Append a = new Append(Bytes.toBytes("to_reject"));
+ a.addColumn(Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"),
+ Bytes.toBytes("reject"));
+ helper.writeUntilViolationAndVerifyViolation(SpaceViolationPolicy.NO_WRITES, a);
+ }
+
+ @Test
+ public void testNoWritesWithIncrement() throws Exception {
+ Increment i = new Increment(Bytes.toBytes("to_reject"));
+ i.addColumn(Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("count"), 0);
+ helper.writeUntilViolationAndVerifyViolation(SpaceViolationPolicy.NO_WRITES, i);
+ }
+
+ @Test
+ public void testNoWritesWithDelete() throws Exception {
+ Delete d = new Delete(Bytes.toBytes("to_reject"));
+ helper.writeUntilViolationAndVerifyViolation(SpaceViolationPolicy.NO_WRITES, d);
+ }
+
+ @Test
+ public void testNoCompactions() throws Exception {
+ Put p = new Put(Bytes.toBytes("to_reject"));
+ p.addColumn(Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"),
+ Bytes.toBytes("reject"));
+ final TableName tn =
+ helper.writeUntilViolationAndVerifyViolation(SpaceViolationPolicy.NO_WRITES_COMPACTIONS, p);
+ // We know the policy is active at this point
+
+ // Major compactions should be rejected
+ try {
+ TEST_UTIL.getAdmin().majorCompact(tn);
+ fail("Expected that invoking the compaction should throw an Exception");
+ } catch (DoNotRetryIOException e) {
+ // Expected!
+ }
+ // Minor compactions should also be rejected.
+ try {
+ TEST_UTIL.getAdmin().compact(tn);
+ fail("Expected that invoking the compaction should throw an Exception");
+ } catch (DoNotRetryIOException e) {
+ // Expected!
+ }
+ }
+
+ @Test
+ public void testNoEnableAfterDisablePolicy() throws Exception {
+ Put p = new Put(Bytes.toBytes("to_reject"));
+ p.addColumn(Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"),
+ Bytes.toBytes("reject"));
+ final TableName tn = helper.writeUntilViolation(SpaceViolationPolicy.DISABLE);
+ final Admin admin = TEST_UTIL.getAdmin();
+ // Disabling a table relies on some external action (over the other policies), so wait a bit
+ // more than the other tests.
+ for (int i = 0; i < NUM_RETRIES * 2; i++) {
+ if (admin.isTableEnabled(tn)) {
+ LOG.info(tn + " is still enabled, expecting it to be disabled. Will wait and re-check.");
+ Thread.sleep(2000);
+ }
+ }
+ assertFalse(tn + " is still enabled but it should be disabled", admin.isTableEnabled(tn));
+ try {
+ admin.enableTable(tn);
+ } catch (AccessDeniedException e) {
+ String exceptionContents = StringUtils.stringifyException(e);
+ final String expectedText = "violated space quota";
+ assertTrue(
+ "Expected the exception to contain " + expectedText + ", but was: " + exceptionContents,
+ exceptionContents.contains(expectedText));
+ }
+ }
+
+ @Test
+ public void testTableQuotaOverridesNamespaceQuota() throws Exception {
+ final SpaceViolationPolicy policy = SpaceViolationPolicy.NO_INSERTS;
+ final TableName tn = helper.createTableWithRegions(10);
+
+ // 2MB limit on the table, 1GB limit on the namespace
+ final long tableLimit = 2L * SpaceQuotaHelperForTests.ONE_MEGABYTE;
+ final long namespaceLimit = 1024L * SpaceQuotaHelperForTests.ONE_MEGABYTE;
+ TEST_UTIL.getAdmin().setQuota(QuotaSettingsFactory.limitTableSpace(tn, tableLimit, policy));
+ TEST_UTIL.getAdmin().setQuota(QuotaSettingsFactory
+ .limitNamespaceSpace(tn.getNamespaceAsString(), namespaceLimit, policy));
+
+ // Write more data than should be allowed and flush it to disk
+ helper.writeData(tn, 3L * SpaceQuotaHelperForTests.ONE_MEGABYTE);
+
+ // This should be sufficient time for the chores to run and see the change.
+ Thread.sleep(5000);
+
+ // The write should be rejected because the table quota takes priority over the namespace
+ Put p = new Put(Bytes.toBytes("to_reject"));
+ p.addColumn(Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"),
+ Bytes.toBytes("reject"));
+ helper.verifyViolation(policy, tn, p);
+ }
+}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestSpaceQuotaDropTable.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestSpaceQuotaDropTable.java
new file mode 100644
index 0000000..cb60d12
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestSpaceQuotaDropTable.java
@@ -0,0 +1,109 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.quotas;
+
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.testclassification.LargeTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.rules.TestName;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+@Category(LargeTests.class)
+public class TestSpaceQuotaDropTable {
+
+ @ClassRule
+ public static final HBaseClassTestRule CLASS_RULE =
+ HBaseClassTestRule.forClass(TestSpaceQuotaDropTable.class);
+
+ private static final Logger LOG = LoggerFactory.getLogger(TestSpaceQuotaDropTable.class);
+ private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+
+ @Rule
+ public TestName testName = new TestName();
+ private SpaceQuotaHelperForTests helper;
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ Configuration conf = TEST_UTIL.getConfiguration();
+ SpaceQuotaHelperForTests.updateConfigForQuotas(conf);
+ TEST_UTIL.startMiniCluster(1);
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ TEST_UTIL.shutdownMiniCluster();
+ }
+
+ @Before
+ public void removeAllQuotas() throws Exception {
+ helper = new SpaceQuotaHelperForTests(TEST_UTIL, testName, new AtomicLong(0));
+ helper.removeAllQuotas();
+ }
+
+ @Test
+ public void testSetQuotaAndThenDropTableWithNoInserts() throws Exception {
+ setQuotaAndThenDropTable(SpaceViolationPolicy.NO_INSERTS);
+ }
+
+ @Test
+ public void testSetQuotaAndThenDropTableWithNoWrite() throws Exception {
+ setQuotaAndThenDropTable(SpaceViolationPolicy.NO_WRITES);
+ }
+
+ @Test
+ public void testSetQuotaAndThenDropTableWithNoWritesCompactions() throws Exception {
+ setQuotaAndThenDropTable(SpaceViolationPolicy.NO_WRITES_COMPACTIONS);
+ }
+
+ @Test
+ public void testSetQuotaAndThenDropTableWithDisable() throws Exception {
+ setQuotaAndThenDropTable(SpaceViolationPolicy.DISABLE);
+ }
+
+ private void setQuotaAndThenDropTable(SpaceViolationPolicy policy) throws Exception {
+ Put put = new Put(Bytes.toBytes("to_reject"));
+ put.addColumn(Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"),
+ Bytes.toBytes("reject"));
+
+ // Do puts until we violate space policy
+ final TableName tn = helper.writeUntilViolationAndVerifyViolation(policy, put);
+
+ // Now, drop the table
+ TEST_UTIL.deleteTable(tn);
+ LOG.debug("Successfully deleted table ", tn);
+
+ // Now re-create the table
+ TEST_UTIL.createTable(tn, Bytes.toBytes(SpaceQuotaHelperForTests.F1));
+ LOG.debug("Successfully re-created table ", tn);
+
+ // Put some rows now: should not violate as table/quota was dropped
+ helper.verifyNoViolation(tn, put);
+ }
+}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestSpaceQuotaIncrease.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestSpaceQuotaIncrease.java
new file mode 100644
index 0000000..5e6ca0e
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestSpaceQuotaIncrease.java
@@ -0,0 +1,101 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.quotas;
+
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.testclassification.LargeTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.rules.TestName;
+
+@Category(LargeTests.class)
+public class TestSpaceQuotaIncrease {
+
+ @ClassRule
+ public static final HBaseClassTestRule CLASS_RULE =
+ HBaseClassTestRule.forClass(TestSpaceQuotaIncrease.class);
+
+ private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+
+ @Rule
+ public TestName testName = new TestName();
+ private SpaceQuotaHelperForTests helper;
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ Configuration conf = TEST_UTIL.getConfiguration();
+ SpaceQuotaHelperForTests.updateConfigForQuotas(conf);
+ TEST_UTIL.startMiniCluster(1);
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ TEST_UTIL.shutdownMiniCluster();
+ }
+
+ @Before
+ public void removeAllQuotas() throws Exception {
+ helper = new SpaceQuotaHelperForTests(TEST_UTIL, testName, new AtomicLong(0));
+ helper.removeAllQuotas();
+ }
+
+ @Test
+ public void testSetQuotaAndThenIncreaseQuotaWithNoInserts() throws Exception {
+ setQuotaAndThenIncreaseQuota(SpaceViolationPolicy.NO_INSERTS);
+ }
+
+ @Test
+ public void testSetQuotaAndThenIncreaseQuotaWithNoWrite() throws Exception {
+ setQuotaAndThenIncreaseQuota(SpaceViolationPolicy.NO_WRITES);
+ }
+
+ @Test
+ public void testSetQuotaAndThenIncreaseQuotaWithNoWritesCompactions() throws Exception {
+ setQuotaAndThenIncreaseQuota(SpaceViolationPolicy.NO_WRITES_COMPACTIONS);
+ }
+
+ @Test
+ public void testSetQuotaAndThenIncreaseQuotaWithDisable() throws Exception {
+ setQuotaAndThenIncreaseQuota(SpaceViolationPolicy.DISABLE);
+ }
+
+ private void setQuotaAndThenIncreaseQuota(SpaceViolationPolicy policy) throws Exception {
+ Put put = new Put(Bytes.toBytes("to_reject"));
+ put.addColumn(Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"),
+ Bytes.toBytes("reject"));
+
+ // Do puts until we violate space policy
+ final TableName tn = helper.writeUntilViolationAndVerifyViolation(policy, put);
+
+ // Now, increase limit and perform put
+ helper.setQuotaLimit(tn, policy, 4L);
+
+ // Put some row now: should not violate as quota limit increased
+ helper.verifyNoViolation(tn, put);
+ }
+}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestSpaceQuotaOnBulkLoad.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestSpaceQuotaOnBulkLoad.java
new file mode 100644
index 0000000..ce4bd4b
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestSpaceQuotaOnBulkLoad.java
@@ -0,0 +1,190 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.quotas;
+
+import static org.hamcrest.CoreMatchers.instanceOf;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.RegionInfo;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.master.HMaster;
+import org.apache.hadoop.hbase.quotas.policies.DefaultViolationPolicyEnforcement;
+import org.apache.hadoop.hbase.regionserver.HRegionServer;
+import org.apache.hadoop.hbase.testclassification.LargeTests;
+import org.apache.hadoop.hbase.tool.BulkLoadHFiles;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.rules.TestName;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+@Category(LargeTests.class)
+public class TestSpaceQuotaOnBulkLoad {
+
+ @ClassRule
+ public static final HBaseClassTestRule CLASS_RULE =
+ HBaseClassTestRule.forClass(TestSpaceQuotaOnBulkLoad.class);
+
+ private static final Logger LOG = LoggerFactory.getLogger(TestSpaceQuotaOnBulkLoad.class);
+ private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+
+ @Rule
+ public TestName testName = new TestName();
+ private SpaceQuotaHelperForTests helper;
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ Configuration conf = TEST_UTIL.getConfiguration();
+ SpaceQuotaHelperForTests.updateConfigForQuotas(conf);
+ TEST_UTIL.startMiniCluster(1);
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ TEST_UTIL.shutdownMiniCluster();
+ }
+
+ @Before
+ public void removeAllQuotas() throws Exception {
+ helper = new SpaceQuotaHelperForTests(TEST_UTIL, testName, new AtomicLong(0));
+ helper.removeAllQuotas();
+ }
+
+ @Test
+ public void testNoBulkLoadsWithNoWrites() throws Exception {
+ Put p = new Put(Bytes.toBytes("to_reject"));
+ p.addColumn(Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"),
+ Bytes.toBytes("reject"));
+ TableName tableName =
+ helper.writeUntilViolationAndVerifyViolation(SpaceViolationPolicy.NO_WRITES, p);
+
+ // The table is now in violation. Try to do a bulk load
+ Map<byte[], List<Path>> family2Files = helper.generateFileToLoad(tableName, 1, 50);
+ try {
+ BulkLoadHFiles.create(TEST_UTIL.getConfiguration()).bulkLoad(tableName, family2Files);
+ fail("Expected the bulk load call to fail!");
+ } catch (IOException e) {
+ // Pass
+ assertThat(e.getCause(), instanceOf(SpaceLimitingException.class));
+ LOG.trace("Caught expected exception", e);
+ }
+ }
+
+ @Test
+ public void testAtomicBulkLoadUnderQuota() throws Exception {
+ // Need to verify that if the batch of hfiles cannot be loaded, none are loaded.
+ TableName tn = helper.createTableWithRegions(10);
+
+ final long sizeLimit = 50L * SpaceQuotaHelperForTests.ONE_KILOBYTE;
+ QuotaSettings settings =
+ QuotaSettingsFactory.limitTableSpace(tn, sizeLimit, SpaceViolationPolicy.NO_INSERTS);
+ TEST_UTIL.getAdmin().setQuota(settings);
+
+ HRegionServer rs = TEST_UTIL.getMiniHBaseCluster().getRegionServer(0);
+ RegionServerSpaceQuotaManager spaceQuotaManager = rs.getRegionServerSpaceQuotaManager();
+ Map<TableName, SpaceQuotaSnapshot> snapshots = spaceQuotaManager.copyQuotaSnapshots();
+ Map<RegionInfo, Long> regionSizes = getReportedSizesForTable(tn);
+ while (true) {
+ SpaceQuotaSnapshot snapshot = snapshots.get(tn);
+ if (snapshot != null && snapshot.getLimit() > 0) {
+ break;
+ }
+ LOG.debug("Snapshot does not yet realize quota limit: " + snapshots + ", regionsizes: "
+ + regionSizes);
+ Thread.sleep(3000);
+ snapshots = spaceQuotaManager.copyQuotaSnapshots();
+ regionSizes = getReportedSizesForTable(tn);
+ }
+ // Our quota limit should be reflected in the latest snapshot
+ SpaceQuotaSnapshot snapshot = snapshots.get(tn);
+ assertEquals(0L, snapshot.getUsage());
+ assertEquals(sizeLimit, snapshot.getLimit());
+
+ // We would also not have a "real" policy in violation
+ ActivePolicyEnforcement activePolicies = spaceQuotaManager.getActiveEnforcements();
+ SpaceViolationPolicyEnforcement enforcement = activePolicies.getPolicyEnforcement(tn);
+ assertTrue("Expected to find Noop policy, but got " + enforcement.getClass().getSimpleName(),
+ enforcement instanceof DefaultViolationPolicyEnforcement);
+
+ // Should generate two files, each of which is over 25KB each
+ Map<byte[], List<Path>> family2Files = helper.generateFileToLoad(tn, 2, 525);
+ FileSystem fs = TEST_UTIL.getTestFileSystem();
+ FileStatus[] files =
+ fs.listStatus(new Path(fs.getHomeDirectory(), testName.getMethodName() + "_files"));
+ for (FileStatus file : files) {
+ assertTrue(
+ "Expected the file, " + file.getPath() + ", length to be larger than 25KB, but was "
+ + file.getLen(), file.getLen() > 25 * SpaceQuotaHelperForTests.ONE_KILOBYTE);
+ LOG.debug(file.getPath() + " -> " + file.getLen() + "B");
+ }
+
+ try {
+ BulkLoadHFiles.create(TEST_UTIL.getConfiguration()).bulkLoad(tn, family2Files);
+ fail("Expected the bulk load call to fail!");
+ } catch (IOException e) {
+ // Pass
+ assertThat(e.getCause(), instanceOf(SpaceLimitingException.class));
+ LOG.trace("Caught expected exception", e);
+ }
+ // Verify that we have no data in the table because neither file should have been
+ // loaded even though one of the files could have.
+ Table table = TEST_UTIL.getConnection().getTable(tn);
+ ResultScanner scanner = table.getScanner(new Scan());
+ try {
+ assertNull("Expected no results", scanner.next());
+ } finally {
+ scanner.close();
+ }
+ }
+
+ private Map<RegionInfo, Long> getReportedSizesForTable(TableName tn) {
+ HMaster master = TEST_UTIL.getMiniHBaseCluster().getMaster();
+ MasterQuotaManager quotaManager = master.getMasterQuotaManager();
+ Map<RegionInfo, Long> filteredRegionSizes = new HashMap<>();
+ for (Map.Entry<RegionInfo, Long> entry : quotaManager.snapshotRegionSizes().entrySet()) {
+ if (entry.getKey().getTable().equals(tn)) {
+ filteredRegionSizes.put(entry.getKey(), entry.getValue());
+ }
+ }
+ return filteredRegionSizes;
+ }
+}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestSpaceQuotaOnNonExistingTables.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestSpaceQuotaOnNonExistingTables.java
new file mode 100644
index 0000000..2ae321c
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestSpaceQuotaOnNonExistingTables.java
@@ -0,0 +1,86 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.quotas;
+
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.testclassification.LargeTests;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.rules.TestName;
+
+@Category(LargeTests.class)
+public class TestSpaceQuotaOnNonExistingTables {
+
+ @ClassRule
+ public static final HBaseClassTestRule CLASS_RULE =
+ HBaseClassTestRule.forClass(TestSpaceQuotaOnNonExistingTables.class);
+
+ private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+
+ private final TableName NON_EXISTENT_TABLE = TableName.valueOf("NON_EXISTENT_TABLE");
+
+ @Rule
+ public TestName testName = new TestName();
+ private SpaceQuotaHelperForTests helper;
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ Configuration conf = TEST_UTIL.getConfiguration();
+ SpaceQuotaHelperForTests.updateConfigForQuotas(conf);
+ TEST_UTIL.startMiniCluster(1);
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ TEST_UTIL.shutdownMiniCluster();
+ }
+
+ @Before
+ public void removeAllQuotas() throws Exception {
+ helper = new SpaceQuotaHelperForTests(TEST_UTIL, testName, new AtomicLong(0));
+ helper.removeAllQuotas();
+ }
+
+ @Test
+ public void testSetQuotaOnNonExistingTableWithNoInserts() throws Exception {
+ helper.setQuotaLimit(NON_EXISTENT_TABLE, SpaceViolationPolicy.NO_INSERTS, 2L);
+ }
+
+ @Test
+ public void testSetQuotaOnNonExistingTableWithNoWrites() throws Exception {
+ helper.setQuotaLimit(NON_EXISTENT_TABLE, SpaceViolationPolicy.NO_WRITES, 2L);
+ }
+
+ @Test
+ public void testSetQuotaOnNonExistingTableWithNoWritesCompaction() throws Exception {
+ helper.setQuotaLimit(NON_EXISTENT_TABLE, SpaceViolationPolicy.NO_WRITES_COMPACTIONS, 2L);
+ }
+
+ @Test
+ public void testSetQuotaOnNonExistingTableWithDisable() throws Exception {
+ helper.setQuotaLimit(NON_EXISTENT_TABLE, SpaceViolationPolicy.DISABLE, 2L);
+ }
+}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestSpaceQuotaRemoval.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestSpaceQuotaRemoval.java
new file mode 100644
index 0000000..ba89990
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestSpaceQuotaRemoval.java
@@ -0,0 +1,190 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.quotas;
+
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.testclassification.LargeTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.rules.TestName;
+
+@Category(LargeTests.class)
+public class TestSpaceQuotaRemoval {
+
+ @ClassRule
+ public static final HBaseClassTestRule CLASS_RULE =
+ HBaseClassTestRule.forClass(TestSpaceQuotaRemoval.class);
+
+ private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+
+ @Rule
+ public TestName testName = new TestName();
+ private SpaceQuotaHelperForTests helper;
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ Configuration conf = TEST_UTIL.getConfiguration();
+ SpaceQuotaHelperForTests.updateConfigForQuotas(conf);
+ TEST_UTIL.startMiniCluster(1);
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ TEST_UTIL.shutdownMiniCluster();
+ }
+
+ @Before
+ public void removeAllQuotas() throws Exception {
+ helper = new SpaceQuotaHelperForTests(TEST_UTIL, testName, new AtomicLong(0));
+ helper.removeAllQuotas();
+ }
+
+ @Test
+ public void testSetQuotaAndThenRemoveInOneWithNoInserts() throws Exception {
+ setQuotaAndThenRemoveInOneAmongTwoTables(SpaceViolationPolicy.NO_INSERTS);
+ }
+
+ @Test
+ public void testSetQuotaAndThenRemoveInOneWithNoWrite() throws Exception {
+ setQuotaAndThenRemoveInOneAmongTwoTables(SpaceViolationPolicy.NO_WRITES);
+ }
+
+ @Test
+ public void testSetQuotaAndThenRemoveInOneWithNoWritesCompaction() throws Exception {
+ setQuotaAndThenRemoveInOneAmongTwoTables(SpaceViolationPolicy.NO_WRITES_COMPACTIONS);
+ }
+
+ @Test
+ public void testSetQuotaAndThenRemoveInOneWithDisable() throws Exception {
+ setQuotaAndThenRemoveInOneAmongTwoTables(SpaceViolationPolicy.DISABLE);
+ }
+
+ @Test
+ public void testSetQuotaAndThenRemoveWithNoInserts() throws Exception {
+ setQuotaAndThenRemove(SpaceViolationPolicy.NO_INSERTS);
+ }
+
+ @Test
+ public void testSetQuotaAndThenRemoveWithNoWrite() throws Exception {
+ setQuotaAndThenRemove(SpaceViolationPolicy.NO_WRITES);
+ }
+
+ @Test
+ public void testSetQuotaAndThenRemoveWithNoWritesCompactions() throws Exception {
+ setQuotaAndThenRemove(SpaceViolationPolicy.NO_WRITES_COMPACTIONS);
+ }
+
+ @Test
+ public void testSetQuotaAndThenRemoveWithDisable() throws Exception {
+ setQuotaAndThenRemove(SpaceViolationPolicy.DISABLE);
+ }
+
+ @Test
+ public void testSetQuotaAndThenDisableIncrEnableWithNoInserts() throws Exception {
+ setQuotaNextDisableThenIncreaseFinallyEnable(SpaceViolationPolicy.NO_INSERTS);
+ }
+
+ @Test
+ public void testSetQuotaAndThenDisableIncrEnableWithNoWrite() throws Exception {
+ setQuotaNextDisableThenIncreaseFinallyEnable(SpaceViolationPolicy.NO_WRITES);
+ }
+
+ @Test
+ public void testSetQuotaAndThenDisableIncrEnableWithNoWritesCompaction() throws Exception {
+ setQuotaNextDisableThenIncreaseFinallyEnable(SpaceViolationPolicy.NO_WRITES_COMPACTIONS);
+ }
+
+ @Test
+ public void testSetQuotaAndThenDisableIncrEnableWithDisable() throws Exception {
+ setQuotaNextDisableThenIncreaseFinallyEnable(SpaceViolationPolicy.DISABLE);
+ }
+
+ private void setQuotaAndThenRemove(SpaceViolationPolicy policy) throws Exception {
+ Put put = new Put(Bytes.toBytes("to_reject"));
+ put.addColumn(Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"),
+ Bytes.toBytes("reject"));
+
+ // Do puts until we violate space policy
+ final TableName tn = helper.writeUntilViolationAndVerifyViolation(policy, put);
+
+ // Now, remove the quota
+ helper.removeQuotaFromtable(tn);
+
+ // Put some rows now: should not violate as quota settings removed
+ helper.verifyNoViolation(tn, put);
+ }
+
+ private void setQuotaAndThenRemoveInOneAmongTwoTables(SpaceViolationPolicy policy)
+ throws Exception {
+ Put put = new Put(Bytes.toBytes("to_reject"));
+ put.addColumn(Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"),
+ Bytes.toBytes("reject"));
+
+ // Do puts until we violate space policy on table tn1
+ final TableName tn1 = helper.writeUntilViolationAndVerifyViolation(policy, put);
+
+ // Do puts until we violate space policy on table tn2
+ final TableName tn2 = helper.writeUntilViolationAndVerifyViolation(policy, put);
+
+ // Now, remove the quota from table tn1
+ helper.removeQuotaFromtable(tn1);
+
+ // Put a new row now on tn1: should not violate as quota settings removed
+ helper.verifyNoViolation(tn1, put);
+ // Put a new row now on tn2: should violate as quota settings exists
+ helper.verifyViolation(policy, tn2, put);
+ }
+
+ private void setQuotaNextDisableThenIncreaseFinallyEnable(SpaceViolationPolicy policy)
+ throws Exception {
+ Put put = new Put(Bytes.toBytes("to_reject"));
+ put.addColumn(Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"),
+ Bytes.toBytes("reject"));
+
+ // Do puts until we violate space policy
+ final TableName tn = helper.writeUntilViolationAndVerifyViolation(policy, put);
+
+ // Disable the table; in case of SpaceViolationPolicy.DISABLE already disabled
+ if (!policy.equals(SpaceViolationPolicy.DISABLE)) {
+ TEST_UTIL.getAdmin().disableTable(tn);
+ TEST_UTIL.waitTableDisabled(tn, 10000);
+ }
+
+ // Now, increase limit and perform put
+ helper.setQuotaLimit(tn, policy, 4L);
+
+ // in case of disable policy quota manager will enable it
+ if (!policy.equals(SpaceViolationPolicy.DISABLE)) {
+ TEST_UTIL.getAdmin().enableTable(tn);
+ }
+ TEST_UTIL.waitTableEnabled(tn, 10000);
+
+ // Put some row now: should not violate as quota limit increased
+ helper.verifyNoViolation(tn, put);
+ }
+}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestSpaceQuotaSwitchPolicies.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestSpaceQuotaSwitchPolicies.java
new file mode 100644
index 0000000..3d4a276
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestSpaceQuotaSwitchPolicies.java
@@ -0,0 +1,117 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.quotas;
+
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.testclassification.LargeTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.rules.TestName;
+
+@Category(LargeTests.class)
+public class TestSpaceQuotaSwitchPolicies {
+
+ @ClassRule
+ public static final HBaseClassTestRule CLASS_RULE =
+ HBaseClassTestRule.forClass(TestSpaceQuotaSwitchPolicies.class);
+
+ private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+
+ @Rule
+ public TestName testName = new TestName();
+ private SpaceQuotaHelperForTests helper;
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ Configuration conf = TEST_UTIL.getConfiguration();
+ SpaceQuotaHelperForTests.updateConfigForQuotas(conf);
+ TEST_UTIL.startMiniCluster(1);
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ TEST_UTIL.shutdownMiniCluster();
+ }
+
+ @Before
+ public void removeAllQuotas() throws Exception {
+ helper = new SpaceQuotaHelperForTests(TEST_UTIL, testName, new AtomicLong(0));
+ helper.removeAllQuotas();
+ }
+
+ @Test
+ public void testSetQuotaFirstWithDisableNextNoWrites() throws Exception {
+ setQuotaAndViolateNextSwitchPoliciesAndValidate(SpaceViolationPolicy.DISABLE,
+ SpaceViolationPolicy.NO_WRITES);
+ }
+
+ @Test
+ public void testSetQuotaFirstWithDisableNextAgainDisable() throws Exception {
+ setQuotaAndViolateNextSwitchPoliciesAndValidate(SpaceViolationPolicy.DISABLE,
+ SpaceViolationPolicy.DISABLE);
+ }
+
+ @Test
+ public void testSetQuotaFirstWithDisableNextNoInserts() throws Exception {
+ setQuotaAndViolateNextSwitchPoliciesAndValidate(SpaceViolationPolicy.DISABLE,
+ SpaceViolationPolicy.NO_INSERTS);
+ }
+
+ @Test
+ public void testSetQuotaFirstWithDisableNextNoWritesCompaction() throws Exception {
+ setQuotaAndViolateNextSwitchPoliciesAndValidate(SpaceViolationPolicy.DISABLE,
+ SpaceViolationPolicy.NO_WRITES_COMPACTIONS);
+ }
+
+ @Test
+ public void testSetQuotaFirstWithNoWritesNextWithDisable() throws Exception {
+ setQuotaAndViolateNextSwitchPoliciesAndValidate(SpaceViolationPolicy.NO_WRITES,
+ SpaceViolationPolicy.DISABLE);
+ }
+
+ private void setQuotaAndViolateNextSwitchPoliciesAndValidate(SpaceViolationPolicy policy1,
+ SpaceViolationPolicy policy2) throws Exception {
+ Put put = new Put(Bytes.toBytes("to_reject"));
+ put.addColumn(Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"),
+ Bytes.toBytes("reject"));
+
+ // Do puts until we violate space violation policy1
+ final TableName tn = helper.writeUntilViolationAndVerifyViolation(policy1, put);
+
+ // Now, change violation policy to policy2
+ helper.setQuotaLimit(tn, policy2, 2L);
+
+ // The table should be in enabled state on changing violation policy
+ if (policy1.equals(SpaceViolationPolicy.DISABLE) && !policy1.equals(policy2)) {
+ TEST_UTIL.waitTableEnabled(tn, 20000);
+ }
+ // Put some row now: should still violate as quota limit still violated
+ helper.verifyViolation(policy2, tn, put);
+ }
+}
+
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestSpaceQuotas.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestSpaceQuotas.java
deleted file mode 100644
index fca5453..0000000
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestSpaceQuotas.java
+++ /dev/null
@@ -1,748 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.quotas;
-
-import static org.hamcrest.CoreMatchers.instanceOf;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertThat;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.concurrent.atomic.AtomicLong;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.DoNotRetryIOException;
-import org.apache.hadoop.hbase.HBaseClassTestRule;
-import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.TableNotEnabledException;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.Append;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.client.Delete;
-import org.apache.hadoop.hbase.client.Increment;
-import org.apache.hadoop.hbase.client.Mutation;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.RegionInfo;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.client.Table;
-import org.apache.hadoop.hbase.master.HMaster;
-import org.apache.hadoop.hbase.quotas.policies.DefaultViolationPolicyEnforcement;
-import org.apache.hadoop.hbase.regionserver.HRegionServer;
-import org.apache.hadoop.hbase.security.AccessDeniedException;
-import org.apache.hadoop.hbase.testclassification.LargeTests;
-import org.apache.hadoop.hbase.tool.BulkLoadHFiles;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.util.StringUtils;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.ClassRule;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-import org.junit.rules.TestName;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * End-to-end test class for filesystem space quotas.
- */
-@Category(LargeTests.class)
-public class TestSpaceQuotas {
-
- @ClassRule
- public static final HBaseClassTestRule CLASS_RULE =
- HBaseClassTestRule.forClass(TestSpaceQuotas.class);
-
- private static final Logger LOG = LoggerFactory.getLogger(TestSpaceQuotas.class);
- private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
- // Global for all tests in the class
- private static final AtomicLong COUNTER = new AtomicLong(0);
- private static final int NUM_RETRIES = 10;
-
- @Rule
- public TestName testName = new TestName();
- private SpaceQuotaHelperForTests helper;
- private final TableName NON_EXISTENT_TABLE = TableName.valueOf("NON_EXISTENT_TABLE");
-
- @BeforeClass
- public static void setUp() throws Exception {
- Configuration conf = TEST_UTIL.getConfiguration();
- SpaceQuotaHelperForTests.updateConfigForQuotas(conf);
- TEST_UTIL.startMiniCluster(1);
- }
-
- @AfterClass
- public static void tearDown() throws Exception {
- TEST_UTIL.shutdownMiniCluster();
- }
-
- @Before
- public void removeAllQuotas() throws Exception {
- final Connection conn = TEST_UTIL.getConnection();
- if (helper == null) {
- helper = new SpaceQuotaHelperForTests(TEST_UTIL, testName, COUNTER);
- }
- // Wait for the quota table to be created
- if (!conn.getAdmin().tableExists(QuotaUtil.QUOTA_TABLE_NAME)) {
- helper.waitForQuotaTable(conn);
- } else {
- // Or, clean up any quotas from previous test runs.
- helper.removeAllQuotas(conn);
- assertEquals(0, helper.listNumDefinedQuotas(conn));
- }
- }
-
- @Test
- public void testNoInsertsWithPut() throws Exception {
- Put p = new Put(Bytes.toBytes("to_reject"));
- p.addColumn(
- Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"), Bytes.toBytes("reject"));
- writeUntilViolationAndVerifyViolation(SpaceViolationPolicy.NO_INSERTS, p);
- }
-
- @Test
- public void testNoInsertsWithAppend() throws Exception {
- Append a = new Append(Bytes.toBytes("to_reject"));
- a.addColumn(
- Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"), Bytes.toBytes("reject"));
- writeUntilViolationAndVerifyViolation(SpaceViolationPolicy.NO_INSERTS, a);
- }
-
- @Test
- public void testNoInsertsWithIncrement() throws Exception {
- Increment i = new Increment(Bytes.toBytes("to_reject"));
- i.addColumn(Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("count"), 0);
- writeUntilViolationAndVerifyViolation(SpaceViolationPolicy.NO_INSERTS, i);
- }
-
- @Test
- public void testDeletesAfterNoInserts() throws Exception {
- final TableName tn = writeUntilViolation(SpaceViolationPolicy.NO_INSERTS);
- // Try a couple of times to verify that the quota never gets enforced, same as we
- // do when we're trying to catch the failure.
- Delete d = new Delete(Bytes.toBytes("should_not_be_rejected"));
- for (int i = 0; i < NUM_RETRIES; i++) {
- try (Table t = TEST_UTIL.getConnection().getTable(tn)) {
- t.delete(d);
- }
- }
- }
-
- @Test
- public void testNoWritesWithPut() throws Exception {
- Put p = new Put(Bytes.toBytes("to_reject"));
- p.addColumn(
- Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"), Bytes.toBytes("reject"));
- writeUntilViolationAndVerifyViolation(SpaceViolationPolicy.NO_WRITES, p);
- }
-
- @Test
- public void testNoWritesWithAppend() throws Exception {
- Append a = new Append(Bytes.toBytes("to_reject"));
- a.addColumn(
- Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"), Bytes.toBytes("reject"));
- writeUntilViolationAndVerifyViolation(SpaceViolationPolicy.NO_WRITES, a);
- }
-
- @Test
- public void testNoWritesWithIncrement() throws Exception {
- Increment i = new Increment(Bytes.toBytes("to_reject"));
- i.addColumn(Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("count"), 0);
- writeUntilViolationAndVerifyViolation(SpaceViolationPolicy.NO_WRITES, i);
- }
-
- @Test
- public void testNoWritesWithDelete() throws Exception {
- Delete d = new Delete(Bytes.toBytes("to_reject"));
- writeUntilViolationAndVerifyViolation(SpaceViolationPolicy.NO_WRITES, d);
- }
-
- @Test
- public void testNoCompactions() throws Exception {
- Put p = new Put(Bytes.toBytes("to_reject"));
- p.addColumn(
- Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"), Bytes.toBytes("reject"));
- final TableName tn = writeUntilViolationAndVerifyViolation(
- SpaceViolationPolicy.NO_WRITES_COMPACTIONS, p);
- // We know the policy is active at this point
-
- // Major compactions should be rejected
- try {
- TEST_UTIL.getAdmin().majorCompact(tn);
- fail("Expected that invoking the compaction should throw an Exception");
- } catch (DoNotRetryIOException e) {
- // Expected!
- }
- // Minor compactions should also be rejected.
- try {
- TEST_UTIL.getAdmin().compact(tn);
- fail("Expected that invoking the compaction should throw an Exception");
- } catch (DoNotRetryIOException e) {
- // Expected!
- }
- }
-
- @Test
- public void testNoEnableAfterDisablePolicy() throws Exception {
- Put p = new Put(Bytes.toBytes("to_reject"));
- p.addColumn(
- Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"), Bytes.toBytes("reject"));
- final TableName tn = writeUntilViolation(SpaceViolationPolicy.DISABLE);
- final Admin admin = TEST_UTIL.getAdmin();
- // Disabling a table relies on some external action (over the other policies), so wait a bit
- // more than the other tests.
- for (int i = 0; i < NUM_RETRIES * 2; i++) {
- if (admin.isTableEnabled(tn)) {
- LOG.info(tn + " is still enabled, expecting it to be disabled. Will wait and re-check.");
- Thread.sleep(2000);
- }
- }
- assertFalse(tn + " is still enabled but it should be disabled", admin.isTableEnabled(tn));
- try {
- admin.enableTable(tn);
- } catch (AccessDeniedException e) {
- String exceptionContents = StringUtils.stringifyException(e);
- final String expectedText = "violated space quota";
- assertTrue("Expected the exception to contain " + expectedText + ", but was: "
- + exceptionContents, exceptionContents.contains(expectedText));
- }
- }
-
- @Test
- public void testNoBulkLoadsWithNoWrites() throws Exception {
- Put p = new Put(Bytes.toBytes("to_reject"));
- p.addColumn(Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"),
- Bytes.toBytes("reject"));
- TableName tableName = writeUntilViolationAndVerifyViolation(SpaceViolationPolicy.NO_WRITES, p);
-
- // The table is now in violation. Try to do a bulk load
- Map<byte[], List<Path>> family2Files = helper.generateFileToLoad(tableName, 1, 50);
- try {
- BulkLoadHFiles.create(TEST_UTIL.getConfiguration()).bulkLoad(tableName, family2Files);
- fail("Expected the bulk load call to fail!");
- } catch (IOException e) {
- // Pass
- assertThat(e.getCause(), instanceOf(SpaceLimitingException.class));
- LOG.trace("Caught expected exception", e);
- }
- }
-
- @Test
- public void testAtomicBulkLoadUnderQuota() throws Exception {
- // Need to verify that if the batch of hfiles cannot be loaded, none are loaded.
- TableName tn = helper.createTableWithRegions(10);
-
- final long sizeLimit = 50L * SpaceQuotaHelperForTests.ONE_KILOBYTE;
- QuotaSettings settings = QuotaSettingsFactory.limitTableSpace(
- tn, sizeLimit, SpaceViolationPolicy.NO_INSERTS);
- TEST_UTIL.getAdmin().setQuota(settings);
-
- HRegionServer rs = TEST_UTIL.getMiniHBaseCluster().getRegionServer(0);
- RegionServerSpaceQuotaManager spaceQuotaManager = rs.getRegionServerSpaceQuotaManager();
- Map<TableName,SpaceQuotaSnapshot> snapshots = spaceQuotaManager.copyQuotaSnapshots();
- Map<RegionInfo,Long> regionSizes = getReportedSizesForTable(tn);
- while (true) {
- SpaceQuotaSnapshot snapshot = snapshots.get(tn);
- if (snapshot != null && snapshot.getLimit() > 0) {
- break;
- }
- LOG.debug(
- "Snapshot does not yet realize quota limit: " + snapshots + ", regionsizes: " +
- regionSizes);
- Thread.sleep(3000);
- snapshots = spaceQuotaManager.copyQuotaSnapshots();
- regionSizes = getReportedSizesForTable(tn);
- }
- // Our quota limit should be reflected in the latest snapshot
- SpaceQuotaSnapshot snapshot = snapshots.get(tn);
- assertEquals(0L, snapshot.getUsage());
- assertEquals(sizeLimit, snapshot.getLimit());
-
- // We would also not have a "real" policy in violation
- ActivePolicyEnforcement activePolicies = spaceQuotaManager.getActiveEnforcements();
- SpaceViolationPolicyEnforcement enforcement = activePolicies.getPolicyEnforcement(tn);
- assertTrue(
- "Expected to find Noop policy, but got " + enforcement.getClass().getSimpleName(),
- enforcement instanceof DefaultViolationPolicyEnforcement);
-
- // Should generate two files, each of which is over 25KB each
- Map<byte[], List<Path>> family2Files = helper.generateFileToLoad(tn, 2, 525);
- FileSystem fs = TEST_UTIL.getTestFileSystem();
- FileStatus[] files = fs.listStatus(
- new Path(fs.getHomeDirectory(), testName.getMethodName() + "_files"));
- for (FileStatus file : files) {
- assertTrue(
- "Expected the file, " + file.getPath() + ", length to be larger than 25KB, but was "
- + file.getLen(),
- file.getLen() > 25 * SpaceQuotaHelperForTests.ONE_KILOBYTE);
- LOG.debug(file.getPath() + " -> " + file.getLen() +"B");
- }
-
- try {
- BulkLoadHFiles.create(TEST_UTIL.getConfiguration()).bulkLoad(tn, family2Files);
- fail("Expected the bulk load call to fail!");
- } catch (IOException e) {
- // Pass
- assertThat(e.getCause(), instanceOf(SpaceLimitingException.class));
- LOG.trace("Caught expected exception", e);
- }
- // Verify that we have no data in the table because neither file should have been
- // loaded even though one of the files could have.
- Table table = TEST_UTIL.getConnection().getTable(tn);
- ResultScanner scanner = table.getScanner(new Scan());
- try {
- assertNull("Expected no results", scanner.next());
- } finally{
- scanner.close();
- }
- }
-
- @Test
- public void testTableQuotaOverridesNamespaceQuota() throws Exception {
- final SpaceViolationPolicy policy = SpaceViolationPolicy.NO_INSERTS;
- final TableName tn = helper.createTableWithRegions(10);
-
- // 2MB limit on the table, 1GB limit on the namespace
- final long tableLimit = 2L * SpaceQuotaHelperForTests.ONE_MEGABYTE;
- final long namespaceLimit = 1024L * SpaceQuotaHelperForTests.ONE_MEGABYTE;
- TEST_UTIL.getAdmin().setQuota(QuotaSettingsFactory.limitTableSpace(tn, tableLimit, policy));
- TEST_UTIL.getAdmin().setQuota(QuotaSettingsFactory.limitNamespaceSpace(
- tn.getNamespaceAsString(), namespaceLimit, policy));
-
- // Write more data than should be allowed and flush it to disk
- helper.writeData(tn, 3L * SpaceQuotaHelperForTests.ONE_MEGABYTE);
-
- // This should be sufficient time for the chores to run and see the change.
- Thread.sleep(5000);
-
- // The write should be rejected because the table quota takes priority over the namespace
- Put p = new Put(Bytes.toBytes("to_reject"));
- p.addColumn(
- Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"), Bytes.toBytes("reject"));
- verifyViolation(policy, tn, p);
- }
-
- @Test
- public void testSetQuotaAndThenRemoveWithNoInserts() throws Exception {
- setQuotaAndThenRemove(SpaceViolationPolicy.NO_INSERTS);
- }
-
- @Test
- public void testSetQuotaAndThenRemoveWithNoWrite() throws Exception {
- setQuotaAndThenRemove(SpaceViolationPolicy.NO_WRITES);
- }
-
- @Test
- public void testSetQuotaAndThenRemoveWithNoWritesCompactions() throws Exception {
- setQuotaAndThenRemove(SpaceViolationPolicy.NO_WRITES_COMPACTIONS);
- }
-
- @Test
- public void testSetQuotaAndThenRemoveWithDisable() throws Exception {
- setQuotaAndThenRemove(SpaceViolationPolicy.DISABLE);
- }
-
- @Test
- public void testSetQuotaAndThenDropTableWithNoInserts() throws Exception {
- setQuotaAndThenDropTable(SpaceViolationPolicy.NO_INSERTS);
- }
-
- @Test
- public void testSetQuotaAndThenDropTableWithNoWrite() throws Exception {
- setQuotaAndThenDropTable(SpaceViolationPolicy.NO_WRITES);
- }
-
- @Test
- public void testSetQuotaAndThenDropTableWithNoWritesCompactions() throws Exception {
- setQuotaAndThenDropTable(SpaceViolationPolicy.NO_WRITES_COMPACTIONS);
- }
-
- @Test
- public void testSetQuotaAndThenDropTableWithDisable() throws Exception {
- setQuotaAndThenDropTable(SpaceViolationPolicy.DISABLE);
- }
-
- @Test
- public void testSetQuotaAndThenIncreaseQuotaWithNoInserts() throws Exception {
- setQuotaAndThenIncreaseQuota(SpaceViolationPolicy.NO_INSERTS);
- }
-
- @Test
- public void testSetQuotaAndThenIncreaseQuotaWithNoWrite() throws Exception {
- setQuotaAndThenIncreaseQuota(SpaceViolationPolicy.NO_WRITES);
- }
-
- @Test
- public void testSetQuotaAndThenIncreaseQuotaWithNoWritesCompactions() throws Exception {
- setQuotaAndThenIncreaseQuota(SpaceViolationPolicy.NO_WRITES_COMPACTIONS);
- }
-
- @Test
- public void testSetQuotaAndThenIncreaseQuotaWithDisable() throws Exception {
- setQuotaAndThenIncreaseQuota(SpaceViolationPolicy.DISABLE);
- }
-
- @Test
- public void testSetQuotaAndThenDisableIncrEnableWithDisable() throws Exception {
- setQuotaNextDisableThenIncreaseFinallyEnable(SpaceViolationPolicy.DISABLE);
- }
-
- @Test
- public void testSetQuotaAndThenRemoveInOneWithNoInserts() throws Exception {
- setQuotaAndThenRemoveInOneAmongTwoTables(SpaceViolationPolicy.NO_INSERTS);
- }
-
- @Test
- public void testSetQuotaAndThenRemoveInOneWithNoWrite() throws Exception {
- setQuotaAndThenRemoveInOneAmongTwoTables(SpaceViolationPolicy.NO_WRITES);
- }
-
- @Test
- public void testSetQuotaAndThenRemoveInOneWithNoWritesCompaction() throws Exception {
- setQuotaAndThenRemoveInOneAmongTwoTables(SpaceViolationPolicy.NO_WRITES_COMPACTIONS);
- }
-
- @Test
- public void testSetQuotaAndThenRemoveInOneWithDisable() throws Exception {
- setQuotaAndThenRemoveInOneAmongTwoTables(SpaceViolationPolicy.DISABLE);
- }
-
- @Test
- public void testSetQuotaFirstWithDisableNextNoWrites() throws Exception {
- setQuotaAndViolateNextSwitchPoliciesAndValidate(SpaceViolationPolicy.DISABLE,
- SpaceViolationPolicy.NO_WRITES);
- }
-
- @Test
- public void testSetQuotaFirstWithDisableNextAgainDisable() throws Exception {
- setQuotaAndViolateNextSwitchPoliciesAndValidate(SpaceViolationPolicy.DISABLE,
- SpaceViolationPolicy.DISABLE);
- }
-
- @Test
- public void testSetQuotaFirstWithDisableNextNoInserts() throws Exception {
- setQuotaAndViolateNextSwitchPoliciesAndValidate(SpaceViolationPolicy.DISABLE,
- SpaceViolationPolicy.NO_INSERTS);
- }
-
- @Test
- public void testSetQuotaFirstWithDisableNextNoWritesCompaction() throws Exception {
- setQuotaAndViolateNextSwitchPoliciesAndValidate(SpaceViolationPolicy.DISABLE,
- SpaceViolationPolicy.NO_WRITES_COMPACTIONS);
- }
-
- @Test
- public void testSetQuotaFirstWithNoWritesNextWithDisable() throws Exception {
- setQuotaAndViolateNextSwitchPoliciesAndValidate(SpaceViolationPolicy.NO_WRITES,
- SpaceViolationPolicy.DISABLE);
- }
-
- @Test
- public void testSetQuotaOnNonExistingTableWithNoInserts() throws Exception {
- setQuotaLimit(NON_EXISTENT_TABLE, SpaceViolationPolicy.NO_INSERTS, 2L);
- }
-
- @Test
- public void testSetQuotaOnNonExistingTableWithNoWrites() throws Exception {
- setQuotaLimit(NON_EXISTENT_TABLE, SpaceViolationPolicy.NO_WRITES, 2L);
- }
-
- @Test
- public void testSetQuotaOnNonExistingTableWithNoWritesCompaction() throws Exception {
- setQuotaLimit(NON_EXISTENT_TABLE, SpaceViolationPolicy.NO_WRITES_COMPACTIONS, 2L);
- }
-
- @Test
- public void testSetQuotaOnNonExistingTableWithDisable() throws Exception {
- setQuotaLimit(NON_EXISTENT_TABLE, SpaceViolationPolicy.DISABLE, 2L);
- }
-
- public void setQuotaAndViolateNextSwitchPoliciesAndValidate(SpaceViolationPolicy policy1,
- SpaceViolationPolicy policy2) throws Exception {
- Put put = new Put(Bytes.toBytes("to_reject"));
- put.addColumn(Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"),
- Bytes.toBytes("reject"));
-
- // Do puts until we violate space violation policy1
- final TableName tn = writeUntilViolationAndVerifyViolation(policy1, put);
-
- // Now, change violation policy to policy2
- setQuotaLimit(tn, policy2, 2L);
-
- // The table should be in enabled state on changing violation policy
- if (policy1.equals(SpaceViolationPolicy.DISABLE) && !policy1.equals(policy2)) {
- TEST_UTIL.waitTableEnabled(tn, 20000);
- }
- // Put some row now: should still violate as quota limit still violated
- verifyViolation(policy2, tn, put);
- }
-
- private void setQuotaAndThenRemove(SpaceViolationPolicy policy) throws Exception {
- Put put = new Put(Bytes.toBytes("to_reject"));
- put.addColumn(Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"),
- Bytes.toBytes("reject"));
-
- // Do puts until we violate space policy
- final TableName tn = writeUntilViolationAndVerifyViolation(policy, put);
-
- // Now, remove the quota
- removeQuotaFromtable(tn);
-
- // Put some rows now: should not violate as quota settings removed
- verifyNoViolation(policy, tn, put);
- }
-
- private void setQuotaAndThenDropTable(SpaceViolationPolicy policy) throws Exception {
- Put put = new Put(Bytes.toBytes("to_reject"));
- put.addColumn(Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"),
- Bytes.toBytes("reject"));
-
- // Do puts until we violate space policy
- final TableName tn = writeUntilViolationAndVerifyViolation(policy, put);
-
- // Now, drop the table
- TEST_UTIL.deleteTable(tn);
- LOG.debug("Successfully deleted table ", tn);
-
- // Now re-create the table
- TEST_UTIL.createTable(tn, Bytes.toBytes(SpaceQuotaHelperForTests.F1));
- LOG.debug("Successfully re-created table ", tn);
-
- // Put some rows now: should not violate as table/quota was dropped
- verifyNoViolation(policy, tn, put);
- }
-
- private void setQuotaAndThenIncreaseQuota(SpaceViolationPolicy policy) throws Exception {
- Put put = new Put(Bytes.toBytes("to_reject"));
- put.addColumn(Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"),
- Bytes.toBytes("reject"));
-
- // Do puts until we violate space policy
- final TableName tn = writeUntilViolationAndVerifyViolation(policy, put);
-
- // Now, increase limit and perform put
- setQuotaLimit(tn, policy, 4L);
-
- // Put some row now: should not violate as quota limit increased
- verifyNoViolation(policy, tn, put);
- }
-
- private void setQuotaNextDisableThenIncreaseFinallyEnable(SpaceViolationPolicy policy)
- throws Exception {
- Put put = new Put(Bytes.toBytes("to_reject"));
- put.addColumn(Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"),
- Bytes.toBytes("reject"));
-
- // Do puts until we violate space policy
- final TableName tn = writeUntilViolationAndVerifyViolation(policy, put);
-
- // Disable the table; in case of SpaceViolationPolicy.DISABLE already disabled
- if (!policy.equals(SpaceViolationPolicy.DISABLE)) {
- TEST_UTIL.getAdmin().disableTable(tn);
- TEST_UTIL.waitTableDisabled(tn, 10000);
- }
-
- // Now, increase limit and perform put
- setQuotaLimit(tn, policy, 4L);
-
- // in case of disable policy quota manager will enable it
- if (!policy.equals(SpaceViolationPolicy.DISABLE)) {
- TEST_UTIL.getAdmin().enableTable(tn);
- }
- TEST_UTIL.waitTableEnabled(tn, 10000);
-
- // Put some row now: should not violate as quota limit increased
- verifyNoViolation(policy, tn, put);
- }
-
- public void setQuotaAndThenRemoveInOneAmongTwoTables(SpaceViolationPolicy policy)
- throws Exception {
- Put put = new Put(Bytes.toBytes("to_reject"));
- put.addColumn(Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"),
- Bytes.toBytes("reject"));
-
- // Do puts until we violate space policy on table tn1
- final TableName tn1 = writeUntilViolationAndVerifyViolation(policy, put);
-
- // Do puts until we violate space policy on table tn2
- final TableName tn2 = writeUntilViolationAndVerifyViolation(policy, put);
-
- // Now, remove the quota from table tn1
- removeQuotaFromtable(tn1);
-
- // Put a new row now on tn1: should not violate as quota settings removed
- verifyNoViolation(policy, tn1, put);
- // Put a new row now on tn2: should violate as quota settings exists
- verifyViolation(policy, tn2, put);
- }
-
- private void removeQuotaFromtable(final TableName tn) throws Exception {
- QuotaSettings removeQuota = QuotaSettingsFactory.removeTableSpaceLimit(tn);
- TEST_UTIL.getAdmin().setQuota(removeQuota);
- LOG.debug("Space quota settings removed from the table ", tn);
- }
-
- private void setQuotaLimit(final TableName tn, SpaceViolationPolicy policy, long sizeInMBs)
- throws Exception {
- final long sizeLimit = sizeInMBs * SpaceQuotaHelperForTests.ONE_MEGABYTE;
- QuotaSettings settings = QuotaSettingsFactory.limitTableSpace(tn, sizeLimit, policy);
- TEST_UTIL.getAdmin().setQuota(settings);
- LOG.debug("Quota limit set for table = {}, limit = {}", tn, sizeLimit);
- }
-
- private Map<RegionInfo,Long> getReportedSizesForTable(TableName tn) {
- HMaster master = TEST_UTIL.getMiniHBaseCluster().getMaster();
- MasterQuotaManager quotaManager = master.getMasterQuotaManager();
- Map<RegionInfo,Long> filteredRegionSizes = new HashMap<>();
- for (Entry<RegionInfo,Long> entry : quotaManager.snapshotRegionSizes().entrySet()) {
- if (entry.getKey().getTable().equals(tn)) {
- filteredRegionSizes.put(entry.getKey(), entry.getValue());
- }
- }
- return filteredRegionSizes;
- }
-
- private TableName writeUntilViolation(SpaceViolationPolicy policyToViolate) throws Exception {
- TableName tn = helper.createTableWithRegions(10);
- setQuotaLimit(tn, policyToViolate, 2L);
- // Write more data than should be allowed and flush it to disk
- helper.writeData(tn, 3L * SpaceQuotaHelperForTests.ONE_MEGABYTE);
-
- // This should be sufficient time for the chores to run and see the change.
- Thread.sleep(5000);
-
- return tn;
- }
-
- private TableName writeUntilViolationAndVerifyViolation(
- SpaceViolationPolicy policyToViolate, Mutation m) throws Exception {
- final TableName tn = writeUntilViolation(policyToViolate);
- verifyViolation(policyToViolate, tn, m);
- return tn;
- }
-
- private void verifyViolation(
- SpaceViolationPolicy policyToViolate, TableName tn, Mutation m) throws Exception {
- // But let's try a few times to get the exception before failing
- boolean sawError = false;
- String msg = "";
- for (int i = 0; i < NUM_RETRIES && !sawError; i++) {
- try (Table table = TEST_UTIL.getConnection().getTable(tn)) {
- if (m instanceof Put) {
- table.put((Put) m);
- } else if (m instanceof Delete) {
- table.delete((Delete) m);
- } else if (m instanceof Append) {
- table.append((Append) m);
- } else if (m instanceof Increment) {
- table.increment((Increment) m);
- } else {
- fail(
- "Failed to apply " + m.getClass().getSimpleName() +
- " to the table. Programming error");
- }
- LOG.info("Did not reject the " + m.getClass().getSimpleName() + ", will sleep and retry");
- Thread.sleep(2000);
- } catch (Exception e) {
- msg = StringUtils.stringifyException(e);
- if ((policyToViolate.equals(SpaceViolationPolicy.DISABLE)
- && e instanceof TableNotEnabledException) || msg.contains(policyToViolate.name())) {
- LOG.info("Got the expected exception={}", msg);
- sawError = true;
- break;
- } else {
- LOG.warn("Did not get the expected exception, will sleep and retry", e);
- Thread.sleep(2000);
- }
- }
- }
- if (!sawError) {
- try (Table quotaTable = TEST_UTIL.getConnection().getTable(QuotaUtil.QUOTA_TABLE_NAME)) {
- ResultScanner scanner = quotaTable.getScanner(new Scan());
- Result result = null;
- LOG.info("Dumping contents of hbase:quota table");
- while ((result = scanner.next()) != null) {
- LOG.info(Bytes.toString(result.getRow()) + " => " + result.toString());
- }
- scanner.close();
- }
- } else {
- if (policyToViolate.equals(SpaceViolationPolicy.DISABLE)) {
- assertTrue(
- msg.contains("TableNotEnabledException") || msg.contains(policyToViolate.name()));
- } else {
- assertTrue("Expected exception message to contain the word '" + policyToViolate.name()
- + "', but was " + msg,
- msg.contains(policyToViolate.name()));
- }
- }
- assertTrue(
- "Expected to see an exception writing data to a table exceeding its quota", sawError);
- }
-
- private void verifyNoViolation(SpaceViolationPolicy policyToViolate, TableName tn, Mutation m)
- throws Exception {
- // But let's try a few times to write data before failing
- boolean sawSuccess = false;
- for (int i = 0; i < NUM_RETRIES && !sawSuccess; i++) {
- try (Table table = TEST_UTIL.getConnection().getTable(tn)) {
- if (m instanceof Put) {
- table.put((Put) m);
- } else if (m instanceof Delete) {
- table.delete((Delete) m);
- } else if (m instanceof Append) {
- table.append((Append) m);
- } else if (m instanceof Increment) {
- table.increment((Increment) m);
- } else {
- fail(
- "Failed to apply " + m.getClass().getSimpleName() + " to the table. Programming error");
- }
- sawSuccess = true;
- } catch (Exception e) {
- LOG.info("Rejected the " + m.getClass().getSimpleName() + ", will sleep and retry");
- Thread.sleep(2000);
- }
- }
- if (!sawSuccess) {
- try (Table quotaTable = TEST_UTIL.getConnection().getTable(QuotaUtil.QUOTA_TABLE_NAME)) {
- ResultScanner scanner = quotaTable.getScanner(new Scan());
- Result result = null;
- LOG.info("Dumping contents of hbase:quota table");
- while ((result = scanner.next()) != null) {
- LOG.info(Bytes.toString(result.getRow()) + " => " + result.toString());
- }
- scanner.close();
- }
- }
- assertTrue("Expected to succeed in writing data to a table not having quota ", sawSuccess);
- }
-}