You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by el...@apache.org on 2017/05/22 17:09:06 UTC
[19/50] [abbrv] hbase git commit: HBASE-16998 Implement Master-side
analysis of region space reports
http://git-wip-us.apache.org/repos/asf/hbase/blob/616bca5a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaObserverChoreWithMiniCluster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaObserverChoreWithMiniCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaObserverChoreWithMiniCluster.java
new file mode 100644
index 0000000..98236c2
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaObserverChoreWithMiniCluster.java
@@ -0,0 +1,596 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.quotas;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Random;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.NamespaceDescriptor;
+import org.apache.hadoop.hbase.NamespaceNotFoundException;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.master.HMaster;
+import org.apache.hadoop.hbase.quotas.QuotaObserverChore.TablesWithQuotas;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota;
+import org.apache.hadoop.hbase.testclassification.LargeTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.rules.TestName;
+
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Multimap;
+
+/**
+ * Test class for {@link QuotaObserverChore} that uses a live HBase cluster.
+ */
+@Category(LargeTests.class)
+public class TestQuotaObserverChoreWithMiniCluster {
+ private static final Log LOG = LogFactory.getLog(TestQuotaObserverChoreWithMiniCluster.class);
+ private static final int SIZE_PER_VALUE = 256;
+ private static final String F1 = "f1";
+ private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+ private static final AtomicLong COUNTER = new AtomicLong(0);
+ private static final long ONE_MEGABYTE = 1024L * 1024L;
+ private static final long DEFAULT_WAIT_MILLIS = 500;
+
+ @Rule
+ public TestName testName = new TestName();
+
+ private HMaster master;
+ private QuotaObserverChore chore;
+ private SpaceQuotaViolationNotifierForTest violationNotifier;
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ Configuration conf = TEST_UTIL.getConfiguration();
+ conf.setInt(FileSystemUtilizationChore.FS_UTILIZATION_CHORE_DELAY_KEY, 1000);
+ conf.setInt(FileSystemUtilizationChore.FS_UTILIZATION_CHORE_PERIOD_KEY, 1000);
+ conf.setInt(QuotaObserverChore.VIOLATION_OBSERVER_CHORE_DELAY_KEY, 1000);
+ conf.setInt(QuotaObserverChore.VIOLATION_OBSERVER_CHORE_PERIOD_KEY, 1000);
+ conf.setBoolean(QuotaUtil.QUOTA_CONF_KEY, true);
+ TEST_UTIL.startMiniCluster(1);
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ TEST_UTIL.shutdownMiniCluster();
+ }
+
+ @Before
+ public void removeAllQuotas() throws Exception {
+ final Connection conn = TEST_UTIL.getConnection();
+ // Wait for the quota table to be created
+ if (!conn.getAdmin().tableExists(QuotaUtil.QUOTA_TABLE_NAME)) {
+ do {
+ LOG.debug("Quota table does not yet exist");
+ Thread.sleep(DEFAULT_WAIT_MILLIS);
+ } while (!conn.getAdmin().tableExists(QuotaUtil.QUOTA_TABLE_NAME));
+ } else {
+ // Or, clean up any quotas from previous test runs.
+ QuotaRetriever scanner = QuotaRetriever.open(TEST_UTIL.getConfiguration());
+ for (QuotaSettings quotaSettings : scanner) {
+ final String namespace = quotaSettings.getNamespace();
+ final TableName tableName = quotaSettings.getTableName();
+ if (null != namespace) {
+ LOG.debug("Deleting quota for namespace: " + namespace);
+ QuotaUtil.deleteNamespaceQuota(conn, namespace);
+ } else {
+ assert null != tableName;
+ LOG.debug("Deleting quota for table: "+ tableName);
+ QuotaUtil.deleteTableQuota(conn, tableName);
+ }
+ }
+ }
+
+ master = TEST_UTIL.getMiniHBaseCluster().getMaster();
+ violationNotifier =
+ (SpaceQuotaViolationNotifierForTest) master.getSpaceQuotaViolationNotifier();
+ violationNotifier.clearTableViolations();
+ chore = master.getQuotaObserverChore();
+ }
+
+ @Test
+ public void testTableViolatesQuota() throws Exception {
+ TableName tn = createTableWithRegions(10);
+
+ final long sizeLimit = 2L * ONE_MEGABYTE;
+ final SpaceViolationPolicy violationPolicy = SpaceViolationPolicy.NO_INSERTS;
+ QuotaSettings settings = QuotaSettingsFactory.limitTableSpace(tn, sizeLimit, violationPolicy);
+ TEST_UTIL.getAdmin().setQuota(settings);
+
+ // Write more data than should be allowed
+ writeData(tn, 3L * ONE_MEGABYTE);
+
+ Map<TableName,SpaceViolationPolicy> violatedQuotas = violationNotifier.snapshotTablesInViolation();
+ while (violatedQuotas.isEmpty()) {
+ LOG.info("Found no violated quotas, sleeping and retrying. Current reports: "
+ + master.getMasterQuotaManager().snapshotRegionSizes());
+ try {
+ Thread.sleep(DEFAULT_WAIT_MILLIS);
+ } catch (InterruptedException e) {
+ LOG.debug("Interrupted while sleeping.", e);
+ Thread.currentThread().interrupt();
+ }
+ violatedQuotas = violationNotifier.snapshotTablesInViolation();
+ }
+
+ Entry<TableName,SpaceViolationPolicy> entry = Iterables.getOnlyElement(violatedQuotas.entrySet());
+ assertEquals(tn, entry.getKey());
+ assertEquals(violationPolicy, entry.getValue());
+ }
+
+ @Test
+ public void testNamespaceViolatesQuota() throws Exception {
+ final String namespace = testName.getMethodName();
+ final Admin admin = TEST_UTIL.getAdmin();
+ // Ensure the namespace exists
+ try {
+ admin.getNamespaceDescriptor(namespace);
+ } catch (NamespaceNotFoundException e) {
+ NamespaceDescriptor desc = NamespaceDescriptor.create(namespace).build();
+ admin.createNamespace(desc);
+ }
+
+ TableName tn1 = createTableWithRegions(namespace, 5);
+ TableName tn2 = createTableWithRegions(namespace, 5);
+ TableName tn3 = createTableWithRegions(namespace, 5);
+
+ final long sizeLimit = 5L * ONE_MEGABYTE;
+ final SpaceViolationPolicy violationPolicy = SpaceViolationPolicy.DISABLE;
+ QuotaSettings settings = QuotaSettingsFactory.limitNamespaceSpace(namespace, sizeLimit, violationPolicy);
+ admin.setQuota(settings);
+
+ writeData(tn1, 2L * ONE_MEGABYTE);
+ admin.flush(tn1);
+ Map<TableName,SpaceViolationPolicy> violatedQuotas = violationNotifier.snapshotTablesInViolation();
+ for (int i = 0; i < 5; i++) {
+ // Check a few times to make sure we don't prematurely move to violation
+ assertEquals("Should not see any quota violations after writing 2MB of data", 0, violatedQuotas.size());
+ try {
+ Thread.sleep(DEFAULT_WAIT_MILLIS);
+ } catch (InterruptedException e) {
+ LOG.debug("Interrupted while sleeping." , e);
+ }
+ violatedQuotas = violationNotifier.snapshotTablesInViolation();
+ }
+
+ writeData(tn2, 2L * ONE_MEGABYTE);
+ admin.flush(tn2);
+ violatedQuotas = violationNotifier.snapshotTablesInViolation();
+ for (int i = 0; i < 5; i++) {
+ // Check a few times to make sure we don't prematurely move to violation
+ assertEquals("Should not see any quota violations after writing 4MB of data", 0,
+ violatedQuotas.size());
+ try {
+ Thread.sleep(DEFAULT_WAIT_MILLIS);
+ } catch (InterruptedException e) {
+ LOG.debug("Interrupted while sleeping." , e);
+ }
+ violatedQuotas = violationNotifier.snapshotTablesInViolation();
+ }
+
+ // Writing the final 2MB of data will push the namespace over the 5MB limit (6MB in total)
+ // and should push all three tables in the namespace into violation.
+ writeData(tn3, 2L * ONE_MEGABYTE);
+ admin.flush(tn3);
+ violatedQuotas = violationNotifier.snapshotTablesInViolation();
+ while (violatedQuotas.size() < 3) {
+ LOG.debug("Saw fewer violations than desired (expected 3): " + violatedQuotas
+ + ". Current reports: " + master.getMasterQuotaManager().snapshotRegionSizes());
+ try {
+ Thread.sleep(DEFAULT_WAIT_MILLIS);
+ } catch (InterruptedException e) {
+ LOG.debug("Interrupted while sleeping.", e);
+ Thread.currentThread().interrupt();
+ }
+ violatedQuotas = violationNotifier.snapshotTablesInViolation();
+ }
+
+ SpaceViolationPolicy vp1 = violatedQuotas.remove(tn1);
+ assertNotNull("tn1 should be in violation", vp1);
+ assertEquals(violationPolicy, vp1);
+ SpaceViolationPolicy vp2 = violatedQuotas.remove(tn2);
+ assertNotNull("tn2 should be in violation", vp2);
+ assertEquals(violationPolicy, vp2);
+ SpaceViolationPolicy vp3 = violatedQuotas.remove(tn3);
+ assertNotNull("tn3 should be in violation", vp3);
+ assertEquals(violationPolicy, vp3);
+ assertTrue("Unexpected additional quota violations: " + violatedQuotas, violatedQuotas.isEmpty());
+ }
+
+ @Test
+ public void testTableQuotaOverridesNamespaceQuota() throws Exception {
+ final String namespace = testName.getMethodName();
+ final Admin admin = TEST_UTIL.getAdmin();
+ // Ensure the namespace exists
+ try {
+ admin.getNamespaceDescriptor(namespace);
+ } catch (NamespaceNotFoundException e) {
+ NamespaceDescriptor desc = NamespaceDescriptor.create(namespace).build();
+ admin.createNamespace(desc);
+ }
+
+ TableName tn1 = createTableWithRegions(namespace, 5);
+ TableName tn2 = createTableWithRegions(namespace, 5);
+
+ final long namespaceSizeLimit = 3L * ONE_MEGABYTE;
+ final SpaceViolationPolicy namespaceViolationPolicy = SpaceViolationPolicy.DISABLE;
+ QuotaSettings namespaceSettings = QuotaSettingsFactory.limitNamespaceSpace(namespace,
+ namespaceSizeLimit, namespaceViolationPolicy);
+ admin.setQuota(namespaceSettings);
+
+ writeData(tn1, 2L * ONE_MEGABYTE);
+ admin.flush(tn1);
+ Map<TableName,SpaceViolationPolicy> violatedQuotas = violationNotifier.snapshotTablesInViolation();
+ for (int i = 0; i < 5; i++) {
+ // Check a few times to make sure we don't prematurely move to violation
+ assertEquals("Should not see any quota violations after writing 2MB of data", 0,
+ violatedQuotas.size());
+ try {
+ Thread.sleep(DEFAULT_WAIT_MILLIS);
+ } catch (InterruptedException e) {
+ LOG.debug("Interrupted while sleeping." , e);
+ }
+ violatedQuotas = violationNotifier.snapshotTablesInViolation();
+ }
+
+ writeData(tn2, 2L * ONE_MEGABYTE);
+ admin.flush(tn2);
+ violatedQuotas = violationNotifier.snapshotTablesInViolation();
+ while (violatedQuotas.size() < 2) {
+ LOG.debug("Saw fewer violations than desired (expected 2): " + violatedQuotas
+ + ". Current reports: " + master.getMasterQuotaManager().snapshotRegionSizes());
+ try {
+ Thread.sleep(DEFAULT_WAIT_MILLIS);
+ } catch (InterruptedException e) {
+ LOG.debug("Interrupted while sleeping.", e);
+ Thread.currentThread().interrupt();
+ }
+ violatedQuotas = violationNotifier.snapshotTablesInViolation();
+ }
+
+ SpaceViolationPolicy actualPolicyTN1 = violatedQuotas.get(tn1);
+ assertNotNull("Expected to see violation policy for tn1", actualPolicyTN1);
+ assertEquals(namespaceViolationPolicy, actualPolicyTN1);
+ SpaceViolationPolicy actualPolicyTN2 = violatedQuotas.get(tn2);
+ assertNotNull("Expected to see violation policy for tn2", actualPolicyTN2);
+ assertEquals(namespaceViolationPolicy, actualPolicyTN2);
+
+ // Override the namespace quota with a table quota
+ final long tableSizeLimit = ONE_MEGABYTE;
+ final SpaceViolationPolicy tableViolationPolicy = SpaceViolationPolicy.NO_INSERTS;
+ QuotaSettings tableSettings = QuotaSettingsFactory.limitTableSpace(tn1, tableSizeLimit,
+ tableViolationPolicy);
+ admin.setQuota(tableSettings);
+
+ // Keep checking for the table quota policy to override the namespace quota
+ while (true) {
+ violatedQuotas = violationNotifier.snapshotTablesInViolation();
+ SpaceViolationPolicy actualTableViolationPolicy = violatedQuotas.get(tn1);
+ assertNotNull("Violation policy should never be null", actualTableViolationPolicy);
+ if (tableViolationPolicy != actualTableViolationPolicy) {
+ LOG.debug("Saw unexpected table violation policy, waiting and re-checking.");
+ try {
+ Thread.sleep(DEFAULT_WAIT_MILLIS);
+ } catch (InterruptedException e) {
+ LOG.debug("Interrupted while sleeping");
+ Thread.currentThread().interrupt();
+ }
+ continue;
+ }
+ assertEquals(tableViolationPolicy, actualTableViolationPolicy);
+ break;
+ }
+
+ // This should not change with the introduction of the table quota for tn1
+ actualPolicyTN2 = violatedQuotas.get(tn2);
+ assertNotNull("Expected to see violation policy for tn2", actualPolicyTN2);
+ assertEquals(namespaceViolationPolicy, actualPolicyTN2);
+ }
+
+ @Test
+ public void testGetAllTablesWithQuotas() throws Exception {
+ final Multimap<TableName, QuotaSettings> quotas = createTablesWithSpaceQuotas();
+ Set<TableName> tablesWithQuotas = new HashSet<>();
+ Set<TableName> namespaceTablesWithQuotas = new HashSet<>();
+ // Partition the tables with quotas by table and ns quota
+ partitionTablesByQuotaTarget(quotas, tablesWithQuotas, namespaceTablesWithQuotas);
+
+ TablesWithQuotas tables = chore.fetchAllTablesWithQuotasDefined();
+ assertEquals("Found tables: " + tables, tablesWithQuotas, tables.getTableQuotaTables());
+ assertEquals("Found tables: " + tables, namespaceTablesWithQuotas, tables.getNamespaceQuotaTables());
+ }
+
+ @Test
+ public void testRpcQuotaTablesAreFiltered() throws Exception {
+ final Multimap<TableName, QuotaSettings> quotas = createTablesWithSpaceQuotas();
+ Set<TableName> tablesWithQuotas = new HashSet<>();
+ Set<TableName> namespaceTablesWithQuotas = new HashSet<>();
+ // Partition the tables with quotas by table and ns quota
+ partitionTablesByQuotaTarget(quotas, tablesWithQuotas, namespaceTablesWithQuotas);
+
+ TableName rpcQuotaTable = createTable();
+ TEST_UTIL.getAdmin().setQuota(QuotaSettingsFactory
+ .throttleTable(rpcQuotaTable, ThrottleType.READ_NUMBER, 6, TimeUnit.MINUTES));
+
+ // The `rpcQuotaTable` should not be included in this Set
+ TablesWithQuotas tables = chore.fetchAllTablesWithQuotasDefined();
+ assertEquals("Found tables: " + tables, tablesWithQuotas, tables.getTableQuotaTables());
+ assertEquals("Found tables: " + tables, namespaceTablesWithQuotas, tables.getNamespaceQuotaTables());
+ }
+
+ @Test
+ public void testFilterRegions() throws Exception {
+ Map<TableName,Integer> mockReportedRegions = new HashMap<>();
+ // Can't mock because of primitive int as a return type -- Mockito
+ // can only handle an Integer.
+ TablesWithQuotas tables = new TablesWithQuotas(TEST_UTIL.getConnection(),
+ TEST_UTIL.getConfiguration()) {
+ @Override
+ int getNumReportedRegions(TableName table, QuotaViolationStore<TableName> tableStore) {
+ Integer i = mockReportedRegions.get(table);
+ if (null == i) {
+ return 0;
+ }
+ return i;
+ }
+ };
+
+ // Create the tables
+ TableName tn1 = createTableWithRegions(20);
+ TableName tn2 = createTableWithRegions(20);
+ TableName tn3 = createTableWithRegions(20);
+
+ // Add them to the Tables with Quotas object
+ tables.addTableQuotaTable(tn1);
+ tables.addTableQuotaTable(tn2);
+ tables.addTableQuotaTable(tn3);
+
+ // Mock the number of regions reported
+ mockReportedRegions.put(tn1, 10); // 50%
+ mockReportedRegions.put(tn2, 19); // 95%
+ mockReportedRegions.put(tn3, 20); // 100%
+
+ // Argument is un-used
+ tables.filterInsufficientlyReportedTables(null);
+ // The default of 95% reported should prevent tn1 from appearing
+ assertEquals(new HashSet<>(Arrays.asList(tn2, tn3)), tables.getTableQuotaTables());
+ }
+
+ @Test
+ public void testFetchSpaceQuota() throws Exception {
+ Multimap<TableName,QuotaSettings> tables = createTablesWithSpaceQuotas();
+ // Can pass in an empty map, we're not consulting it.
+ chore.initializeViolationStores(Collections.emptyMap());
+ // All tables that were created should have a quota defined.
+ for (Entry<TableName,QuotaSettings> entry : tables.entries()) {
+ final TableName table = entry.getKey();
+ final QuotaSettings qs = entry.getValue();
+
+ assertTrue("QuotaSettings was an instance of " + qs.getClass(),
+ qs instanceof SpaceLimitSettings);
+
+ SpaceQuota spaceQuota = null;
+ if (null != qs.getTableName()) {
+ spaceQuota = chore.getTableViolationStore().getSpaceQuota(table);
+ assertNotNull("Could not find table space quota for " + table, spaceQuota);
+ } else if (null != qs.getNamespace()) {
+ spaceQuota = chore.getNamespaceViolationStore().getSpaceQuota(table.getNamespaceAsString());
+ assertNotNull("Could not find namespace space quota for " + table.getNamespaceAsString(), spaceQuota);
+ } else {
+ fail("Expected table or namespace space quota");
+ }
+
+ final SpaceLimitSettings sls = (SpaceLimitSettings) qs;
+ assertEquals(sls.getProto().getQuota(), spaceQuota);
+ }
+
+ TableName tableWithoutQuota = createTable();
+ assertNull(chore.getTableViolationStore().getSpaceQuota(tableWithoutQuota));
+ }
+
+ //
+ // Helpers
+ //
+
+ void writeData(TableName tn, long sizeInBytes) throws IOException {
+ final Connection conn = TEST_UTIL.getConnection();
+ final Table table = conn.getTable(tn);
+ try {
+ List<Put> updates = new ArrayList<>();
+ long bytesToWrite = sizeInBytes;
+ long rowKeyId = 0L;
+ final StringBuilder sb = new StringBuilder();
+ final Random r = new Random();
+ while (bytesToWrite > 0L) {
+ sb.setLength(0);
+ sb.append(Long.toString(rowKeyId));
+ // Use the reverse counter as the rowKey to get even spread across all regions
+ Put p = new Put(Bytes.toBytes(sb.reverse().toString()));
+ byte[] value = new byte[SIZE_PER_VALUE];
+ r.nextBytes(value);
+ p.addColumn(Bytes.toBytes(F1), Bytes.toBytes("q1"), value);
+ updates.add(p);
+
+ // Batch 50K worth of updates
+ if (updates.size() > 50) {
+ table.put(updates);
+ updates.clear();
+ }
+
+ // Just count the value size, ignore the size of rowkey + column
+ bytesToWrite -= SIZE_PER_VALUE;
+ rowKeyId++;
+ }
+
+ // Write the final batch
+ if (!updates.isEmpty()) {
+ table.put(updates);
+ }
+
+ LOG.debug("Data was written to HBase");
+ // Push the data to disk.
+ TEST_UTIL.getAdmin().flush(tn);
+ LOG.debug("Data flushed to disk");
+ } finally {
+ table.close();
+ }
+ }
+
+ Multimap<TableName, QuotaSettings> createTablesWithSpaceQuotas() throws Exception {
+ final Admin admin = TEST_UTIL.getAdmin();
+ final Multimap<TableName, QuotaSettings> tablesWithQuotas = HashMultimap.create();
+
+ final TableName tn1 = createTable();
+ final TableName tn2 = createTable();
+
+ NamespaceDescriptor nd = NamespaceDescriptor.create("ns" + COUNTER.getAndIncrement()).build();
+ admin.createNamespace(nd);
+ final TableName tn3 = createTableInNamespace(nd);
+ final TableName tn4 = createTableInNamespace(nd);
+ final TableName tn5 = createTableInNamespace(nd);
+
+ final long sizeLimit1 = 1024L * 1024L * 1024L * 1024L * 5L; // 5TB
+ final SpaceViolationPolicy violationPolicy1 = SpaceViolationPolicy.NO_WRITES;
+ QuotaSettings qs1 = QuotaSettingsFactory.limitTableSpace(tn1, sizeLimit1, violationPolicy1);
+ tablesWithQuotas.put(tn1, qs1);
+ admin.setQuota(qs1);
+
+ final long sizeLimit2 = 1024L * 1024L * 1024L * 200L; // 200GB
+ final SpaceViolationPolicy violationPolicy2 = SpaceViolationPolicy.NO_WRITES_COMPACTIONS;
+ QuotaSettings qs2 = QuotaSettingsFactory.limitTableSpace(tn2, sizeLimit2, violationPolicy2);
+ tablesWithQuotas.put(tn2, qs2);
+ admin.setQuota(qs2);
+
+ final long sizeLimit3 = 1024L * 1024L * 1024L * 1024L * 100L; // 100TB
+ final SpaceViolationPolicy violationPolicy3 = SpaceViolationPolicy.NO_INSERTS;
+ QuotaSettings qs3 = QuotaSettingsFactory.limitNamespaceSpace(nd.getName(), sizeLimit3, violationPolicy3);
+ tablesWithQuotas.put(tn3, qs3);
+ tablesWithQuotas.put(tn4, qs3);
+ tablesWithQuotas.put(tn5, qs3);
+ admin.setQuota(qs3);
+
+ final long sizeLimit4 = 1024L * 1024L * 1024L * 5L; // 5GB
+ final SpaceViolationPolicy violationPolicy4 = SpaceViolationPolicy.NO_INSERTS;
+ QuotaSettings qs4 = QuotaSettingsFactory.limitTableSpace(tn5, sizeLimit4, violationPolicy4);
+ // Override the ns quota for tn5, import edge-case to catch table quota taking
+ // precedence over ns quota.
+ tablesWithQuotas.put(tn5, qs4);
+ admin.setQuota(qs4);
+
+ return tablesWithQuotas;
+ }
+
+ TableName createTable() throws Exception {
+ return createTableWithRegions(1);
+ }
+
+ TableName createTableWithRegions(int numRegions) throws Exception {
+ return createTableWithRegions(NamespaceDescriptor.DEFAULT_NAMESPACE_NAME_STR, numRegions);
+ }
+
+ TableName createTableWithRegions(String namespace, int numRegions) throws Exception {
+ final Admin admin = TEST_UTIL.getAdmin();
+ final TableName tn = TableName.valueOf(namespace, testName.getMethodName() + COUNTER.getAndIncrement());
+
+ // Delete the old table
+ if (admin.tableExists(tn)) {
+ admin.disableTable(tn);
+ admin.deleteTable(tn);
+ }
+
+ // Create the table
+ HTableDescriptor tableDesc = new HTableDescriptor(tn);
+ tableDesc.addFamily(new HColumnDescriptor(F1));
+ if (numRegions == 1) {
+ admin.createTable(tableDesc);
+ } else {
+ admin.createTable(tableDesc, Bytes.toBytes("0"), Bytes.toBytes("9"), numRegions);
+ }
+ return tn;
+ }
+
+ TableName createTableInNamespace(NamespaceDescriptor nd) throws Exception {
+ final Admin admin = TEST_UTIL.getAdmin();
+ final TableName tn = TableName.valueOf(nd.getName(),
+ testName.getMethodName() + COUNTER.getAndIncrement());
+
+ // Delete the old table
+ if (admin.tableExists(tn)) {
+ admin.disableTable(tn);
+ admin.deleteTable(tn);
+ }
+
+ // Create the table
+ HTableDescriptor tableDesc = new HTableDescriptor(tn);
+ tableDesc.addFamily(new HColumnDescriptor(F1));
+
+ admin.createTable(tableDesc);
+ return tn;
+ }
+
+ void partitionTablesByQuotaTarget(Multimap<TableName,QuotaSettings> quotas,
+ Set<TableName> tablesWithTableQuota, Set<TableName> tablesWithNamespaceQuota) {
+ // Partition the tables with quotas by table and ns quota
+ for (Entry<TableName, QuotaSettings> entry : quotas.entries()) {
+ SpaceLimitSettings settings = (SpaceLimitSettings) entry.getValue();
+ TableName tn = entry.getKey();
+ if (null != settings.getTableName()) {
+ tablesWithTableQuota.add(tn);
+ }
+ if (null != settings.getNamespace()) {
+ tablesWithNamespaceQuota.add(tn);
+ }
+
+ if (null == settings.getTableName() && null == settings.getNamespace()) {
+ fail("Unexpected table name with null tableName and namespace: " + tn);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/616bca5a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaTableUtil.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaTableUtil.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaTableUtil.java
index 3b276ad..238c4c0 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaTableUtil.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaTableUtil.java
@@ -23,14 +23,11 @@ import static org.junit.Assert.assertEquals;
import java.io.IOException;
import java.util.concurrent.TimeUnit;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
-import org.apache.hadoop.hbase.protobuf.generated.QuotaProtos;
import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.Quotas;
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.Throttle;
@@ -50,7 +47,6 @@ import org.junit.rules.TestName;
*/
@Category({MasterTests.class, MediumTests.class})
public class TestQuotaTableUtil {
- private static final Log LOG = LogFactory.getLog(TestQuotaTableUtil.class);
private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
private Connection connection;
http://git-wip-us.apache.org/repos/asf/hbase/blob/616bca5a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestTableQuotaViolationStore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestTableQuotaViolationStore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestTableQuotaViolationStore.java
new file mode 100644
index 0000000..efc046b
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestTableQuotaViolationStore.java
@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.quotas;
+
+import static com.google.common.collect.Iterables.size;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicReference;
+
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.quotas.QuotaViolationStore.ViolationState;
+import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.Quotas;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota;
+import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
+
+/**
+ * Test class for {@link TableQuotaViolationStore}.
+ */
+@Category(SmallTests.class)
+public class TestTableQuotaViolationStore {
+ private static final long ONE_MEGABYTE = 1024L * 1024L;
+
+ private Connection conn;
+ private QuotaObserverChore chore;
+ private Map<HRegionInfo, Long> regionReports;
+ private TableQuotaViolationStore store;
+
+ @Before
+ public void setup() {
+ conn = mock(Connection.class);
+ chore = mock(QuotaObserverChore.class);
+ regionReports = new HashMap<>();
+ store = new TableQuotaViolationStore(conn, chore, regionReports);
+ }
+
+ @Test
+ public void testFilterRegionsByTable() throws Exception {
+ TableName tn1 = TableName.valueOf("foo");
+ TableName tn2 = TableName.valueOf("bar");
+ TableName tn3 = TableName.valueOf("ns", "foo");
+
+ assertEquals(0, size(store.filterBySubject(tn1)));
+
+ for (int i = 0; i < 5; i++) {
+ regionReports.put(new HRegionInfo(tn1, Bytes.toBytes(i), Bytes.toBytes(i+1)), 0L);
+ }
+ for (int i = 0; i < 3; i++) {
+ regionReports.put(new HRegionInfo(tn2, Bytes.toBytes(i), Bytes.toBytes(i+1)), 0L);
+ }
+ for (int i = 0; i < 10; i++) {
+ regionReports.put(new HRegionInfo(tn3, Bytes.toBytes(i), Bytes.toBytes(i+1)), 0L);
+ }
+ assertEquals(18, regionReports.size());
+ assertEquals(5, size(store.filterBySubject(tn1)));
+ assertEquals(3, size(store.filterBySubject(tn2)));
+ assertEquals(10, size(store.filterBySubject(tn3)));
+ }
+
+ @Test
+ public void testTargetViolationState() {
+ TableName tn1 = TableName.valueOf("violation1");
+ TableName tn2 = TableName.valueOf("observance1");
+ TableName tn3 = TableName.valueOf("observance2");
+ SpaceQuota quota = SpaceQuota.newBuilder()
+ .setSoftLimit(1024L * 1024L)
+ .setViolationPolicy(ProtobufUtil.toProtoViolationPolicy(SpaceViolationPolicy.DISABLE))
+ .build();
+
+ // Create some junk data to filter. Makes sure it's so large that it would
+ // immediately violate the quota.
+ for (int i = 0; i < 3; i++) {
+ regionReports.put(new HRegionInfo(tn2, Bytes.toBytes(i), Bytes.toBytes(i + 1)),
+ 5L * ONE_MEGABYTE);
+ regionReports.put(new HRegionInfo(tn3, Bytes.toBytes(i), Bytes.toBytes(i + 1)),
+ 5L * ONE_MEGABYTE);
+ }
+
+ regionReports.put(new HRegionInfo(tn1, Bytes.toBytes(0), Bytes.toBytes(1)), 1024L * 512L);
+ regionReports.put(new HRegionInfo(tn1, Bytes.toBytes(1), Bytes.toBytes(2)), 1024L * 256L);
+
+ // Below the quota
+ assertEquals(ViolationState.IN_OBSERVANCE, store.getTargetState(tn1, quota));
+
+ regionReports.put(new HRegionInfo(tn1, Bytes.toBytes(2), Bytes.toBytes(3)), 1024L * 256L);
+
+ // Equal to the quota is still in observance
+ assertEquals(ViolationState.IN_OBSERVANCE, store.getTargetState(tn1, quota));
+
+ regionReports.put(new HRegionInfo(tn1, Bytes.toBytes(3), Bytes.toBytes(4)), 1024L);
+
+ // Exceeds the quota, should be in violation
+ assertEquals(ViolationState.IN_VIOLATION, store.getTargetState(tn1, quota));
+ }
+
+ @Test
+ public void testGetSpaceQuota() throws Exception {
+ TableQuotaViolationStore mockStore = mock(TableQuotaViolationStore.class);
+ when(mockStore.getSpaceQuota(any(TableName.class))).thenCallRealMethod();
+
+ Quotas quotaWithSpace = Quotas.newBuilder().setSpace(
+ SpaceQuota.newBuilder()
+ .setSoftLimit(1024L)
+ .setViolationPolicy(QuotaProtos.SpaceViolationPolicy.DISABLE)
+ .build())
+ .build();
+ Quotas quotaWithoutSpace = Quotas.newBuilder().build();
+
+ AtomicReference<Quotas> quotaRef = new AtomicReference<>();
+ when(mockStore.getQuotaForTable(any(TableName.class))).then(new Answer<Quotas>() {
+ @Override
+ public Quotas answer(InvocationOnMock invocation) throws Throwable {
+ return quotaRef.get();
+ }
+ });
+
+ quotaRef.set(quotaWithSpace);
+ assertEquals(quotaWithSpace.getSpace(), mockStore.getSpaceQuota(TableName.valueOf("foo")));
+ quotaRef.set(quotaWithoutSpace);
+ assertNull(mockStore.getSpaceQuota(TableName.valueOf("foo")));
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/616bca5a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestTablesWithQuotas.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestTablesWithQuotas.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestTablesWithQuotas.java
new file mode 100644
index 0000000..bb8d5cd
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestTablesWithQuotas.java
@@ -0,0 +1,198 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.quotas;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.quotas.QuotaObserverChore.TablesWithQuotas;
+import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import com.google.common.collect.Multimap;
+
+/**
+ * Non-HBase cluster unit tests for {@link TablesWithQuotas}.
+ */
+@Category(SmallTests.class)
+public class TestTablesWithQuotas {
+ private Connection conn;
+ private Configuration conf;
+
+ @Before
+ public void setup() throws Exception {
+ conn = mock(Connection.class);
+ conf = HBaseConfiguration.create();
+ }
+
+ @Test
+ public void testImmutableGetters() {
+ Set<TableName> tablesWithTableQuotas = new HashSet<>();
+ Set<TableName> tablesWithNamespaceQuotas = new HashSet<>();
+ final TablesWithQuotas tables = new TablesWithQuotas(conn, conf);
+ for (int i = 0; i < 5; i++) {
+ TableName tn = TableName.valueOf("tn" + i);
+ tablesWithTableQuotas.add(tn);
+ tables.addTableQuotaTable(tn);
+ }
+ for (int i = 0; i < 3; i++) {
+ TableName tn = TableName.valueOf("tn_ns" + i);
+ tablesWithNamespaceQuotas.add(tn);
+ tables.addNamespaceQuotaTable(tn);
+ }
+ Set<TableName> actualTableQuotaTables = tables.getTableQuotaTables();
+ Set<TableName> actualNamespaceQuotaTables = tables.getNamespaceQuotaTables();
+ assertEquals(tablesWithTableQuotas, actualTableQuotaTables);
+ assertEquals(tablesWithNamespaceQuotas, actualNamespaceQuotaTables);
+ try {
+ actualTableQuotaTables.add(null);
+ fail("Should not be able to add an element");
+ } catch (UnsupportedOperationException e) {
+ // pass
+ }
+ try {
+ actualNamespaceQuotaTables.add(null);
+ fail("Should not be able to add an element");
+ } catch (UnsupportedOperationException e) {
+ // pass
+ }
+ }
+
+ @Test
+ public void testInsufficientlyReportedTableFiltering() throws Exception {
+ final Map<TableName,Integer> reportedRegions = new HashMap<>();
+ final Map<TableName,Integer> actualRegions = new HashMap<>();
+ final Configuration conf = HBaseConfiguration.create();
+ conf.setDouble(QuotaObserverChore.VIOLATION_OBSERVER_CHORE_REPORT_PERCENT_KEY, 0.95);
+
+ TableName tooFewRegionsTable = TableName.valueOf("tn1");
+ TableName sufficientRegionsTable = TableName.valueOf("tn2");
+ TableName tooFewRegionsNamespaceTable = TableName.valueOf("ns1", "tn2");
+ TableName sufficientRegionsNamespaceTable = TableName.valueOf("ns1", "tn2");
+ final TablesWithQuotas tables = new TablesWithQuotas(conn, conf) {
+ @Override
+ Configuration getConfiguration() {
+ return conf;
+ }
+
+ @Override
+ int getNumRegions(TableName tableName) {
+ return actualRegions.get(tableName);
+ }
+
+ @Override
+ int getNumReportedRegions(TableName table, QuotaViolationStore<TableName> tableStore) {
+ return reportedRegions.get(table);
+ }
+ };
+ tables.addTableQuotaTable(tooFewRegionsTable);
+ tables.addTableQuotaTable(sufficientRegionsTable);
+ tables.addNamespaceQuotaTable(tooFewRegionsNamespaceTable);
+ tables.addNamespaceQuotaTable(sufficientRegionsNamespaceTable);
+
+ reportedRegions.put(tooFewRegionsTable, 5);
+ actualRegions.put(tooFewRegionsTable, 10);
+ reportedRegions.put(sufficientRegionsTable, 19);
+ actualRegions.put(sufficientRegionsTable, 20);
+ reportedRegions.put(tooFewRegionsNamespaceTable, 9);
+ actualRegions.put(tooFewRegionsNamespaceTable, 10);
+ reportedRegions.put(sufficientRegionsNamespaceTable, 98);
+ actualRegions.put(sufficientRegionsNamespaceTable, 100);
+
+ // Unused argument
+ tables.filterInsufficientlyReportedTables(null);
+ Set<TableName> filteredTablesWithTableQuotas = tables.getTableQuotaTables();
+ assertEquals(Collections.singleton(sufficientRegionsTable), filteredTablesWithTableQuotas);
+ Set<TableName> filteredTablesWithNamespaceQutoas = tables.getNamespaceQuotaTables();
+ assertEquals(Collections.singleton(sufficientRegionsNamespaceTable), filteredTablesWithNamespaceQutoas);
+ }
+
+ @Test
+ public void testGetTablesByNamespace() {
+ final TablesWithQuotas tables = new TablesWithQuotas(conn, conf);
+ tables.addTableQuotaTable(TableName.valueOf("ignored1"));
+ tables.addTableQuotaTable(TableName.valueOf("ignored2"));
+ tables.addNamespaceQuotaTable(TableName.valueOf("ns1", "t1"));
+ tables.addNamespaceQuotaTable(TableName.valueOf("ns1", "t2"));
+ tables.addNamespaceQuotaTable(TableName.valueOf("ns1", "t3"));
+ tables.addNamespaceQuotaTable(TableName.valueOf("ns2", "t1"));
+ tables.addNamespaceQuotaTable(TableName.valueOf("ns2", "t2"));
+
+ Multimap<String,TableName> tablesByNamespace = tables.getTablesByNamespace();
+ Collection<TableName> tablesInNs = tablesByNamespace.get("ns1");
+ assertEquals(3, tablesInNs.size());
+ assertTrue("Unexpected results for ns1: " + tablesInNs,
+ tablesInNs.containsAll(Arrays.asList(
+ TableName.valueOf("ns1", "t1"),
+ TableName.valueOf("ns1", "t2"),
+ TableName.valueOf("ns1", "t3"))));
+ tablesInNs = tablesByNamespace.get("ns2");
+ assertEquals(2, tablesInNs.size());
+ assertTrue("Unexpected results for ns2: " + tablesInNs,
+ tablesInNs.containsAll(Arrays.asList(
+ TableName.valueOf("ns2", "t1"),
+ TableName.valueOf("ns2", "t2"))));
+ }
+
+ @Test
+ public void testFilteringMissingTables() throws Exception {
+ final TableName missingTable = TableName.valueOf("doesNotExist");
+ // Set up Admin to return null (match the implementation)
+ Admin admin = mock(Admin.class);
+ when(conn.getAdmin()).thenReturn(admin);
+ when(admin.getTableRegions(missingTable)).thenReturn(null);
+
+ QuotaObserverChore chore = mock(QuotaObserverChore.class);
+ Map<HRegionInfo,Long> regionUsage = new HashMap<>();
+ TableQuotaViolationStore store = new TableQuotaViolationStore(conn, chore, regionUsage);
+
+ // A super dirty hack to verify that, after getting no regions for our table,
+ // we bail out and start processing the next element (which there is none).
+ final TablesWithQuotas tables = new TablesWithQuotas(conn, conf) {
+ @Override
+ int getNumReportedRegions(TableName table, QuotaViolationStore<TableName> tableStore) {
+ throw new RuntimeException("Should should not reach here");
+ }
+ };
+ tables.addTableQuotaTable(missingTable);
+
+ tables.filterInsufficientlyReportedTables(store);
+
+ final Set<TableName> tablesWithQuotas = tables.getTableQuotaTables();
+ assertTrue(
+ "Expected to find no tables, but found " + tablesWithQuotas, tablesWithQuotas.isEmpty());
+ }
+}