You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by el...@apache.org on 2017/03/20 22:50:22 UTC
[49/54] [abbrv] hbase git commit: HBASE-17002 JMX metrics and some UI
additions for space quotas
http://git-wip-us.apache.org/repos/asf/hbase/blob/6d1558c8/hbase-protocol-shaded/src/main/protobuf/Master.proto
----------------------------------------------------------------------
diff --git a/hbase-protocol-shaded/src/main/protobuf/Master.proto b/hbase-protocol-shaded/src/main/protobuf/Master.proto
index 58e6f77..90a7f07 100644
--- a/hbase-protocol-shaded/src/main/protobuf/Master.proto
+++ b/hbase-protocol-shaded/src/main/protobuf/Master.proto
@@ -930,7 +930,11 @@ service MasterService {
rpc removeDrainFromRegionServers(RemoveDrainFromRegionServersRequest)
returns(RemoveDrainFromRegionServersResponse);
- /** Fetches the Master's view of space quotas */
+ /** Fetches the Master's view of space utilization */
rpc GetSpaceQuotaRegionSizes(GetSpaceQuotaRegionSizesRequest)
returns(GetSpaceQuotaRegionSizesResponse);
+
+ /** Fetches the Master's view of quotas */
+ rpc GetQuotaStates(GetQuotaStatesRequest)
+ returns(GetQuotaStatesResponse);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/6d1558c8/hbase-protocol-shaded/src/main/protobuf/Quota.proto
----------------------------------------------------------------------
diff --git a/hbase-protocol-shaded/src/main/protobuf/Quota.proto b/hbase-protocol-shaded/src/main/protobuf/Quota.proto
index 2d7e5f5..1a6d5ed 100644
--- a/hbase-protocol-shaded/src/main/protobuf/Quota.proto
+++ b/hbase-protocol-shaded/src/main/protobuf/Quota.proto
@@ -119,6 +119,7 @@ message GetSpaceQuotaRegionSizesResponse {
message RegionSizes {
optional TableName table_name = 1;
optional uint64 size = 2;
+
}
repeated RegionSizes sizes = 1;
}
@@ -146,3 +147,19 @@ message GetSpaceQuotaEnforcementsResponse {
}
repeated TableViolationPolicy violation_policies = 1;
}
+
+message GetQuotaStatesRequest {
+}
+
+message GetQuotaStatesResponse {
+ message TableQuotaSnapshot {
+ optional TableName table_name = 1;
+ optional SpaceQuotaSnapshot snapshot = 2;
+ }
+ message NamespaceQuotaSnapshot {
+ optional string namespace = 1;
+ optional SpaceQuotaSnapshot snapshot = 2;
+ }
+ repeated TableQuotaSnapshot table_snapshots = 1;
+ repeated NamespaceQuotaSnapshot ns_snapshots = 2;
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/6d1558c8/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index bc4987a..6cfddaa 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -907,7 +907,7 @@ public class HMaster extends HRegionServer implements MasterServices {
// Create the quota snapshot notifier
spaceQuotaSnapshotNotifier = createQuotaSnapshotNotifier();
spaceQuotaSnapshotNotifier.initialize(getClusterConnection());
- this.quotaObserverChore = new QuotaObserverChore(this);
+ this.quotaObserverChore = new QuotaObserverChore(this, getMasterMetrics());
// Start the chore to read the region FS space reports and act on them
getChoreService().scheduleChore(quotaObserverChore);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/6d1558c8/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
index eb2711c..1c4abc0 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
@@ -62,7 +62,9 @@ import org.apache.hadoop.hbase.procedure.MasterProcedureManager;
import org.apache.hadoop.hbase.procedure2.Procedure;
import org.apache.hadoop.hbase.procedure2.ProcedureUtil;
import org.apache.hadoop.hbase.quotas.MasterQuotaManager;
+import org.apache.hadoop.hbase.quotas.QuotaObserverChore;
import org.apache.hadoop.hbase.quotas.QuotaUtil;
+import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot;
import org.apache.hadoop.hbase.regionserver.RSRpcServices;
import org.apache.hadoop.hbase.replication.ReplicationException;
import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
@@ -214,8 +216,12 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.TruncateTa
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.TruncateTableResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.UnassignRegionRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.UnassignRegionResponse;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesRequest;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesResponse;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesResponse.NamespaceQuotaSnapshot;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesResponse.TableQuotaSnapshot;
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesResponse.RegionSizes;
import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.GetLastFlushedSequenceIdRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.GetLastFlushedSequenceIdResponse;
@@ -2071,4 +2077,36 @@ public class MasterRpcServices extends RSRpcServices
throw new ServiceException(e);
}
}
+
+ @Override
+ public GetQuotaStatesResponse getQuotaStates(
+ RpcController controller, GetQuotaStatesRequest request) throws ServiceException {
+ try {
+ master.checkInitialized();
+ QuotaObserverChore quotaChore = this.master.getQuotaObserverChore();
+ GetQuotaStatesResponse.Builder builder = GetQuotaStatesResponse.newBuilder();
+ if (null != quotaChore) {
+ // The "current" view of all tables with quotas
+ Map<TableName, SpaceQuotaSnapshot> tableSnapshots = quotaChore.getTableQuotaSnapshots();
+ for (Entry<TableName, SpaceQuotaSnapshot> entry : tableSnapshots.entrySet()) {
+ builder.addTableSnapshots(
+ TableQuotaSnapshot.newBuilder()
+ .setTableName(ProtobufUtil.toProtoTableName(entry.getKey()))
+ .setSnapshot(SpaceQuotaSnapshot.toProtoSnapshot(entry.getValue())).build());
+ }
+ // The "current" view of all namespaces with quotas
+ Map<String, SpaceQuotaSnapshot> nsSnapshots = quotaChore.getNamespaceQuotaSnapshots();
+ for (Entry<String, SpaceQuotaSnapshot> entry : nsSnapshots.entrySet()) {
+ builder.addNsSnapshots(
+ NamespaceQuotaSnapshot.newBuilder()
+ .setNamespace(entry.getKey())
+ .setSnapshot(SpaceQuotaSnapshot.toProtoSnapshot(entry.getValue())).build());
+ }
+ return builder.build();
+ }
+ return builder.build();
+ } catch (Exception e) {
+ throw new ServiceException(e);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/6d1558c8/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMaster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMaster.java
index d055853..b5bc3d7 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMaster.java
@@ -37,11 +37,14 @@ public class MetricsMaster {
private static final Log LOG = LogFactory.getLog(MetricsMaster.class);
private MetricsMasterSource masterSource;
private MetricsMasterProcSource masterProcSource;
+ private MetricsMasterQuotaSource masterQuotaSource;
public MetricsMaster(MetricsMasterWrapper masterWrapper) {
masterSource = CompatibilitySingletonFactory.getInstance(MetricsMasterSourceFactory.class).create(masterWrapper);
masterProcSource =
CompatibilitySingletonFactory.getInstance(MetricsMasterProcSourceFactory.class).create(masterWrapper);
+ masterQuotaSource =
+ CompatibilitySingletonFactory.getInstance(MetricsMasterQuotaSourceFactory.class).create(masterWrapper);
}
// for unit-test usage
@@ -53,10 +56,49 @@ public class MetricsMaster {
return masterProcSource;
}
+ public MetricsMasterQuotaSource getMetricsQuotaSource() {
+ return masterQuotaSource;
+ }
+
/**
* @param inc How much to add to requests.
*/
public void incrementRequests(final long inc) {
masterSource.incRequests(inc);
}
+
+ /**
+ * Sets the number of space quotas defined.
+ */
+ public void setNumSpaceQuotas(final long numSpaceQuotas) {
+ masterQuotaSource.updateNumSpaceQuotas(numSpaceQuotas);
+ }
+
+ /**
+ * Sets the number of table in violation of a space quota.
+ */
+ public void setNumTableInSpaceQuotaViolation(final long numTablesInViolation) {
+ masterQuotaSource.updateNumTablesInSpaceQuotaViolation(numTablesInViolation);
+ }
+
+ /**
+ * Sets the number of namespaces in violation of a space quota.
+ */
+ public void setNumNamespacesInSpaceQuotaViolation(final long numNamespacesInViolation) {
+ masterQuotaSource.updateNumNamespacesInSpaceQuotaViolation(numNamespacesInViolation);
+ }
+
+ /**
+ * Sets the number of region size reports the master has seen.
+ */
+ public void setNumRegionSizeReports(final long numRegionReports) {
+ masterQuotaSource.updateNumCurrentSpaceQuotaRegionSizeReports(numRegionReports);
+ }
+
+ /**
+ * Sets the execution time of a period of the QuotaObserverChore.
+ */
+ public void incrementQuotaObserverTime(final long executionTime) {
+ masterQuotaSource.incrementSpaceQuotaObserverChoreTime(executionTime);
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/6d1558c8/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapperImpl.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapperImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapperImpl.java
index 4cff28b..cbf7ba5 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapperImpl.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapperImpl.java
@@ -17,9 +17,18 @@
*/
package org.apache.hadoop.hbase.master;
+import java.util.AbstractMap.SimpleImmutableEntry;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+
import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.quotas.QuotaObserverChore;
+import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot;
import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
/**
@@ -134,4 +143,35 @@ public class MetricsMasterWrapperImpl implements MetricsMasterWrapper {
return master.getNumWALFiles();
}
+ @Override
+ public Map<String,Entry<Long,Long>> getTableSpaceUtilization() {
+ QuotaObserverChore quotaChore = master.getQuotaObserverChore();
+ if (null == quotaChore) {
+ return Collections.emptyMap();
+ }
+ Map<TableName,SpaceQuotaSnapshot> tableSnapshots = quotaChore.getTableQuotaSnapshots();
+ Map<String,Entry<Long,Long>> convertedData = new HashMap<>();
+ for (Entry<TableName,SpaceQuotaSnapshot> entry : tableSnapshots.entrySet()) {
+ convertedData.put(entry.getKey().toString(), convertSnapshot(entry.getValue()));
+ }
+ return convertedData;
+ }
+
+ @Override
+ public Map<String,Entry<Long,Long>> getNamespaceSpaceUtilization() {
+ QuotaObserverChore quotaChore = master.getQuotaObserverChore();
+ if (null == quotaChore) {
+ return Collections.emptyMap();
+ }
+ Map<String,SpaceQuotaSnapshot> namespaceSnapshots = quotaChore.getNamespaceQuotaSnapshots();
+ Map<String,Entry<Long,Long>> convertedData = new HashMap<>();
+ for (Entry<String,SpaceQuotaSnapshot> entry : namespaceSnapshots.entrySet()) {
+ convertedData.put(entry.getKey(), convertSnapshot(entry.getValue()));
+ }
+ return convertedData;
+ }
+
+ Entry<Long,Long> convertSnapshot(SpaceQuotaSnapshot snapshot) {
+ return new SimpleImmutableEntry<Long,Long>(snapshot.getUsage(), snapshot.getLimit());
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/6d1558c8/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
index 79effe7..94c5c87 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
@@ -18,12 +18,12 @@ package org.apache.hadoop.hbase.quotas;
import java.io.IOException;
import java.util.Collections;
-import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
import org.apache.commons.logging.Log;
@@ -37,6 +37,7 @@ import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.master.HMaster;
+import org.apache.hadoop.hbase.master.MetricsMaster;
import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot;
import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot.SpaceQuotaStatus;
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota;
@@ -78,6 +79,7 @@ public class QuotaObserverChore extends ScheduledChore {
private final Connection conn;
private final Configuration conf;
private final MasterQuotaManager quotaManager;
+ private final MetricsMaster metrics;
/*
* Callback that changes in quota snapshots are passed to.
*/
@@ -87,7 +89,9 @@ public class QuotaObserverChore extends ScheduledChore {
* Preserves the state of quota snapshots for tables and namespaces
*/
private final Map<TableName,SpaceQuotaSnapshot> tableQuotaSnapshots;
+ private final Map<TableName,SpaceQuotaSnapshot> readOnlyTableQuotaSnapshots;
private final Map<String,SpaceQuotaSnapshot> namespaceQuotaSnapshots;
+ private final Map<String,SpaceQuotaSnapshot> readOnlyNamespaceSnapshots;
// The time, in millis, that region reports should be kept by the master
private final long regionReportLifetimeMillis;
@@ -98,25 +102,28 @@ public class QuotaObserverChore extends ScheduledChore {
private QuotaSnapshotStore<TableName> tableSnapshotStore;
private QuotaSnapshotStore<String> namespaceSnapshotStore;
- public QuotaObserverChore(HMaster master) {
+ public QuotaObserverChore(HMaster master, MetricsMaster metrics) {
this(
master.getConnection(), master.getConfiguration(),
master.getSpaceQuotaSnapshotNotifier(), master.getMasterQuotaManager(),
- master);
+ master, metrics);
}
QuotaObserverChore(
Connection conn, Configuration conf, SpaceQuotaSnapshotNotifier snapshotNotifier,
- MasterQuotaManager quotaManager, Stoppable stopper) {
+ MasterQuotaManager quotaManager, Stoppable stopper, MetricsMaster metrics) {
super(
QuotaObserverChore.class.getSimpleName(), stopper, getPeriod(conf),
getInitialDelay(conf), getTimeUnit(conf));
this.conn = conn;
this.conf = conf;
+ this.metrics = metrics;
this.quotaManager = quotaManager;
this.snapshotNotifier = Objects.requireNonNull(snapshotNotifier);
- this.tableQuotaSnapshots = new HashMap<>();
- this.namespaceQuotaSnapshots = new HashMap<>();
+ this.tableQuotaSnapshots = new ConcurrentHashMap<>();
+ this.readOnlyTableQuotaSnapshots = Collections.unmodifiableMap(tableQuotaSnapshots);
+ this.namespaceQuotaSnapshots = new ConcurrentHashMap<>();
+ this.readOnlyNamespaceSnapshots = Collections.unmodifiableMap(namespaceQuotaSnapshots);
this.regionReportLifetimeMillis = conf.getLong(
REGION_REPORT_RETENTION_DURATION_KEY, REGION_REPORT_RETENTION_DURATION_DEFAULT);
}
@@ -127,7 +134,11 @@ public class QuotaObserverChore extends ScheduledChore {
if (LOG.isTraceEnabled()) {
LOG.trace("Refreshing space quotas in RegionServer");
}
+ long start = System.nanoTime();
_chore();
+ if (null != metrics) {
+ metrics.incrementQuotaObserverTime((System.nanoTime() - start) / 1_000_000);
+ }
} catch (IOException e) {
LOG.warn("Failed to process quota reports and update quota state. Will retry.", e);
}
@@ -141,6 +152,12 @@ public class QuotaObserverChore extends ScheduledChore {
LOG.trace("Found following tables with quotas: " + tablesWithQuotas);
}
+ if (null != metrics) {
+ // Set the number of namespaces and tables with quotas defined
+ metrics.setNumSpaceQuotas(tablesWithQuotas.getTableQuotaTables().size()
+ + tablesWithQuotas.getNamespacesWithQuotas().size());
+ }
+
// The current "view" of region space use. Used henceforth.
final Map<HRegionInfo,Long> reportedRegionSpaceUse = quotaManager.snapshotRegionSizes();
if (LOG.isTraceEnabled()) {
@@ -152,6 +169,10 @@ public class QuotaObserverChore extends ScheduledChore {
// Create the stores to track table and namespace snapshots
initializeSnapshotStores(reportedRegionSpaceUse);
+ // Report the number of (non-expired) region size reports
+ if (null != metrics) {
+ metrics.setNumRegionSizeReports(reportedRegionSpaceUse.size());
+ }
// Filter out tables for which we don't have adequate regionspace reports yet.
// Important that we do this after we instantiate the stores above
@@ -215,6 +236,7 @@ public class QuotaObserverChore extends ScheduledChore {
* @param tablesWithTableQuotas The HBase tables which have quotas defined
*/
void processTablesWithQuotas(final Set<TableName> tablesWithTableQuotas) throws IOException {
+ long numTablesInViolation = 0L;
for (TableName table : tablesWithTableQuotas) {
final SpaceQuota spaceQuota = tableSnapshotStore.getSpaceQuota(table);
if (null == spaceQuota) {
@@ -227,9 +249,18 @@ public class QuotaObserverChore extends ScheduledChore {
final SpaceQuotaSnapshot currentSnapshot = tableSnapshotStore.getCurrentState(table);
final SpaceQuotaSnapshot targetSnapshot = tableSnapshotStore.getTargetState(table, spaceQuota);
if (LOG.isTraceEnabled()) {
- LOG.trace("Processing " + table + " with current=" + currentSnapshot + ", target=" + targetSnapshot);
+ LOG.trace("Processing " + table + " with current=" + currentSnapshot + ", target="
+ + targetSnapshot);
}
updateTableQuota(table, currentSnapshot, targetSnapshot);
+
+ if (targetSnapshot.getQuotaStatus().isInViolation()) {
+ numTablesInViolation++;
+ }
+ }
+ // Report the number of tables in violation
+ if (null != metrics) {
+ metrics.setNumTableInSpaceQuotaViolation(numTablesInViolation);
}
}
@@ -246,6 +277,7 @@ public class QuotaObserverChore extends ScheduledChore {
void processNamespacesWithQuotas(
final Set<String> namespacesWithQuotas,
final Multimap<String,TableName> tablesByNamespace) throws IOException {
+ long numNamespacesInViolation = 0L;
for (String namespace : namespacesWithQuotas) {
// Get the quota definition for the namespace
final SpaceQuota spaceQuota = namespaceSnapshotStore.getSpaceQuota(namespace);
@@ -257,8 +289,22 @@ public class QuotaObserverChore extends ScheduledChore {
continue;
}
final SpaceQuotaSnapshot currentSnapshot = namespaceSnapshotStore.getCurrentState(namespace);
- final SpaceQuotaSnapshot targetSnapshot = namespaceSnapshotStore.getTargetState(namespace, spaceQuota);
+ final SpaceQuotaSnapshot targetSnapshot = namespaceSnapshotStore.getTargetState(
+ namespace, spaceQuota);
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("Processing " + namespace + " with current=" + currentSnapshot + ", target="
+ + targetSnapshot);
+ }
updateNamespaceQuota(namespace, currentSnapshot, targetSnapshot, tablesByNamespace);
+
+ if (targetSnapshot.getQuotaStatus().isInViolation()) {
+ numNamespacesInViolation++;
+ }
+ }
+
+ // Report the number of namespaces in violation
+ if (null != metrics) {
+ metrics.setNumNamespacesInSpaceQuotaViolation(numNamespacesInViolation);
}
}
@@ -285,14 +331,16 @@ public class QuotaObserverChore extends ScheduledChore {
}
} else if (LOG.isDebugEnabled()) {
// We're either moving into violation or changing violation policies
- LOG.debug(table + " moving into violation of table space quota with policy of " + targetStatus.getPolicy());
+ LOG.debug(table + " moving into violation of table space quota with policy of "
+ + targetStatus.getPolicy());
}
this.snapshotNotifier.transitionTable(table, targetSnapshot);
// Update it in memory
tableSnapshotStore.setCurrentState(table, targetSnapshot);
} else if (LOG.isTraceEnabled()) {
- // Policies are the same, so we have nothing to do except log this. Don't need to re-update the quota table
+ // Policies are the same, so we have nothing to do except log this. Don't need to re-update
+ // the quota table
if (!currentStatus.isInViolation()) {
LOG.trace(table + " remains in observance of quota.");
} else {
@@ -347,11 +395,14 @@ public class QuotaObserverChore extends ScheduledChore {
}
} else {
// No table quota present or a table quota present that is not in violation
- LOG.info(tableInNS + " moving into violation of namespace space quota with policy " + targetStatus.getPolicy());
+ LOG.info(tableInNS + " moving into violation of namespace space quota with policy "
+ + targetStatus.getPolicy());
this.snapshotNotifier.transitionTable(tableInNS, targetSnapshot);
}
}
}
+ // Update the new state in memory for this namespace
+ namespaceSnapshotStore.setCurrentState(namespace, targetSnapshot);
} else {
// Policies are the same
if (!targetStatus.isInViolation()) {
@@ -360,7 +411,8 @@ public class QuotaObserverChore extends ScheduledChore {
LOG.trace(namespace + " remains in observance of quota.");
}
} else {
- // Namespace quota is still in violation, need to enact if the table quota is not taking priority.
+ // Namespace quota is still in violation, need to enact if the table quota is not
+ // taking priority.
for (TableName tableInNS : tablesByNamespace.get(namespace)) {
// Does a table policy exist
if (tableSnapshotStore.getCurrentState(tableInNS).getQuotaStatus().isInViolation()) {
@@ -451,6 +503,22 @@ public class QuotaObserverChore extends ScheduledChore {
}
/**
+ * Returns an unmodifiable view over the current {@link SpaceQuotaSnapshot} objects
+ * for each HBase table with a quota.
+ */
+ public Map<TableName,SpaceQuotaSnapshot> getTableQuotaSnapshots() {
+ return readOnlyTableQuotaSnapshots;
+ }
+
+ /**
+ * Returns an unmodifiable view over the current {@link SpaceQuotaSnapshot} objects
+ * for each HBase namespace with a quota.
+ */
+ public Map<String,SpaceQuotaSnapshot> getNamespaceQuotaSnapshots() {
+ return readOnlyNamespaceSnapshots;
+ }
+
+ /**
* Fetches the {@link SpaceQuotaSnapshot} for the given table.
*/
SpaceQuotaSnapshot getTableQuotaSnapshot(TableName table) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/6d1558c8/hbase-server/src/main/resources/hbase-webapps/master/table.jsp
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/resources/hbase-webapps/master/table.jsp b/hbase-server/src/main/resources/hbase-webapps/master/table.jsp
index 0f8a289..5c52601 100644
--- a/hbase-server/src/main/resources/hbase-webapps/master/table.jsp
+++ b/hbase-server/src/main/resources/hbase-webapps/master/table.jsp
@@ -42,10 +42,14 @@
import="org.apache.hadoop.hbase.HConstants"
import="org.apache.hadoop.hbase.master.HMaster"
import="org.apache.hadoop.hbase.zookeeper.MetaTableLocator"
+ import="org.apache.hadoop.hbase.quotas.QuotaTableUtil"
+ import="org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot"
import="org.apache.hadoop.hbase.util.Bytes"
import="org.apache.hadoop.hbase.util.FSUtils"
import="org.apache.hadoop.hbase.shaded.protobuf.generated.ClusterStatusProtos"
import="org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos"
+ import="org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.Quotas"
+ import="org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota"
import="org.apache.hadoop.hbase.TableName"
import="org.apache.hadoop.hbase.HColumnDescriptor"
import="org.apache.hadoop.hbase.HBaseConfiguration"
@@ -87,6 +91,7 @@
if (showFragmentation) {
frags = FSUtils.getTableFragmentation(master);
}
+ boolean quotasEnabled = conf.getBoolean("hbase.quota.enabled", false);
String action = request.getParameter("action");
String key = request.getParameter("key");
String left = request.getParameter("left");
@@ -328,6 +333,60 @@ if ( fqtn != null ) {
<td>How fragmented is the table. After a major compaction it is 0%.</td>
</tr>
<% } %>
+<%
+ if (quotasEnabled) {
+ TableName tn = TableName.valueOf(fqtn);
+ SpaceQuotaSnapshot masterSnapshot = null;
+ Quotas quota = QuotaTableUtil.getTableQuota(master.getConnection(), tn);
+ if (null == quota || !quota.hasSpace()) {
+ quota = QuotaTableUtil.getNamespaceQuota(master.getConnection(), tn.getNamespaceAsString());
+ if (null != quota) {
+ masterSnapshot = QuotaTableUtil.getCurrentSnapshot(master.getConnection(), tn.getNamespaceAsString());
+ }
+ } else {
+ masterSnapshot = QuotaTableUtil.getCurrentSnapshot(master.getConnection(), tn);
+ }
+ if (null != quota && quota.hasSpace()) {
+ SpaceQuota spaceQuota = quota.getSpace();
+%>
+ <tr>
+ <td>Space Quota</td>
+ <td>
+ <table>
+ <tr>
+ <th>Property</th>
+ <th>Value</th>
+ </tr>
+ <tr>
+ <td>Limit</td>
+ <td><%= StringUtils.byteDesc(spaceQuota.getSoftLimit()) %></td>
+ </tr>
+ <tr>
+ <td>Policy</td>
+ <td><%= spaceQuota.getViolationPolicy() %></td>
+ </tr>
+<%
+ if (null != masterSnapshot) {
+%>
+ <tr>
+ <td>Usage</td>
+ <td><%= StringUtils.byteDesc(masterSnapshot.getUsage()) %></td>
+ </tr>
+ <tr>
+ <td>State</td>
+ <td><%= masterSnapshot.getQuotaStatus().isInViolation() ? "In Violation" : "In Observance" %></td>
+ </tr>
+<%
+ }
+%>
+ </table>
+ </td>
+ <td>Information about a Space Quota on this table, if set.</td>
+ </tr>
+<%
+ }
+ }
+%>
</table>
<h2>Table Schema</h2>
<table class="table table-striped">
http://git-wip-us.apache.org/repos/asf/hbase/blob/6d1558c8/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetricsWrapper.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetricsWrapper.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetricsWrapper.java
index 02f3721..8c1138a 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetricsWrapper.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetricsWrapper.java
@@ -19,9 +19,14 @@ package org.apache.hadoop.hbase.master;
import static org.junit.Assert.*;
+import java.util.AbstractMap.SimpleImmutableEntry;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot;
+import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot.SpaceQuotaStatus;
+import org.apache.hadoop.hbase.quotas.SpaceViolationPolicy;
import org.apache.hadoop.hbase.testclassification.MasterTests;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.util.Threads;
@@ -77,4 +82,16 @@ public class TestMasterMetricsWrapper {
assertEquals(1, info.getNumDeadRegionServers());
assertEquals(1, info.getNumWALFiles());
}
+
+ @Test
+ public void testQuotaSnapshotConversion() {
+ MetricsMasterWrapperImpl info = new MetricsMasterWrapperImpl(
+ TEST_UTIL.getHBaseCluster().getMaster());
+ assertEquals(new SimpleImmutableEntry<Long,Long>(1024L, 2048L),
+ info.convertSnapshot(new SpaceQuotaSnapshot(
+ SpaceQuotaStatus.notInViolation(), 1024L, 2048L)));
+ assertEquals(new SimpleImmutableEntry<Long,Long>(4096L, 2048L),
+ info.convertSnapshot(new SpaceQuotaSnapshot(
+ new SpaceQuotaStatus(SpaceViolationPolicy.NO_INSERTS), 4096L, 2048L)));
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/6d1558c8/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaStatusRPCs.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaStatusRPCs.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaStatusRPCs.java
index 5a00aaf..38dbf66 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaStatusRPCs.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaStatusRPCs.java
@@ -24,6 +24,7 @@ import static org.junit.Assert.assertTrue;
import java.util.Map;
import java.util.Map.Entry;
import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -33,6 +34,7 @@ import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.Waiter;
import org.apache.hadoop.hbase.Waiter.Predicate;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.master.HMaster;
import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot.SpaceQuotaStatus;
import org.apache.hadoop.hbase.regionserver.HRegionServer;
@@ -187,6 +189,87 @@ public class TestQuotaStatusRPCs {
assertEquals(SpaceViolationPolicy.NO_INSERTS, policy);
}
+ @Test
+ public void testQuotaStatusFromMaster() throws Exception {
+ final long sizeLimit = 1024L * 10L; // 10KB
+ final long tableSize = 1024L * 5; // 5KB
+ final long nsLimit = Long.MAX_VALUE;
+ final int numRegions = 10;
+ final TableName tn = helper.createTableWithRegions(numRegions);
+
+ // Define the quota
+ QuotaSettings settings = QuotaSettingsFactory.limitTableSpace(
+ tn, sizeLimit, SpaceViolationPolicy.NO_INSERTS);
+ TEST_UTIL.getAdmin().setQuota(settings);
+ QuotaSettings nsSettings = QuotaSettingsFactory.limitNamespaceSpace(
+ tn.getNamespaceAsString(), nsLimit, SpaceViolationPolicy.NO_INSERTS);
+ TEST_UTIL.getAdmin().setQuota(nsSettings);
+
+ // Write at least `tableSize` data
+ helper.writeData(tn, tableSize);
+
+ final Connection conn = TEST_UTIL.getConnection();
+ // Make sure the master has a snapshot for our table
+ Waiter.waitFor(TEST_UTIL.getConfiguration(), 30 * 1000, new Predicate<Exception>() {
+ @Override
+ public boolean evaluate() throws Exception {
+ SpaceQuotaSnapshot snapshot = QuotaTableUtil.getCurrentSnapshot(conn, tn);
+ LOG.info("Table snapshot after initial ingest: " + snapshot);
+ if (null == snapshot) {
+ return false;
+ }
+ return snapshot.getLimit() == sizeLimit && snapshot.getUsage() > 0L;
+ }
+ });
+ final AtomicReference<Long> nsUsage = new AtomicReference<>();
+ // If we saw the table snapshot, we should also see the namespace snapshot
+ Waiter.waitFor(TEST_UTIL.getConfiguration(), 30 * 1000 * 1000, new Predicate<Exception>() {
+ @Override
+ public boolean evaluate() throws Exception {
+ SpaceQuotaSnapshot snapshot = QuotaTableUtil.getCurrentSnapshot(
+ conn, tn.getNamespaceAsString());
+ LOG.debug("Namespace snapshot after initial ingest: " + snapshot);
+ if (null == snapshot) {
+ return false;
+ }
+ nsUsage.set(snapshot.getUsage());
+ return snapshot.getLimit() == nsLimit && snapshot.getUsage() > 0;
+ }
+ });
+
+ try {
+ helper.writeData(tn, tableSize * 2L);
+ } catch (SpaceLimitingException e) {
+ // Pass
+ }
+
+ // Wait for the status to move to violation
+ Waiter.waitFor(TEST_UTIL.getConfiguration(), 30 * 1000, new Predicate<Exception>() {
+ @Override
+ public boolean evaluate() throws Exception {
+ SpaceQuotaSnapshot snapshot = QuotaTableUtil.getCurrentSnapshot(conn, tn);
+ LOG.info("Table snapshot after second ingest: " + snapshot);
+ if (null == snapshot) {
+ return false;
+ }
+ return snapshot.getQuotaStatus().isInViolation();
+ }
+ });
+ // The namespace should still not be in violation, but have a larger usage than previously
+ Waiter.waitFor(TEST_UTIL.getConfiguration(), 30 * 1000, new Predicate<Exception>() {
+ @Override
+ public boolean evaluate() throws Exception {
+ SpaceQuotaSnapshot snapshot = QuotaTableUtil.getCurrentSnapshot(
+ conn, tn.getNamespaceAsString());
+ LOG.debug("Namespace snapshot after second ingest: " + snapshot);
+ if (null == snapshot) {
+ return false;
+ }
+ return snapshot.getUsage() > nsUsage.get() && !snapshot.getQuotaStatus().isInViolation();
+ }
+ });
+ }
+
private int countRegionsForTable(TableName tn, Map<HRegionInfo,Long> regionSizes) {
int size = 0;
for (HRegionInfo regionInfo : regionSizes.keySet()) {