You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by mm...@apache.org on 2019/07/10 20:09:26 UTC
[accumulo] branch 2.0 updated: Improve error handling in Monitor.
Fixes #1258 (#1262)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch 2.0
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.0 by this push:
new 477a9cd Improve error handling in Monitor. Fixes #1258 (#1262)
477a9cd is described below
commit 477a9cd275243763e9f1029cb91b4707be714450
Author: Mike Miller <mm...@apache.org>
AuthorDate: Wed Jul 10 16:09:22 2019 -0400
Improve error handling in Monitor. Fixes #1258 (#1262)
* Make rest calls fail fast and return empty objects
* This will make the Monitor more stable when Master is down
---
.../java/org/apache/accumulo/monitor/Monitor.java | 1 +
.../rest/bulkImports/BulkImportResource.java | 9 +++--
.../monitor/rest/master/MasterResource.java | 6 +++-
.../accumulo/monitor/rest/scans/ScansResource.java | 8 +++--
.../monitor/rest/status/StatusResource.java | 10 +++---
.../monitor/rest/tables/TablesResource.java | 38 ++++++++++++----------
.../rest/tservers/TabletServerResource.java | 13 ++++----
7 files changed, 52 insertions(+), 33 deletions(-)
diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java b/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
index 929d1c0..4785671 100644
--- a/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
@@ -256,6 +256,7 @@ public class Monitor extends AbstractServer implements HighlyAvailableService {
retry = false;
} else {
mmi = null;
+ log.error("Unable to get info from Master");
}
gcStatus = fetchGcStatus();
} catch (Exception e) {
diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/bulkImports/BulkImportResource.java b/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/bulkImports/BulkImportResource.java
index d337e53..3b4a3a6 100644
--- a/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/bulkImports/BulkImportResource.java
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/bulkImports/BulkImportResource.java
@@ -25,6 +25,7 @@ import javax.ws.rs.Produces;
import javax.ws.rs.core.MediaType;
import org.apache.accumulo.core.master.thrift.BulkImportStatus;
+import org.apache.accumulo.core.master.thrift.MasterMonitorInfo;
import org.apache.accumulo.core.master.thrift.TabletServerStatus;
import org.apache.accumulo.monitor.Monitor;
@@ -48,17 +49,19 @@ public class BulkImportResource {
*/
@GET
public BulkImport getTables() {
-
BulkImport bulkImport = new BulkImport();
+ MasterMonitorInfo mmi = monitor.getMmi();
+ if (mmi == null)
+ return bulkImport;
// Generating Bulk Import and adding it to the return object
- for (BulkImportStatus bulk : monitor.getMmi().bulkImports) {
+ for (BulkImportStatus bulk : mmi.bulkImports) {
bulkImport
.addBulkImport(new BulkImportInformation(bulk.filename, bulk.startTime, bulk.state));
}
// Generating TServer Bulk Import and adding it to the return object
- for (TabletServerStatus tserverInfo : monitor.getMmi().getTServerInfo()) {
+ for (TabletServerStatus tserverInfo : mmi.getTServerInfo()) {
int size = 0;
long oldest = 0L;
diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/master/MasterResource.java b/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/master/MasterResource.java
index d29912e..e937868 100644
--- a/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/master/MasterResource.java
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/master/MasterResource.java
@@ -232,9 +232,13 @@ public class MasterResource {
* @return servers shutting down list
*/
public static ServersShuttingDown getServersShuttingDown(Monitor monitor) {
+ MasterMonitorInfo mmi = monitor.getMmi();
ServersShuttingDown servers = new ServersShuttingDown();
+ if (mmi == null)
+ return servers;
+
// Add new servers to the list
- for (String server : monitor.getMmi().serversShuttingDown) {
+ for (String server : mmi.serversShuttingDown) {
servers.addServerShuttingDown(new ServerShuttingDownInformation(server));
}
return servers;
diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/scans/ScansResource.java b/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/scans/ScansResource.java
index 7b211d5..738f90f 100644
--- a/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/scans/ScansResource.java
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/scans/ScansResource.java
@@ -24,6 +24,7 @@ import javax.ws.rs.Path;
import javax.ws.rs.Produces;
import javax.ws.rs.core.MediaType;
+import org.apache.accumulo.core.master.thrift.MasterMonitorInfo;
import org.apache.accumulo.core.master.thrift.TabletServerStatus;
import org.apache.accumulo.core.util.HostAndPort;
import org.apache.accumulo.monitor.Monitor;
@@ -48,13 +49,16 @@ public class ScansResource {
*/
@GET
public Scans getTables() {
-
Scans scans = new Scans();
+ MasterMonitorInfo mmi = monitor.getMmi();
+ if (mmi == null) {
+ return scans;
+ }
Map<HostAndPort,ScanStats> entry = monitor.getScans();
// Adds new scans to the array
- for (TabletServerStatus tserverInfo : monitor.getMmi().getTServerInfo()) {
+ for (TabletServerStatus tserverInfo : mmi.getTServerInfo()) {
ScanStats stats = entry.get(HostAndPort.fromString(tserverInfo.name));
if (stats != null) {
scans.addScan(new ScanInformation(tserverInfo, stats.scanCount, stats.oldestScan));
diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/status/StatusResource.java b/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/status/StatusResource.java
index c89b370..8ac76e9 100644
--- a/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/status/StatusResource.java
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/status/StatusResource.java
@@ -24,6 +24,7 @@ import javax.ws.rs.Path;
import javax.ws.rs.Produces;
import javax.ws.rs.core.MediaType;
+import org.apache.accumulo.core.master.thrift.MasterMonitorInfo;
import org.apache.accumulo.monitor.Monitor;
import org.apache.accumulo.server.monitor.DedupedLogEvent;
import org.apache.accumulo.server.monitor.LogService;
@@ -57,8 +58,9 @@ public class StatusResource {
Status masterStatus;
Status gcStatus;
Status tServerStatus = Status.ERROR;
+ MasterMonitorInfo mmi = monitor.getMmi();
- if (monitor.getMmi() != null) {
+ if (mmi != null) {
if (monitor.getGcStatus() != null) {
gcStatus = Status.OK;
} else {
@@ -68,9 +70,9 @@ public class StatusResource {
List<String> masters = monitor.getContext().getMasterLocations();
masterStatus = masters.size() == 0 ? Status.ERROR : Status.OK;
- int tServerUp = monitor.getMmi().getTServerInfoSize();
- int tServerDown = monitor.getMmi().getDeadTabletServersSize();
- int tServerBad = monitor.getMmi().getBadTServersSize();
+ int tServerUp = mmi.getTServerInfoSize();
+ int tServerDown = mmi.getDeadTabletServersSize();
+ int tServerBad = mmi.getBadTServersSize();
/*
* If there are no dead or bad servers and there are tservers up, status is OK, if there are
diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/tables/TablesResource.java b/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/tables/TablesResource.java
index ccabfc4..949ea32 100644
--- a/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/tables/TablesResource.java
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/tables/TablesResource.java
@@ -38,6 +38,7 @@ import org.apache.accumulo.core.clientImpl.Tables;
import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.TableId;
import org.apache.accumulo.core.master.state.tables.TableState;
+import org.apache.accumulo.core.master.thrift.MasterMonitorInfo;
import org.apache.accumulo.core.master.thrift.TableInfo;
import org.apache.accumulo.core.master.thrift.TabletServerStatus;
import org.apache.accumulo.core.metadata.MetadataTable;
@@ -79,15 +80,19 @@ public class TablesResource {
public static TableInformationList getTables(Monitor monitor) {
TableInformationList tableList = new TableInformationList();
+ MasterMonitorInfo mmi = monitor.getMmi();
+ if (mmi == null) {
+ return tableList;
+ }
SortedMap<TableId,TableInfo> tableStats = new TreeMap<>();
- if (monitor.getMmi() != null && monitor.getMmi().tableMap != null) {
- for (Map.Entry<String,TableInfo> te : monitor.getMmi().tableMap.entrySet()) {
+ if (mmi.tableMap != null) {
+ for (Map.Entry<String,TableInfo> te : mmi.tableMap.entrySet()) {
tableStats.put(TableId.of(te.getKey()), te.getValue());
}
}
- Map<String,Double> compactingByTable = TableInfoUtil.summarizeTableStats(monitor.getMmi());
+ Map<String,Double> compactingByTable = TableInfoUtil.summarizeTableStats(mmi);
TableManager tableManager = monitor.getContext().getTableManager();
// Add tables to the list
@@ -125,8 +130,13 @@ public class TablesResource {
regexp = ALPHA_NUM_REGEX_TABLE_ID) String tableIdStr) {
String rootTabletLocation = monitor.getContext().getRootTabletLocation();
TableId tableId = TableId.of(tableIdStr);
+ MasterMonitorInfo mmi = monitor.getMmi();
+ // fail fast if unable to get monitor info
+ if (mmi == null) {
+ return new TabletServers();
+ }
- TabletServers tabletServers = new TabletServers(monitor.getMmi().tServerInfo.size());
+ TabletServers tabletServers = new TabletServers(mmi.tServerInfo.size());
if (StringUtils.isBlank(tableIdStr)) {
return tabletServers;
@@ -158,15 +168,13 @@ public class TablesResource {
}
List<TabletServerStatus> tservers = new ArrayList<>();
- if (monitor.getMmi() != null) {
- for (TabletServerStatus tss : monitor.getMmi().tServerInfo) {
- try {
- if (tss.name != null && locs.contains(tss.name)) {
- tservers.add(tss);
- }
- } catch (Exception ex) {
- return tabletServers;
+ for (TabletServerStatus tss : mmi.tServerInfo) {
+ try {
+ if (tss.name != null && locs.contains(tss.name)) {
+ tservers.add(tss);
}
+ } catch (Exception ex) {
+ return tabletServers;
}
}
@@ -175,10 +183,7 @@ public class TablesResource {
if (status == null) {
status = NO_STATUS;
}
- TableInfo summary = TableInfoUtil.summarizeTableStats(status);
- if (tableId != null) {
- summary = status.tableMap.get(tableId.canonical());
- }
+ TableInfo summary = status.tableMap.get(tableId.canonical());
if (summary == null) {
continue;
}
@@ -190,7 +195,6 @@ public class TablesResource {
}
return tabletServers;
-
}
}
diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/tservers/TabletServerResource.java b/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/tservers/TabletServerResource.java
index df80133..e3ef081 100644
--- a/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/tservers/TabletServerResource.java
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/tservers/TabletServerResource.java
@@ -34,9 +34,7 @@ import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
import javax.ws.rs.QueryParam;
-import javax.ws.rs.WebApplicationException;
import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.Response.Status;
import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.clientImpl.ClientContext;
@@ -83,7 +81,7 @@ public class TabletServerResource {
public TabletServers getTserverSummary() {
MasterMonitorInfo mmi = monitor.getMmi();
if (mmi == null) {
- throw new WebApplicationException(Status.INTERNAL_SERVER_ERROR);
+ return new TabletServers();
}
TabletServers tserverInfo = new TabletServers(mmi.tServerInfo.size());
@@ -123,7 +121,7 @@ public class TabletServerResource {
MasterMonitorInfo mmi = monitor.getMmi();
if (mmi == null) {
- throw new WebApplicationException(Status.INTERNAL_SERVER_ERROR);
+ return new TabletServersRecovery();
}
for (TabletServerStatus server : mmi.tServerInfo) {
@@ -154,9 +152,12 @@ public class TabletServerResource {
public TabletServerSummary getTserverDetails(
@PathParam("address") @NotNull @Pattern(regexp = HOSTNAME_PORT_REGEX) String tserverAddress)
throws Exception {
+ MasterMonitorInfo mmi = monitor.getMmi();
+ if (mmi == null)
+ return new TabletServerSummary();
boolean tserverExists = false;
- for (TabletServerStatus ts : monitor.getMmi().getTServerInfo()) {
+ for (TabletServerStatus ts : mmi.getTServerInfo()) {
if (tserverAddress.equals(ts.getName())) {
tserverExists = true;
break;
@@ -188,7 +189,7 @@ public class TabletServerResource {
TabletClientService.Client client =
ThriftUtil.getClient(new TabletClientService.Client.Factory(), address, context);
try {
- for (String tableId : monitor.getMmi().tableMap.keySet()) {
+ for (String tableId : mmi.tableMap.keySet()) {
tsStats.addAll(client.getTabletStats(TraceUtil.traceInfo(), context.rpcCreds(), tableId));
}
historical = client.getHistoricalStats(TraceUtil.traceInfo(), context.rpcCreds());