You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by ec...@apache.org on 2014/09/11 21:21:57 UTC
[1/2] git commit: ACCUMULO-2645 add (very) basic data about running
scans to identify run-away queries
Repository: accumulo
Updated Branches:
refs/heads/master 6104c98b9 -> e3b0412e1
ACCUMULO-2645 add (very) basic data about running scans to identify run-away queries
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/7f329a9c
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/7f329a9c
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/7f329a9c
Branch: refs/heads/master
Commit: 7f329a9c4dbf6f249a14a1f4bde43f261bf12875
Parents: 560c268
Author: Eric C. Newton <er...@gmail.com>
Authored: Thu Sep 11 15:17:39 2014 -0400
Committer: Eric C. Newton <er...@gmail.com>
Committed: Thu Sep 11 15:17:39 2014 -0400
----------------------------------------------------------------------
.../org/apache/accumulo/monitor/Monitor.java | 65 +++++++++++++++++-
.../accumulo/monitor/servlets/BasicServlet.java | 1 +
.../accumulo/monitor/servlets/ScanServlet.java | 72 ++++++++++++++++++++
3 files changed, 136 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7f329a9c/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
----------------------------------------------------------------------
diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java b/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
index 0e6b37f..b0124a9 100644
--- a/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
@@ -21,13 +21,18 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
+import java.util.TreeMap;
+import org.apache.accumulo.monitor.servlets.ScanServlet;
+import com.google.common.net.HostAndPort;
import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.Instance;
import org.apache.accumulo.core.client.impl.MasterClient;
import org.apache.accumulo.core.conf.AccumuloConfiguration;
@@ -39,6 +44,8 @@ import org.apache.accumulo.core.master.thrift.MasterMonitorInfo;
import org.apache.accumulo.core.master.thrift.TableInfo;
import org.apache.accumulo.core.master.thrift.TabletServerStatus;
import org.apache.accumulo.core.security.SecurityUtil;
+import org.apache.accumulo.core.tabletserver.thrift.ActiveScan;
+import org.apache.accumulo.core.tabletserver.thrift.TabletClientService.Client;
import org.apache.accumulo.core.util.Daemon;
import org.apache.accumulo.core.util.LoggingRunnable;
import org.apache.accumulo.core.util.Pair;
@@ -83,8 +90,6 @@ import org.apache.accumulo.trace.instrument.Tracer;
import org.apache.log4j.Logger;
import org.apache.zookeeper.KeeperException;
-import com.google.common.net.HostAndPort;
-
/**
* Serve master statistics with an embedded web server.
*/
@@ -436,6 +441,7 @@ public class Monitor {
server.addServlet(XMLServlet.class, "/xml");
server.addServlet(JSONServlet.class, "/json");
server.addServlet(VisServlet.class, "/vis");
+ server.addServlet(ScanServlet.class, "/scans");
server.addServlet(Summary.class, "/trace/summary");
server.addServlet(ListType.class, "/trace/listType");
server.addServlet(ShowTrace.class, "/trace/show");
@@ -482,6 +488,61 @@ public class Monitor {
}
}), "Data fetcher").start();
+
+ new Daemon(new LoggingRunnable(log, new Runnable() {
+ @Override
+ public void run() {
+ while (true) {
+ try {
+ Monitor.fetchScans();
+ } catch (Exception e) {
+ log.warn(e.getMessage(), e);
+ }
+ UtilWaitThread.sleep(5000);
+ }
+ }
+ }), "Scan scanner").start();
+ }
+
+ public static class ScanStats {
+ public final List<ActiveScan> scans;
+ public final long fetched;
+ ScanStats(List<ActiveScan> active) {
+ this.scans = active;
+ this.fetched = System.currentTimeMillis();
+ }
+ }
+ static final Map<String, ScanStats> allScans = new HashMap<String, ScanStats>();
+ public static Map<String, ScanStats> getScans() {
+ synchronized (allScans) {
+ return new TreeMap<String, ScanStats>(allScans);
+ }
+ }
+
+ protected static void fetchScans() throws Exception {
+ if (instance == null)
+ return;
+ Connector c = instance.getConnector(SystemCredentials.get().getPrincipal(), SystemCredentials.get().getToken());
+ for (String server : c.instanceOperations().getTabletServers()) {
+ Client tserver = ThriftUtil.getTServerClient(server, Monitor.getSystemConfiguration());
+ try {
+ List<ActiveScan> scans = tserver.getActiveScans(null, SystemCredentials.get().toThrift(instance));
+ synchronized (allScans) {
+ allScans.put(server, new ScanStats(scans));
+ }
+ } catch (Exception ex) {
+ log.debug(ex, ex);
+ }
+ }
+ // Age off old scan information
+ Iterator<Entry<String,ScanStats>> entryIter = allScans.entrySet().iterator();
+ long now = System.currentTimeMillis();
+ while (entryIter.hasNext()) {
+ Entry<String,ScanStats> entry = entryIter.next();
+ if (now - entry.getValue().fetched > 5 * 60 * 1000) {
+ entryIter.remove();
+ }
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7f329a9c/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/BasicServlet.java
----------------------------------------------------------------------
diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/BasicServlet.java b/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/BasicServlet.java
index 603f55f..d896040 100644
--- a/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/BasicServlet.java
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/BasicServlet.java
@@ -184,6 +184,7 @@ abstract public class BasicServlet extends HttpServlet {
sb.append("<hr />\n");
sb.append("<a href='/master'>Master Server</a><br />\n");
sb.append("<a href='/tservers'>Tablet Servers</a><br />\n");
+ sb.append("<a href='/scans'>Active Scans</a><br />\n");
sb.append("<a href='/vis'>Server Activity</a><br />\n");
sb.append("<a href='/gc'>Garbage Collector</a><br />\n");
sb.append("<a href='/tables'>Tables</a><br />\n");
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7f329a9c/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/ScanServlet.java
----------------------------------------------------------------------
diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/ScanServlet.java b/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/ScanServlet.java
new file mode 100644
index 0000000..12b0871
--- /dev/null
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/ScanServlet.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.monitor.servlets;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import org.apache.accumulo.core.master.thrift.TabletServerStatus;
+
+import org.apache.accumulo.monitor.util.celltypes.TServerLinkType;
+import org.apache.accumulo.core.tabletserver.thrift.ActiveScan;
+import org.apache.accumulo.monitor.Monitor;
+import org.apache.accumulo.monitor.Monitor.ScanStats;
+import org.apache.accumulo.monitor.util.Table;
+import org.apache.accumulo.monitor.util.TableRow;
+import org.apache.accumulo.monitor.util.celltypes.DurationType;
+import org.apache.accumulo.monitor.util.celltypes.StringType;
+
+public class ScanServlet extends BasicServlet {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ protected String getTitle(HttpServletRequest req) {
+ return "Scans";
+ }
+
+ @Override
+ protected void pageBody(HttpServletRequest req, HttpServletResponse response, StringBuilder sb) throws IOException {
+ Map<String,ScanStats> scans = Monitor.getScans();
+ Table scanTable = new Table("scanStatus", "Scan Status");
+ scanTable.addSortableColumn("Server", new TServerLinkType(), null);
+ scanTable.addSortableColumn("#", new PreciseNumberType(0, 20, 0, 100), "Number of scans presently running");
+ scanTable.addSortableColumn("Oldest Age", new DurationType(0l, 5 * 60 * 100l), "The age of the oldest scan on this server.");
+ for (TabletServerStatus tserverInfo : Monitor.getMmi().getTServerInfo()) {
+ ScanStats stats = scans.get(tserverInfo.name);
+ long count = 0;
+ long oldest = 0;
+ if (stats != null) {
+ count = stats.scans.size();
+ for (ActiveScan scan : stats.scans) {
+ oldest = Math.max(scan.age, oldest);
+ }
+ }
+ TableRow row = scanTable.prepareRow();
+ row.add(tserverInfo);
+ row.add(count);
+ row.add(oldest);
+ scanTable.addRow(row);
+ }
+ scanTable.generate(req, sb);
+ }
+
+}
[2/2] git commit: Merge branch '1.6.1-SNAPSHOT'
Posted by ec...@apache.org.
Merge branch '1.6.1-SNAPSHOT'
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/e3b0412e
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/e3b0412e
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/e3b0412e
Branch: refs/heads/master
Commit: e3b0412e1e29d8e66b621ceb44845e5ec7e40186
Parents: 6104c98 7f329a9
Author: Eric C. Newton <er...@gmail.com>
Authored: Thu Sep 11 15:18:06 2014 -0400
Committer: Eric C. Newton <er...@gmail.com>
Committed: Thu Sep 11 15:18:06 2014 -0400
----------------------------------------------------------------------
.../org/apache/accumulo/monitor/Monitor.java | 65 +++++++++++++++++-
.../accumulo/monitor/servlets/BasicServlet.java | 1 +
.../accumulo/monitor/servlets/ScanServlet.java | 72 ++++++++++++++++++++
3 files changed, 136 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e3b0412e/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e3b0412e/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/BasicServlet.java
----------------------------------------------------------------------