You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by ec...@apache.org on 2013/12/19 21:33:26 UTC
[01/13] git commit: ACCUMULO-2065 Using the hostname for the system
running the monitor instead of what was passed via --address.
Updated Branches:
refs/heads/1.4.5-SNAPSHOT 40df8cdbd -> 2235b5038
refs/heads/1.5.1-SNAPSHOT a480f6395 -> 76842b294
refs/heads/1.6.0-SNAPSHOT ba0d7d343 -> fa31b0c84
refs/heads/master 3be435e9f -> 52f25e2cd
ACCUMULO-2065 Using the hostname for the system running the monitor instead of what was passed via --address.
The only downside here is if DNS is not set up. However, if DNS is not set up, you're likely running on a single box, at
which point localhost is all you need anyways.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/a480f639
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/a480f639
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/a480f639
Branch: refs/heads/master
Commit: a480f63958cee884091fa1f0ad3c542fec5c385a
Parents: a55f5d3
Author: Josh Elser <el...@apache.org>
Authored: Thu Dec 19 14:05:24 2013 -0500
Committer: Josh Elser <el...@apache.org>
Committed: Thu Dec 19 14:05:24 2013 -0500
----------------------------------------------------------------------
.../main/java/org/apache/accumulo/server/monitor/Monitor.java | 6 ++++++
1 file changed, 6 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a480f639/server/src/main/java/org/apache/accumulo/server/monitor/Monitor.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/accumulo/server/monitor/Monitor.java b/server/src/main/java/org/apache/accumulo/server/monitor/Monitor.java
index c373610..3836916 100644
--- a/server/src/main/java/org/apache/accumulo/server/monitor/Monitor.java
+++ b/server/src/main/java/org/apache/accumulo/server/monitor/Monitor.java
@@ -16,6 +16,7 @@
*/
package org.apache.accumulo.server.monitor;
+import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.util.ArrayList;
import java.util.Collections;
@@ -481,7 +482,12 @@ public class Monitor {
server.addServlet(ShellServlet.class, "/shell");
server.start();
+
try {
+ hostname = InetAddress.getLocalHost().getHostName();
+
+ log.debug("Using " + hostname + " to advertise monitor location in ZooKeeper");
+
String monitorAddress = org.apache.accumulo.core.util.AddressUtil.toString(new InetSocketAddress(hostname, server.getPort()));
ZooReaderWriter.getInstance().putPersistentData(ZooUtil.getRoot(instance) + Constants.ZMONITOR, monitorAddress.getBytes(),
NodeExistsPolicy.OVERWRITE);
[02/13] git commit: Merge branch '1.5.1-SNAPSHOT' into 1.6.0-SNAPSHOT
Posted by ec...@apache.org.
Merge branch '1.5.1-SNAPSHOT' into 1.6.0-SNAPSHOT
Conflicts:
server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/fda6d9c1
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/fda6d9c1
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/fda6d9c1
Branch: refs/heads/master
Commit: fda6d9c1996b045153f159b055ff620a895655b5
Parents: 152f194 a480f63
Author: Josh Elser <el...@apache.org>
Authored: Thu Dec 19 14:25:50 2013 -0500
Committer: Josh Elser <el...@apache.org>
Committed: Thu Dec 19 14:38:01 2013 -0500
----------------------------------------------------------------------
.../src/main/java/org/apache/accumulo/monitor/Monitor.java | 7 +++++++
1 file changed, 7 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/fda6d9c1/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
----------------------------------------------------------------------
diff --cc server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
index bce6461,0000000..3a3692e
mode 100644,000000..100644
--- a/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
@@@ -1,612 -1,0 +1,619 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.monitor;
+
++import java.net.InetAddress;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.impl.MasterClient;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.gc.thrift.GCMonitorService;
+import org.apache.accumulo.core.gc.thrift.GCStatus;
+import org.apache.accumulo.core.master.thrift.MasterClientService;
+import org.apache.accumulo.core.master.thrift.MasterMonitorInfo;
+import org.apache.accumulo.core.master.thrift.TableInfo;
+import org.apache.accumulo.core.master.thrift.TabletServerStatus;
+import org.apache.accumulo.core.security.SecurityUtil;
+import org.apache.accumulo.core.util.Daemon;
+import org.apache.accumulo.core.util.LoggingRunnable;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.accumulo.core.util.ServerServices;
+import org.apache.accumulo.core.util.ServerServices.Service;
+import org.apache.accumulo.core.util.ThriftUtil;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
+import org.apache.accumulo.monitor.servlets.DefaultServlet;
+import org.apache.accumulo.monitor.servlets.GcStatusServlet;
+import org.apache.accumulo.monitor.servlets.JSONServlet;
+import org.apache.accumulo.monitor.servlets.LogServlet;
+import org.apache.accumulo.monitor.servlets.MasterServlet;
+import org.apache.accumulo.monitor.servlets.OperationServlet;
+import org.apache.accumulo.monitor.servlets.ProblemServlet;
+import org.apache.accumulo.monitor.servlets.ShellServlet;
+import org.apache.accumulo.monitor.servlets.TServersServlet;
+import org.apache.accumulo.monitor.servlets.TablesServlet;
+import org.apache.accumulo.monitor.servlets.VisServlet;
+import org.apache.accumulo.monitor.servlets.XMLServlet;
+import org.apache.accumulo.monitor.servlets.trace.ListType;
+import org.apache.accumulo.monitor.servlets.trace.ShowTrace;
+import org.apache.accumulo.monitor.servlets.trace.Summary;
+import org.apache.accumulo.server.Accumulo;
+import org.apache.accumulo.server.ServerOpts;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.conf.ServerConfiguration;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.server.fs.VolumeManagerImpl;
+import org.apache.accumulo.server.monitor.LogService;
+import org.apache.accumulo.server.problems.ProblemReports;
+import org.apache.accumulo.server.problems.ProblemType;
+import org.apache.accumulo.server.security.SystemCredentials;
+import org.apache.accumulo.server.util.TableInfoUtil;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.accumulo.trace.instrument.Tracer;
+import org.apache.log4j.Logger;
+
+import com.google.common.net.HostAndPort;
+
+/**
+ * Serve master statistics with an embedded web server.
+ */
+public class Monitor {
+ private static final Logger log = Logger.getLogger(Monitor.class);
+
+ private static final int REFRESH_TIME = 5;
+ private static long lastRecalc = 0L;
+ private static double totalIngestRate = 0.0;
+ private static double totalIngestByteRate = 0.0;
+ private static double totalQueryRate = 0.0;
+ private static double totalScanRate = 0.0;
+ private static double totalQueryByteRate = 0.0;
+ private static long totalEntries = 0L;
+ private static int totalTabletCount = 0;
+ private static int onlineTabletCount = 0;
+ private static long totalHoldTime = 0;
+ private static long totalLookups = 0;
+ private static int totalTables = 0;
+
+ private static class MaxList<T> extends LinkedList<Pair<Long,T>> {
+ private static final long serialVersionUID = 1L;
+
+ private long maxDelta;
+
+ public MaxList(long maxDelta) {
+ this.maxDelta = maxDelta;
+ }
+
+ @Override
+ public boolean add(Pair<Long,T> obj) {
+ boolean result = super.add(obj);
+
+ if (obj.getFirst() - get(0).getFirst() > maxDelta)
+ remove(0);
+
+ return result;
+ }
+
+ }
+
+ private static final int MAX_TIME_PERIOD = 60 * 60 * 1000;
+ private static final List<Pair<Long,Double>> loadOverTime = Collections.synchronizedList(new MaxList<Double>(MAX_TIME_PERIOD));
+ private static final List<Pair<Long,Double>> ingestRateOverTime = Collections.synchronizedList(new MaxList<Double>(MAX_TIME_PERIOD));
+ private static final List<Pair<Long,Double>> ingestByteRateOverTime = Collections.synchronizedList(new MaxList<Double>(MAX_TIME_PERIOD));
+ private static final List<Pair<Long,Integer>> recoveriesOverTime = Collections.synchronizedList(new MaxList<Integer>(MAX_TIME_PERIOD));
+ private static final List<Pair<Long,Integer>> minorCompactionsOverTime = Collections.synchronizedList(new MaxList<Integer>(MAX_TIME_PERIOD));
+ private static final List<Pair<Long,Integer>> majorCompactionsOverTime = Collections.synchronizedList(new MaxList<Integer>(MAX_TIME_PERIOD));
+ private static final List<Pair<Long,Double>> lookupsOverTime = Collections.synchronizedList(new MaxList<Double>(MAX_TIME_PERIOD));
+ private static final List<Pair<Long,Integer>> queryRateOverTime = Collections.synchronizedList(new MaxList<Integer>(MAX_TIME_PERIOD));
+ private static final List<Pair<Long,Integer>> scanRateOverTime = Collections.synchronizedList(new MaxList<Integer>(MAX_TIME_PERIOD));
+ private static final List<Pair<Long,Double>> queryByteRateOverTime = Collections.synchronizedList(new MaxList<Double>(MAX_TIME_PERIOD));
+ private static final List<Pair<Long,Double>> indexCacheHitRateOverTime = Collections.synchronizedList(new MaxList<Double>(MAX_TIME_PERIOD));
+ private static final List<Pair<Long,Double>> dataCacheHitRateOverTime = Collections.synchronizedList(new MaxList<Double>(MAX_TIME_PERIOD));
+ private static EventCounter lookupRateTracker = new EventCounter();
+ private static EventCounter indexCacheHitTracker = new EventCounter();
+ private static EventCounter indexCacheRequestTracker = new EventCounter();
+ private static EventCounter dataCacheHitTracker = new EventCounter();
+ private static EventCounter dataCacheRequestTracker = new EventCounter();
+
+ private static volatile boolean fetching = false;
+ private static MasterMonitorInfo mmi;
+ private static Map<String,Map<ProblemType,Integer>> problemSummary = Collections.emptyMap();
+ private static Exception problemException;
+ private static GCStatus gcStatus;
+
+ private static Instance instance;
+
+ private static ServerConfiguration config;
+
+ private static EmbeddedWebServer server;
+
+ private static class EventCounter {
+
+ Map<String,Pair<Long,Long>> prevSamples = new HashMap<String,Pair<Long,Long>>();
+ Map<String,Pair<Long,Long>> samples = new HashMap<String,Pair<Long,Long>>();
+ Set<String> serversUpdated = new HashSet<String>();
+
+ void startingUpdates() {
+ serversUpdated.clear();
+ }
+
+ void updateTabletServer(String name, long sampleTime, long numEvents) {
+ Pair<Long,Long> newSample = new Pair<Long,Long>(sampleTime, numEvents);
+ Pair<Long,Long> lastSample = samples.get(name);
+
+ if (lastSample == null || !lastSample.equals(newSample)) {
+ samples.put(name, newSample);
+ if (lastSample != null) {
+ prevSamples.put(name, lastSample);
+ }
+ }
+ serversUpdated.add(name);
+ }
+
+ void finishedUpdating() {
+ // remove any tablet servers not updated
+ samples.keySet().retainAll(serversUpdated);
+ prevSamples.keySet().retainAll(serversUpdated);
+ }
+
+ double calculateRate() {
+ double totalRate = 0;
+
+ for (Entry<String,Pair<Long,Long>> entry : prevSamples.entrySet()) {
+ Pair<Long,Long> prevSample = entry.getValue();
+ Pair<Long,Long> sample = samples.get(entry.getKey());
+
+ totalRate += (sample.getSecond() - prevSample.getSecond()) / ((sample.getFirst() - prevSample.getFirst()) / (double) 1000);
+ }
+
+ return totalRate;
+ }
+
+ long calculateCount() {
+ long count = 0;
+
+ for (Entry<String,Pair<Long,Long>> entry : prevSamples.entrySet()) {
+ Pair<Long,Long> prevSample = entry.getValue();
+ Pair<Long,Long> sample = samples.get(entry.getKey());
+
+ count += sample.getSecond() - prevSample.getSecond();
+ }
+
+ return count;
+ }
+ }
+
+ public static void fetchData() {
+ double totalIngestRate = 0.;
+ double totalIngestByteRate = 0.;
+ double totalQueryRate = 0.;
+ double totalQueryByteRate = 0.;
+ double totalScanRate = 0.;
+ long totalEntries = 0;
+ int totalTabletCount = 0;
+ int onlineTabletCount = 0;
+ long totalHoldTime = 0;
+ long totalLookups = 0;
+ boolean retry = true;
+
+ // only recalc every so often
+ long currentTime = System.currentTimeMillis();
+ if (currentTime - lastRecalc < REFRESH_TIME * 1000)
+ return;
+
+ synchronized (Monitor.class) {
+ if (fetching)
+ return;
+ fetching = true;
+ }
+
+ try {
+ while (retry) {
+ MasterClientService.Iface client = null;
+ try {
+ client = MasterClient.getConnection(HdfsZooInstance.getInstance());
+ if (client != null) {
+ mmi = client.getMasterStats(Tracer.traceInfo(), SystemCredentials.get().toThrift(HdfsZooInstance.getInstance()));
+ retry = false;
+ } else {
+ mmi = null;
+ }
+ Monitor.gcStatus = fetchGcStatus();
+ } catch (Exception e) {
+ mmi = null;
+ log.info("Error fetching stats: " + e);
+ } finally {
+ if (client != null) {
+ MasterClient.close(client);
+ }
+ }
+ if (mmi == null)
+ UtilWaitThread.sleep(1000);
+ }
+ if (mmi != null) {
+ int majorCompactions = 0;
+ int minorCompactions = 0;
+
+ lookupRateTracker.startingUpdates();
+ indexCacheHitTracker.startingUpdates();
+ indexCacheRequestTracker.startingUpdates();
+ dataCacheHitTracker.startingUpdates();
+ dataCacheRequestTracker.startingUpdates();
+
+ for (TabletServerStatus server : mmi.tServerInfo) {
+ TableInfo summary = TableInfoUtil.summarizeTableStats(server);
+ totalIngestRate += summary.ingestRate;
+ totalIngestByteRate += summary.ingestByteRate;
+ totalQueryRate += summary.queryRate;
+ totalScanRate += summary.scanRate;
+ totalQueryByteRate += summary.queryByteRate;
+ totalEntries += summary.recs;
+ totalHoldTime += server.holdTime;
+ totalLookups += server.lookups;
+ majorCompactions += summary.majors.running;
+ minorCompactions += summary.minors.running;
+ lookupRateTracker.updateTabletServer(server.name, server.lastContact, server.lookups);
+ indexCacheHitTracker.updateTabletServer(server.name, server.lastContact, server.indexCacheHits);
+ indexCacheRequestTracker.updateTabletServer(server.name, server.lastContact, server.indexCacheRequest);
+ dataCacheHitTracker.updateTabletServer(server.name, server.lastContact, server.dataCacheHits);
+ dataCacheRequestTracker.updateTabletServer(server.name, server.lastContact, server.dataCacheRequest);
+ }
+
+ lookupRateTracker.finishedUpdating();
+ indexCacheHitTracker.finishedUpdating();
+ indexCacheRequestTracker.finishedUpdating();
+ dataCacheHitTracker.finishedUpdating();
+ dataCacheRequestTracker.finishedUpdating();
+
+ int totalTables = 0;
+ for (TableInfo tInfo : mmi.tableMap.values()) {
+ totalTabletCount += tInfo.tablets;
+ onlineTabletCount += tInfo.onlineTablets;
+ totalTables++;
+ }
+ Monitor.totalIngestRate = totalIngestRate;
+ Monitor.totalTables = totalTables;
+ totalIngestByteRate = totalIngestByteRate / 1000000.0;
+ Monitor.totalIngestByteRate = totalIngestByteRate;
+ Monitor.totalQueryRate = totalQueryRate;
+ Monitor.totalScanRate = totalScanRate;
+ totalQueryByteRate = totalQueryByteRate / 1000000.0;
+ Monitor.totalQueryByteRate = totalQueryByteRate;
+ Monitor.totalEntries = totalEntries;
+ Monitor.totalTabletCount = totalTabletCount;
+ Monitor.onlineTabletCount = onlineTabletCount;
+ Monitor.totalHoldTime = totalHoldTime;
+ Monitor.totalLookups = totalLookups;
+
+ ingestRateOverTime.add(new Pair<Long,Double>(currentTime, totalIngestRate));
+ ingestByteRateOverTime.add(new Pair<Long,Double>(currentTime, totalIngestByteRate));
+
+ double totalLoad = 0.;
+ for (TabletServerStatus status : mmi.tServerInfo) {
+ if (status != null)
+ totalLoad += status.osLoad;
+ }
+ loadOverTime.add(new Pair<Long,Double>(currentTime, totalLoad));
+
+ minorCompactionsOverTime.add(new Pair<Long,Integer>(currentTime, minorCompactions));
+ majorCompactionsOverTime.add(new Pair<Long,Integer>(currentTime, majorCompactions));
+
+ lookupsOverTime.add(new Pair<Long,Double>(currentTime, lookupRateTracker.calculateRate()));
+
+ queryRateOverTime.add(new Pair<Long,Integer>(currentTime, (int) totalQueryRate));
+ queryByteRateOverTime.add(new Pair<Long,Double>(currentTime, totalQueryByteRate));
+
+ scanRateOverTime.add(new Pair<Long,Integer>(currentTime, (int) totalScanRate));
+
+ calcCacheHitRate(indexCacheHitRateOverTime, currentTime, indexCacheHitTracker, indexCacheRequestTracker);
+ calcCacheHitRate(dataCacheHitRateOverTime, currentTime, dataCacheHitTracker, dataCacheRequestTracker);
+ }
+ try {
+ Monitor.problemSummary = ProblemReports.getInstance().summarize();
+ Monitor.problemException = null;
+ } catch (Exception e) {
+ log.info("Failed to obtain problem reports ", e);
+ Monitor.problemSummary = Collections.emptyMap();
+ Monitor.problemException = e;
+ }
+
+ } finally {
+ synchronized (Monitor.class) {
+ fetching = false;
+ lastRecalc = currentTime;
+ }
+ }
+ }
+
+ private static void calcCacheHitRate(List<Pair<Long,Double>> hitRate, long currentTime, EventCounter cacheHits, EventCounter cacheReq) {
+ long req = cacheReq.calculateCount();
+ if (req > 0)
+ hitRate.add(new Pair<Long,Double>(currentTime, cacheHits.calculateCount() / (double) cacheReq.calculateCount()));
+ else
+ hitRate.add(new Pair<Long,Double>(currentTime, null));
+ }
+
+ private static GCStatus fetchGcStatus() {
+ GCStatus result = null;
+ HostAndPort address = null;
+ try {
+ // Read the gc location from its lock
+ ZooReaderWriter zk = ZooReaderWriter.getInstance();
+ String path = ZooUtil.getRoot(instance) + Constants.ZGC_LOCK;
+ List<String> locks = zk.getChildren(path, null);
+ if (locks != null && locks.size() > 0) {
+ Collections.sort(locks);
+ address = new ServerServices(new String(zk.getData(path + "/" + locks.get(0), null))).getAddress(Service.GC_CLIENT);
+ GCMonitorService.Client client = ThriftUtil.getClient(new GCMonitorService.Client.Factory(), address, config.getConfiguration());
+ try {
+ result = client.getStatus(Tracer.traceInfo(), SystemCredentials.get().toThrift(instance));
+ } finally {
+ ThriftUtil.returnClient(client);
+ }
+ }
+ } catch (Exception ex) {
+ log.warn("Unable to contact the garbage collector at " + address, ex);
+ }
+ return result;
+ }
+
+ public static void main(String[] args) throws Exception {
+ SecurityUtil.serverLogin();
+
+ VolumeManager fs = VolumeManagerImpl.get();
+ ServerOpts opts = new ServerOpts();
+ opts.parseArgs("monitor", args);
+ String hostname = opts.getAddress();
+ instance = HdfsZooInstance.getInstance();
+ config = new ServerConfiguration(instance);
+ Accumulo.init(fs, config, "monitor");
+ Monitor monitor = new Monitor();
+ Accumulo.enableTracing(hostname, "monitor");
+ monitor.run(hostname);
+ }
+
+ private static long START_TIME;
+
+ public void run(String hostname) {
+ Monitor.START_TIME = System.currentTimeMillis();
+ int port = config.getConfiguration().getPort(Property.MONITOR_PORT);
+ try {
+ log.debug("Creating monitor on port " + port);
+ server = new EmbeddedWebServer(hostname, port);
+ } catch (Throwable ex) {
+ log.error("Unable to start embedded web server", ex);
+ throw new RuntimeException(ex);
+ }
+
+ server.addServlet(DefaultServlet.class, "/");
+ server.addServlet(OperationServlet.class, "/op");
+ server.addServlet(MasterServlet.class, "/master");
+ server.addServlet(TablesServlet.class, "/tables");
+ server.addServlet(TServersServlet.class, "/tservers");
+ server.addServlet(ProblemServlet.class, "/problems");
+ server.addServlet(GcStatusServlet.class, "/gc");
+ server.addServlet(LogServlet.class, "/log");
+ server.addServlet(XMLServlet.class, "/xml");
+ server.addServlet(JSONServlet.class, "/json");
+ server.addServlet(VisServlet.class, "/vis");
+ server.addServlet(Summary.class, "/trace/summary");
+ server.addServlet(ListType.class, "/trace/listType");
+ server.addServlet(ShowTrace.class, "/trace/show");
+ if (server.isUsingSsl())
+ server.addServlet(ShellServlet.class, "/shell");
+ server.start();
+
++
+ try {
++ hostname = InetAddress.getLocalHost().getHostName();
++
++ log.debug("Using " + hostname + " to advertise monitor location in ZooKeeper");
++
+ String monitorAddress = HostAndPort.fromParts(hostname, server.getPort()).toString();
++
+ ZooReaderWriter.getInstance().putPersistentData(ZooUtil.getRoot(instance) + Constants.ZMONITOR, monitorAddress.getBytes(),
+ NodeExistsPolicy.OVERWRITE);
+ log.info("Set monitor address in zookeeper to " + monitorAddress);
+ } catch (Exception ex) {
+ log.error("Unable to set monitor address in zookeeper");
+ }
+ LogService.startLogListener(Monitor.getSystemConfiguration(), instance.getInstanceID());
+
+ new Daemon(new LoggingRunnable(log, new ZooKeeperStatus()), "ZooKeeperStatus").start();
+
+ // need to regularly fetch data so plot data is updated
+ new Daemon(new LoggingRunnable(log, new Runnable() {
+
+ @Override
+ public void run() {
+ while (true) {
+ try {
+ Monitor.fetchData();
+ } catch (Exception e) {
+ log.warn(e.getMessage(), e);
+ }
+
+ UtilWaitThread.sleep(333);
+ }
+
+ }
+ }), "Data fetcher").start();
+ }
+
+ public static MasterMonitorInfo getMmi() {
+ return mmi;
+ }
+
+ public static int getTotalTables() {
+ return totalTables;
+ }
+
+ public static int getTotalTabletCount() {
+ return totalTabletCount;
+ }
+
+ public static int getOnlineTabletCount() {
+ return onlineTabletCount;
+ }
+
+ public static long getTotalEntries() {
+ return totalEntries;
+ }
+
+ public static double getTotalIngestRate() {
+ return totalIngestRate;
+ }
+
+ public static double getTotalIngestByteRate() {
+ return totalIngestByteRate;
+ }
+
+ public static double getTotalQueryRate() {
+ return totalQueryRate;
+ }
+
+ public static double getTotalScanRate() {
+ return totalScanRate;
+ }
+
+ public static double getTotalQueryByteRate() {
+ return totalQueryByteRate;
+ }
+
+ public static long getTotalHoldTime() {
+ return totalHoldTime;
+ }
+
+ public static Exception getProblemException() {
+ return problemException;
+ }
+
+ public static Map<String,Map<ProblemType,Integer>> getProblemSummary() {
+ return problemSummary;
+ }
+
+ public static GCStatus getGcStatus() {
+ return gcStatus;
+ }
+
+ public static long getTotalLookups() {
+ return totalLookups;
+ }
+
+ public static long getStartTime() {
+ return START_TIME;
+ }
+
+ public static List<Pair<Long,Double>> getLoadOverTime() {
+ synchronized (loadOverTime) {
+ return new ArrayList<Pair<Long,Double>>(loadOverTime);
+ }
+ }
+
+ public static List<Pair<Long,Double>> getIngestRateOverTime() {
+ synchronized (ingestRateOverTime) {
+ return new ArrayList<Pair<Long,Double>>(ingestRateOverTime);
+ }
+ }
+
+ public static List<Pair<Long,Double>> getIngestByteRateOverTime() {
+ synchronized (ingestByteRateOverTime) {
+ return new ArrayList<Pair<Long,Double>>(ingestByteRateOverTime);
+ }
+ }
+
+ public static List<Pair<Long,Integer>> getRecoveriesOverTime() {
+ synchronized (recoveriesOverTime) {
+ return new ArrayList<Pair<Long,Integer>>(recoveriesOverTime);
+ }
+ }
+
+ public static List<Pair<Long,Integer>> getMinorCompactionsOverTime() {
+ synchronized (minorCompactionsOverTime) {
+ return new ArrayList<Pair<Long,Integer>>(minorCompactionsOverTime);
+ }
+ }
+
+ public static List<Pair<Long,Integer>> getMajorCompactionsOverTime() {
+ synchronized (majorCompactionsOverTime) {
+ return new ArrayList<Pair<Long,Integer>>(majorCompactionsOverTime);
+ }
+ }
+
+ public static List<Pair<Long,Double>> getLookupsOverTime() {
+ synchronized (lookupsOverTime) {
+ return new ArrayList<Pair<Long,Double>>(lookupsOverTime);
+ }
+ }
+
+ public static double getLookupRate() {
+ return lookupRateTracker.calculateRate();
+ }
+
+ public static List<Pair<Long,Integer>> getQueryRateOverTime() {
+ synchronized (queryRateOverTime) {
+ return new ArrayList<Pair<Long,Integer>>(queryRateOverTime);
+ }
+ }
+
+ public static List<Pair<Long,Integer>> getScanRateOverTime() {
+ synchronized (scanRateOverTime) {
+ return new ArrayList<Pair<Long,Integer>>(scanRateOverTime);
+ }
+ }
+
+ public static List<Pair<Long,Double>> getQueryByteRateOverTime() {
+ synchronized (queryByteRateOverTime) {
+ return new ArrayList<Pair<Long,Double>>(queryByteRateOverTime);
+ }
+ }
+
+ public static List<Pair<Long,Double>> getIndexCacheHitRateOverTime() {
+ synchronized (indexCacheHitRateOverTime) {
+ return new ArrayList<Pair<Long,Double>>(indexCacheHitRateOverTime);
+ }
+ }
+
+ public static List<Pair<Long,Double>> getDataCacheHitRateOverTime() {
+ synchronized (dataCacheHitRateOverTime) {
+ return new ArrayList<Pair<Long,Double>>(dataCacheHitRateOverTime);
+ }
+ }
+
+ public static AccumuloConfiguration getSystemConfiguration() {
+ return config.getConfiguration();
+ }
+
+ public static Instance getInstance() {
+ return instance;
+ }
+
+ public static boolean isUsingSsl() {
+ return server.isUsingSsl();
+ }
+}
[09/13] git commit: ACCUMULO-2057 handle duplicate last locations
gracefully
Posted by ec...@apache.org.
ACCUMULO-2057 handle duplicate last locations gracefully
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/76842b29
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/76842b29
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/76842b29
Branch: refs/heads/1.6.0-SNAPSHOT
Commit: 76842b29458571fdd66e5b3440f7fd0075d10542
Parents: a480f63 2235b50
Author: Eric Newton <er...@gmail.com>
Authored: Thu Dec 19 15:31:59 2013 -0500
Committer: Eric Newton <er...@gmail.com>
Committed: Thu Dec 19 15:31:59 2013 -0500
----------------------------------------------------------------------
.../accumulo/server/master/state/MetaDataTableScanner.java | 8 +++-----
1 file changed, 3 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/76842b29/server/src/main/java/org/apache/accumulo/server/master/state/MetaDataTableScanner.java
----------------------------------------------------------------------
diff --cc server/src/main/java/org/apache/accumulo/server/master/state/MetaDataTableScanner.java
index bf1da22,0000000..2458a07
mode 100644,000000..100644
--- a/server/src/main/java/org/apache/accumulo/server/master/state/MetaDataTableScanner.java
+++ b/server/src/main/java/org/apache/accumulo/server/master/state/MetaDataTableScanner.java
@@@ -1,185 -1,0 +1,183 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.master.state;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map.Entry;
+import java.util.SortedMap;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.BatchScanner;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.ScannerBase;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.user.WholeRowIterator;
+import org.apache.accumulo.core.security.CredentialHelper;
+import org.apache.accumulo.core.security.thrift.TCredentials;
+import org.apache.accumulo.server.master.state.TabletLocationState.BadLocationStateException;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Logger;
+
+public class MetaDataTableScanner implements Iterator<TabletLocationState> {
+ private static final Logger log = Logger.getLogger(MetaDataTableScanner.class);
+
+ BatchScanner mdScanner;
+ Iterator<Entry<Key,Value>> iter;
+
+ public MetaDataTableScanner(Instance instance, TCredentials auths, Range range, CurrentState state) {
+ // scan over metadata table, looking for tablets in the wrong state based on the live servers and online tables
+ try {
+ Connector connector = instance.getConnector(auths.getPrincipal(), CredentialHelper.extractToken(auths));
+ mdScanner = connector.createBatchScanner(Constants.METADATA_TABLE_NAME, Constants.NO_AUTHS, 8);
+ configureScanner(mdScanner, state);
+ mdScanner.setRanges(Collections.singletonList(range));
+ iter = mdScanner.iterator();
+ } catch (Exception ex) {
+ mdScanner.close();
+ throw new RuntimeException(ex);
+ }
+ }
+
+ static public void configureScanner(ScannerBase scanner, CurrentState state) {
+ Constants.METADATA_PREV_ROW_COLUMN.fetch(scanner);
+ scanner.fetchColumnFamily(Constants.METADATA_CURRENT_LOCATION_COLUMN_FAMILY);
+ scanner.fetchColumnFamily(Constants.METADATA_FUTURE_LOCATION_COLUMN_FAMILY);
+ scanner.fetchColumnFamily(Constants.METADATA_LAST_LOCATION_COLUMN_FAMILY);
+ scanner.fetchColumnFamily(Constants.METADATA_LOG_COLUMN_FAMILY);
+ scanner.fetchColumnFamily(Constants.METADATA_CHOPPED_COLUMN_FAMILY);
+ scanner.addScanIterator(new IteratorSetting(1000, "wholeRows", WholeRowIterator.class));
+ IteratorSetting tabletChange = new IteratorSetting(1001, "tabletChange", TabletStateChangeIterator.class);
+ if (state != null) {
+ TabletStateChangeIterator.setCurrentServers(tabletChange, state.onlineTabletServers());
+ TabletStateChangeIterator.setOnlineTables(tabletChange, state.onlineTables());
+ TabletStateChangeIterator.setMerges(tabletChange, state.merges());
+ }
+ scanner.addScanIterator(tabletChange);
+ }
+
+ public MetaDataTableScanner(Instance instance, TCredentials auths, Range range) {
+ this(instance, auths, range, null);
+ }
+
+ public void close() {
+ if (iter != null) {
+ mdScanner.close();
+ iter = null;
+ }
+ }
+
+ public void finalize() {
+ close();
+ }
+
+ @Override
+ public boolean hasNext() {
+ if (iter == null)
+ return false;
+ boolean result = iter.hasNext();
+ if (!result) {
+ close();
+ }
+ return result;
+ }
+
+ @Override
+ public TabletLocationState next() {
+ try {
+ return fetch();
+ } catch (RuntimeException ex) {
+ // something is wrong with the records in the !METADATA table, just skip over it
+ log.error(ex, ex);
+ mdScanner.close();
+ return null;
+ }
+ }
+
+ public static TabletLocationState createTabletLocationState(Key k, Value v) throws IOException, BadLocationStateException {
+ final SortedMap<Key,Value> decodedRow = WholeRowIterator.decodeRow(k, v);
+ KeyExtent extent = null;
+ TServerInstance future = null;
+ TServerInstance current = null;
+ TServerInstance last = null;
++ long lastTimestamp = 0;
+ List<Collection<String>> walogs = new ArrayList<Collection<String>>();
+ boolean chopped = false;
+
+ for (Entry<Key,Value> entry : decodedRow.entrySet()) {
+ Key key = entry.getKey();
+ Text row = key.getRow();
+ Text cf = key.getColumnFamily();
+ Text cq = key.getColumnQualifier();
+
+ if (cf.compareTo(Constants.METADATA_FUTURE_LOCATION_COLUMN_FAMILY) == 0) {
+ TServerInstance location = new TServerInstance(entry.getValue(), cq);
+ if (future != null) {
+ throw new BadLocationStateException("found two assignments for the same extent " + key.getRow() + ": " + future + " and " + location);
+ }
+ future = location;
+ } else if (cf.compareTo(Constants.METADATA_CURRENT_LOCATION_COLUMN_FAMILY) == 0) {
+ TServerInstance location = new TServerInstance(entry.getValue(), cq);
+ if (current != null) {
+ throw new BadLocationStateException("found two locations for the same extent " + key.getRow() + ": " + current + " and " + location);
+ }
+ current = location;
+ } else if (cf.compareTo(Constants.METADATA_LOG_COLUMN_FAMILY) == 0) {
+ String[] split = entry.getValue().toString().split("\\|")[0].split(";");
+ walogs.add(Arrays.asList(split));
+ } else if (cf.compareTo(Constants.METADATA_LAST_LOCATION_COLUMN_FAMILY) == 0) {
- TServerInstance location = new TServerInstance(entry.getValue(), cq);
- if (last != null) {
- throw new BadLocationStateException("found two last locations for the same extent " + key.getRow() + ": " + last + " and " + location);
- }
- last = new TServerInstance(entry.getValue(), cq);
++ if (lastTimestamp < entry.getKey().getTimestamp())
++ last = new TServerInstance(entry.getValue(), cq);
+ } else if (cf.compareTo(Constants.METADATA_CHOPPED_COLUMN_FAMILY) == 0) {
+ chopped = true;
+ } else if (Constants.METADATA_PREV_ROW_COLUMN.equals(cf, cq)) {
+ extent = new KeyExtent(row, entry.getValue());
+ }
+ }
+ if (extent == null) {
+ log.warn("No prev-row for key extent: " + decodedRow);
+ return null;
+ }
+ return new TabletLocationState(extent, future, current, last, walogs, chopped);
+ }
+
+ private TabletLocationState fetch() {
+ try {
+ Entry<Key,Value> e = iter.next();
+ return createTabletLocationState(e.getKey(), e.getValue());
+ } catch (IOException ex) {
+ throw new RuntimeException(ex);
+ } catch (BadLocationStateException ex) {
+ throw new RuntimeException(ex);
+ }
+ }
+
+ @Override
+ public void remove() {
+ throw new RuntimeException("Unimplemented");
+ }
+}
[07/13] git commit: ACCUMULO-2057 handle multiple last locations
gracefully
Posted by ec...@apache.org.
ACCUMULO-2057 handle multiple last locations gracefully
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/2235b503
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/2235b503
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/2235b503
Branch: refs/heads/master
Commit: 2235b5038a85147cae3495d4d0368e0bd9f38098
Parents: 40df8cd
Author: Eric Newton <er...@gmail.com>
Authored: Thu Dec 19 15:00:22 2013 -0500
Committer: Eric Newton <er...@gmail.com>
Committed: Thu Dec 19 15:17:02 2013 -0500
----------------------------------------------------------------------
.../accumulo/server/master/state/MetaDataTableScanner.java | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/2235b503/src/server/src/main/java/org/apache/accumulo/server/master/state/MetaDataTableScanner.java
----------------------------------------------------------------------
diff --git a/src/server/src/main/java/org/apache/accumulo/server/master/state/MetaDataTableScanner.java b/src/server/src/main/java/org/apache/accumulo/server/master/state/MetaDataTableScanner.java
index e10b1b3..157cb76 100644
--- a/src/server/src/main/java/org/apache/accumulo/server/master/state/MetaDataTableScanner.java
+++ b/src/server/src/main/java/org/apache/accumulo/server/master/state/MetaDataTableScanner.java
@@ -127,6 +127,7 @@ public class MetaDataTableScanner implements Iterator<TabletLocationState> {
TServerInstance future = null;
TServerInstance current = null;
TServerInstance last = null;
+ long lastTimestamp = 0;
List<Collection<String>> walogs = new ArrayList<Collection<String>>();
boolean chopped = false;
@@ -144,7 +145,8 @@ public class MetaDataTableScanner implements Iterator<TabletLocationState> {
String[] split = entry.getValue().toString().split("\\|")[0].split(";");
walogs.add(Arrays.asList(split));
} else if (cf.compareTo(Constants.METADATA_LAST_LOCATION_COLUMN_FAMILY) == 0) {
- last = new TServerInstance(entry.getValue(), cq);
+ if (lastTimestamp < entry.getKey().getTimestamp())
+ last = new TServerInstance(entry.getValue(), cq);
} else if (cf.compareTo(Constants.METADATA_CHOPPED_COLUMN_FAMILY) == 0) {
chopped = true;
} else if (Constants.METADATA_PREV_ROW_COLUMN.equals(cf, cq)) {
[12/13] git commit: ACCUMULO-2057 handle duplicate last locations
gracefully
Posted by ec...@apache.org.
ACCUMULO-2057 handle duplicate last locations gracefully
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/fa31b0c8
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/fa31b0c8
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/fa31b0c8
Branch: refs/heads/1.6.0-SNAPSHOT
Commit: fa31b0c844d09ee66d8f1286f6360cbcb9165e1f
Parents: ba0d7d3 76842b2
Author: Eric Newton <er...@gmail.com>
Authored: Thu Dec 19 15:33:22 2013 -0500
Committer: Eric Newton <er...@gmail.com>
Committed: Thu Dec 19 15:33:22 2013 -0500
----------------------------------------------------------------------
.../accumulo/server/master/state/MetaDataTableScanner.java | 8 +++-----
1 file changed, 3 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/fa31b0c8/server/base/src/main/java/org/apache/accumulo/server/master/state/MetaDataTableScanner.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/master/state/MetaDataTableScanner.java
index 0d8f572,0000000..c03b5b4
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/master/state/MetaDataTableScanner.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/master/state/MetaDataTableScanner.java
@@@ -1,200 -1,0 +1,198 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.master.state;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map.Entry;
+import java.util.SortedMap;
+
+import org.apache.accumulo.core.client.BatchScanner;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.ScannerBase;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.user.WholeRowIterator;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ChoppedColumnFamily;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.LogColumnFamily;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.Credentials;
+import org.apache.accumulo.server.master.state.TabletLocationState.BadLocationStateException;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Logger;
+
+public class MetaDataTableScanner implements Iterator<TabletLocationState> {
+ private static final Logger log = Logger.getLogger(MetaDataTableScanner.class);
+
+ BatchScanner mdScanner = null;
+ Iterator<Entry<Key,Value>> iter = null;
+
+ public MetaDataTableScanner(Instance instance, Credentials credentials, Range range, CurrentState state) {
+ this(instance, credentials, range, state, MetadataTable.NAME);
+ }
+
+ MetaDataTableScanner(Instance instance, Credentials credentials, Range range, CurrentState state, String tableName) {
+ // scan over metadata table, looking for tablets in the wrong state based on the live servers and online tables
+ try {
+ Connector connector = instance.getConnector(credentials.getPrincipal(), credentials.getToken());
+ mdScanner = connector.createBatchScanner(tableName, Authorizations.EMPTY, 8);
+ configureScanner(mdScanner, state);
+ mdScanner.setRanges(Collections.singletonList(range));
+ iter = mdScanner.iterator();
+ } catch (Exception ex) {
+ if (mdScanner != null)
+ mdScanner.close();
+ iter = null;
+ mdScanner = null;
+ throw new RuntimeException(ex);
+ }
+ }
+
+ static public void configureScanner(ScannerBase scanner, CurrentState state) {
+ TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.fetch(scanner);
+ scanner.fetchColumnFamily(TabletsSection.CurrentLocationColumnFamily.NAME);
+ scanner.fetchColumnFamily(TabletsSection.FutureLocationColumnFamily.NAME);
+ scanner.fetchColumnFamily(TabletsSection.LastLocationColumnFamily.NAME);
+ scanner.fetchColumnFamily(LogColumnFamily.NAME);
+ scanner.fetchColumnFamily(ChoppedColumnFamily.NAME);
+ scanner.addScanIterator(new IteratorSetting(1000, "wholeRows", WholeRowIterator.class));
+ IteratorSetting tabletChange = new IteratorSetting(1001, "tabletChange", TabletStateChangeIterator.class);
+ if (state != null) {
+ TabletStateChangeIterator.setCurrentServers(tabletChange, state.onlineTabletServers());
+ TabletStateChangeIterator.setOnlineTables(tabletChange, state.onlineTables());
+ TabletStateChangeIterator.setMerges(tabletChange, state.merges());
+ }
+ scanner.addScanIterator(tabletChange);
+ }
+
+ public MetaDataTableScanner(Instance instance, Credentials credentials, Range range) {
+ this(instance, credentials, range, MetadataTable.NAME);
+ }
+
+ public MetaDataTableScanner(Instance instance, Credentials credentials, Range range, String tableName) {
+ this(instance, credentials, range, null, tableName);
+ }
+
+ public void close() {
+ if (iter != null) {
+ mdScanner.close();
+ iter = null;
+ }
+ }
+
+ @Override
+ public void finalize() {
+ close();
+ }
+
+ @Override
+ public boolean hasNext() {
+ if (iter == null)
+ return false;
+ boolean result = iter.hasNext();
+ if (!result) {
+ close();
+ }
+ return result;
+ }
+
+ @Override
+ public TabletLocationState next() {
+ try {
+ return fetch();
+ } catch (RuntimeException ex) {
+ // something is wrong with the metadata records, just skip over it
+ log.error(ex, ex);
+ mdScanner.close();
+ return null;
+ }
+ }
+
+ public static TabletLocationState createTabletLocationState(Key k, Value v) throws IOException, BadLocationStateException {
+ final SortedMap<Key,Value> decodedRow = WholeRowIterator.decodeRow(k, v);
+ KeyExtent extent = null;
+ TServerInstance future = null;
+ TServerInstance current = null;
+ TServerInstance last = null;
++ long lastTimestamp = 0;
+ List<Collection<String>> walogs = new ArrayList<Collection<String>>();
+ boolean chopped = false;
+
+ for (Entry<Key,Value> entry : decodedRow.entrySet()) {
+ Key key = entry.getKey();
+ Text row = key.getRow();
+ Text cf = key.getColumnFamily();
+ Text cq = key.getColumnQualifier();
+
+ if (cf.compareTo(TabletsSection.FutureLocationColumnFamily.NAME) == 0) {
+ TServerInstance location = new TServerInstance(entry.getValue(), cq);
+ if (future != null) {
+ throw new BadLocationStateException("found two assignments for the same extent " + key.getRow() + ": " + future + " and " + location);
+ }
+ future = location;
+ } else if (cf.compareTo(TabletsSection.CurrentLocationColumnFamily.NAME) == 0) {
+ TServerInstance location = new TServerInstance(entry.getValue(), cq);
+ if (current != null) {
+ throw new BadLocationStateException("found two locations for the same extent " + key.getRow() + ": " + current + " and " + location);
+ }
+ current = location;
+ } else if (cf.compareTo(LogColumnFamily.NAME) == 0) {
+ String[] split = entry.getValue().toString().split("\\|")[0].split(";");
+ walogs.add(Arrays.asList(split));
+ } else if (cf.compareTo(TabletsSection.LastLocationColumnFamily.NAME) == 0) {
- TServerInstance location = new TServerInstance(entry.getValue(), cq);
- if (last != null) {
- throw new BadLocationStateException("found two last locations for the same extent " + key.getRow() + ": " + last + " and " + location);
- }
- last = new TServerInstance(entry.getValue(), cq);
++ if (lastTimestamp < entry.getKey().getTimestamp())
++ last = new TServerInstance(entry.getValue(), cq);
+ } else if (cf.compareTo(ChoppedColumnFamily.NAME) == 0) {
+ chopped = true;
+ } else if (TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.equals(cf, cq)) {
+ extent = new KeyExtent(row, entry.getValue());
+ }
+ }
+ if (extent == null) {
+ log.warn("No prev-row for key extent: " + decodedRow);
+ return null;
+ }
+ return new TabletLocationState(extent, future, current, last, walogs, chopped);
+ }
+
+ private TabletLocationState fetch() {
+ try {
+ Entry<Key,Value> e = iter.next();
+ return createTabletLocationState(e.getKey(), e.getValue());
+ } catch (IOException ex) {
+ throw new RuntimeException(ex);
+ } catch (BadLocationStateException ex) {
+ throw new RuntimeException(ex);
+ }
+ }
+
+ @Override
+ public void remove() {
+ throw new RuntimeException("Unimplemented");
+ }
+}
[04/13] git commit: ACCUMULO-2057 handle multiple last locations
gracefully
Posted by ec...@apache.org.
ACCUMULO-2057 handle multiple last locations gracefully
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/2235b503
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/2235b503
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/2235b503
Branch: refs/heads/1.6.0-SNAPSHOT
Commit: 2235b5038a85147cae3495d4d0368e0bd9f38098
Parents: 40df8cd
Author: Eric Newton <er...@gmail.com>
Authored: Thu Dec 19 15:00:22 2013 -0500
Committer: Eric Newton <er...@gmail.com>
Committed: Thu Dec 19 15:17:02 2013 -0500
----------------------------------------------------------------------
.../accumulo/server/master/state/MetaDataTableScanner.java | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/2235b503/src/server/src/main/java/org/apache/accumulo/server/master/state/MetaDataTableScanner.java
----------------------------------------------------------------------
diff --git a/src/server/src/main/java/org/apache/accumulo/server/master/state/MetaDataTableScanner.java b/src/server/src/main/java/org/apache/accumulo/server/master/state/MetaDataTableScanner.java
index e10b1b3..157cb76 100644
--- a/src/server/src/main/java/org/apache/accumulo/server/master/state/MetaDataTableScanner.java
+++ b/src/server/src/main/java/org/apache/accumulo/server/master/state/MetaDataTableScanner.java
@@ -127,6 +127,7 @@ public class MetaDataTableScanner implements Iterator<TabletLocationState> {
TServerInstance future = null;
TServerInstance current = null;
TServerInstance last = null;
+ long lastTimestamp = 0;
List<Collection<String>> walogs = new ArrayList<Collection<String>>();
boolean chopped = false;
@@ -144,7 +145,8 @@ public class MetaDataTableScanner implements Iterator<TabletLocationState> {
String[] split = entry.getValue().toString().split("\\|")[0].split(";");
walogs.add(Arrays.asList(split));
} else if (cf.compareTo(Constants.METADATA_LAST_LOCATION_COLUMN_FAMILY) == 0) {
- last = new TServerInstance(entry.getValue(), cq);
+ if (lastTimestamp < entry.getKey().getTimestamp())
+ last = new TServerInstance(entry.getValue(), cq);
} else if (cf.compareTo(Constants.METADATA_CHOPPED_COLUMN_FAMILY) == 0) {
chopped = true;
} else if (Constants.METADATA_PREV_ROW_COLUMN.equals(cf, cq)) {
[11/13] git commit: ACCUMULO-2057 handle duplicate last locations
gracefully
Posted by ec...@apache.org.
ACCUMULO-2057 handle duplicate last locations gracefully
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/fa31b0c8
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/fa31b0c8
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/fa31b0c8
Branch: refs/heads/master
Commit: fa31b0c844d09ee66d8f1286f6360cbcb9165e1f
Parents: ba0d7d3 76842b2
Author: Eric Newton <er...@gmail.com>
Authored: Thu Dec 19 15:33:22 2013 -0500
Committer: Eric Newton <er...@gmail.com>
Committed: Thu Dec 19 15:33:22 2013 -0500
----------------------------------------------------------------------
.../accumulo/server/master/state/MetaDataTableScanner.java | 8 +++-----
1 file changed, 3 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/fa31b0c8/server/base/src/main/java/org/apache/accumulo/server/master/state/MetaDataTableScanner.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/master/state/MetaDataTableScanner.java
index 0d8f572,0000000..c03b5b4
mode 100644,000000..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/master/state/MetaDataTableScanner.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/master/state/MetaDataTableScanner.java
@@@ -1,200 -1,0 +1,198 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.master.state;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map.Entry;
+import java.util.SortedMap;
+
+import org.apache.accumulo.core.client.BatchScanner;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.ScannerBase;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.user.WholeRowIterator;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ChoppedColumnFamily;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.LogColumnFamily;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.Credentials;
+import org.apache.accumulo.server.master.state.TabletLocationState.BadLocationStateException;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Logger;
+
+public class MetaDataTableScanner implements Iterator<TabletLocationState> {
+ private static final Logger log = Logger.getLogger(MetaDataTableScanner.class);
+
+ BatchScanner mdScanner = null;
+ Iterator<Entry<Key,Value>> iter = null;
+
+ public MetaDataTableScanner(Instance instance, Credentials credentials, Range range, CurrentState state) {
+ this(instance, credentials, range, state, MetadataTable.NAME);
+ }
+
+ MetaDataTableScanner(Instance instance, Credentials credentials, Range range, CurrentState state, String tableName) {
+ // scan over metadata table, looking for tablets in the wrong state based on the live servers and online tables
+ try {
+ Connector connector = instance.getConnector(credentials.getPrincipal(), credentials.getToken());
+ mdScanner = connector.createBatchScanner(tableName, Authorizations.EMPTY, 8);
+ configureScanner(mdScanner, state);
+ mdScanner.setRanges(Collections.singletonList(range));
+ iter = mdScanner.iterator();
+ } catch (Exception ex) {
+ if (mdScanner != null)
+ mdScanner.close();
+ iter = null;
+ mdScanner = null;
+ throw new RuntimeException(ex);
+ }
+ }
+
+ static public void configureScanner(ScannerBase scanner, CurrentState state) {
+ TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.fetch(scanner);
+ scanner.fetchColumnFamily(TabletsSection.CurrentLocationColumnFamily.NAME);
+ scanner.fetchColumnFamily(TabletsSection.FutureLocationColumnFamily.NAME);
+ scanner.fetchColumnFamily(TabletsSection.LastLocationColumnFamily.NAME);
+ scanner.fetchColumnFamily(LogColumnFamily.NAME);
+ scanner.fetchColumnFamily(ChoppedColumnFamily.NAME);
+ scanner.addScanIterator(new IteratorSetting(1000, "wholeRows", WholeRowIterator.class));
+ IteratorSetting tabletChange = new IteratorSetting(1001, "tabletChange", TabletStateChangeIterator.class);
+ if (state != null) {
+ TabletStateChangeIterator.setCurrentServers(tabletChange, state.onlineTabletServers());
+ TabletStateChangeIterator.setOnlineTables(tabletChange, state.onlineTables());
+ TabletStateChangeIterator.setMerges(tabletChange, state.merges());
+ }
+ scanner.addScanIterator(tabletChange);
+ }
+
+ public MetaDataTableScanner(Instance instance, Credentials credentials, Range range) {
+ this(instance, credentials, range, MetadataTable.NAME);
+ }
+
+ public MetaDataTableScanner(Instance instance, Credentials credentials, Range range, String tableName) {
+ this(instance, credentials, range, null, tableName);
+ }
+
+ public void close() {
+ if (iter != null) {
+ mdScanner.close();
+ iter = null;
+ }
+ }
+
+ @Override
+ public void finalize() {
+ close();
+ }
+
+ @Override
+ public boolean hasNext() {
+ if (iter == null)
+ return false;
+ boolean result = iter.hasNext();
+ if (!result) {
+ close();
+ }
+ return result;
+ }
+
+ @Override
+ public TabletLocationState next() {
+ try {
+ return fetch();
+ } catch (RuntimeException ex) {
+ // something is wrong with the metadata records, just skip over it
+ log.error(ex, ex);
+ mdScanner.close();
+ return null;
+ }
+ }
+
+ public static TabletLocationState createTabletLocationState(Key k, Value v) throws IOException, BadLocationStateException {
+ final SortedMap<Key,Value> decodedRow = WholeRowIterator.decodeRow(k, v);
+ KeyExtent extent = null;
+ TServerInstance future = null;
+ TServerInstance current = null;
+ TServerInstance last = null;
++ long lastTimestamp = 0;
+ List<Collection<String>> walogs = new ArrayList<Collection<String>>();
+ boolean chopped = false;
+
+ for (Entry<Key,Value> entry : decodedRow.entrySet()) {
+ Key key = entry.getKey();
+ Text row = key.getRow();
+ Text cf = key.getColumnFamily();
+ Text cq = key.getColumnQualifier();
+
+ if (cf.compareTo(TabletsSection.FutureLocationColumnFamily.NAME) == 0) {
+ TServerInstance location = new TServerInstance(entry.getValue(), cq);
+ if (future != null) {
+ throw new BadLocationStateException("found two assignments for the same extent " + key.getRow() + ": " + future + " and " + location);
+ }
+ future = location;
+ } else if (cf.compareTo(TabletsSection.CurrentLocationColumnFamily.NAME) == 0) {
+ TServerInstance location = new TServerInstance(entry.getValue(), cq);
+ if (current != null) {
+ throw new BadLocationStateException("found two locations for the same extent " + key.getRow() + ": " + current + " and " + location);
+ }
+ current = location;
+ } else if (cf.compareTo(LogColumnFamily.NAME) == 0) {
+ String[] split = entry.getValue().toString().split("\\|")[0].split(";");
+ walogs.add(Arrays.asList(split));
+ } else if (cf.compareTo(TabletsSection.LastLocationColumnFamily.NAME) == 0) {
- TServerInstance location = new TServerInstance(entry.getValue(), cq);
- if (last != null) {
- throw new BadLocationStateException("found two last locations for the same extent " + key.getRow() + ": " + last + " and " + location);
- }
- last = new TServerInstance(entry.getValue(), cq);
++ if (lastTimestamp < entry.getKey().getTimestamp())
++ last = new TServerInstance(entry.getValue(), cq);
+ } else if (cf.compareTo(ChoppedColumnFamily.NAME) == 0) {
+ chopped = true;
+ } else if (TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.equals(cf, cq)) {
+ extent = new KeyExtent(row, entry.getValue());
+ }
+ }
+ if (extent == null) {
+ log.warn("No prev-row for key extent: " + decodedRow);
+ return null;
+ }
+ return new TabletLocationState(extent, future, current, last, walogs, chopped);
+ }
+
+ private TabletLocationState fetch() {
+ try {
+ Entry<Key,Value> e = iter.next();
+ return createTabletLocationState(e.getKey(), e.getValue());
+ } catch (IOException ex) {
+ throw new RuntimeException(ex);
+ } catch (BadLocationStateException ex) {
+ throw new RuntimeException(ex);
+ }
+ }
+
+ @Override
+ public void remove() {
+ throw new RuntimeException("Unimplemented");
+ }
+}
[13/13] git commit: Merge branch '1.6.0-SNAPSHOT'
Posted by ec...@apache.org.
Merge branch '1.6.0-SNAPSHOT'
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/52f25e2c
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/52f25e2c
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/52f25e2c
Branch: refs/heads/master
Commit: 52f25e2cd705c3c516aa4a0034db4dbcd7134ebd
Parents: 3be435e fa31b0c
Author: Eric Newton <er...@gmail.com>
Authored: Thu Dec 19 15:33:37 2013 -0500
Committer: Eric Newton <er...@gmail.com>
Committed: Thu Dec 19 15:33:37 2013 -0500
----------------------------------------------------------------------
.../org/apache/accumulo/core/util/shell/ShellOptionsJC.java | 2 --
.../accumulo/server/master/state/MetaDataTableScanner.java | 8 +++-----
.../src/main/java/org/apache/accumulo/monitor/Monitor.java | 7 +++++++
3 files changed, 10 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
[03/13] git commit: ACCUMULO-2068 Remove deprecation warning as we're
preserving backwards compatibility for functionality that has existed for a
very long time.
Posted by ec...@apache.org.
ACCUMULO-2068 Remove deprecation warning as we're preserving backwards compatibility for functionality that has existed
for a very long time.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/ba0d7d34
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/ba0d7d34
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/ba0d7d34
Branch: refs/heads/master
Commit: ba0d7d34395140f147484198652a0c25bbf0f165
Parents: fda6d9c
Author: Josh Elser <el...@apache.org>
Authored: Thu Dec 19 15:01:52 2013 -0500
Committer: Josh Elser <el...@apache.org>
Committed: Thu Dec 19 15:01:52 2013 -0500
----------------------------------------------------------------------
.../java/org/apache/accumulo/core/util/shell/ShellOptionsJC.java | 2 --
1 file changed, 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/ba0d7d34/core/src/main/java/org/apache/accumulo/core/util/shell/ShellOptionsJC.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/util/shell/ShellOptionsJC.java b/core/src/main/java/org/apache/accumulo/core/util/shell/ShellOptionsJC.java
index 547da48..85e77e0 100644
--- a/core/src/main/java/org/apache/accumulo/core/util/shell/ShellOptionsJC.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/shell/ShellOptionsJC.java
@@ -107,8 +107,6 @@ public class ShellOptionsJC {
}
}
- // No match found, assume it is the password for compatibility
- log.warn("Specifying a raw password is deprecated.");
return value;
}
}
[08/13] git commit: ACCUMULO-2057 handle duplicate last locations
gracefully
Posted by ec...@apache.org.
ACCUMULO-2057 handle duplicate last locations gracefully
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/76842b29
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/76842b29
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/76842b29
Branch: refs/heads/1.5.1-SNAPSHOT
Commit: 76842b29458571fdd66e5b3440f7fd0075d10542
Parents: a480f63 2235b50
Author: Eric Newton <er...@gmail.com>
Authored: Thu Dec 19 15:31:59 2013 -0500
Committer: Eric Newton <er...@gmail.com>
Committed: Thu Dec 19 15:31:59 2013 -0500
----------------------------------------------------------------------
.../accumulo/server/master/state/MetaDataTableScanner.java | 8 +++-----
1 file changed, 3 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/76842b29/server/src/main/java/org/apache/accumulo/server/master/state/MetaDataTableScanner.java
----------------------------------------------------------------------
diff --cc server/src/main/java/org/apache/accumulo/server/master/state/MetaDataTableScanner.java
index bf1da22,0000000..2458a07
mode 100644,000000..100644
--- a/server/src/main/java/org/apache/accumulo/server/master/state/MetaDataTableScanner.java
+++ b/server/src/main/java/org/apache/accumulo/server/master/state/MetaDataTableScanner.java
@@@ -1,185 -1,0 +1,183 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.master.state;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map.Entry;
+import java.util.SortedMap;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.BatchScanner;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.ScannerBase;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.user.WholeRowIterator;
+import org.apache.accumulo.core.security.CredentialHelper;
+import org.apache.accumulo.core.security.thrift.TCredentials;
+import org.apache.accumulo.server.master.state.TabletLocationState.BadLocationStateException;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Logger;
+
+public class MetaDataTableScanner implements Iterator<TabletLocationState> {
+ private static final Logger log = Logger.getLogger(MetaDataTableScanner.class);
+
+ BatchScanner mdScanner;
+ Iterator<Entry<Key,Value>> iter;
+
+ public MetaDataTableScanner(Instance instance, TCredentials auths, Range range, CurrentState state) {
+ // scan over metadata table, looking for tablets in the wrong state based on the live servers and online tables
+ try {
+ Connector connector = instance.getConnector(auths.getPrincipal(), CredentialHelper.extractToken(auths));
+ mdScanner = connector.createBatchScanner(Constants.METADATA_TABLE_NAME, Constants.NO_AUTHS, 8);
+ configureScanner(mdScanner, state);
+ mdScanner.setRanges(Collections.singletonList(range));
+ iter = mdScanner.iterator();
+ } catch (Exception ex) {
+ mdScanner.close();
+ throw new RuntimeException(ex);
+ }
+ }
+
+ static public void configureScanner(ScannerBase scanner, CurrentState state) {
+ Constants.METADATA_PREV_ROW_COLUMN.fetch(scanner);
+ scanner.fetchColumnFamily(Constants.METADATA_CURRENT_LOCATION_COLUMN_FAMILY);
+ scanner.fetchColumnFamily(Constants.METADATA_FUTURE_LOCATION_COLUMN_FAMILY);
+ scanner.fetchColumnFamily(Constants.METADATA_LAST_LOCATION_COLUMN_FAMILY);
+ scanner.fetchColumnFamily(Constants.METADATA_LOG_COLUMN_FAMILY);
+ scanner.fetchColumnFamily(Constants.METADATA_CHOPPED_COLUMN_FAMILY);
+ scanner.addScanIterator(new IteratorSetting(1000, "wholeRows", WholeRowIterator.class));
+ IteratorSetting tabletChange = new IteratorSetting(1001, "tabletChange", TabletStateChangeIterator.class);
+ if (state != null) {
+ TabletStateChangeIterator.setCurrentServers(tabletChange, state.onlineTabletServers());
+ TabletStateChangeIterator.setOnlineTables(tabletChange, state.onlineTables());
+ TabletStateChangeIterator.setMerges(tabletChange, state.merges());
+ }
+ scanner.addScanIterator(tabletChange);
+ }
+
+ public MetaDataTableScanner(Instance instance, TCredentials auths, Range range) {
+ this(instance, auths, range, null);
+ }
+
+ public void close() {
+ if (iter != null) {
+ mdScanner.close();
+ iter = null;
+ }
+ }
+
+ public void finalize() {
+ close();
+ }
+
+ @Override
+ public boolean hasNext() {
+ if (iter == null)
+ return false;
+ boolean result = iter.hasNext();
+ if (!result) {
+ close();
+ }
+ return result;
+ }
+
+ @Override
+ public TabletLocationState next() {
+ try {
+ return fetch();
+ } catch (RuntimeException ex) {
+ // something is wrong with the records in the !METADATA table, just skip over it
+ log.error(ex, ex);
+ mdScanner.close();
+ return null;
+ }
+ }
+
+ public static TabletLocationState createTabletLocationState(Key k, Value v) throws IOException, BadLocationStateException {
+ final SortedMap<Key,Value> decodedRow = WholeRowIterator.decodeRow(k, v);
+ KeyExtent extent = null;
+ TServerInstance future = null;
+ TServerInstance current = null;
+ TServerInstance last = null;
++ long lastTimestamp = 0;
+ List<Collection<String>> walogs = new ArrayList<Collection<String>>();
+ boolean chopped = false;
+
+ for (Entry<Key,Value> entry : decodedRow.entrySet()) {
+ Key key = entry.getKey();
+ Text row = key.getRow();
+ Text cf = key.getColumnFamily();
+ Text cq = key.getColumnQualifier();
+
+ if (cf.compareTo(Constants.METADATA_FUTURE_LOCATION_COLUMN_FAMILY) == 0) {
+ TServerInstance location = new TServerInstance(entry.getValue(), cq);
+ if (future != null) {
+ throw new BadLocationStateException("found two assignments for the same extent " + key.getRow() + ": " + future + " and " + location);
+ }
+ future = location;
+ } else if (cf.compareTo(Constants.METADATA_CURRENT_LOCATION_COLUMN_FAMILY) == 0) {
+ TServerInstance location = new TServerInstance(entry.getValue(), cq);
+ if (current != null) {
+ throw new BadLocationStateException("found two locations for the same extent " + key.getRow() + ": " + current + " and " + location);
+ }
+ current = location;
+ } else if (cf.compareTo(Constants.METADATA_LOG_COLUMN_FAMILY) == 0) {
+ String[] split = entry.getValue().toString().split("\\|")[0].split(";");
+ walogs.add(Arrays.asList(split));
+ } else if (cf.compareTo(Constants.METADATA_LAST_LOCATION_COLUMN_FAMILY) == 0) {
- TServerInstance location = new TServerInstance(entry.getValue(), cq);
- if (last != null) {
- throw new BadLocationStateException("found two last locations for the same extent " + key.getRow() + ": " + last + " and " + location);
- }
- last = new TServerInstance(entry.getValue(), cq);
++ if (lastTimestamp < entry.getKey().getTimestamp())
++ last = new TServerInstance(entry.getValue(), cq);
+ } else if (cf.compareTo(Constants.METADATA_CHOPPED_COLUMN_FAMILY) == 0) {
+ chopped = true;
+ } else if (Constants.METADATA_PREV_ROW_COLUMN.equals(cf, cq)) {
+ extent = new KeyExtent(row, entry.getValue());
+ }
+ }
+ if (extent == null) {
+ log.warn("No prev-row for key extent: " + decodedRow);
+ return null;
+ }
+ return new TabletLocationState(extent, future, current, last, walogs, chopped);
+ }
+
+ private TabletLocationState fetch() {
+ try {
+ Entry<Key,Value> e = iter.next();
+ return createTabletLocationState(e.getKey(), e.getValue());
+ } catch (IOException ex) {
+ throw new RuntimeException(ex);
+ } catch (BadLocationStateException ex) {
+ throw new RuntimeException(ex);
+ }
+ }
+
+ @Override
+ public void remove() {
+ throw new RuntimeException("Unimplemented");
+ }
+}
[06/13] git commit: ACCUMULO-2057 handle multiple last locations
gracefully
Posted by ec...@apache.org.
ACCUMULO-2057 handle multiple last locations gracefully
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/2235b503
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/2235b503
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/2235b503
Branch: refs/heads/1.4.5-SNAPSHOT
Commit: 2235b5038a85147cae3495d4d0368e0bd9f38098
Parents: 40df8cd
Author: Eric Newton <er...@gmail.com>
Authored: Thu Dec 19 15:00:22 2013 -0500
Committer: Eric Newton <er...@gmail.com>
Committed: Thu Dec 19 15:17:02 2013 -0500
----------------------------------------------------------------------
.../accumulo/server/master/state/MetaDataTableScanner.java | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/2235b503/src/server/src/main/java/org/apache/accumulo/server/master/state/MetaDataTableScanner.java
----------------------------------------------------------------------
diff --git a/src/server/src/main/java/org/apache/accumulo/server/master/state/MetaDataTableScanner.java b/src/server/src/main/java/org/apache/accumulo/server/master/state/MetaDataTableScanner.java
index e10b1b3..157cb76 100644
--- a/src/server/src/main/java/org/apache/accumulo/server/master/state/MetaDataTableScanner.java
+++ b/src/server/src/main/java/org/apache/accumulo/server/master/state/MetaDataTableScanner.java
@@ -127,6 +127,7 @@ public class MetaDataTableScanner implements Iterator<TabletLocationState> {
TServerInstance future = null;
TServerInstance current = null;
TServerInstance last = null;
+ long lastTimestamp = 0;
List<Collection<String>> walogs = new ArrayList<Collection<String>>();
boolean chopped = false;
@@ -144,7 +145,8 @@ public class MetaDataTableScanner implements Iterator<TabletLocationState> {
String[] split = entry.getValue().toString().split("\\|")[0].split(";");
walogs.add(Arrays.asList(split));
} else if (cf.compareTo(Constants.METADATA_LAST_LOCATION_COLUMN_FAMILY) == 0) {
- last = new TServerInstance(entry.getValue(), cq);
+ if (lastTimestamp < entry.getKey().getTimestamp())
+ last = new TServerInstance(entry.getValue(), cq);
} else if (cf.compareTo(Constants.METADATA_CHOPPED_COLUMN_FAMILY) == 0) {
chopped = true;
} else if (Constants.METADATA_PREV_ROW_COLUMN.equals(cf, cq)) {
[05/13] git commit: ACCUMULO-2057 handle multiple last locations
gracefully
Posted by ec...@apache.org.
ACCUMULO-2057 handle multiple last locations gracefully
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/2235b503
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/2235b503
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/2235b503
Branch: refs/heads/1.5.1-SNAPSHOT
Commit: 2235b5038a85147cae3495d4d0368e0bd9f38098
Parents: 40df8cd
Author: Eric Newton <er...@gmail.com>
Authored: Thu Dec 19 15:00:22 2013 -0500
Committer: Eric Newton <er...@gmail.com>
Committed: Thu Dec 19 15:17:02 2013 -0500
----------------------------------------------------------------------
.../accumulo/server/master/state/MetaDataTableScanner.java | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/2235b503/src/server/src/main/java/org/apache/accumulo/server/master/state/MetaDataTableScanner.java
----------------------------------------------------------------------
diff --git a/src/server/src/main/java/org/apache/accumulo/server/master/state/MetaDataTableScanner.java b/src/server/src/main/java/org/apache/accumulo/server/master/state/MetaDataTableScanner.java
index e10b1b3..157cb76 100644
--- a/src/server/src/main/java/org/apache/accumulo/server/master/state/MetaDataTableScanner.java
+++ b/src/server/src/main/java/org/apache/accumulo/server/master/state/MetaDataTableScanner.java
@@ -127,6 +127,7 @@ public class MetaDataTableScanner implements Iterator<TabletLocationState> {
TServerInstance future = null;
TServerInstance current = null;
TServerInstance last = null;
+ long lastTimestamp = 0;
List<Collection<String>> walogs = new ArrayList<Collection<String>>();
boolean chopped = false;
@@ -144,7 +145,8 @@ public class MetaDataTableScanner implements Iterator<TabletLocationState> {
String[] split = entry.getValue().toString().split("\\|")[0].split(";");
walogs.add(Arrays.asList(split));
} else if (cf.compareTo(Constants.METADATA_LAST_LOCATION_COLUMN_FAMILY) == 0) {
- last = new TServerInstance(entry.getValue(), cq);
+ if (lastTimestamp < entry.getKey().getTimestamp())
+ last = new TServerInstance(entry.getValue(), cq);
} else if (cf.compareTo(Constants.METADATA_CHOPPED_COLUMN_FAMILY) == 0) {
chopped = true;
} else if (Constants.METADATA_PREV_ROW_COLUMN.equals(cf, cq)) {
[10/13] git commit: ACCUMULO-2057 handle duplicate last locations
gracefully
Posted by ec...@apache.org.
ACCUMULO-2057 handle duplicate last locations gracefully
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/76842b29
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/76842b29
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/76842b29
Branch: refs/heads/master
Commit: 76842b29458571fdd66e5b3440f7fd0075d10542
Parents: a480f63 2235b50
Author: Eric Newton <er...@gmail.com>
Authored: Thu Dec 19 15:31:59 2013 -0500
Committer: Eric Newton <er...@gmail.com>
Committed: Thu Dec 19 15:31:59 2013 -0500
----------------------------------------------------------------------
.../accumulo/server/master/state/MetaDataTableScanner.java | 8 +++-----
1 file changed, 3 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/76842b29/server/src/main/java/org/apache/accumulo/server/master/state/MetaDataTableScanner.java
----------------------------------------------------------------------
diff --cc server/src/main/java/org/apache/accumulo/server/master/state/MetaDataTableScanner.java
index bf1da22,0000000..2458a07
mode 100644,000000..100644
--- a/server/src/main/java/org/apache/accumulo/server/master/state/MetaDataTableScanner.java
+++ b/server/src/main/java/org/apache/accumulo/server/master/state/MetaDataTableScanner.java
@@@ -1,185 -1,0 +1,183 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.master.state;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map.Entry;
+import java.util.SortedMap;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.BatchScanner;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.ScannerBase;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.user.WholeRowIterator;
+import org.apache.accumulo.core.security.CredentialHelper;
+import org.apache.accumulo.core.security.thrift.TCredentials;
+import org.apache.accumulo.server.master.state.TabletLocationState.BadLocationStateException;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Logger;
+
+public class MetaDataTableScanner implements Iterator<TabletLocationState> {
+ private static final Logger log = Logger.getLogger(MetaDataTableScanner.class);
+
+ BatchScanner mdScanner;
+ Iterator<Entry<Key,Value>> iter;
+
+ public MetaDataTableScanner(Instance instance, TCredentials auths, Range range, CurrentState state) {
+ // scan over metadata table, looking for tablets in the wrong state based on the live servers and online tables
+ try {
+ Connector connector = instance.getConnector(auths.getPrincipal(), CredentialHelper.extractToken(auths));
+ mdScanner = connector.createBatchScanner(Constants.METADATA_TABLE_NAME, Constants.NO_AUTHS, 8);
+ configureScanner(mdScanner, state);
+ mdScanner.setRanges(Collections.singletonList(range));
+ iter = mdScanner.iterator();
+ } catch (Exception ex) {
+ mdScanner.close();
+ throw new RuntimeException(ex);
+ }
+ }
+
+ static public void configureScanner(ScannerBase scanner, CurrentState state) {
+ Constants.METADATA_PREV_ROW_COLUMN.fetch(scanner);
+ scanner.fetchColumnFamily(Constants.METADATA_CURRENT_LOCATION_COLUMN_FAMILY);
+ scanner.fetchColumnFamily(Constants.METADATA_FUTURE_LOCATION_COLUMN_FAMILY);
+ scanner.fetchColumnFamily(Constants.METADATA_LAST_LOCATION_COLUMN_FAMILY);
+ scanner.fetchColumnFamily(Constants.METADATA_LOG_COLUMN_FAMILY);
+ scanner.fetchColumnFamily(Constants.METADATA_CHOPPED_COLUMN_FAMILY);
+ scanner.addScanIterator(new IteratorSetting(1000, "wholeRows", WholeRowIterator.class));
+ IteratorSetting tabletChange = new IteratorSetting(1001, "tabletChange", TabletStateChangeIterator.class);
+ if (state != null) {
+ TabletStateChangeIterator.setCurrentServers(tabletChange, state.onlineTabletServers());
+ TabletStateChangeIterator.setOnlineTables(tabletChange, state.onlineTables());
+ TabletStateChangeIterator.setMerges(tabletChange, state.merges());
+ }
+ scanner.addScanIterator(tabletChange);
+ }
+
+ public MetaDataTableScanner(Instance instance, TCredentials auths, Range range) {
+ this(instance, auths, range, null);
+ }
+
+ public void close() {
+ if (iter != null) {
+ mdScanner.close();
+ iter = null;
+ }
+ }
+
+ public void finalize() {
+ close();
+ }
+
+ @Override
+ public boolean hasNext() {
+ if (iter == null)
+ return false;
+ boolean result = iter.hasNext();
+ if (!result) {
+ close();
+ }
+ return result;
+ }
+
+ @Override
+ public TabletLocationState next() {
+ try {
+ return fetch();
+ } catch (RuntimeException ex) {
+ // something is wrong with the records in the !METADATA table, just skip over it
+ log.error(ex, ex);
+ mdScanner.close();
+ return null;
+ }
+ }
+
+ public static TabletLocationState createTabletLocationState(Key k, Value v) throws IOException, BadLocationStateException {
+ final SortedMap<Key,Value> decodedRow = WholeRowIterator.decodeRow(k, v);
+ KeyExtent extent = null;
+ TServerInstance future = null;
+ TServerInstance current = null;
+ TServerInstance last = null;
++ long lastTimestamp = 0;
+ List<Collection<String>> walogs = new ArrayList<Collection<String>>();
+ boolean chopped = false;
+
+ for (Entry<Key,Value> entry : decodedRow.entrySet()) {
+ Key key = entry.getKey();
+ Text row = key.getRow();
+ Text cf = key.getColumnFamily();
+ Text cq = key.getColumnQualifier();
+
+ if (cf.compareTo(Constants.METADATA_FUTURE_LOCATION_COLUMN_FAMILY) == 0) {
+ TServerInstance location = new TServerInstance(entry.getValue(), cq);
+ if (future != null) {
+ throw new BadLocationStateException("found two assignments for the same extent " + key.getRow() + ": " + future + " and " + location);
+ }
+ future = location;
+ } else if (cf.compareTo(Constants.METADATA_CURRENT_LOCATION_COLUMN_FAMILY) == 0) {
+ TServerInstance location = new TServerInstance(entry.getValue(), cq);
+ if (current != null) {
+ throw new BadLocationStateException("found two locations for the same extent " + key.getRow() + ": " + current + " and " + location);
+ }
+ current = location;
+ } else if (cf.compareTo(Constants.METADATA_LOG_COLUMN_FAMILY) == 0) {
+ String[] split = entry.getValue().toString().split("\\|")[0].split(";");
+ walogs.add(Arrays.asList(split));
+ } else if (cf.compareTo(Constants.METADATA_LAST_LOCATION_COLUMN_FAMILY) == 0) {
- TServerInstance location = new TServerInstance(entry.getValue(), cq);
- if (last != null) {
- throw new BadLocationStateException("found two last locations for the same extent " + key.getRow() + ": " + last + " and " + location);
- }
- last = new TServerInstance(entry.getValue(), cq);
++ if (lastTimestamp < entry.getKey().getTimestamp())
++ last = new TServerInstance(entry.getValue(), cq);
+ } else if (cf.compareTo(Constants.METADATA_CHOPPED_COLUMN_FAMILY) == 0) {
+ chopped = true;
+ } else if (Constants.METADATA_PREV_ROW_COLUMN.equals(cf, cq)) {
+ extent = new KeyExtent(row, entry.getValue());
+ }
+ }
+ if (extent == null) {
+ log.warn("No prev-row for key extent: " + decodedRow);
+ return null;
+ }
+ return new TabletLocationState(extent, future, current, last, walogs, chopped);
+ }
+
+ private TabletLocationState fetch() {
+ try {
+ Entry<Key,Value> e = iter.next();
+ return createTabletLocationState(e.getKey(), e.getValue());
+ } catch (IOException ex) {
+ throw new RuntimeException(ex);
+ } catch (BadLocationStateException ex) {
+ throw new RuntimeException(ex);
+ }
+ }
+
+ @Override
+ public void remove() {
+ throw new RuntimeException("Unimplemented");
+ }
+}