You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by bi...@apache.org on 2015/04/11 02:15:38 UTC
[2/3] accumulo git commit: ACCUMULO-3715 adjust,
document and standardize span receiver configuration properties
ACCUMULO-3715 adjust, document and standardize span receiver configuration properties
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/7e72d563
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/7e72d563
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/7e72d563
Branch: refs/heads/master
Commit: 7e72d56331d84500d3887f8f304d1a0668f3aa89
Parents: 281c2f3
Author: Billie Rinaldi <bi...@gmail.com>
Authored: Wed Apr 8 11:54:22 2015 -0700
Committer: Billie Rinaldi <bi...@gmail.com>
Committed: Fri Apr 10 17:01:00 2015 -0700
----------------------------------------------------------------------
.../org/apache/accumulo/core/conf/Property.java | 2 +-
.../accumulo/core/trace/CountSampler.java | 3 +
.../accumulo/core/trace/DistributedTrace.java | 1 -
.../accumulo/core/trace/ProbabilitySampler.java | 3 +
.../main/asciidoc/chapters/administration.txt | 58 ++++---
.../accumulo/tracer/AsyncSpanReceiver.java | 15 +-
.../apache/accumulo/tracer/TraceTableStats.java | 157 +++++++++++++++++++
7 files changed, 217 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7e72d563/core/src/main/java/org/apache/accumulo/core/conf/Property.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
index f9d8662..08ba3a2 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@ -382,7 +382,7 @@ public enum Property {
TRACE_SPAN_RECEIVERS("trace.span.receivers", "org.apache.accumulo.tracer.ZooTraceClient", PropertyType.CLASSNAMELIST,
"A list of span receiver classes to send trace spans"),
TRACE_SPAN_RECEIVER_PREFIX("trace.span.receiver.", null, PropertyType.PREFIX, "Prefix for span receiver configuration properties"),
- TRACE_ZK_PATH("trace.span.receiver.zookeeper.path", Constants.ZTRACERS, PropertyType.STRING, "The zookeeper node where tracers are registered"),
+ TRACE_ZK_PATH("trace.zookeeper.path", Constants.ZTRACERS, PropertyType.STRING, "The zookeeper node where tracers are registered"),
TRACE_PORT("trace.port.client", "12234", PropertyType.PORT, "The listening port for the trace server"),
TRACE_TABLE("trace.table", "trace", PropertyType.STRING, "The name of the table to store distributed traces"),
TRACE_USER("trace.user", "root", PropertyType.STRING, "The name of the user to store distributed traces"),
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7e72d563/core/src/main/java/org/apache/accumulo/core/trace/CountSampler.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/trace/CountSampler.java b/core/src/main/java/org/apache/accumulo/core/trace/CountSampler.java
index 852fa9d..a4f206a 100644
--- a/core/src/main/java/org/apache/accumulo/core/trace/CountSampler.java
+++ b/core/src/main/java/org/apache/accumulo/core/trace/CountSampler.java
@@ -20,6 +20,9 @@ import org.apache.htrace.HTraceConfiguration;
import java.util.Collections;
+/**
+ * This wrapper intended for internal Accumulo tracing makes creating a CountSampler easier.
+ */
public class CountSampler extends org.apache.htrace.impl.CountSampler {
public CountSampler(long frequency) {
super(HTraceConfiguration.fromMap(Collections.singletonMap(CountSampler.SAMPLER_FREQUENCY_CONF_KEY, Long.toString(frequency))));
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7e72d563/core/src/main/java/org/apache/accumulo/core/trace/DistributedTrace.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/trace/DistributedTrace.java b/core/src/main/java/org/apache/accumulo/core/trace/DistributedTrace.java
index c5b6eac..14886f1 100644
--- a/core/src/main/java/org/apache/accumulo/core/trace/DistributedTrace.java
+++ b/core/src/main/java/org/apache/accumulo/core/trace/DistributedTrace.java
@@ -47,7 +47,6 @@ public class DistributedTrace {
public static final String TRACE_HOST_PROPERTY = "trace.host";
public static final String TRACE_SERVICE_PROPERTY = "trace.service";
- public static final String TRACE_QUEUE_SIZE_PROPERTY = "trace.queue.size";
public static final String TRACER_ZK_HOST = "tracer.zookeeper.host";
public static final String TRACER_ZK_TIMEOUT = "tracer.zookeeper.timeout";
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7e72d563/core/src/main/java/org/apache/accumulo/core/trace/ProbabilitySampler.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/trace/ProbabilitySampler.java b/core/src/main/java/org/apache/accumulo/core/trace/ProbabilitySampler.java
index ff02fd7..7204e38 100644
--- a/core/src/main/java/org/apache/accumulo/core/trace/ProbabilitySampler.java
+++ b/core/src/main/java/org/apache/accumulo/core/trace/ProbabilitySampler.java
@@ -20,6 +20,9 @@ import org.apache.htrace.HTraceConfiguration;
import java.util.Collections;
+/**
+ * This wrapper intended for internal Accumulo tracing makes creating a ProbabilitySampler easier.
+ */
public class ProbabilitySampler extends org.apache.htrace.impl.ProbabilitySampler {
public ProbabilitySampler(double d) {
super(HTraceConfiguration.fromMap(Collections.singletonMap(ProbabilitySampler.SAMPLER_FRACTION_CONF_KEY, Double.toString(d))));
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7e72d563/docs/src/main/asciidoc/chapters/administration.txt
----------------------------------------------------------------------
diff --git a/docs/src/main/asciidoc/chapters/administration.txt b/docs/src/main/asciidoc/chapters/administration.txt
index 0382934..520a07c 100644
--- a/docs/src/main/asciidoc/chapters/administration.txt
+++ b/docs/src/main/asciidoc/chapters/administration.txt
@@ -495,8 +495,14 @@ the following properties
Other tracer configuration properties include
- trace.port.client
- trace.table
+ trace.port.client - port tracer listens on
+ trace.table - table tracer writes to
+ trace.zookeeper.path - zookeeper path where tracers register
+
+The zookeeper path is configured to /tracers by default. If
+multiple Accumulo instances are sharing the same ZooKeeper
+quorum, take care to configure Accumulo with unique values for
+this property.
==== Configuring Tracing
Traces are collected via SpanReceivers. The default SpanReceiver
@@ -510,36 +516,52 @@ comma-separated list, by modifying the property
Individual span receivers may require their own configuration
parameters, which are grouped under the trace.span.receiver.*
-prefix. The ZooTraceClient requires the following property that
-indicates where the tracer servers will register themselves in
-ZooKeeper.
-
- trace.span.receiver.zookeeper.path
-
-This is configured to /tracers by default. If multiple Accumulo
-instances are sharing the same ZooKeeper quorum, take care to
-configure Accumulo with unique values for this property.
+prefix. ZooTraceClient uses the following properties. The first
+three properties are populated from other Accumulo properties,
+while the remaining ones should be prefixed with
+trace.span.receiver. when set in the Accumulo configuration.
+
+ tracer.zookeeper.host - populated from instance.zookeepers
+ tracer.zookeeper.timeout - populated from instance.zookeeper.timeout
+ tracer.zookeeper.path - populated from trace.zookeeper.path
+ tracer.send.timer.millis - timer for flushing send queue (in ms, default 1000)
+ tracer.queue.size - max queue size (default 5000)
+ tracer.span.min.ms - minimum span length to store (in ms, default 1)
+
+Note that to configure an Accumulo client for tracing, including
+the Accumulo shell, the client configuration must be given the same
+trace.span.receivers, trace.span.receiver.*, and trace.zookeeper.path
+properties as the servers have.
Hadoop can also be configured to send traces to Accumulo, as of
-Hadoop 2.6.0, by setting the following properties in Hadoop's
-core-site.xml file (the path property is optional if left as the
-default).
+Hadoop 2.6.0, by setting properties in Hadoop's core-site.xml
+file. Instead of using the trace.span.receiver.* prefix, Hadoop
+uses hadoop.htrace.*. The Hadoop configuration does not have
+access to Accumulo's properties, so the
+hadoop.htrace.tracer.zookeeper.host property must be specified.
+The zookeeper timeout defaults to 30000 (30 seconds), and the
+zookeeper path defaults to /tracers. An example of configuring
+Hadoop to send traces to ZooTraceClient is
<property>
<name>hadoop.htrace.spanreceiver.classes</name>
<value>org.apache.accumulo.core.trace.ZooTraceClient</value>
</property>
<property>
- <name>hadoop.tracer.zookeeper.host</name>
+ <name>hadoop.htrace.tracer.zookeeper.host</name>
<value>zookeeperHost:2181</value>
</property>
<property>
- <name>hadoop.tracer.zookeeper.path</name>
+ <name>hadoop.htrace.tracer.zookeeper.path</name>
<value>/tracers</value>
</property>
+ <property>
+ <name>hadoop.htrace.tracer.span.min.ms</name>
+ <value>1</value>
+ </property>
-The accumulo-core, accumulo-trace, and libthrift jars must also
-be placed on Hadoop's classpath.
+The accumulo-core, accumulo-tracer, accumulo-fate and libthrift
+jars must also be placed on Hadoop's classpath.
==== Instrumenting a Client
Tracing can be used to measure a client operation, such as a scan, as
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7e72d563/server/tracer/src/main/java/org/apache/accumulo/tracer/AsyncSpanReceiver.java
----------------------------------------------------------------------
diff --git a/server/tracer/src/main/java/org/apache/accumulo/tracer/AsyncSpanReceiver.java b/server/tracer/src/main/java/org/apache/accumulo/tracer/AsyncSpanReceiver.java
index dbcb335..34a75d5 100644
--- a/server/tracer/src/main/java/org/apache/accumulo/tracer/AsyncSpanReceiver.java
+++ b/server/tracer/src/main/java/org/apache/accumulo/tracer/AsyncSpanReceiver.java
@@ -51,7 +51,9 @@ public abstract class AsyncSpanReceiver<SpanKey,Destination> implements SpanRece
private static final Logger log = LoggerFactory.getLogger(AsyncSpanReceiver.class);
- public static final String SEND_TIMER_MILLIS = "send.timer.millis";
+ public static final String SEND_TIMER_MILLIS = "tracer.send.timer.millis";
+ public static final String QUEUE_SIZE = "tracer.queue.size";
+ private static final String SPAN_MIN_MS = "tracer.span.min.ms";
private final Map<SpanKey,Destination> clients = new HashMap<SpanKey,Destination>();
@@ -68,6 +70,7 @@ public abstract class AsyncSpanReceiver<SpanKey,Destination> implements SpanRece
protected final AbstractQueue<RemoteSpan> sendQueue = new ConcurrentLinkedQueue<RemoteSpan>();
int maxQueueSize = 5000;
long lastNotificationOfDroppedSpans = 0;
+ int minSpanSize = 1;
// Visible for testing
AsyncSpanReceiver() {}
@@ -82,7 +85,8 @@ public abstract class AsyncSpanReceiver<SpanKey,Destination> implements SpanRece
}
}
service = conf.get(DistributedTrace.TRACE_SERVICE_PROPERTY, service);
- maxQueueSize = conf.getInt(DistributedTrace.TRACE_QUEUE_SIZE_PROPERTY, maxQueueSize);
+ maxQueueSize = conf.getInt(QUEUE_SIZE, maxQueueSize);
+ minSpanSize = conf.getInt(SPAN_MIN_MS, minSpanSize);
int millis = conf.getInt(SEND_TIMER_MILLIS, 1000);
timer.schedule(new TimerTask() {
@@ -102,6 +106,13 @@ public abstract class AsyncSpanReceiver<SpanKey,Destination> implements SpanRece
while (!sendQueue.isEmpty()) {
boolean sent = false;
RemoteSpan s = sendQueue.peek();
+ if (s.stop - s.start < minSpanSize) {
+ synchronized (sendQueue) {
+ sendQueue.remove();
+ sendQueue.notifyAll();
+ }
+ continue;
+ }
SpanKey dest = getSpanKey(s.data);
Destination client = clients.get(dest);
if (client == null) {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7e72d563/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceTableStats.java
----------------------------------------------------------------------
diff --git a/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceTableStats.java b/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceTableStats.java
new file mode 100644
index 0000000..63c806f
--- /dev/null
+++ b/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceTableStats.java
@@ -0,0 +1,157 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.tracer;
+
+import org.apache.accumulo.core.cli.ClientOnDefaultTable;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.tracer.thrift.RemoteSpan;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.TreeMap;
+
+/**
+ * Reads the trace table and prints out some stats about the spans found.
+ */
+public class TraceTableStats {
+ static class Opts extends ClientOnDefaultTable {
+ public Opts() {
+ super("trace");
+ }
+ }
+
+ static class SpanTypeCount {
+ String type;
+ long nonzeroCount = 0l;
+ long zeroCount = 0l;
+ ArrayList<Long> log10SpanLength = new ArrayList<>();
+ Set<Long> traceIds = new HashSet<>();
+
+ public SpanTypeCount() {
+ for (int i = 0; i < 7; i++)
+ log10SpanLength.add(0l);
+ }
+
+ @Override
+ public String toString() {
+ return "{" +
+ "type='" + type + '\'' +
+ ", nonzeroCount=" + nonzeroCount +
+ ", zeroCount=" + zeroCount +
+ ", numTraces=" + traceIds.size() +
+ ", log10SpanLength=" + log10SpanLength +
+ '}';
+ }
+ }
+
+ public static void main(String[] args) throws Exception {
+ TraceTableStats stats = new TraceTableStats();
+ Opts opts = new Opts();
+ opts.parseArgs(TraceTableStats.class.getName(), args);
+ stats.count(opts);
+ }
+
+ public void count(Opts opts) throws AccumuloSecurityException, AccumuloException, TableNotFoundException {
+ Connector conn = opts.getConnector();
+ Scanner scanner = conn.createScanner(opts.getTableName(), Authorizations.EMPTY);
+ scanner.setRange(new Range(null, true, "idx:", false));
+ Map<String, SpanTypeCount> counts = new TreeMap<>();
+ final SpanTypeCount hdfs = new SpanTypeCount();
+ hdfs.type = "HDFS";
+ final SpanTypeCount accumulo = new SpanTypeCount();
+ accumulo.type = "Accumulo";
+ long numSpans = 0;
+ double maxSpanLength = 0;
+ double maxSpanLengthMS = 0;
+ for (Entry<Key,Value> entry : scanner) {
+ numSpans++;
+ RemoteSpan span = TraceFormatter.getRemoteSpan(entry);
+ String id = span.getSvc()+":"+span.getDescription().replaceAll("[0-9][0-9][0-9]+", "");
+ SpanTypeCount stc = counts.get(id);
+ if (stc == null) {
+ stc = new SpanTypeCount();
+ counts.put(id, stc);
+ if (span.description.startsWith("org.apache.hadoop") || span.svc.equals("NameNode") || span.svc.equals("DataNode")
+ || span.description.contains("DFSOutputStream") || span.description.contains("DFSInputStream")
+ || span.description.contains("BlockReader")) {
+ stc.type = hdfs.type;
+ } else {
+ stc.type = accumulo.type;
+ }
+ }
+ increment(stc, span);
+ if (stc.type.equals(hdfs.type)) {
+ increment(hdfs, span);
+ } else {
+ increment(accumulo, span);
+ }
+ maxSpanLength = Math.max(maxSpanLength, Math.log10(span.stop-span.start));
+ maxSpanLengthMS = Math.max(maxSpanLengthMS, span.stop-span.start);
+ }
+ System.out.println();
+ System.out.println("log10 max span length "+maxSpanLength+" "+maxSpanLengthMS);
+ System.out.println("Total spans "+numSpans);
+ System.out.println("Percentage Accumulo nonzero of total " + accumulo.nonzeroCount + "/" + numSpans + " = " + (accumulo.nonzeroCount * 1.0 / numSpans));
+ System.out.println(hdfs + ", total " + (hdfs.nonzeroCount + hdfs.zeroCount));
+ System.out.println(accumulo+ ", total " + (accumulo.nonzeroCount + accumulo.zeroCount));
+ System.out.println();
+ System.out.println("source:desc={stats}");
+ for (Entry<String, SpanTypeCount> c : counts.entrySet()) {
+ System.out.println(c);
+ }
+ }
+
+ private static void increment(SpanTypeCount stc, RemoteSpan span) {
+ stc.traceIds.add(span.getTraceId());
+ if (span.stop == span.start) {
+ stc.zeroCount++;
+ incrementIndex(stc.log10SpanLength, 0);
+ } else {
+ stc.nonzeroCount++;
+ long ms = span.stop - span.start;
+ if (ms <= 10)
+ incrementIndex(stc.log10SpanLength, 1);
+ else if (ms <= 100)
+ incrementIndex(stc.log10SpanLength, 2);
+ else if (ms <= 1000)
+ incrementIndex(stc.log10SpanLength, 3);
+ else if (ms <= 10000)
+ incrementIndex(stc.log10SpanLength, 4);
+ else if (ms <= 100000)
+ incrementIndex(stc.log10SpanLength, 5);
+ else if (ms <= 1000000)
+ incrementIndex(stc.log10SpanLength, 6);
+ else
+ throw new IllegalArgumentException("unexpected span length "+ms);
+ }
+ }
+
+ private static void incrementIndex(ArrayList<Long> hist, int i) {
+ hist.set(i, hist.get(i) + 1);
+ }
+}