You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by zj...@apache.org on 2015/06/03 01:47:51 UTC
[19/50] [abbrv] hadoop git commit: HADOOP-11894. Bump the version of
Apache HTrace to 3.2.0-incubating (Masatake Iwasaki via Colin P. McCabe)
HADOOP-11894. Bump the version of Apache HTrace to 3.2.0-incubating (Masatake Iwasaki via Colin P. McCabe)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/81f0adb7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/81f0adb7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/81f0adb7
Branch: refs/heads/YARN-2928
Commit: 81f0adb7c4c8b2310e2bba02643b76544a0d1d9c
Parents: 3d5c0d4
Author: Colin Patrick Mccabe <cm...@cloudera.com>
Authored: Thu May 28 12:00:55 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Jun 2 16:12:55 2015 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 ++
.../apache/hadoop/tracing/SpanReceiverHost.java | 45 ++------------------
.../hadoop-common/src/site/markdown/Tracing.md | 12 +++---
.../java/org/apache/hadoop/hdfs/DFSClient.java | 16 ++-----
hadoop-project/pom.xml | 2 +-
5 files changed, 18 insertions(+), 60 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/81f0adb7/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 51eff78..2b85c1c 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -620,6 +620,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-12030. test-patch should only report on newly introduced
findbugs warnings. (Sean Busbey via aw)
+ HADOOP-11894. Bump the version of Apache HTrace to 3.2.0-incubating
+ (Masatake Iwasaki via Colin P. McCabe)
+
OPTIMIZATIONS
HADOOP-11785. Reduce the number of listStatus operation in distcp
http://git-wip-us.apache.org/repos/asf/hadoop/blob/81f0adb7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tracing/SpanReceiverHost.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tracing/SpanReceiverHost.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tracing/SpanReceiverHost.java
index bf9479b..9a99e05 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tracing/SpanReceiverHost.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tracing/SpanReceiverHost.java
@@ -17,33 +17,23 @@
*/
package org.apache.hadoop.tracing;
-import java.io.BufferedReader;
-import java.io.DataInputStream;
-import java.io.EOFException;
-import java.io.File;
-import java.io.FileInputStream;
import java.io.IOException;
-import java.io.InputStreamReader;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
-import java.util.UUID;
-import org.apache.commons.io.Charsets;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.tracing.SpanReceiverInfo.ConfigurationPair;
-import org.apache.hadoop.tracing.TraceUtils;
-import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.ShutdownHookManager;
import org.apache.htrace.SpanReceiver;
import org.apache.htrace.SpanReceiverBuilder;
import org.apache.htrace.Trace;
+import org.apache.htrace.impl.LocalFileSpanReceiver;
/**
* This class provides functions for reading the names of SpanReceivers from
@@ -54,7 +44,7 @@ import org.apache.htrace.Trace;
@InterfaceAudience.Private
public class SpanReceiverHost implements TraceAdminProtocol {
public static final String SPAN_RECEIVERS_CONF_SUFFIX =
- "spanreceiver.classes";
+ "spanreceiver.classes";
private static final Log LOG = LogFactory.getLog(SpanReceiverHost.class);
private static final HashMap<String, SpanReceiverHost> hosts =
new HashMap<String, SpanReceiverHost>(1);
@@ -88,33 +78,6 @@ public class SpanReceiverHost implements TraceAdminProtocol {
private static List<ConfigurationPair> EMPTY = Collections.emptyList();
- private static String getUniqueLocalTraceFileName() {
- String tmp = System.getProperty("java.io.tmpdir", "/tmp");
- String nonce = null;
- BufferedReader reader = null;
- try {
- // On Linux we can get a unique local file name by reading the process id
- // out of /proc/self/stat. (There isn't any portable way to get the
- // process ID from Java.)
- reader = new BufferedReader(
- new InputStreamReader(new FileInputStream("/proc/self/stat"),
- Charsets.UTF_8));
- String line = reader.readLine();
- if (line == null) {
- throw new EOFException();
- }
- nonce = line.split(" ")[0];
- } catch (IOException e) {
- } finally {
- IOUtils.cleanup(LOG, reader);
- }
- if (nonce == null) {
- // If we can't use the process ID, use a random nonce.
- nonce = UUID.randomUUID().toString();
- }
- return new File(tmp, nonce).getAbsolutePath();
- }
-
private SpanReceiverHost(String confPrefix) {
this.confPrefix = confPrefix;
}
@@ -143,7 +106,7 @@ public class SpanReceiverHost implements TraceAdminProtocol {
// testing.
String pathKey = confPrefix + LOCAL_FILE_SPAN_RECEIVER_PATH_SUFFIX;
if (config.get(pathKey) == null) {
- String uniqueFile = getUniqueLocalTraceFileName();
+ String uniqueFile = LocalFileSpanReceiver.getUniqueLocalTraceFileName();
config.set(pathKey, uniqueFile);
if (LOG.isTraceEnabled()) {
LOG.trace("Set " + pathKey + " to " + uniqueFile);
@@ -191,7 +154,7 @@ public class SpanReceiverHost implements TraceAdminProtocol {
public synchronized SpanReceiverInfo[] listSpanReceivers()
throws IOException {
- SpanReceiverInfo info[] = new SpanReceiverInfo[receivers.size()];
+ SpanReceiverInfo[] info = new SpanReceiverInfo[receivers.size()];
int i = 0;
for(Map.Entry<Long, SpanReceiver> entry : receivers.entrySet()) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/81f0adb7/hadoop-common-project/hadoop-common/src/site/markdown/Tracing.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/Tracing.md b/hadoop-common-project/hadoop-common/src/site/markdown/Tracing.md
index 76b5ee0..bf5685c 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/Tracing.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/Tracing.md
@@ -78,6 +78,10 @@ You also need to add the jar bundling SpanReceiver to the classpath of Hadoop
on each node. (LocalFileSpanReceiver in the example above is included in the
jar of htrace-core which is bundled with Hadoop.)
+```
+ $ cp htrace-htraced/target/htrace-htraced-3.2.0-incubating.jar $HADOOP_HOME/share/hadoop/common/lib/
+```
+
### Dynamic update of tracing configuration
You can use `hadoop trace` command to see and update the tracing configuration of each servers.
@@ -149,6 +153,7 @@ which start tracing span before invoking HDFS shell command.
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FsShell;
import org.apache.hadoop.hdfs.DFSConfigKeys;
+ import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.tracing.SpanReceiverHost;
import org.apache.hadoop.util.ToolRunner;
import org.apache.htrace.Sampler;
@@ -157,19 +162,16 @@ which start tracing span before invoking HDFS shell command.
public class TracingFsShell {
public static void main(String argv[]) throws Exception {
- Configuration conf = new Configuration();
+ Configuration conf = new HdfsConfiguration();
FsShell shell = new FsShell();
conf.setQuietMode(false);
shell.setConf(conf);
SpanReceiverHost.get(conf, DFSConfigKeys.DFS_SERVER_HTRACE_PREFIX);
int res = 0;
- TraceScope ts = null;
- try {
- ts = Trace.startSpan("FsShell", Sampler.ALWAYS);
+ try (TraceScope ts = Trace.startSpan("FsShell", Sampler.ALWAYS)) {
res = ToolRunner.run(shell, argv);
} finally {
shell.close();
- if (ts != null) ts.close();
}
System.exit(res);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/81f0adb7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index 60e5577..fc1cd26 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -37,7 +37,6 @@ import java.net.Socket;
import java.net.SocketAddress;
import java.net.URI;
import java.net.UnknownHostException;
-import java.nio.charset.Charset;
import java.security.GeneralSecurityException;
import java.util.ArrayList;
import java.util.Collections;
@@ -3223,35 +3222,26 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
return saslClient;
}
- private static final byte[] PATH = "path".getBytes(Charset.forName("UTF-8"));
-
TraceScope getPathTraceScope(String description, String path) {
TraceScope scope = Trace.startSpan(description, traceSampler);
Span span = scope.getSpan();
if (span != null) {
if (path != null) {
- span.addKVAnnotation(PATH,
- path.getBytes(Charset.forName("UTF-8")));
+ span.addKVAnnotation("path", path);
}
}
return scope;
}
- private static final byte[] SRC = "src".getBytes(Charset.forName("UTF-8"));
-
- private static final byte[] DST = "dst".getBytes(Charset.forName("UTF-8"));
-
TraceScope getSrcDstTraceScope(String description, String src, String dst) {
TraceScope scope = Trace.startSpan(description, traceSampler);
Span span = scope.getSpan();
if (span != null) {
if (src != null) {
- span.addKVAnnotation(SRC,
- src.getBytes(Charset.forName("UTF-8")));
+ span.addKVAnnotation("src", src);
}
if (dst != null) {
- span.addKVAnnotation(DST,
- dst.getBytes(Charset.forName("UTF-8")));
+ span.addKVAnnotation("dst", dst);
}
}
return scope;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/81f0adb7/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 73e29f8..94e1f5e 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -820,7 +820,7 @@
<dependency>
<groupId>org.apache.htrace</groupId>
<artifactId>htrace-core</artifactId>
- <version>3.1.0-incubating</version>
+ <version>3.2.0-incubating</version>
</dependency>
<dependency>
<groupId>org.jdom</groupId>