You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by jz...@apache.org on 2017/04/19 04:27:19 UTC
hadoop git commit: HDFS-11531. Expose hedged read metrics via libHDFS
API. Contributed by Sailesh Mukil.
Repository: hadoop
Updated Branches:
refs/heads/trunk af8e9842d -> 8c81a16a1
HDFS-11531. Expose hedged read metrics via libHDFS API. Contributed by Sailesh Mukil.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8c81a16a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8c81a16a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8c81a16a
Branch: refs/heads/trunk
Commit: 8c81a16a1fa40d3a3796b1c3633328f1340883ca
Parents: af8e984
Author: John Zhuge <jz...@apache.org>
Authored: Tue Apr 18 21:11:07 2017 -0700
Committer: John Zhuge <jz...@apache.org>
Committed: Tue Apr 18 21:11:07 2017 -0700
----------------------------------------------------------------------
.../hadoop/hdfs/DistributedFileSystem.java | 9 +++
.../libhdfs-tests/test_libhdfs_threaded.c | 10 +++
.../src/main/native/libhdfs/hdfs.c | 81 ++++++++++++++++++++
.../src/main/native/libhdfs/include/hdfs/hdfs.h | 29 +++++++
4 files changed, 129 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8c81a16a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
index 11d7eb8..e1c2105 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
@@ -201,6 +201,15 @@ public class DistributedFileSystem extends FileSystem {
}
/**
+ * Returns the hedged read metrics object for this client.
+ *
+ * @return object of DFSHedgedReadMetrics
+ */
+ public DFSHedgedReadMetrics getHedgedReadMetrics() {
+ return dfs.getHedgedReadMetrics();
+ }
+
+ /**
* Checks that the passed URI belongs to this filesystem and returns
* just the path component. Expects a URI with an absolute path.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8c81a16a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/test_libhdfs_threaded.c
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/test_libhdfs_threaded.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/test_libhdfs_threaded.c
index 032acbf..ee1b080 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/test_libhdfs_threaded.c
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/test_libhdfs_threaded.c
@@ -145,6 +145,7 @@ static int doTestHdfsOperations(struct tlhThreadInfo *ti, hdfsFS fs,
int ret, expected, numEntries;
hdfsFileInfo *fileInfo;
struct hdfsReadStatistics *readStats = NULL;
+ struct hdfsHedgedReadMetrics *hedgedMetrics = NULL;
if (hdfsExists(fs, paths->prefix) == 0) {
EXPECT_ZERO(hdfsDelete(fs, paths->prefix, 1));
@@ -204,6 +205,15 @@ static int doTestHdfsOperations(struct tlhThreadInfo *ti, hdfsFS fs,
EXPECT_UINT64_EQ(UINT64_C(0), readStats->totalLocalBytesRead);
EXPECT_UINT64_EQ(UINT64_C(0), readStats->totalShortCircuitBytesRead);
hdfsFileFreeReadStatistics(readStats);
+
+ /* Verify that we can retrieve the hedged read metrics */
+ EXPECT_ZERO(hdfsGetHedgedReadMetrics(fs, &hedgedMetrics));
+ errno = 0;
+ EXPECT_UINT64_EQ(UINT64_C(0), hedgedMetrics->hedgedReadOps);
+ EXPECT_UINT64_EQ(UINT64_C(0), hedgedMetrics->hedgedReadOpsWin);
+ EXPECT_UINT64_EQ(UINT64_C(0), hedgedMetrics->hedgedReadOpsInCurThread);
+ hdfsFreeHedgedReadMetrics(hedgedMetrics);
+
/* TODO: implement readFully and use it here */
ret = hdfsRead(fs, file, tmp, sizeof(tmp));
if (ret < 0) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8c81a16a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/hdfs.c
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/hdfs.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/hdfs.c
index 1dcc768..36c9583 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/hdfs.c
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/hdfs.c
@@ -93,6 +93,87 @@ int hdfsFileIsOpenForRead(hdfsFile file)
return (file->type == HDFS_STREAM_INPUT);
}
+int hdfsGetHedgedReadMetrics(hdfsFS fs, struct hdfsHedgedReadMetrics **metrics)
+{
+ jthrowable jthr;
+ jobject hedgedReadMetrics = NULL;
+ jvalue jVal;
+ struct hdfsHedgedReadMetrics *m = NULL;
+ int ret;
+ jobject jFS = (jobject)fs;
+ JNIEnv* env = getJNIEnv();
+
+ if (env == NULL) {
+ errno = EINTERNAL;
+ return -1;
+ }
+
+ jthr = invokeMethod(env, &jVal, INSTANCE, jFS,
+ HADOOP_DFS,
+ "getHedgedReadMetrics",
+ "()Lorg/apache/hadoop/hdfs/DFSHedgedReadMetrics;");
+ if (jthr) {
+ ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+ "hdfsGetHedgedReadMetrics: getHedgedReadMetrics failed");
+ goto done;
+ }
+ hedgedReadMetrics = jVal.l;
+
+ m = malloc(sizeof(struct hdfsHedgedReadMetrics));
+ if (!m) {
+ ret = ENOMEM;
+ goto done;
+ }
+
+ jthr = invokeMethod(env, &jVal, INSTANCE, hedgedReadMetrics,
+ "org/apache/hadoop/hdfs/DFSHedgedReadMetrics",
+ "getHedgedReadOps", "()J");
+ if (jthr) {
+ ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+ "hdfsGetHedgedReadStatistics: getHedgedReadOps failed");
+ goto done;
+ }
+ m->hedgedReadOps = jVal.j;
+
+ jthr = invokeMethod(env, &jVal, INSTANCE, hedgedReadMetrics,
+ "org/apache/hadoop/hdfs/DFSHedgedReadMetrics",
+ "getHedgedReadWins", "()J");
+ if (jthr) {
+ ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+ "hdfsGetHedgedReadStatistics: getHedgedReadWins failed");
+ goto done;
+ }
+ m->hedgedReadOpsWin = jVal.j;
+
+ jthr = invokeMethod(env, &jVal, INSTANCE, hedgedReadMetrics,
+ "org/apache/hadoop/hdfs/DFSHedgedReadMetrics",
+ "getHedgedReadOpsInCurThread", "()J");
+ if (jthr) {
+ ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+ "hdfsGetHedgedReadStatistics: getHedgedReadOpsInCurThread failed");
+ goto done;
+ }
+ m->hedgedReadOpsInCurThread = jVal.j;
+
+ *metrics = m;
+ m = NULL;
+ ret = 0;
+
+done:
+ destroyLocalReference(env, hedgedReadMetrics);
+ free(m);
+ if (ret) {
+ errno = ret;
+ return -1;
+ }
+ return 0;
+}
+
+void hdfsFreeHedgedReadMetrics(struct hdfsHedgedReadMetrics *metrics)
+{
+ free(metrics);
+}
+
int hdfsFileGetReadStatistics(hdfsFile file,
struct hdfsReadStatistics **stats)
{
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8c81a16a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/include/hdfs/hdfs.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/include/hdfs/hdfs.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/include/hdfs/hdfs.h
index 83c1c59..71fa2c9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/include/hdfs/hdfs.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/include/hdfs/hdfs.h
@@ -169,6 +169,35 @@ extern "C" {
LIBHDFS_EXTERNAL
void hdfsFileFreeReadStatistics(struct hdfsReadStatistics *stats);
+ struct hdfsHedgedReadMetrics {
+ uint64_t hedgedReadOps;
+ uint64_t hedgedReadOpsWin;
+ uint64_t hedgedReadOpsInCurThread;
+ };
+
+ /**
+ * Get cluster wide hedged read metrics.
+ *
+ * @param fs The configured filesystem handle
+ * @param metrics (out parameter) on a successful return, the hedged read
+ * metrics. Unchanged otherwise. You must free the returned
+ * statistics with hdfsFreeHedgedReadMetrics.
+ * @return 0 if the metrics were successfully returned, -1 otherwise.
+ * On a failure, please check errno against
+ * ENOTSUP. webhdfs, LocalFilesystem, and so forth may
+ * not support hedged read metrics.
+ */
+ LIBHDFS_EXTERNAL
+ int hdfsGetHedgedReadMetrics(hdfsFS fs, struct hdfsHedgedReadMetrics **metrics);
+
+ /**
+ * Free HDFS Hedged read metrics.
+ *
+ * @param metrics The HDFS Hedged read metrics to free
+ */
+ LIBHDFS_EXTERNAL
+ void hdfsFreeHedgedReadMetrics(struct hdfsHedgedReadMetrics *metrics);
+
/**
* hdfsConnectAsUser - Connect to a hdfs file system as a specific user
* Connect to the hdfs.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org