mesos git commit: Accept the 3-field version of HDFS du output.

[ji...@apache.org]

Repository: mesos
Updated Branches:
  refs/heads/master b5df7cc09 -> 4eba365a9


Accept the 3-field version of HDFS du output.

The HDFS du command can output 3 fields in later Hadoop versions.
We should accept both 2-field and 3-field versions for better
compatibility.

Review: https://reviews.apache.org/r/42370/


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/4eba365a
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/4eba365a
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/4eba365a

Branch: refs/heads/master
Commit: 4eba365a9ec59cce2901598c0aca43467cce5355
Parents: b5df7cc
Author: James Peach <jp...@apache.org>
Authored: Fri Jan 15 16:06:57 2016 -0800
Committer: Jie Yu <yu...@gmail.com>
Committed: Fri Jan 15 16:07:53 2016 -0800

----------------------------------------------------------------------
 src/hdfs/hdfs.cpp        |  6 +++++-
 src/tests/hdfs_tests.cpp | 26 ++++++++++++++++++++++++++
 2 files changed, 31 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/mesos/blob/4eba365a/src/hdfs/hdfs.cpp
----------------------------------------------------------------------
diff --git a/src/hdfs/hdfs.cpp b/src/hdfs/hdfs.cpp
index 51f016b..9c91d1f 100644
--- a/src/hdfs/hdfs.cpp
+++ b/src/hdfs/hdfs.cpp
@@ -197,7 +197,11 @@ Future<Bytes> HDFS::du(const string& _path)
         // fields can be delimited by multiple spaces.
         vector<string> fields = strings::tokenize(line, " \t");
 
-        if (fields.size() == 2 && fields[1] == path) {
+        // There might be 2 or 3 fields, see HADOOP-6857. The 2-field
+        // version contains object size and path, the 3-field version
+        // contains object size, object storage size and path.
+        if ((fields.size() == 2 || fields.size() == 3) &&
+            fields.back() == path) {
           Result<size_t> size = numify<size_t>(fields[0]);
           if (size.isSome()) {
             return Bytes(size.get());

http://git-wip-us.apache.org/repos/asf/mesos/blob/4eba365a/src/tests/hdfs_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/hdfs_tests.cpp b/src/tests/hdfs_tests.cpp
index 29f1560..e7154c7 100644
--- a/src/tests/hdfs_tests.cpp
+++ b/src/tests/hdfs_tests.cpp
@@ -116,6 +116,32 @@ TEST_F(HdfsTest, Du)
 }
 
 
+// This is the same test as HdfsTest::Du except it emulates a HDFS
+// version that returns 3 fields for the du subcommand.
+TEST_F(HdfsTest, ThreeFieldDu)
+{
+  // The script emulating 'hadoop fs -du <path>'.
+  // NOTE: We emulate a version call here which is exercised when
+  // creating the HDFS client.
+  ASSERT_SOME(os::write(
+      hadoop,
+      "#!/bin/sh\n"
+      "if [ \"$1\" = \"version\" ]; then\n"
+      "  exit 0\n"
+      "fi\n"
+      "du $3 | awk '{printf \"%s  %s  %s\\n\", $1, $1, $2}' \n"));
+
+  Try<Owned<HDFS>> hdfs = HDFS::create(hadoop);
+  ASSERT_SOME(hdfs);
+
+  Future<Bytes> bytes = hdfs.get()->du(hadoop);
+  AWAIT_READY(bytes);
+
+  bytes = hdfs.get()->du(path::join(os::getcwd(), "Invalid"));
+  AWAIT_FAILED(bytes);
+}
+
+
 // This test verifies the 'HDFS::rm(path)' method. We emulate the
 // hadoop client by removing a file on the local filesystem.
 TEST_F(HdfsTest, Rm)