You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by ji...@apache.org on 2016/01/28 03:19:56 UTC
mesos git commit: Added common command utils file.
Repository: mesos
Updated Branches:
refs/heads/master 8f75d3bb8 -> 011121bd6
Added common command utils file.
This common file is a good place to add common command line utilities
like tar, digests(sha256, sha512, etc). Currently this functionality is
spread in the code base and this change would enable all those call
sites to be replaced with a common code.
Review: https://reviews.apache.org/r/42662/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/011121bd
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/011121bd
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/011121bd
Branch: refs/heads/master
Commit: 011121bd686a7c82e9d659d7e1751f72278fb0be
Parents: 8f75d3b
Author: Jojy Varghese <jo...@mesosphere.io>
Authored: Wed Jan 27 18:18:22 2016 -0800
Committer: Jie Yu <yu...@gmail.com>
Committed: Wed Jan 27 18:19:43 2016 -0800
----------------------------------------------------------------------
src/CMakeLists.txt | 1 +
src/Makefile.am | 3 +
src/common/command_utils.cpp | 170 +++++++++++++++++++
src/common/command_utils.hpp | 68 ++++++++
src/tests/common/command_utils_tests.cpp | 227 ++++++++++++++++++++++++++
5 files changed, 469 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/011121bd/src/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 47d0a7c..93caf00 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -99,6 +99,7 @@ set(AUTHORIZER_SRC
set(COMMON_SRC
common/attributes.cpp
+ common/command_utils.cpp
common/date_utils.cpp
common/http.cpp
common/protobuf_utils.cpp
http://git-wip-us.apache.org/repos/asf/mesos/blob/011121bd/src/Makefile.am
----------------------------------------------------------------------
diff --git a/src/Makefile.am b/src/Makefile.am
index bdb3402..fac17f4 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -572,6 +572,7 @@ libmesos_no_3rdparty_la_SOURCES += \
authorizer/authorizer.cpp \
authorizer/local/authorizer.cpp \
common/attributes.cpp \
+ common/command_utils.cpp \
common/date_utils.cpp \
common/http.cpp \
common/protobuf_utils.cpp \
@@ -682,6 +683,7 @@ libmesos_no_3rdparty_la_SOURCES += \
authentication/cram_md5/auxprop.hpp \
authorizer/local/authorizer.hpp \
common/build.hpp \
+ common/command_utils.hpp \
common/date_utils.hpp \
common/http.hpp \
common/parse.hpp \
@@ -1825,6 +1827,7 @@ mesos_tests_SOURCES = \
tests/utils.cpp \
tests/values_tests.cpp \
tests/zookeeper_url_tests.cpp \
+ tests/common/command_utils_tests.cpp \
tests/common/http_tests.cpp \
tests/common/recordio_tests.cpp \
tests/containerizer/composing_containerizer_tests.cpp \
http://git-wip-us.apache.org/repos/asf/mesos/blob/011121bd/src/common/command_utils.cpp
----------------------------------------------------------------------
diff --git a/src/common/command_utils.cpp b/src/common/command_utils.cpp
new file mode 100644
index 0000000..e9d7821
--- /dev/null
+++ b/src/common/command_utils.cpp
@@ -0,0 +1,170 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <tuple>
+#include <vector>
+
+#include <process/collect.hpp>
+#include <process/io.hpp>
+#include <process/subprocess.hpp>
+
+#include <stout/os.hpp>
+
+#include "common/command_utils.hpp"
+#include "common/status_utils.hpp"
+
+using std::string;
+using std::tuple;
+using std::vector;
+
+using process::Failure;
+using process::Future;
+using process::Subprocess;
+
+namespace mesos {
+namespace internal {
+namespace command {
+
+static Future<string> launch(
+ const string& path,
+ const vector<string>& argv)
+{
+ Try<Subprocess> s = subprocess(
+ path,
+ argv,
+ Subprocess::PATH("/dev/null"),
+ Subprocess::PIPE(),
+ Subprocess::PIPE());
+
+ string command = strings::join(
+ ", ",
+ path,
+ strings::join(", ", argv));
+
+ if (s.isError()) {
+ return Failure(
+ "Failed to execute the subprocess '" + command + "': " + s.error());
+ }
+
+ return await(
+ s.get().status(),
+ process::io::read(s.get().out().get()),
+ process::io::read(s.get().err().get()))
+ .then([command](const tuple<
+ Future<Option<int>>,
+ Future<string>,
+ Future<string>>& t) -> Future<string> {
+ Future<Option<int>> status = std::get<0>(t);
+ if (!status.isReady()) {
+ return Failure(
+ "Failed to get the exit status of the subprocess: " +
+ (status.isFailed() ? status.failure() : "discarded"));
+ }
+
+ if (status->isNone()) {
+ return Failure("Failed to reap the subprocess");
+ }
+
+ if (status->get() != 0) {
+ Future<string> error = std::get<2>(t);
+ if (!error.isReady()) {
+ return Failure(
+ "Unexpected result from the subprocess: " +
+ WSTRINGIFY(status->get()) + ", stderr='" +
+ error.get() + "'");
+ }
+
+ return Failure("Subprocess '" + command + "' failed: " + error.get());
+ }
+
+ Future<string> output = std::get<1>(t);
+ if (!output.isReady()) {
+ return Failure(
+ "Failed to read stdout from '" + command + "': " +
+ (output.isFailed() ? output.failure() : "discarded"));
+ }
+
+ return output;
+ });
+}
+
+
+Future<Nothing> tar(
+ const Path& input,
+ const Path& output,
+ const Option<Path>& directory,
+ const Option<Compression>& compression)
+{
+ vector<string> argv = {
+ "tar",
+ "-c", // Create archive.
+ "-f", // Output file.
+ output
+ };
+
+ // Add additional flags.
+ if (directory.isSome()) {
+ argv.emplace_back("-C");
+ argv.emplace_back(directory.get());
+ }
+
+ if (compression.isSome()) {
+ switch (compression.get()) {
+ case Compression::GZIP:
+ argv.emplace_back("-z");
+ break;
+ case Compression::BZIP2:
+ argv.emplace_back("-j");
+ break;
+ case Compression::XZ:
+ argv.emplace_back("-J");
+ break;
+ default:
+ UNREACHABLE();
+ }
+ }
+
+ argv.emplace_back(input);
+
+ return launch("tar", argv)
+ .then([] () {return Nothing();});
+}
+
+
+Future<Nothing> untar(
+ const Path& input,
+ const Option<Path>& directory)
+{
+ vector<string> argv = {
+ "tar",
+ "-x", // Extract/unarchive.
+ "-f", // Input file to extract/unarchive.
+ input
+ };
+
+ // Add additional flags.
+ if (directory.isSome()) {
+ argv.emplace_back("-C");
+ argv.emplace_back(directory.get());
+ }
+
+ return launch("tar", argv)
+ .then([] () {return Nothing();});
+}
+
+} // namespace command {
+} // namespace internal {
+} // namespace mesos {
http://git-wip-us.apache.org/repos/asf/mesos/blob/011121bd/src/common/command_utils.hpp
----------------------------------------------------------------------
diff --git a/src/common/command_utils.hpp b/src/common/command_utils.hpp
new file mode 100644
index 0000000..bfab345
--- /dev/null
+++ b/src/common/command_utils.hpp
@@ -0,0 +1,68 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef __COMMON_COMMAND_UTILS_HPP__
+#define __COMMON_COMMAND_UTILS_HPP__
+
+#include <process/process.hpp>
+
+#include <stout/option.hpp>
+#include <stout/path.hpp>
+
+namespace mesos {
+namespace internal {
+namespace command {
+
+enum class Compression
+{
+ GZIP,
+ BZIP2,
+ XZ
+};
+
+
+/**
+ * Tar(archive) the file/directory to produce output file.
+ *
+ * @param input file or directory that will be archived.
+ * @param output output archive name.
+ * @param directory change to this directory before archiving.
+ * @param compression compression type if the archive has to be compressed.
+ */
+process::Future<Nothing> tar(
+ const Path& input,
+ const Path& output,
+ const Option<Path>& directory = None(),
+ const Option<Compression>& compression = None());
+
+
+/**
+ * Untar(unarchive) the given file.
+ *
+ * @param input file or directory that will be unarchived.
+ * @param directory change to this directory before unarchiving.
+ */
+process::Future<Nothing> untar(
+ const Path& input,
+ const Option<Path>& directory = None());
+
+// TODO(Jojy): Add more overloads/options for untar (eg., keep existing files)
+
+} // namespace command {
+} // namespace internal {
+} // namespace mesos {
+
+#endif // __COMMON_COMMAND_UTILS_HPP__
http://git-wip-us.apache.org/repos/asf/mesos/blob/011121bd/src/tests/common/command_utils_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/common/command_utils_tests.cpp b/src/tests/common/command_utils_tests.cpp
new file mode 100644
index 0000000..938f399
--- /dev/null
+++ b/src/tests/common/command_utils_tests.cpp
@@ -0,0 +1,227 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <string>
+
+#include <gmock/gmock.h>
+
+#include <gtest/gtest.h>
+
+#include <process/io.hpp>
+
+#include <stout/os.hpp>
+#include <stout/path.hpp>
+
+#include "common/command_utils.hpp"
+
+#include "tests/mesos.hpp"
+#include "tests/utils.hpp"
+
+using std::string;
+
+using process::Future;
+
+namespace mesos {
+namespace internal {
+namespace tests {
+
+class TarTest : public TemporaryDirectoryTest
+{
+protected:
+ Try<Nothing> createTestFile(
+ const Path& file,
+ const Option<Path>& directory = None())
+ {
+ const Path testFile(
+ directory.isSome() ?
+ path::join(directory.get(), file.value): file);
+
+ const string& testFileDir = testFile.dirname();
+ if (!os::exists(testFileDir)) {
+ Try<Nothing> mkdir = os::mkdir(testFileDir, true);
+ if (mkdir.isError()) {
+ return Error("Failed to create test directory: " + mkdir.error());
+ }
+ }
+
+ Try<Nothing> write = os::write(testFile, "test");
+ if (write.isError()) {
+ return Error("Failed to create to test file: " + write.error());
+ }
+
+ return Nothing();
+ }
+};
+
+
+// Tests the archive APIs for a simple file.
+TEST_F(TarTest, File)
+{
+ // Create a test file.
+ const Path testFile("testfile");
+ ASSERT_SOME(createTestFile(testFile));
+
+ // Archive the test file.
+ const Path outputTarFile("test.tar");
+ AWAIT_ASSERT_READY(command::tar(testFile, outputTarFile, None()));
+ ASSERT_TRUE(os::exists(outputTarFile));
+
+ // Remove the test file to make sure untar process creates new test file.
+ ASSERT_SOME(os::rm(testFile));
+ ASSERT_FALSE(os::exists(testFile));
+
+ // Untar the tarball and verify that the original file is created.
+ AWAIT_ASSERT_READY(command::untar(outputTarFile));
+ ASSERT_TRUE(os::exists(testFile));
+
+ // Verify that the content is same as original file.
+ EXPECT_SOME_EQ("test", os::read(testFile));
+}
+
+
+// Tests the archive APIs for a simple directory.
+TEST_F(TarTest, Directory)
+{
+ const Path testDir("test_dir");
+ const Path testFile(path::join(testDir, "testfile"));
+
+ // Create a test file in the test directory.
+ ASSERT_SOME(createTestFile(testFile));
+
+ // Archive the test directory.
+ const Path outputTarFile("test_dir.tar");
+ AWAIT_ASSERT_READY(command::tar(testDir, outputTarFile, None()));
+ ASSERT_TRUE(os::exists(outputTarFile));
+
+ // Remove the test directory to make sure untar process creates new test file.
+ ASSERT_SOME(os::rmdir(testDir));
+ ASSERT_FALSE(os::exists(testDir));
+
+ // Untar the tarball and verify that the original directory is created.
+ AWAIT_ASSERT_READY(command::untar(outputTarFile));
+ ASSERT_TRUE(os::exists(testDir));
+
+ // Verify that the content is same as original file.
+ EXPECT_SOME_EQ("test", os::read(testFile));
+}
+
+
+// Tests the archive APIs for archiving after changing directory.
+TEST_F(TarTest, ChangeDirectory)
+{
+ // Create a top directory where the directory to be archived will be placed.
+ const Path topDir("top_dir");
+ const Path testDir("test_dir");
+
+ const Path testFile(path::join(testDir, "testfile"));
+
+ // Create a test file in the test directory.
+ ASSERT_SOME(createTestFile(testFile, topDir));
+
+ // Archive the test directory.
+ const Path outputTarFile("test_dir.tar");
+ AWAIT_ASSERT_READY(command::tar(testDir, outputTarFile, topDir));
+ ASSERT_TRUE(os::exists(outputTarFile));
+
+ // Remove the top directory to make sure untar process creates new directory.
+ ASSERT_SOME(os::rmdir(topDir));
+ ASSERT_FALSE(os::exists(topDir));
+
+ // Untar the tarball and verify that the original file is created.
+ AWAIT_ASSERT_READY(command::untar(outputTarFile));
+ ASSERT_TRUE(os::exists(testDir));
+
+ // Verify that the top directory was not created.
+ ASSERT_FALSE(os::exists(topDir));
+
+ // Verify that the content is same as original file.
+ EXPECT_SOME_EQ("test", os::read(testFile));
+}
+
+
+// Tests the archive APIs for archiving/unarchiving a simple file for BZIP2
+// compression.
+TEST_F(TarTest, BZIP2CompressFile)
+{
+ // Create a test file.
+ const Path testFile("testfile");
+ ASSERT_SOME(createTestFile(testFile));
+
+ // Archive the test file.
+ const Path outputTarFile("test.tar");
+ AWAIT_ASSERT_READY(command::tar(
+ testFile,
+ outputTarFile,
+ None(),
+ command::Compression::BZIP2));
+
+ ASSERT_TRUE(os::exists(outputTarFile));
+
+ // Remove the test file to make sure untar process creates new test file.
+ ASSERT_SOME(os::rm(testFile));
+ ASSERT_FALSE(os::exists(testFile));
+
+ // Untar the tarball and verify that the original file is created.
+ AWAIT_ASSERT_READY(command::untar(outputTarFile));
+ ASSERT_TRUE(os::exists(testFile));
+
+ // Verify that the content is same as original file.
+ EXPECT_SOME_EQ("test", os::read(testFile));
+}
+
+
+// Tests the archive APIs for archiving/unarchiving a directory for GZIP
+// compression. This test specifically tests for the use case of changing
+// directory before archiving.
+TEST_F(TarTest, GZIPChangeDirectory)
+{
+ // Create a top directory where the directory to be archived will be placed.
+ const Path topDir("top_dir");
+ const Path testDir("test_dir");
+
+ const Path testFile(path::join(testDir, "testfile"));
+
+ // Create a test file in the test directory.
+ ASSERT_SOME(createTestFile(testFile, topDir));
+
+ // Archive the test directory.
+ const Path outputTarFile("test_dir.tar");
+ AWAIT_ASSERT_READY(command::tar(
+ testDir,
+ outputTarFile,
+ topDir,
+ command::Compression::GZIP));
+
+ ASSERT_TRUE(os::exists(outputTarFile));
+
+ // Remove the top directory to make sure untar process creates new directory.
+ ASSERT_SOME(os::rmdir(topDir));
+ ASSERT_FALSE(os::exists(topDir));
+
+ // Untar the tarball and verify that the original file is created.
+ AWAIT_ASSERT_READY(command::untar(outputTarFile));
+ ASSERT_TRUE(os::exists(testDir));
+
+ // Verify that the top directory was not created.
+ ASSERT_FALSE(os::exists(topDir));
+
+ // Verify that the content is same as original file.
+ EXPECT_SOME_EQ("test", os::read(testFile));
+}
+
+} // namespace tests {
+} // namespace internal {
+} // namespace mesos {