You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ga...@apache.org on 2022/03/05 17:35:57 UTC
[hadoop] branch trunk updated: HDFS-16462. Make HDFS get tool cross platform (#4003)
This is an automated email from the ASF dual-hosted git repository.
gaurava pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 8f07081 HDFS-16462. Make HDFS get tool cross platform (#4003)
8f07081 is described below
commit 8f07081789e4690e2c5023c0f07ca6088d937632
Author: Gautham B A <ga...@gmail.com>
AuthorDate: Sat Mar 5 23:05:14 2022 +0530
HDFS-16462. Make HDFS get tool cross platform (#4003)
---
.../native/libhdfspp/tests/tools/CMakeLists.txt | 3 +
.../native/libhdfspp/tests/tools/hdfs-get-mock.cc | 56 ++++++++++++++
.../native/libhdfspp/tests/tools/hdfs-get-mock.h | 68 +++++++++++++++++
.../libhdfspp/tests/tools/hdfs-tool-tests.cc | 14 ++++
.../src/main/native/libhdfspp/tools/CMakeLists.txt | 3 +-
.../tools/hdfs-copy-to-local/hdfs-copy-to-local.cc | 14 +++-
.../tools/hdfs-copy-to-local/hdfs-copy-to-local.h | 5 ++
.../native/libhdfspp/tools/hdfs-get/CMakeLists.txt | 27 +++++++
.../native/libhdfspp/tools/hdfs-get/hdfs-get.cc | 25 ++++++
.../native/libhdfspp/tools/hdfs-get/hdfs-get.h | 47 ++++++++++++
.../main/native/libhdfspp/tools/hdfs-get/main.cc | 52 +++++++++++++
.../src/main/native/libhdfspp/tools/hdfs_get.cc | 88 ----------------------
12 files changed, 308 insertions(+), 94 deletions(-)
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/CMakeLists.txt
index 56755ad..769e5da 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/CMakeLists.txt
@@ -35,6 +35,7 @@ add_executable(hdfs_tool_tests
hdfs-count-mock.cc
hdfs-mkdir-mock.cc
hdfs-rm-mock.cc
+ hdfs-get-mock.cc
main.cc)
target_include_directories(hdfs_tool_tests PRIVATE
../tools
@@ -54,6 +55,7 @@ target_include_directories(hdfs_tool_tests PRIVATE
../../tools/hdfs-count
../../tools/hdfs-mkdir
../../tools/hdfs-rm
+ ../../tools/hdfs-get
../../tools/hdfs-cat)
target_link_libraries(hdfs_tool_tests PRIVATE
gmock_main
@@ -72,5 +74,6 @@ target_link_libraries(hdfs_tool_tests PRIVATE
hdfs_count_lib
hdfs_mkdir_lib
hdfs_rm_lib
+ hdfs_get_lib
hdfs_cat_lib)
add_test(hdfs_tool_tests hdfs_tool_tests)
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-get-mock.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-get-mock.cc
new file mode 100644
index 0000000..713564e
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-get-mock.cc
@@ -0,0 +1,56 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <functional>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include <gmock/gmock.h>
+#include <gtest/gtest.h>
+
+#include "hdfs-get-mock.h"
+#include "hdfs-tool-tests.h"
+
+namespace hdfs::tools::test {
+GetMock::~GetMock() = default;
+
+void GetMock::SetExpectations(
+ std::function<std::unique_ptr<GetMock>()> test_case,
+ const std::vector<std::string> &args) const {
+ // Get the pointer to the function that defines the test case
+ const auto test_case_func =
+ test_case.target<std::unique_ptr<GetMock> (*)()>();
+ ASSERT_NE(test_case_func, nullptr);
+
+ // Set the expected method calls and their corresponding arguments for each
+ // test case
+ if (*test_case_func == &CallHelp<GetMock>) {
+ EXPECT_CALL(*this, HandleHelp()).Times(1).WillOnce(testing::Return(true));
+ return;
+ }
+
+ if (*test_case_func == &Pass2Paths<GetMock>) {
+ const auto arg1 = args[0];
+ const auto arg2 = args[1];
+ EXPECT_CALL(*this, HandlePath(arg1, arg2))
+ .Times(1)
+ .WillOnce(testing::Return(true));
+ }
+}
+} // namespace hdfs::tools::test
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-get-mock.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-get-mock.h
new file mode 100644
index 0000000..535f715
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-get-mock.h
@@ -0,0 +1,68 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef LIBHDFSPP_TOOLS_HDFS_GET_MOCK
+#define LIBHDFSPP_TOOLS_HDFS_GET_MOCK
+
+#include <functional>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include <gmock/gmock.h>
+
+#include "hdfs-get.h"
+
+namespace hdfs::tools::test {
+/**
+ * {@class GetMock} is an {@class Get} whereby it mocks the
+ * HandleHelp and HandlePath methods for testing their functionality.
+ */
+class GetMock : public hdfs::tools::Get {
+public:
+ /**
+ * {@inheritdoc}
+ */
+ GetMock(const int argc, char **argv) : Get(argc, argv) {}
+
+ // Abiding to the Rule of 5
+ GetMock(const GetMock &) = delete;
+ GetMock(GetMock &&) = delete;
+ GetMock &operator=(const GetMock &) = delete;
+ GetMock &operator=(GetMock &&) = delete;
+ ~GetMock() override;
+
+ /**
+ * Defines the methods and the corresponding arguments that are expected
+ * to be called on this instance of {@link HdfsTool} for the given test case.
+ *
+ * @param test_case An {@link std::function} object that points to the
+ * function defining the test case
+ * @param args The arguments that are passed to this test case
+ */
+ void SetExpectations(std::function<std::unique_ptr<GetMock>()> test_case,
+ const std::vector<std::string> &args = {}) const;
+
+ MOCK_METHOD(bool, HandleHelp, (), (const, override));
+
+ MOCK_METHOD(bool, HandlePath, (const std::string &, const std::string &),
+ (const, override));
+};
+} // namespace hdfs::tools::test
+
+#endif
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-tool-tests.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-tool-tests.cc
index 7678834..50d555a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-tool-tests.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-tool-tests.cc
@@ -31,6 +31,7 @@
#include "hdfs-df-mock.h"
#include "hdfs-disallow-snapshot-mock.h"
#include "hdfs-du-mock.h"
+#include "hdfs-get-mock.h"
#include "hdfs-mkdir-mock.h"
#include "hdfs-move-to-local-mock.h"
#include "hdfs-rename-snapshot-mock.h"
@@ -109,6 +110,11 @@ INSTANTIATE_TEST_SUITE_P(
Pass2Paths<hdfs::tools::test::CopyToLocalMock>));
INSTANTIATE_TEST_SUITE_P(
+ HdfsGet, HdfsToolBasicTest,
+ testing::Values(CallHelp<hdfs::tools::test::GetMock>,
+ Pass2Paths<hdfs::tools::test::GetMock>));
+
+INSTANTIATE_TEST_SUITE_P(
HdfsMoveToLocal, HdfsToolBasicTest,
testing::Values(CallHelp<hdfs::tools::test::MoveToLocalMock>,
Pass2Paths<hdfs::tools::test::MoveToLocalMock>));
@@ -174,6 +180,10 @@ INSTANTIATE_TEST_SUITE_P(
testing::Values(Pass3Paths<hdfs::tools::test::CopyToLocalMock>));
INSTANTIATE_TEST_SUITE_P(
+ HdfsGet, HdfsToolNegativeTestThrows,
+ testing::Values(Pass3Paths<hdfs::tools::test::GetMock>));
+
+INSTANTIATE_TEST_SUITE_P(
HdfsMoveToLocal, HdfsToolNegativeTestThrows,
testing::Values(Pass3Paths<hdfs::tools::test::MoveToLocalMock>));
@@ -221,6 +231,10 @@ INSTANTIATE_TEST_SUITE_P(
testing::Values(PassAPath<hdfs::tools::test::CopyToLocalMock>));
INSTANTIATE_TEST_SUITE_P(
+ HdfsGet, HdfsToolNegativeTestNoThrow,
+ testing::Values(PassAPath<hdfs::tools::test::GetMock>));
+
+INSTANTIATE_TEST_SUITE_P(
HdfsDeleteSnapshot, HdfsToolNegativeTestNoThrow,
testing::Values(PassAPath<hdfs::tools::test::DeleteSnapshotMock>));
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/CMakeLists.txt
index aa82c01..bed78b7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/CMakeLists.txt
@@ -60,8 +60,7 @@ add_subdirectory(hdfs-df)
add_subdirectory(hdfs-du)
-add_executable(hdfs_get hdfs_get.cc)
-target_link_libraries(hdfs_get tools_common hdfspp_static)
+add_subdirectory(hdfs-get)
add_subdirectory(hdfs-copy-to-local)
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-copy-to-local/hdfs-copy-to-local.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-copy-to-local/hdfs-copy-to-local.cc
index 9219b36..7affa1f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-copy-to-local/hdfs-copy-to-local.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-copy-to-local/hdfs-copy-to-local.cc
@@ -64,7 +64,8 @@ bool CopyToLocal::ValidateConstraints() const {
std::string CopyToLocal::GetDescription() const {
std::stringstream desc;
- desc << "Usage: hdfs_copyToLocal [OPTION] SRC_FILE DST_FILE" << std::endl
+ desc << "Usage: hdfs_" << GetToolName() << " [OPTION] SRC_FILE DST_FILE"
+ << std::endl
<< std::endl
<< "Copy SRC_FILE from hdfs to DST_FILE on the local file system."
<< std::endl
@@ -72,16 +73,19 @@ std::string CopyToLocal::GetDescription() const {
<< " -h display this help and exit" << std::endl
<< std::endl
<< "Examples:" << std::endl
- << "hdfs_copyToLocal hdfs://localhost.localdomain:8020/dir/file "
+ << "hdfs_" << GetToolName()
+ << " hdfs://localhost.localdomain:8020/dir/file "
"/home/usr/myfile"
<< std::endl
- << "hdfs_copyToLocal /dir/file /home/usr/dir/file" << std::endl;
+ << "hdfs_" << GetToolName() << " /dir/file /home/usr/dir/file"
+ << std::endl;
return desc.str();
}
bool CopyToLocal::Do() {
if (!Initialize()) {
- std::cerr << "Unable to initialize HDFS copyToLocal tool" << std::endl;
+ std::cerr << "Unable to initialize HDFS " << GetToolName() << " tool"
+ << std::endl;
return false;
}
@@ -129,4 +133,6 @@ bool CopyToLocal::HandlePath(const std::string &source,
std::fclose(dst_file);
return true;
}
+
+std::string CopyToLocal::GetToolName() const { return "copyToLocal"; }
} // namespace hdfs::tools
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-copy-to-local/hdfs-copy-to-local.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-copy-to-local/hdfs-copy-to-local.h
index 6eb8cf1..0137f1e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-copy-to-local/hdfs-copy-to-local.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-copy-to-local/hdfs-copy-to-local.h
@@ -81,6 +81,11 @@ protected:
[[nodiscard]] virtual bool HandlePath(const std::string &source,
const std::string &target) const;
+ /**
+ * @return The name of the tool.
+ */
+ [[nodiscard]] virtual std::string GetToolName() const;
+
private:
/**
* A boost data-structure containing the description of positional arguments
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-get/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-get/CMakeLists.txt
new file mode 100644
index 0000000..367bca6
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-get/CMakeLists.txt
@@ -0,0 +1,27 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+add_library(hdfs_get_lib STATIC $<TARGET_OBJECTS:hdfs_tool_obj> hdfs-get.cc)
+target_include_directories(hdfs_get_lib PRIVATE ../../tools hdfs-get ${Boost_INCLUDE_DIRS})
+target_link_libraries(hdfs_get_lib PRIVATE Boost::boost Boost::program_options tools_common hdfspp_static hdfs_copyToLocal_lib)
+
+add_executable(hdfs_get main.cc)
+target_include_directories(hdfs_get PRIVATE ../../tools)
+target_link_libraries(hdfs_get PRIVATE hdfs_get_lib)
+
+install(TARGETS hdfs_get RUNTIME DESTINATION bin)
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-get/hdfs-get.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-get/hdfs-get.cc
new file mode 100644
index 0000000..e45c1d0
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-get/hdfs-get.cc
@@ -0,0 +1,25 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "hdfs-get.h"
+
+namespace hdfs::tools {
+Get::Get(const int argc, char **argv) : CopyToLocal(argc, argv) {}
+
+std::string Get::GetToolName() const { return "get"; }
+} // namespace hdfs::tools
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-get/hdfs-get.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-get/hdfs-get.h
new file mode 100644
index 0000000..8153264
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-get/hdfs-get.h
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef LIBHDFSPP_TOOLS_HDFS_GET
+#define LIBHDFSPP_TOOLS_HDFS_GET
+
+#include "hdfs-copy-to-local/hdfs-copy-to-local.h"
+
+namespace hdfs::tools {
+class Get : public CopyToLocal {
+public:
+ /**
+ * {@inheritdoc}
+ */
+ Get(int argc, char **argv);
+
+ // Abiding to the Rule of 5
+ Get(const Get &) = default;
+ Get(Get &&) = default;
+ Get &operator=(const Get &) = delete;
+ Get &operator=(Get &&) = delete;
+ ~Get() override = default;
+
+protected:
+ /**
+ * {@inheritdoc}
+ */
+ [[nodiscard]] std::string GetToolName() const override;
+};
+} // namespace hdfs::tools
+
+#endif
\ No newline at end of file
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-get/main.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-get/main.cc
new file mode 100644
index 0000000..916e60a
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-get/main.cc
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <cstdlib>
+#include <exception>
+#include <iostream>
+
+#include <google/protobuf/stubs/common.h>
+
+#include "hdfs-get.h"
+
+int main(int argc, char *argv[]) {
+ const auto result = std::atexit([]() -> void {
+ // Clean up static data on exit and prevent valgrind memory leaks
+ google::protobuf::ShutdownProtobufLibrary();
+ });
+ if (result != 0) {
+ std::cerr << "Error: Unable to schedule clean-up tasks for HDFS "
+ "get tool, exiting"
+ << std::endl;
+ std::exit(EXIT_FAILURE);
+ }
+
+ hdfs::tools::Get get(argc, argv);
+ auto success = false;
+
+ try {
+ success = get.Do();
+ } catch (const std::exception &e) {
+ std::cerr << "Error: " << e.what() << std::endl;
+ }
+
+ if (!success) {
+ std::exit(EXIT_FAILURE);
+ }
+ return 0;
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_get.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_get.cc
deleted file mode 100644
index 16dd72d..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_get.cc
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-*/
-
-#include <google/protobuf/stubs/common.h>
-#include <unistd.h>
-#include "tools_common.h"
-
-void usage(){
- std::cout << "Usage: hdfs_get [OPTION] SRC_FILE DST_FILE"
- << std::endl
- << std::endl << "Copy SRC_FILE from hdfs to DST_FILE on the local file system."
- << std::endl
- << std::endl << " -h display this help and exit"
- << std::endl
- << std::endl << "Examples:"
- << std::endl << "hdfs_get hdfs://localhost.localdomain:8020/dir/file /home/usr/myfile"
- << std::endl << "hdfs_get /dir/file /home/usr/dir/file"
- << std::endl;
-}
-
-int main(int argc, char *argv[]) {
- if (argc > 4) {
- usage();
- exit(EXIT_FAILURE);
- }
-
- int input;
-
- //Using GetOpt to read in the values
- opterr = 0;
- while ((input = getopt(argc, argv, "h")) != -1) {
- switch (input)
- {
- case 'h':
- usage();
- exit(EXIT_SUCCESS);
- case '?':
- if (isprint(optopt))
- std::cerr << "Unknown option `-" << (char) optopt << "'." << std::endl;
- else
- std::cerr << "Unknown option character `" << (char) optopt << "'." << std::endl;
- usage();
- exit(EXIT_FAILURE);
- default:
- exit(EXIT_FAILURE);
- }
- }
-
- std::string uri_path = argv[optind];
- std::string dest = argv[optind+1];
-
- //Building a URI object from the given uri_path
- hdfs::URI uri = hdfs::parse_path_or_exit(uri_path);
-
- std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri, false);
- if (!fs) {
- std::cerr << "Could not connect the file system. " << std::endl;
- exit(EXIT_FAILURE);
- }
-
- std::FILE* dst_file = std::fopen(dest.c_str(), "wb");
- if(!dst_file){
- std::cerr << "Unable to open the destination file: " << dest << std::endl;
- exit(EXIT_FAILURE);
- }
- readFile(fs, uri.get_path(), 0, dst_file, false);
- std::fclose(dst_file);
-
- // Clean up static data and prevent valgrind memory leaks
- google::protobuf::ShutdownProtobufLibrary();
- return 0;
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org