You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@quickstep.apache.org by ji...@apache.org on 2017/02/21 03:37:59 UTC
[16/50] [abbrv] incubator-quickstep git commit: Moved hdfs-related
flags into a file.
Moved hdfs-related flags into a file.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/dda085c7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/dda085c7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/dda085c7
Branch: refs/heads/LIP-time-decomposition
Commit: dda085c7aeaa4f7f353c3392b547ca788d1507b8
Parents: 43a9b39
Author: Zuyu Zhang <zu...@apache.org>
Authored: Sat Feb 4 14:46:44 2017 -0800
Committer: Zuyu Zhang <zu...@apache.org>
Committed: Sat Feb 4 16:25:03 2017 -0800
----------------------------------------------------------------------
CMakeLists.txt | 1 +
cli/QuickstepCli.cpp | 1 +
storage/CMakeLists.txt | 13 ++++++---
storage/FileManagerHdfs.cpp | 35 +----------------------
storage/FileManagerHdfs.hpp | 6 ----
storage/Flags.cpp | 62 ++++++++++++++++++++++++++++++++++++++++
storage/Flags.hpp | 51 +++++++++++++++++++++++++++++++++
storage/StorageManager.cpp | 5 +---
storage/StorageManager.hpp | 4 ---
9 files changed, 126 insertions(+), 52 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/dda085c7/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 85210ef..e74a274 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -789,6 +789,7 @@ target_link_libraries(quickstep_cli_shell
quickstep_queryexecution_WorkerDirectory
quickstep_queryoptimizer_QueryHandle
quickstep_queryoptimizer_QueryProcessor
+ quickstep_storage_Flags
quickstep_storage_PreloaderThread
quickstep_storage_StorageConstants
quickstep_storage_StorageManager
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/dda085c7/cli/QuickstepCli.cpp
----------------------------------------------------------------------
diff --git a/cli/QuickstepCli.cpp b/cli/QuickstepCli.cpp
index 9db7577..02ec4ec 100644
--- a/cli/QuickstepCli.cpp
+++ b/cli/QuickstepCli.cpp
@@ -58,6 +58,7 @@ typedef quickstep::LineReaderDumb LineReaderImpl;
#include "query_execution/WorkerDirectory.hpp"
#include "query_optimizer/QueryHandle.hpp"
#include "query_optimizer/QueryProcessor.hpp"
+#include "storage/Flags.hpp"
#include "storage/StorageConfig.h" // For QUICKSTEP_HAVE_FILE_MANAGER_HDFS.
#ifdef QUICKSTEP_HAVE_FILE_MANAGER_HDFS
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/dda085c7/storage/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/storage/CMakeLists.txt b/storage/CMakeLists.txt
index fddea1f..a44c3a7 100644
--- a/storage/CMakeLists.txt
+++ b/storage/CMakeLists.txt
@@ -207,6 +207,7 @@ if (QUICKSTEP_HAVE_FILE_MANAGER_POSIX)
elseif (QUICKSTEP_HAVE_FILE_MANAGER_WINDOWS)
add_library(quickstep_storage_FileManagerWindows FileManagerWindows.cpp FileManagerWindows.hpp)
endif()
+add_library(quickstep_storage_Flags Flags.cpp Flags.hpp)
add_library(quickstep_storage_HashTable ../empty_src.cpp HashTable.hpp)
add_library(quickstep_storage_HashTable_proto ${storage_HashTable_proto_srcs})
add_library(quickstep_storage_HashTableBase ../empty_src.cpp HashTableBase.hpp)
@@ -679,8 +680,8 @@ target_link_libraries(quickstep_storage_FileManager
if (QUICKSTEP_HAVE_FILE_MANAGER_HDFS)
target_link_libraries(quickstep_storage_FileManagerHdfs
glog
- ${GFLAGS_LIB_NAME}
quickstep_storage_FileManager
+ quickstep_storage_Flags
quickstep_storage_StorageBlockInfo
quickstep_storage_StorageConstants
quickstep_storage_StorageErrors
@@ -711,6 +712,8 @@ elseif (QUICKSTEP_HAVE_FILE_MANAGER_WINDOWS)
quickstep_utility_Macros
quickstep_utility_StringUtil)
endif()
+target_link_libraries(quickstep_storage_Flags
+ ${GFLAGS_LIB_NAME})
target_link_libraries(quickstep_storage_HashTable
quickstep_catalog_CatalogTypedefs
quickstep_storage_HashTableBase
@@ -1000,6 +1003,7 @@ target_link_libraries(quickstep_storage_StorageManager
quickstep_storage_EvictionPolicy
quickstep_storage_FileManager
quickstep_storage_FileManagerLocal
+ quickstep_storage_Flags
quickstep_storage_StorageBlob
quickstep_storage_StorageBlock
quickstep_storage_StorageBlockBase
@@ -1121,17 +1125,18 @@ target_link_libraries(quickstep_storage
quickstep_storage_CompressedTupleStorageSubBlock
quickstep_storage_CountedReference
quickstep_storage_EvictionPolicy
- quickstep_storage_FileManager
- quickstep_storage_FileManagerLocal
quickstep_storage_FastHashTable
quickstep_storage_FastHashTableFactory
quickstep_storage_FastSeparateChainingHashTable
+ quickstep_storage_FileManager
+ quickstep_storage_FileManagerLocal
+ quickstep_storage_Flags
quickstep_storage_HashTable
- quickstep_storage_HashTable_proto
quickstep_storage_HashTableBase
quickstep_storage_HashTableFactory
quickstep_storage_HashTableKeyManager
quickstep_storage_HashTablePool
+ quickstep_storage_HashTable_proto
quickstep_storage_IndexSubBlock
quickstep_storage_IndexSubBlockDescriptionFactory
quickstep_storage_InsertDestination
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/dda085c7/storage/FileManagerHdfs.cpp
----------------------------------------------------------------------
diff --git a/storage/FileManagerHdfs.cpp b/storage/FileManagerHdfs.cpp
index e28ed3b..937750a 100644
--- a/storage/FileManagerHdfs.cpp
+++ b/storage/FileManagerHdfs.cpp
@@ -31,12 +31,12 @@
#include <cstring>
#include <string>
+#include "storage/Flags.hpp"
#include "storage/StorageBlockInfo.hpp"
#include "storage/StorageConstants.hpp"
#include "storage/StorageErrors.hpp"
#include "utility/StringUtil.hpp"
-#include "gflags/gflags.h"
#include "glog/logging.h"
using std::size_t;
@@ -46,41 +46,8 @@ using std::string;
namespace quickstep {
-DEFINE_string(hdfs_namenode_host, "localhost", "Host name or IP address of HDFS namenode.");
-
-static bool ValidateHdfsNameNodePort(const char *flagname,
- std::int32_t value) {
- if (value > 0 && value < 65536) {
- return true;
- } else {
- std::fprintf(stderr, "--%s must be between 1 and 65535 (inclusive)\n", flagname);
- return false;
- }
-}
-DEFINE_int32(hdfs_namenode_port, 9000, "Port of HDFS namenode.");
-static const bool hdfs_namenode_port_dummy
- = gflags::RegisterFlagValidator(&FLAGS_hdfs_namenode_port, &ValidateHdfsNameNodePort);
-
-static bool ValidateHdfsNumReplications(const char *flagname,
- std::int32_t value) {
- // The default dfs.namenode.replication.min is 1, while the default
- // dfs.replication.max is 512.
- if (value > 0) {
- return true;
- } else {
- std::fprintf(stderr, "--%s must be at least 1\n", flagname);
- return false;
- }
-}
-DEFINE_int32(hdfs_num_replications, 1, "Number of HDFS file replications.");
-static const bool hdfs_num_replications_dummy
- = gflags::RegisterFlagValidator(&FLAGS_hdfs_num_replications, &ValidateHdfsNumReplications);
-
FileManagerHdfs::FileManagerHdfs(const string &storage_path)
: FileManager(storage_path) {
- DCHECK(hdfs_namenode_port_dummy);
- DCHECK(hdfs_num_replications_dummy);
-
struct hdfsBuilder *builder = hdfsNewBuilder();
hdfsBuilderSetNameNode(builder, FLAGS_hdfs_namenode_host.c_str());
hdfsBuilderSetNameNodePort(builder, FLAGS_hdfs_namenode_port);
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/dda085c7/storage/FileManagerHdfs.hpp
----------------------------------------------------------------------
diff --git a/storage/FileManagerHdfs.hpp b/storage/FileManagerHdfs.hpp
index 3c751b5..f47e4a8 100644
--- a/storage/FileManagerHdfs.hpp
+++ b/storage/FileManagerHdfs.hpp
@@ -29,14 +29,8 @@
#include "storage/StorageBlockInfo.hpp"
#include "utility/Macros.hpp"
-#include "gflags/gflags.h"
-
namespace quickstep {
-DECLARE_string(hdfs_namenode_host);
-DECLARE_int32(hdfs_namenode_port);
-DECLARE_int32(hdfs_num_replications);
-
/** \addtogroup Storage
* @{
*/
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/dda085c7/storage/Flags.cpp
----------------------------------------------------------------------
diff --git a/storage/Flags.cpp b/storage/Flags.cpp
new file mode 100644
index 0000000..4672f81
--- /dev/null
+++ b/storage/Flags.cpp
@@ -0,0 +1,62 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ **/
+
+#include "storage/Flags.hpp"
+
+#include "storage/StorageConfig.h"
+
+#include "gflags/gflags.h"
+
+namespace quickstep {
+
+#ifdef QUICKSTEP_HAVE_FILE_MANAGER_HDFS
+DEFINE_bool(use_hdfs, false, "Use HDFS as the persistent storage, instead of the local disk.");
+
+DEFINE_string(hdfs_namenode_host, "localhost", "Host name or IP address of HDFS namenode.");
+
+static bool ValidateHdfsNameNodePort(const char *flagname,
+ std::int32_t value) {
+ if (value > 0 && value < 65536) {
+ return true;
+ } else {
+ std::fprintf(stderr, "--%s must be between 1 and 65535 (inclusive)\n", flagname);
+ return false;
+ }
+}
+DEFINE_int32(hdfs_namenode_port, 9000, "Port of HDFS namenode.");
+static const bool hdfs_namenode_port_dummy
+ = gflags::RegisterFlagValidator(&FLAGS_hdfs_namenode_port, &ValidateHdfsNameNodePort);
+
+static bool ValidateHdfsNumReplications(const char *flagname,
+ std::int32_t value) {
+ // The default dfs.namenode.replication.min is 1, while the default
+ // dfs.replication.max is 512.
+ if (value > 0) {
+ return true;
+ } else {
+ std::fprintf(stderr, "--%s must be at least 1\n", flagname);
+ return false;
+ }
+}
+DEFINE_int32(hdfs_num_replications, 1, "Number of HDFS file replications.");
+static const bool hdfs_num_replications_dummy
+ = gflags::RegisterFlagValidator(&FLAGS_hdfs_num_replications, &ValidateHdfsNumReplications);
+#endif
+
+} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/dda085c7/storage/Flags.hpp
----------------------------------------------------------------------
diff --git a/storage/Flags.hpp b/storage/Flags.hpp
new file mode 100644
index 0000000..1d5527c
--- /dev/null
+++ b/storage/Flags.hpp
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ **/
+
+#ifndef QUICKSTEP_STORAGE_FLAGS_HPP_
+#define QUICKSTEP_STORAGE_FLAGS_HPP_
+
+#include "storage/StorageConfig.h" // For QUICKSTEP_HAVE_FILE_MANAGER_HDFS.
+
+#include "gflags/gflags_declare.h"
+
+namespace quickstep {
+
+/** \addtogroup STORAGE
+ * @{
+ */
+
+/**
+ * @brief A collection of common flags shared by Quickstep STORAGEs in both the
+ * single-node and the distributed version.
+ **/
+
+#ifdef QUICKSTEP_HAVE_FILE_MANAGER_HDFS
+DECLARE_bool(use_hdfs);
+
+DECLARE_string(hdfs_namenode_host);
+DECLARE_int32(hdfs_namenode_port);
+DECLARE_int32(hdfs_num_replications);
+
+#endif
+
+/** @} */
+
+} // namespace quickstep
+
+#endif // QUICKSTEP_STORAGE_FLAGS_HPP_
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/dda085c7/storage/StorageManager.cpp
----------------------------------------------------------------------
diff --git a/storage/StorageManager.cpp b/storage/StorageManager.cpp
index 6299cda..6f7d38b 100644
--- a/storage/StorageManager.cpp
+++ b/storage/StorageManager.cpp
@@ -76,6 +76,7 @@
#include "storage/EvictionPolicy.hpp"
#include "storage/FileManagerLocal.hpp"
+#include "storage/Flags.hpp"
#include "storage/StorageBlob.hpp"
#include "storage/StorageBlock.hpp"
#include "storage/StorageBlockBase.hpp"
@@ -189,10 +190,6 @@ DEFINE_uint64(buffer_pool_slots, 0,
static const volatile bool buffer_pool_slots_dummy
= gflags::RegisterFlagValidator(&FLAGS_buffer_pool_slots, &SetOrValidateBufferPoolSlots);
-#ifdef QUICKSTEP_HAVE_FILE_MANAGER_HDFS
-DEFINE_bool(use_hdfs, false, "Use HDFS as the persistent storage, instead of the local disk.");
-#endif
-
StorageManager::StorageManager(
const std::string &storage_path,
const block_id_domain block_domain,
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/dda085c7/storage/StorageManager.hpp
----------------------------------------------------------------------
diff --git a/storage/StorageManager.hpp b/storage/StorageManager.hpp
index b61f10a..42176ee 100644
--- a/storage/StorageManager.hpp
+++ b/storage/StorageManager.hpp
@@ -63,10 +63,6 @@ namespace quickstep {
DECLARE_int32(block_domain);
DECLARE_uint64(buffer_pool_slots);
-#ifdef QUICKSTEP_HAVE_FILE_MANAGER_HDFS
-DECLARE_bool(use_hdfs);
-#endif
-
class CatalogRelationSchema;
#ifdef QUICKSTEP_DISTRIBUTED