You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@quickstep.apache.org by zu...@apache.org on 2017/01/11 01:01:18 UTC
[26/50] incubator-quickstep git commit: Moved FLAGS_num_workers into
a file for shared flags.
Moved FLAGS_num_workers into a file for shared flags.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/e666a81f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/e666a81f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/e666a81f
Branch: refs/heads/quickstep_partition_parser_support
Commit: e666a81f472e4c8ed81e98995c1914ce655e247a
Parents: 212e6cd
Author: Zuyu Zhang <zu...@apache.org>
Authored: Sun Nov 20 22:42:49 2016 -0800
Committer: Zuyu Zhang <zu...@apache.org>
Committed: Sun Nov 20 22:42:49 2016 -0800
----------------------------------------------------------------------
cli/CMakeLists.txt | 1 +
cli/Flags.cpp | 29 +++++++++++++++++++++++++++++
cli/Flags.hpp | 2 ++
cli/QuickstepCli.cpp | 31 +++++--------------------------
4 files changed, 37 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/e666a81f/cli/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/cli/CMakeLists.txt b/cli/CMakeLists.txt
index 9928f3f..9847787 100644
--- a/cli/CMakeLists.txt
+++ b/cli/CMakeLists.txt
@@ -112,6 +112,7 @@ if(QUICKSTEP_HAVE_LIBNUMA)
${LIBNUMA_LIBRARY})
endif()
target_link_libraries(quickstep_cli_Flags
+ quickstep_cli_DefaultsConfigurator
quickstep_storage_StorageConstants
${GFLAGS_LIB_NAME})
target_link_libraries(quickstep_cli_InputParserUtil
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/e666a81f/cli/Flags.cpp
----------------------------------------------------------------------
diff --git a/cli/Flags.cpp b/cli/Flags.cpp
index 384ef71..87f9f73 100644
--- a/cli/Flags.cpp
+++ b/cli/Flags.cpp
@@ -19,14 +19,43 @@
#include "cli/Flags.hpp"
+#include <cstddef>
+#include <cstdio>
#include <string>
+#include "cli/DefaultsConfigurator.hpp"
#include "storage/StorageConstants.hpp"
#include "gflags/gflags.h"
+using std::fprintf;
+
namespace quickstep {
+static bool ValidateNumWorkers(const char *flagname, int value) {
+ if (value > 0) {
+ return true;
+ }
+
+ // Detect the hardware concurrency level.
+ const std::size_t num_hw_threads =
+ DefaultsConfigurator::GetNumHardwareThreads();
+
+ // Use the command-line value if that was supplied, else use the value
+ // that we computed above, provided it did return a valid value.
+ // TODO(jmp): May need to change this at some point to keep one thread
+ // available for the OS if the hardware concurrency level is high.
+ FLAGS_num_workers = num_hw_threads != 0 ? num_hw_threads : 1;
+
+ return FLAGS_num_workers > 0;
+}
+DEFINE_int32(num_workers, 0, "Number of worker threads. If this value is "
+ "specified and is greater than 0, then this user-supplied value is "
+ "used. Else (i.e. the default case), we examine the reported "
+ "hardware concurrency level, and use that.");
+static const volatile bool num_workers_dummy
+ = gflags::RegisterFlagValidator(&FLAGS_num_workers, &ValidateNumWorkers);
+
static bool ValidateStoragePath(const char *flagname,
const std::string &value) {
if (!value.empty() && value.back() != kPathSeparator) {
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/e666a81f/cli/Flags.hpp
----------------------------------------------------------------------
diff --git a/cli/Flags.hpp b/cli/Flags.hpp
index a623448..b020a3e 100644
--- a/cli/Flags.hpp
+++ b/cli/Flags.hpp
@@ -32,6 +32,8 @@ namespace quickstep {
* @brief A collection of common flags shared by Quickstep CLIs in both the
* single-node and the distributed version.
**/
+DECLARE_int32(num_workers);
+
DECLARE_string(storage_path);
/** @} */
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/e666a81f/cli/QuickstepCli.cpp
----------------------------------------------------------------------
diff --git a/cli/QuickstepCli.cpp b/cli/QuickstepCli.cpp
index 33a9ff4..6a7c500 100644
--- a/cli/QuickstepCli.cpp
+++ b/cli/QuickstepCli.cpp
@@ -109,6 +109,7 @@ using quickstep::AdmitRequestMessage;
using quickstep::CatalogRelation;
using quickstep::DefaultsConfigurator;
using quickstep::DropRelation;
+using quickstep::FLAGS_num_workers;
using quickstep::FLAGS_storage_path;
using quickstep::ForemanSingleNode;
using quickstep::InputParserUtil;
@@ -136,11 +137,6 @@ using tmb::client_id;
namespace quickstep {
-DEFINE_int32(num_workers, 0, "Number of worker threads. If this value is "
- "specified and is greater than 0, then this "
- "user-supplied value is used. Else (i.e. the"
- "default case), we examine the reported "
- "hardware concurrency level, and use that.");
DEFINE_bool(preload_buffer_pool, false,
"If true, pre-load all known blocks into buffer pool before "
"accepting queries (should also set --buffer_pool_slots to be "
@@ -189,25 +185,8 @@ int main(int argc, char* argv[]) {
google::InitGoogleLogging(argv[0]);
gflags::ParseCommandLineFlags(&argc, &argv, true);
- // Detect the hardware concurrency level.
- const std::size_t num_hw_threads =
- DefaultsConfigurator::GetNumHardwareThreads();
-
- // Use the command-line value if that was supplied, else use the value
- // that we computed above, provided it did return a valid value.
- // TODO(jmp): May need to change this at some point to keep one thread
- // available for the OS if the hardware concurrency level is high.
- if (quickstep::FLAGS_num_workers <= 0) {
- LOG(INFO) << "Quickstep expects at least one worker thread, switching to "
- "the default number of worker threads";
- }
- const int real_num_workers = quickstep::FLAGS_num_workers > 0
- ? quickstep::FLAGS_num_workers
- : (num_hw_threads != 0 ? num_hw_threads : 1);
-
- DCHECK_GT(real_num_workers, 0);
- printf("Starting Quickstep with %d worker thread(s) and a %.2f GB buffer pool\n",
- real_num_workers,
+ printf("Starting Quickstep with %d worker thread(s) and a %.2f GB buffer pool.\n",
+ FLAGS_num_workers,
(static_cast<double>(quickstep::FLAGS_buffer_pool_slots) * quickstep::kSlotSizeBytes)/quickstep::kAGigaByte);
#ifdef QUICKSTEP_HAVE_FILE_MANAGER_HDFS
@@ -290,7 +269,7 @@ int main(int argc, char* argv[]) {
// Parse the CPU affinities for workers and the preloader thread, if enabled
// to warm up the buffer pool.
const vector<int> worker_cpu_affinities =
- InputParserUtil::ParseWorkerAffinities(real_num_workers,
+ InputParserUtil::ParseWorkerAffinities(FLAGS_num_workers,
quickstep::FLAGS_worker_affinities);
const std::size_t num_numa_nodes_system = DefaultsConfigurator::GetNumNUMANodes();
@@ -323,7 +302,7 @@ int main(int argc, char* argv[]) {
vector<client_id> worker_client_ids;
// Initialize the worker threads.
- DCHECK_EQ(static_cast<std::size_t>(real_num_workers),
+ DCHECK_EQ(static_cast<std::size_t>(FLAGS_num_workers),
worker_cpu_affinities.size());
for (std::size_t worker_thread_index = 0;
worker_thread_index < worker_cpu_affinities.size();