You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@quickstep.apache.org by zu...@apache.org on 2016/06/15 16:45:56 UTC
[1/3] incubator-quickstep git commit: BugFix: Update
NumQueuedWorkOrders to fix scheduling [Forced Update!]
Repository: incubator-quickstep
Updated Branches:
refs/heads/travis-grpc eace5e095 -> 044877b30 (forced update)
BugFix: Update NumQueuedWorkOrders to fix scheduling
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/49316237
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/49316237
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/49316237
Branch: refs/heads/travis-grpc
Commit: 49316237f54cfff6e2ea92fe3c4333f35d46190f
Parents: dd44958
Author: Navneet Potti <na...@apache.org>
Authored: Tue Jun 14 21:52:25 2016 -0500
Committer: Harshad Deshmukh <hb...@apache.org>
Committed: Wed Jun 15 00:20:34 2016 -0500
----------------------------------------------------------------------
query_execution/CMakeLists.txt | 1 +
query_execution/Foreman.cpp | 8 ++++++--
query_execution/PolicyEnforcer.cpp | 2 ++
query_execution/PolicyEnforcer.hpp | 4 ++++
4 files changed, 13 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/49316237/query_execution/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_execution/CMakeLists.txt b/query_execution/CMakeLists.txt
index 323e4a9..501166e 100644
--- a/query_execution/CMakeLists.txt
+++ b/query_execution/CMakeLists.txt
@@ -93,6 +93,7 @@ target_link_libraries(quickstep_queryexecution_PolicyEnforcer
quickstep_queryexecution_QueryExecutionMessages_proto
quickstep_queryexecution_QueryExecutionTypedefs
quickstep_queryexecution_QueryManager
+ quickstep_queryexecution_WorkerDirectory
quickstep_queryexecution_WorkerMessage
quickstep_queryoptimizer_QueryHandle
quickstep_relationaloperators_WorkOrder
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/49316237/query_execution/Foreman.cpp
----------------------------------------------------------------------
diff --git a/query_execution/Foreman.cpp b/query_execution/Foreman.cpp
index 0577e20..828834d 100644
--- a/query_execution/Foreman.cpp
+++ b/query_execution/Foreman.cpp
@@ -89,6 +89,7 @@ Foreman::Foreman(const tmb::client_id main_thread_client_id,
num_numa_nodes,
catalog_database_,
storage_manager_,
+ worker_directory_,
bus_));
}
@@ -115,6 +116,7 @@ void Foreman::run() {
policy_enforcer_->processMessage(tagged_message);
break;
}
+
case kAdmitRequestMessage: {
const AdmitRequestMessage *msg =
static_cast<const AdmitRequestMessage *>(tagged_message.message());
@@ -195,9 +197,11 @@ void Foreman::dispatchWorkerMessages(const vector<unique_ptr<WorkerMessage>> &me
if (recipient_worker_thread_index != WorkerMessage::kInvalidRecipientIndexHint) {
sendWorkerMessage(static_cast<size_t>(recipient_worker_thread_index),
*message);
+ worker_directory_->incrementNumQueuedWorkOrders(recipient_worker_thread_index);
} else {
- sendWorkerMessage(worker_directory_->getLeastLoadedWorker().first,
- *message);
+ const size_t least_loaded_worker_thread_index = worker_directory_->getLeastLoadedWorker().first;
+ sendWorkerMessage(least_loaded_worker_thread_index, *message);
+ worker_directory_->incrementNumQueuedWorkOrders(least_loaded_worker_thread_index);
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/49316237/query_execution/PolicyEnforcer.cpp
----------------------------------------------------------------------
diff --git a/query_execution/PolicyEnforcer.cpp b/query_execution/PolicyEnforcer.cpp
index 2145429..4501026 100644
--- a/query_execution/PolicyEnforcer.cpp
+++ b/query_execution/PolicyEnforcer.cpp
@@ -27,6 +27,7 @@
#include "catalog/CatalogTypedefs.hpp"
#include "query_execution/QueryExecutionMessages.pb.h"
#include "query_execution/QueryManager.hpp"
+#include "query_execution/WorkerDirectory.hpp"
#include "query_optimizer/QueryHandle.hpp"
#include "relational_operators/WorkOrder.hpp"
@@ -72,6 +73,7 @@ void PolicyEnforcer::processMessage(const TaggedMessage &tagged_message) {
CHECK(proto.ParseFromArray(tagged_message.message(),
tagged_message.message_bytes()));
query_id = proto.query_id();
+ worker_directory_->decrementNumQueuedWorkOrders(proto.worker_thread_index());
break;
}
case kCatalogRelationNewBlockMessage: {
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/49316237/query_execution/PolicyEnforcer.hpp
----------------------------------------------------------------------
diff --git a/query_execution/PolicyEnforcer.hpp b/query_execution/PolicyEnforcer.hpp
index 5915b79..9f87056 100644
--- a/query_execution/PolicyEnforcer.hpp
+++ b/query_execution/PolicyEnforcer.hpp
@@ -40,6 +40,7 @@ namespace quickstep {
class CatalogDatabaseLite;
class QueryHandle;
class StorageManager;
+class WorkerDirectory;
/**
* @brief A class that ensures that a high level policy is maintained
@@ -60,11 +61,13 @@ class PolicyEnforcer {
const std::size_t num_numa_nodes,
CatalogDatabaseLite *catalog_database,
StorageManager *storage_manager,
+ WorkerDirectory *worker_directory,
tmb::MessageBus *bus)
: foreman_client_id_(foreman_client_id),
num_numa_nodes_(num_numa_nodes),
catalog_database_(catalog_database),
storage_manager_(storage_manager),
+ worker_directory_(worker_directory),
bus_(bus) {}
/**
@@ -148,6 +151,7 @@ class PolicyEnforcer {
CatalogDatabaseLite *catalog_database_;
StorageManager *storage_manager_;
+ WorkerDirectory *worker_directory_;
tmb::MessageBus *bus_;
[2/3] incubator-quickstep git commit: CI w/ gRPC to build the
distributed version.
Posted by zu...@apache.org.
CI w/ gRPC to build the distributed version.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/64bed613
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/64bed613
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/64bed613
Branch: refs/heads/travis-grpc
Commit: 64bed6131f1490bd1f319c41679f157ea9276b88
Parents: 4931623
Author: Zuyu Zhang <zz...@pivotal.io>
Authored: Mon May 30 11:57:22 2016 -0700
Committer: Zuyu Zhang <zu...@apache.org>
Committed: Wed Jun 15 09:47:26 2016 -0700
----------------------------------------------------------------------
.travis.yml | 28 +++++++++++++++++++---------
CMakeLists.txt | 8 ++++++++
2 files changed, 27 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/64bed613/.travis.yml
----------------------------------------------------------------------
diff --git a/.travis.yml b/.travis.yml
index fce3ba0..7e2a6ad 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,4 +1,4 @@
-# NOTE(quickstep-team): In Travis-CI, jobs timeout if they take more than 120
+# NOTE(quickstep-team): In Travis-CI, jobs timeout if they take more than 50
# mins or if there is no log output for more than 10 mins. Hence, we use -O0 to
# speed up compilation in release build. Also, jobs can only use upto 20GB of
# disk space. Hence, we minimize the amount of debug symbol using -g0 (DEBUG
@@ -15,10 +15,10 @@ compiler:
- clang
env:
- - BUILD_TYPE=Debug VECTOR_COPY_ELISION_LEVEL=joinwithbinaryexpressions
- - BUILD_TYPE=Release VECTOR_COPY_ELISION_LEVEL=joinwithbinaryexpressions
- BUILD_TYPE=Debug VECTOR_COPY_ELISION_LEVEL=selection
- BUILD_TYPE=Release VECTOR_COPY_ELISION_LEVEL=selection
+ - BUILD_TYPE=Debug VECTOR_COPY_ELISION_LEVEL=none
+ - BUILD_TYPE=Release VECTOR_COPY_ELISION_LEVEL=none
before_install:
- LLVM_VERSION=3.7.1
@@ -29,12 +29,12 @@ before_install:
tar xf $LLVM_ARCHIVE_PATH -C $HOME/clang-$LLVM_VERSION --strip-components 1;
ln -sf $HOME/clang-$LLVM_VERSION/bin/clang++ $HOME/clang-$LLVM_VERSION/bin/clang++-3.7;
export PATH=$HOME/clang-$LLVM_VERSION/bin:$PATH;
- export CPPFLAGS="-I $HOME/clang-$LLVM_VERSION/include/c++/v1";
+ export CPPFLAGS="-I $HOME/clang-$LLVM_VERSION/include/c++/v1 -L$HOME/clang-$LLVM_VERSION/lib -stdlib=libc++";
echo "Using clang at " `which $CC-3.7` " and $CXX at " `which $CXX-3.7`;
fi
install:
- - if [ "$CC" = "gcc" ]; then
+ - if [ "$VECTOR_COPY_ELISION_LEVEL" = "selection" ] && [ "$CC" = "gcc" ]; then
export MAKE_JOBS=1;
else
export MAKE_JOBS=2;
@@ -51,11 +51,22 @@ install:
- export DEBUG_FLAGS="-g0";
- export RELEASE_FLAGS="-O0 -DNDEBUG";
- export LINKER_FLAGS="-s"
+ # Protobuf 3 beta.
+ - wget https://github.com/google/protobuf/releases/download/v3.0.0-beta-3/protobuf-cpp-3.0.0-beta-3.tar.gz
+ - tar -xzvf protobuf-cpp-3.0.0-beta-3.tar.gz
+ - pushd protobuf-3.0.0-beta-3 && ./configure --prefix=/usr && make && sudo make install && popd
+ - rm -rf protobuf*
+ - export PROTOC=`which protoc`
+ # gRPC.
+ - git clone https://github.com/grpc/grpc.git
+ - pushd grpc && git checkout release-0_14_1 && git submodule update --init && make && sudo make install && popd
+ - rm -rf grpc
before_script:
- $CC --version
- $CXX --version
- $CLINKER --version
+ - $PROTOC --version
- (cd build &&
cmake -D CMAKE_BUILD_TYPE=$BUILD_TYPE
-D BUILD_SHARED_LIBS=On
@@ -68,14 +79,15 @@ before_script:
-D CMAKE_CXX_COMPILER=$CXX
-D CMAKE_LINKER=$CLINKER
-D USE_TCMALLOC=0
- -D VECTOR_COPY_ELISION_LEVEL=$VECTOR_COPY_ELISION_LEVEL ..)
+ -D VECTOR_COPY_ELISION_LEVEL=$VECTOR_COPY_ELISION_LEVEL
+ -D ENABLE_DISTRIBUTED=ON ..)
script:
- ./third_party/cpplint/lint_everything.py
- ./validate_cmakelists.py
- ./cyclic_dependency.py
- (cd build && make -j$MAKE_JOBS)
- - (cd build && ctest --output-on-failure -j$TEST_JOBS)
+ - (cd build && ctest -E native_net_client_message_bus_unittest --output-on-failure -j$TEST_JOBS)
after_failure:
- df -h
@@ -95,8 +107,6 @@ addons:
- gcc-5
- g++-5
- binutils-gold
- - libprotobuf-dev
- - protobuf-compiler
- libgtest-dev
- python-networkx
- libnuma-dev
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/64bed613/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 20e1fb9..de6754a 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -281,6 +281,14 @@ else()
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror")
endif()
+ # Clang reports such warning when using Protoc 3.0 beta.
+ if(${CMAKE_CXX_COMPILER_ID} MATCHES "Clang")
+ CHECK_CXX_COMPILER_FLAG("-Wno-extended-offsetof" COMPILER_HAS_WNO_EXTENDED_OFFSETOF)
+ if (COMPILER_HAS_WNO_EXTENDED_OFFSETOF)
+ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-extended-offsetof")
+ endif()
+ endif()
+
# One of the protobuf headers includes a nested anonymous union within
# another anonymous type. Ordinarily we work around this by compiling the
# protobuf libraries themselves with "-Wno-nested-anon-types" and including
[3/3] incubator-quickstep git commit: Disabled shared build.
Posted by zu...@apache.org.
Disabled shared build.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/044877b3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/044877b3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/044877b3
Branch: refs/heads/travis-grpc
Commit: 044877b3095822a0cd04d7f7d82d7b2f039da23d
Parents: 64bed61
Author: Zuyu Zhang <zu...@apache.org>
Authored: Tue Jun 14 20:04:51 2016 -0700
Committer: Zuyu Zhang <zu...@apache.org>
Committed: Wed Jun 15 09:47:27 2016 -0700
----------------------------------------------------------------------
.travis.yml | 1 -
1 file changed, 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/044877b3/.travis.yml
----------------------------------------------------------------------
diff --git a/.travis.yml b/.travis.yml
index 7e2a6ad..9aadab7 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -69,7 +69,6 @@ before_script:
- $PROTOC --version
- (cd build &&
cmake -D CMAKE_BUILD_TYPE=$BUILD_TYPE
- -D BUILD_SHARED_LIBS=On
-D CMAKE_C_FLAGS_DEBUG="$DEBUG_FLAGS"
-D CMAKE_CXX_FLAGS_DEBUG="$DEBUG_FLAGS"
-D CMAKE_C_FLAGS_RELEASE="$RELEASE_FLAGS"