You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@quickstep.apache.org by zu...@apache.org on 2017/03/04 11:19:18 UTC
[1/5] incubator-quickstep git commit: Repeated HDFS reads in TextScan,
if needed. [Forced Update!]
Repository: incubator-quickstep
Updated Branches:
refs/heads/dist-patch c5aa04127 -> 40a404f15 (forced update)
Repeated HDFS reads in TextScan, if needed.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/64bd2c9a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/64bd2c9a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/64bd2c9a
Branch: refs/heads/dist-patch
Commit: 64bd2c9ae5164dd1926366ae58ea8ac0699edf6f
Parents: e37ec54
Author: Zuyu Zhang <zu...@apache.org>
Authored: Sat Mar 4 03:18:40 2017 -0800
Committer: Zuyu Zhang <zu...@apache.org>
Committed: Sat Mar 4 03:18:40 2017 -0800
----------------------------------------------------------------------
relational_operators/TextScanOperator.cpp | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/64bd2c9a/relational_operators/TextScanOperator.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/TextScanOperator.cpp b/relational_operators/TextScanOperator.cpp
index 6333813..a133e0c 100644
--- a/relational_operators/TextScanOperator.cpp
+++ b/relational_operators/TextScanOperator.cpp
@@ -241,9 +241,8 @@ void TextScanWorkOrder::execute() {
}
bytes_read = hdfsRead(hdfs, file_handle, buffer, text_segment_size_);
- if (bytes_read != text_segment_size_) {
- hdfsCloseFile(hdfs, file_handle);
- throw TextScanReadError(filename_);
+ while (bytes_read != text_segment_size_) {
+ bytes_read += hdfsRead(hdfs, file_handle, buffer + bytes_read, text_segment_size_ - bytes_read);
}
}
#endif // QUICKSTEP_HAVE_FILE_MANAGER_HDFS
[3/5] incubator-quickstep git commit: Set the default
tmb_implementation value to purememory.
Posted by zu...@apache.org.
Set the default tmb_implementation value to purememory.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/749d0160
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/749d0160
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/749d0160
Branch: refs/heads/dist-patch
Commit: 749d01607eab692b69cbaa4e1c7ffd49a222a70a
Parents: 64bd2c9
Author: Zuyu Zhang <zu...@apache.org>
Authored: Sat Feb 11 11:17:55 2017 -0800
Committer: Zuyu Zhang <zu...@apache.org>
Committed: Sat Mar 4 03:19:02 2017 -0800
----------------------------------------------------------------------
third_party/src/tmb/src/tmb_net_server.cc | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/749d0160/third_party/src/tmb/src/tmb_net_server.cc
----------------------------------------------------------------------
diff --git a/third_party/src/tmb/src/tmb_net_server.cc b/third_party/src/tmb/src/tmb_net_server.cc
index c2be7ef..abe0efe 100644
--- a/third_party/src/tmb/src/tmb_net_server.cc
+++ b/third_party/src/tmb/src/tmb_net_server.cc
@@ -256,7 +256,7 @@ static bool ValidateTmbImplementation(const char *flagname,
"Invalid value for --%s: %s\n", flagname, value.c_str());
return false;
}
-DEFINE_string(tmb_implementation, "nativelog",
+DEFINE_string(tmb_implementation, "purememory",
"Which underlying TMB implementation to use. Valid choices are "
"leveldb, nativelog, purememory, sqlite, voltdb, and zookeeper. "
"Depending on how the TMB library was built, some of these may "
[5/5] incubator-quickstep git commit: Refactored block loading order
in StorageManager::loadBlockOrBlob.
Posted by zu...@apache.org.
Refactored block loading order in StorageManager::loadBlockOrBlob.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/40a404f1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/40a404f1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/40a404f1
Branch: refs/heads/dist-patch
Commit: 40a404f15998fba7d2e6b26953b05f1c7e9ea194
Parents: fa4ebfd
Author: Zuyu Zhang <zu...@apache.org>
Authored: Sat Feb 11 11:24:54 2017 -0800
Committer: Zuyu Zhang <zu...@apache.org>
Committed: Sat Mar 4 03:19:03 2017 -0800
----------------------------------------------------------------------
storage/StorageManager.cpp | 60 ++++++++++++++++++++---------------------
1 file changed, 29 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/40a404f1/storage/StorageManager.cpp
----------------------------------------------------------------------
diff --git a/storage/StorageManager.cpp b/storage/StorageManager.cpp
index 94e1b67..b920c17 100644
--- a/storage/StorageManager.cpp
+++ b/storage/StorageManager.cpp
@@ -666,45 +666,43 @@ StorageManager::BlockHandle StorageManager::loadBlockOrBlob(
// already loaded before this function gets called.
BlockHandle loaded_handle;
-#ifdef QUICKSTEP_DISTRIBUTED
- // TODO(quickstep-team): Use a cost model to determine whether to load from
- // a remote peer or the disk.
- if (BlockIdUtil::Domain(block) != block_domain_) {
- DLOG(INFO) << "Pulling Block " << BlockIdUtil::ToString(block) << " from a remote peer";
- const vector<string> peer_domain_network_addresses = getPeerDomainNetworkAddresses(block);
- for (const string &peer_domain_network_address : peer_domain_network_addresses) {
- DataExchangerClientAsync client(
- grpc::CreateChannel(peer_domain_network_address, grpc::InsecureChannelCredentials()),
- this);
-
- if (client.Pull(block, numa_node, &loaded_handle)) {
- sendBlockLocationMessage(block, kAddBlockLocationMessage);
- return loaded_handle;
- }
- }
+ const size_t num_slots = file_manager_->numSlots(block);
+ if (num_slots != 0) {
+ void *block_buffer = allocateSlots(num_slots, numa_node);
- DLOG(INFO) << "Failed to pull Block " << BlockIdUtil::ToString(block)
- << " from remote peers, so try to load from disk.";
- }
-#endif
+ const bool status = file_manager_->readBlockOrBlob(block, block_buffer, kSlotSizeBytes * num_slots);
+ CHECK(status) << "Failed to read block from persistent storage: " << block;
- const size_t num_slots = file_manager_->numSlots(block);
- DEBUG_ASSERT(num_slots != 0);
- void *block_buffer = allocateSlots(num_slots, numa_node);
+ loaded_handle.block_memory = block_buffer;
+ loaded_handle.block_memory_size = num_slots;
- const bool status = file_manager_->readBlockOrBlob(block, block_buffer, kSlotSizeBytes * num_slots);
- CHECK(status) << "Failed to read block from persistent storage: " << block;
+#ifdef QUICKSTEP_DISTRIBUTED
+ if (bus_) {
+ sendBlockLocationMessage(block, kAddBlockLocationMessage);
+ }
+#endif
- loaded_handle.block_memory = block_buffer;
- loaded_handle.block_memory_size = num_slots;
+ return loaded_handle;
+ }
#ifdef QUICKSTEP_DISTRIBUTED
- if (bus_) {
- sendBlockLocationMessage(block, kAddBlockLocationMessage);
+ // TODO(quickstep-team): Use a cost model to determine whether to load from
+ // a remote peer or the disk.
+ DLOG(INFO) << "Pulling Block " << BlockIdUtil::ToString(block) << " from a remote peer";
+ const vector<string> peer_domain_network_addresses = getPeerDomainNetworkAddresses(block);
+ for (const string &peer_domain_network_address : peer_domain_network_addresses) {
+ DataExchangerClientAsync client(
+ grpc::CreateChannel(peer_domain_network_address, grpc::InsecureChannelCredentials()),
+ this);
+
+ if (client.Pull(block, numa_node, &loaded_handle)) {
+ sendBlockLocationMessage(block, kAddBlockLocationMessage);
+ return loaded_handle;
+ }
}
+ LOG(FATAL) << "Failed to pull Block " << BlockIdUtil::ToString(block)
+ << " from remote peers.";
#endif
-
- return loaded_handle;
}
void StorageManager::insertBlockHandleAfterLoad(const block_id block,
[4/5] incubator-quickstep git commit: Disabled LIP in the distributed
version.
Posted by zu...@apache.org.
Disabled LIP in the distributed version.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/fa4ebfdc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/fa4ebfdc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/fa4ebfdc
Branch: refs/heads/dist-patch
Commit: fa4ebfdc9927c7c8ebba97c50dc177d230e91139
Parents: 3615ba1
Author: Zuyu Zhang <zu...@apache.org>
Authored: Sun Feb 12 17:48:53 2017 -0800
Committer: Zuyu Zhang <zu...@apache.org>
Committed: Sat Mar 4 03:19:02 2017 -0800
----------------------------------------------------------------------
query_optimizer/PhysicalGenerator.cpp | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/fa4ebfdc/query_optimizer/PhysicalGenerator.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/PhysicalGenerator.cpp b/query_optimizer/PhysicalGenerator.cpp
index ac51c31..ca6db3f 100644
--- a/query_optimizer/PhysicalGenerator.cpp
+++ b/query_optimizer/PhysicalGenerator.cpp
@@ -59,7 +59,7 @@ DEFINE_bool(reorder_hash_joins, true,
"cardinality and selective tables to be joined first, which is suitable "
"for queries on star-schema tables.");
-DEFINE_bool(use_filter_joins, true,
+DEFINE_bool(use_filter_joins, false,
"If true, apply an optimization that strength-reduces HashJoins to "
"FilterJoins (implemented as LIPFilters attached to some anchoring "
"operators. Briefly speaking, in the case that the join attribute has "
@@ -67,7 +67,7 @@ DEFINE_bool(use_filter_joins, true,
"build a BitVector on the build-side attribute and use the BitVector "
"to filter the probe side table.");
-DEFINE_bool(use_lip_filters, true,
+DEFINE_bool(use_lip_filters, false,
"If true, use LIP (Lookahead Information Passing) filters to accelerate "
"query processing. LIP filters are effective for queries on star schema "
"tables (e.g. the SSB benchmark) and snowflake schema tables (e.g. the "
[2/5] incubator-quickstep git commit: Enabled dag viz by default.
Posted by zu...@apache.org.
Enabled dag viz by default.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/3615ba1c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/3615ba1c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/3615ba1c
Branch: refs/heads/dist-patch
Commit: 3615ba1cf043daa369dc8af7d4d664b23ebe566a
Parents: 749d016
Author: Zuyu Zhang <zu...@apache.org>
Authored: Sat Feb 11 11:40:33 2017 -0800
Committer: Zuyu Zhang <zu...@apache.org>
Committed: Sat Mar 4 03:19:02 2017 -0800
----------------------------------------------------------------------
query_execution/QueryManagerBase.cpp | 2 +-
query_optimizer/tests/DistributedExecutionGeneratorTest.cpp | 3 +++
2 files changed, 4 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3615ba1c/query_execution/QueryManagerBase.cpp
----------------------------------------------------------------------
diff --git a/query_execution/QueryManagerBase.cpp b/query_execution/QueryManagerBase.cpp
index f84ad4e..2dd5467 100644
--- a/query_execution/QueryManagerBase.cpp
+++ b/query_execution/QueryManagerBase.cpp
@@ -37,7 +37,7 @@ using std::pair;
namespace quickstep {
-DEFINE_bool(visualize_execution_dag, false,
+DEFINE_bool(visualize_execution_dag, true,
"If true, visualize the execution plan DAG into a graph in DOT "
"format (DOT is a plain text graph description language) which is "
"then printed via stderr.");
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3615ba1c/query_optimizer/tests/DistributedExecutionGeneratorTest.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/tests/DistributedExecutionGeneratorTest.cpp b/query_optimizer/tests/DistributedExecutionGeneratorTest.cpp
index b18b5ec..9b96e12 100644
--- a/query_optimizer/tests/DistributedExecutionGeneratorTest.cpp
+++ b/query_optimizer/tests/DistributedExecutionGeneratorTest.cpp
@@ -43,6 +43,8 @@ using std::make_unique;
QUICKSTEP_GENERATE_TEXT_TEST(DISTRIBUTED_EXECUTION_GENERATOR_TEST);
+namespace quickstep { DECLARE_bool(visualize_execution_dag); }
+
int main(int argc, char** argv) {
google::InitGoogleLogging(argv[0]);
@@ -52,6 +54,7 @@ int main(int argc, char** argv) {
quickstep::optimizer::FLAGS_use_filter_joins = false;
// Honor FLAGS_buffer_pool_slots in StorageManager.
+ quickstep::FLAGS_visualize_execution_dag = false;
gflags::ParseCommandLineFlags(&argc, &argv, true);
if (argc < 4) {