You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@quickstep.apache.org by zu...@apache.org on 2017/01/11 01:00:53 UTC
[01/50] incubator-quickstep git commit: QUICKSTEP-46 fixed [Forced
Update!]
Repository: incubator-quickstep
Updated Branches:
refs/heads/quickstep_partition_parser_support fb919fef9 -> 4890bbe83 (forced update)
QUICKSTEP-46 fixed
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/7f0067b7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/7f0067b7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/7f0067b7
Branch: refs/heads/quickstep_partition_parser_support
Commit: 7f0067b7913d8f2b68e8ac771fb9a87090d773e9
Parents: 1effc79
Author: tarun <ta...@gmail.com>
Authored: Mon Oct 10 20:29:42 2016 -0500
Committer: tarunbansal <ta...@gmail.com>
Committed: Fri Oct 28 10:02:34 2016 -0500
----------------------------------------------------------------------
relational_operators/CMakeLists.txt | 14 ++++
relational_operators/TextScanOperator.cpp | 73 ++++++++++++++++----
relational_operators/TextScanOperator.hpp | 19 ++++-
.../tests/text_scan_faulty_golden_output.txt | 5 ++
.../tests/text_scan_faulty_input.txt | 4 ++
5 files changed, 99 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/7f0067b7/relational_operators/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/relational_operators/CMakeLists.txt b/relational_operators/CMakeLists.txt
index 8dd65d0..0735bce 100644
--- a/relational_operators/CMakeLists.txt
+++ b/relational_operators/CMakeLists.txt
@@ -784,3 +784,17 @@ add_test(TextScanOperator_unittest
${TEXT_SCAN_INPUT_FILE}
${TEXT_SCAN_GOLDEN_OUTPUT_FILE}
${TEXT_SCAN_FAILURE_OUTPUT_FILE})
+file(TO_NATIVE_PATH
+ "${CMAKE_CURRENT_SOURCE_DIR}/tests/text_scan_faulty_input.txt"
+ TEXT_SCAN_FAULTY_INPUT_FILE)
+file(TO_NATIVE_PATH
+ "${CMAKE_CURRENT_SOURCE_DIR}/tests/text_scan_faulty_golden_output.txt"
+ TEXT_SCAN_FAULTY_GOLDEN_OUTPUT_FILE)
+file(TO_NATIVE_PATH
+ "${CMAKE_CURRENT_BINARY_DIR}/text_scan_faulty_failure_output.txt"
+ TEXT_SCAN_FAULTY_FAILURE_OUTPUT_FILE)
+add_test(TextScanOperator_faulty_unittest
+ TextScanOperator_unittest
+ ${TEXT_SCAN_FAULTY_INPUT_FILE}
+ ${TEXT_SCAN_FAULTY_GOLDEN_OUTPUT_FILE}
+ ${TEXT_SCAN_FAULTY_FAILURE_OUTPUT_FILE})
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/7f0067b7/relational_operators/TextScanOperator.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/TextScanOperator.cpp b/relational_operators/TextScanOperator.cpp
index 4151bac..aa734d3 100644
--- a/relational_operators/TextScanOperator.cpp
+++ b/relational_operators/TextScanOperator.cpp
@@ -196,7 +196,9 @@ serialization::WorkOrder* TextScanOperator::createWorkOrderProto(const string &f
void TextScanWorkOrder::execute() {
const CatalogRelationSchema &relation = output_destination_->getRelation();
std::vector<Tuple> tuples;
+ bool is_faulty;
+ std::vector<TypedValue> vector_tuple_returned;
constexpr std::size_t kSmallBufferSize = 0x4000;
char *buffer = reinterpret_cast<char *>(malloc(std::max(text_segment_size_, kSmallBufferSize)));
@@ -218,7 +220,6 @@ void TextScanWorkOrder::execute() {
} else {
--row_ptr;
}
-
if (row_ptr >= buffer_end) {
// This block does not even contain a newline character.
return;
@@ -238,16 +239,23 @@ void TextScanWorkOrder::execute() {
// RIGHT AFTER the LAST newline character in this text segment.
// Process the tuples which are between the first newline character and the
- // last newline character.
+ // last newline character. SKIP any row which is corrupt instead of ABORTING the
+ // whole COPY operation.
while (row_ptr < end_ptr) {
if (*row_ptr == '\r' || *row_ptr == '\n') {
// Skip empty lines.
++row_ptr;
} else {
- tuples.emplace_back(parseRow(&row_ptr, relation));
+ vector_tuple_returned = parseRow(&row_ptr, relation, &is_faulty);
+ if (is_faulty) {
+ // Skip faulty rows
+ LOG(INFO) << "Faulty row found. Hence switching to next row.";
+ } else {
+ // Convert vector returned to tuple only when a valid row is encountered.
+ tuples.emplace_back(Tuple(std::move(vector_tuple_returned)));
+ }
}
}
-
// Process the tuple that is right after the last newline character.
// NOTE(jianqiao): dynamic_read_size is trying to balance between the cases
// that the last tuple is very small / very large.
@@ -279,7 +287,15 @@ void TextScanWorkOrder::execute() {
row_string.push_back('\n');
}
row_ptr = row_string.c_str();
- tuples.emplace_back(parseRow(&row_ptr, relation));
+
+ vector_tuple_returned = parseRow(&row_ptr, relation, &is_faulty);
+ if (is_faulty) {
+ // Skip the faulty row.
+ LOG(INFO) << "Faulty row found. Hence switching to next row.";
+ } else {
+ // Convert vector returned to tuple only when a valid row is encountered.
+ tuples.emplace_back(Tuple(std::move(vector_tuple_returned)));
+ }
}
std::fclose(file);
@@ -312,19 +328,26 @@ void TextScanWorkOrder::execute() {
output_destination_->bulkInsertTuples(&column_vectors);
}
-Tuple TextScanWorkOrder::parseRow(const char **row_ptr,
- const CatalogRelationSchema &relation) const {
+std::vector<TypedValue> TextScanWorkOrder::parseRow(const char **row_ptr,
+ const CatalogRelationSchema &relation, bool *is_faulty) const {
std::vector<TypedValue> attribute_values;
+ // Always assume current row is not faulty initially.
+ *is_faulty = false;
bool is_null_literal;
bool has_reached_end_of_line = false;
std::string value_str;
for (const auto &attr : relation) {
if (has_reached_end_of_line) {
- throw TextScanFormatError("Row has too few fields");
+ // Do not abort if one of the row is faulty.
+ // Set is_faulty to true and SKIP the current row.
+ *is_faulty = true;
+ LOG(INFO) << "Row has too few fields.";
+ return attribute_values;
}
value_str.clear();
+
extractFieldString(row_ptr,
&is_null_literal,
&has_reached_end_of_line,
@@ -333,24 +356,46 @@ Tuple TextScanWorkOrder::parseRow(const char **row_ptr,
if (is_null_literal) {
// NULL literal.
if (!attr.getType().isNullable()) {
- throw TextScanFormatError(
- "NULL literal '\\N' was specified for a column with a "
- "non-nullable Type");
+ *is_faulty = true;
+ LOG(INFO) << "NULL literal '\\N' was specified for a column with a "
+ "non-nullable Type.";
+ skipFaultyRow(row_ptr);
+ return attribute_values;
}
attribute_values.emplace_back(attr.getType().makeNullValue());
} else {
attribute_values.emplace_back();
if (!attr.getType().parseValueFromString(value_str, &(attribute_values.back()))) {
- throw TextScanFormatError("Failed to parse value");
+ // Do not abort if one of the row is faulty.
+ *is_faulty = true;
+ LOG(INFO) << "Failed to parse value.";
+ skipFaultyRow(row_ptr);
+ return attribute_values;
}
}
}
if (!has_reached_end_of_line) {
- throw TextScanFormatError("Row has too many fields");
+ // Do not abort if one of the row is faulty.
+ // Set is_faulty to true and SKIP the current row.
+ *is_faulty = true;
+ LOG(INFO) << "Row has too many fields.";
+ skipFaultyRow(row_ptr);
}
- return Tuple(std::move(attribute_values));
+ return attribute_values;
+}
+
+void TextScanWorkOrder::skipFaultyRow(const char **field_ptr) const {
+ const char *cur_ptr = *field_ptr;
+ // Move row pointer to the end of faulty row.
+ for (;; ++cur_ptr) {
+ const char c = *cur_ptr;
+ if (c == '\n') {
+ break;
+ }
+ }
+ *field_ptr = cur_ptr + 1;
}
void TextScanWorkOrder::extractFieldString(const char **field_ptr,
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/7f0067b7/relational_operators/TextScanOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/TextScanOperator.hpp b/relational_operators/TextScanOperator.hpp
index 24af844..65863b3 100644
--- a/relational_operators/TextScanOperator.hpp
+++ b/relational_operators/TextScanOperator.hpp
@@ -24,6 +24,7 @@
#include <cstddef>
#include <exception>
#include <string>
+#include <vector>
#include "catalog/CatalogRelation.hpp"
#include "catalog/CatalogTypedefs.hpp"
@@ -241,6 +242,18 @@ class TextScanWorkOrder : public WorkOrder {
std::string *field_string) const;
/**
+ * @brief This method helps incorporate fault tolerance while ingesting data.
+ * It is called whenever a faulty row is encountered and it is
+ * required to move \p *field_ptr to the next row.
+ *
+ * @param field_ptr \p *field_ptr points to the current position of the input
+ * char stream while parsing a faulty row. After the call, \p *field_ptr
+ * will be modified to the start position of the NEXT record in the
+ * stream.
+ */
+ void skipFaultyRow(const char **field_ptr) const;
+
+ /**
* @brief Make a tuple by parsing all of the individual fields from a char stream.
*
* @param \p *row_ptr points to the current position of the input char stream
@@ -248,10 +261,12 @@ class TextScanWorkOrder : public WorkOrder {
* After the call, \p *row_ptr will be modified to the start position of
* the NEXT text row.
* @param relation The relation schema for the tuple.
+ * @param is_faulty OUTPUT parameter. Set to true if the row is faulty,
* @return The tuple parsed from the char stream.
*/
- Tuple parseRow(const char **row_ptr,
- const CatalogRelationSchema &relation) const;
+std::vector<TypedValue> parseRow(const char **row_ptr,
+ const CatalogRelationSchema &relation,
+ bool *is_faulty) const;
/**
* @brief Parse up to three octal digits (0-7) starting at \p *literal_ptr as
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/7f0067b7/relational_operators/tests/text_scan_faulty_golden_output.txt
----------------------------------------------------------------------
diff --git a/relational_operators/tests/text_scan_faulty_golden_output.txt b/relational_operators/tests/text_scan_faulty_golden_output.txt
new file mode 100644
index 0000000..e07bedf
--- /dev/null
+++ b/relational_operators/tests/text_scan_faulty_golden_output.txt
@@ -0,0 +1,5 @@
++--------------------+------------------------+--------------------+-----------------------------------------+----------------------------------------+--------------------+
+|long_attr |double_attr |char_attr |datetime_attr |interval_attr |varchar_attr |
++--------------------+------------------------+--------------------+-----------------------------------------+----------------------------------------+--------------------+
+| 1234| 12.34| foo| 1994-04-27T08:20:50| 12 days 00:00:00| right_row|
++--------------------+------------------------+--------------------+-----------------------------------------+----------------------------------------+--------------------+
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/7f0067b7/relational_operators/tests/text_scan_faulty_input.txt
----------------------------------------------------------------------
diff --git a/relational_operators/tests/text_scan_faulty_input.txt b/relational_operators/tests/text_scan_faulty_input.txt
new file mode 100644
index 0000000..aa00d39
--- /dev/null
+++ b/relational_operators/tests/text_scan_faulty_input.txt
@@ -0,0 +1,4 @@
+1234 12.34 foo 1994-04-27T08:20:50 12 days right_row
+1234 abcd foo 1994-04-27T08:20:50 12 days row_with_wrong_datatype_value
+1234 foo 1994-04-27T08:20:50 12 days row_with_less_values
+1234 abcd foo 1994-04-27T08:20:50 12 days bar row_with_more_values
[27/50] incubator-quickstep git commit: IWYU for DefaultsConfigurator.
Posted by zu...@apache.org.
IWYU for DefaultsConfigurator.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/54e0654b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/54e0654b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/54e0654b
Branch: refs/heads/quickstep_partition_parser_support
Commit: 54e0654bfb9a42553129428fa94f78535e2781f4
Parents: e666a81
Author: Zuyu Zhang <zu...@apache.org>
Authored: Sun Nov 20 23:30:42 2016 -0800
Committer: Zuyu Zhang <zu...@apache.org>
Committed: Sun Nov 20 23:30:42 2016 -0800
----------------------------------------------------------------------
cli/DefaultsConfigurator.hpp | 28 +++++++++++++++++-----------
1 file changed, 17 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/54e0654b/cli/DefaultsConfigurator.hpp
----------------------------------------------------------------------
diff --git a/cli/DefaultsConfigurator.hpp b/cli/DefaultsConfigurator.hpp
index 7450d88..4fd62ac 100644
--- a/cli/DefaultsConfigurator.hpp
+++ b/cli/DefaultsConfigurator.hpp
@@ -20,17 +20,23 @@
#ifndef QUICKSTEP_CLI_DEFAULTS_CONFIGURATOR_HPP_
#define QUICKSTEP_CLI_DEFAULTS_CONFIGURATOR_HPP_
+#include "storage/StorageConfig.h" // For QUICKSTEP_HAVE_LIBNUMA.
+
+#ifdef QUICKSTEP_HAVE_LIBNUMA
+#include <numa.h>
+#endif // QUICKSTEP_HAVE_LIBNUMA
+
+#include <cstddef>
#include <thread> // NOLINT(build/c++11)
+
+#ifdef QUICKSTEP_HAVE_LIBNUMA
#include <unordered_map>
+#endif // QUICKSTEP_HAVE_LIBNUMA
+
#include <vector>
-#include "storage/StorageConfig.h"
#include "utility/Macros.hpp"
-#ifdef QUICKSTEP_HAVE_LIBNUMA
-#include <numa.h>
-#endif
-
namespace quickstep {
/** \addtogroup CLI
@@ -60,12 +66,12 @@ class DefaultsConfigurator {
* have libnuma.
**/
static std::size_t GetNumNUMANodes() {
- #ifdef QUICKSTEP_HAVE_LIBNUMA
+#ifdef QUICKSTEP_HAVE_LIBNUMA
// Id of the maximum node.
return numa_max_node() + 1;
- #else
+#else
return 1;
- #endif
+#endif // QUICKSTEP_HAVE_LIBNUMA
}
/**
@@ -80,7 +86,7 @@ class DefaultsConfigurator {
**/
static std::size_t GetNumNUMANodesCoveredByWorkers(const std::vector<int> &worker_cpu_affinities) {
if (!worker_cpu_affinities.empty()) {
- #ifdef QUICKSTEP_HAVE_LIBNUMA
+#ifdef QUICKSTEP_HAVE_LIBNUMA
// Key = NUMA node, value = whether there is at least one worker whose
// affinity is set to a core on the given NUMA node.
std::unordered_map<int, bool> any_worker_on_numa_node;
@@ -93,7 +99,7 @@ class DefaultsConfigurator {
}
}
return any_worker_on_numa_node.size();
- #endif
+#endif // QUICKSTEP_HAVE_LIBNUMA
}
// When libnuma is not available, or worker affinities are not specified,
// the default return value is 1.
@@ -108,9 +114,9 @@ class DefaultsConfigurator {
DISALLOW_COPY_AND_ASSIGN(DefaultsConfigurator);
};
+
/** @} */
} // namespace quickstep
#endif // QUICKSTEP_CLI_DEFAULTS_CONFIGURATOR_HPP_
-
[48/50] incubator-quickstep git commit: Remove PackedRowStoreBlock
Posted by zu...@apache.org.
Remove PackedRowStoreBlock
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/c394405a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/c394405a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/c394405a
Branch: refs/heads/quickstep_partition_parser_support
Commit: c394405abdf6f6fbb3c4f6cdcb3400da7d43fafe
Parents: c0d510e
Author: cramja <ma...@gmail.com>
Authored: Mon Nov 7 13:33:21 2016 -0600
Committer: cramja <ma...@gmail.com>
Committed: Sun Jan 8 14:55:21 2017 -0600
----------------------------------------------------------------------
storage/CMakeLists.txt | 70 ---------------------------------------------
1 file changed, 70 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c394405a/storage/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/storage/CMakeLists.txt b/storage/CMakeLists.txt
index 61a8a99..534630a 100644
--- a/storage/CMakeLists.txt
+++ b/storage/CMakeLists.txt
@@ -229,12 +229,6 @@ add_library(quickstep_storage_InsertDestination_proto
add_library(quickstep_storage_LinearOpenAddressingHashTable
../empty_src.cpp
LinearOpenAddressingHashTable.hpp)
-add_library(quickstep_storage_PackedRowStoreTupleStorageSubBlock
- PackedRowStoreTupleStorageSubBlock.cpp
- PackedRowStoreTupleStorageSubBlock.hpp)
-add_library(quickstep_storage_PackedRowStoreValueAccessor
- ../empty_src.cpp
- PackedRowStoreValueAccessor.hpp)
add_library(quickstep_storage_PartitionedHashTablePool ../empty_src.cpp PartitionedHashTablePool.hpp)
add_library(quickstep_storage_PreloaderThread PreloaderThread.cpp PreloaderThread.hpp)
add_library(quickstep_storage_SMAIndexSubBlock SMAIndexSubBlock.cpp SMAIndexSubBlock.hpp)
@@ -827,35 +821,6 @@ target_link_libraries(quickstep_storage_LinearOpenAddressingHashTable
quickstep_utility_Alignment
quickstep_utility_Macros
quickstep_utility_PrimeNumber)
-target_link_libraries(quickstep_storage_PackedRowStoreTupleStorageSubBlock
- quickstep_catalog_CatalogAttribute
- quickstep_catalog_CatalogRelationSchema
- quickstep_catalog_CatalogTypedefs
- quickstep_expressions_predicate_PredicateCost
- quickstep_storage_PackedRowStoreValueAccessor
- quickstep_storage_StorageBlockInfo
- quickstep_storage_StorageBlockLayout_proto
- quickstep_storage_StorageErrors
- quickstep_storage_SubBlockTypeRegistry
- quickstep_storage_SubBlockTypeRegistryMacros
- quickstep_storage_TupleIdSequence
- quickstep_storage_TupleStorageSubBlock
- quickstep_storage_ValueAccessor
- quickstep_storage_ValueAccessorUtil
- quickstep_types_Type
- quickstep_types_TypedValue
- quickstep_types_containers_Tuple
- quickstep_utility_BitVector
- quickstep_utility_Macros)
-target_link_libraries(quickstep_storage_PackedRowStoreValueAccessor
- quickstep_catalog_CatalogRelationSchema
- quickstep_catalog_CatalogTypedefs
- quickstep_storage_StorageBlockInfo
- quickstep_storage_ValueAccessor
- quickstep_types_Type
- quickstep_types_TypedValue
- quickstep_utility_BitVector
- quickstep_utility_Macros)
target_link_libraries(quickstep_storage_PartitionedHashTablePool
glog
quickstep_expressions_aggregation_AggregationHandle
@@ -984,7 +949,6 @@ target_link_libraries(quickstep_storage_StorageBlock
quickstep_storage_HashTableBase
quickstep_storage_IndexSubBlock
quickstep_storage_InsertDestinationInterface
- quickstep_storage_PackedRowStoreTupleStorageSubBlock
quickstep_storage_SMAIndexSubBlock
quickstep_storage_SplitRowStoreTupleStorageSubBlock
quickstep_storage_StorageBlockBase
@@ -1109,7 +1073,6 @@ target_link_libraries(quickstep_storage_ValueAccessorUtil
quickstep_storage_BasicColumnStoreValueAccessor
quickstep_storage_CompressedColumnStoreValueAccessor
quickstep_storage_CompressedPackedRowStoreValueAccessor
- quickstep_storage_PackedRowStoreValueAccessor
quickstep_storage_SplitRowStoreValueAccessor
quickstep_storage_ValueAccessor
quickstep_types_containers_ColumnVectorsValueAccessor
@@ -1179,8 +1142,6 @@ target_link_libraries(quickstep_storage
quickstep_storage_InsertDestinationInterface
quickstep_storage_InsertDestination_proto
quickstep_storage_LinearOpenAddressingHashTable
- quickstep_storage_PackedRowStoreTupleStorageSubBlock
- quickstep_storage_PackedRowStoreValueAccessor
quickstep_storage_PartitionedHashTablePool
quickstep_storage_PreloaderThread
quickstep_storage_SMAIndexSubBlock
@@ -1600,37 +1561,6 @@ target_link_libraries(LinearOpenAddressingHashTable_unittest
${LIBS})
add_test(LinearOpenAddressingHashTable_unittest LinearOpenAddressingHashTable_unittest)
-add_executable(PackedRowStoreTupleStorageSubBlock_unittest "${CMAKE_CURRENT_SOURCE_DIR}/tests/PackedRowStoreTupleStorageSubBlock_unittest.cpp")
-target_link_libraries(PackedRowStoreTupleStorageSubBlock_unittest
- gtest
- gtest_main
- quickstep_catalog_CatalogAttribute
- quickstep_catalog_CatalogRelation
- quickstep_storage_PackedRowStoreTupleStorageSubBlock
- quickstep_storage_StorageBlockInfo
- quickstep_storage_StorageBlockLayout
- quickstep_storage_StorageBlockLayout_proto
- quickstep_storage_StorageConstants
- quickstep_storage_StorageErrors
- quickstep_storage_TupleIdSequence
- quickstep_storage_TupleStorageSubBlock
- quickstep_types_CharType
- quickstep_types_DoubleType
- quickstep_types_IntType
- quickstep_types_Type
- quickstep_types_TypeFactory
- quickstep_types_TypeID
- quickstep_types_TypedValue
- quickstep_types_containers_Tuple
- quickstep_types_operations_comparisons_Comparison
- quickstep_types_operations_comparisons_ComparisonFactory
- quickstep_types_operations_comparisons_ComparisonID
- quickstep_utility_BitVector
- quickstep_utility_PtrMap
- quickstep_utility_ScopedBuffer
- ${LIBS})
-add_test(PackedRowStoreTupleStorageSubBlock_unittest PackedRowStoreTupleStorageSubBlock_unittest)
-
add_executable(SMAIndexSubBlock_unittest "${CMAKE_CURRENT_SOURCE_DIR}/tests/SMAIndexSubBlock_unittest.cpp")
target_link_libraries(SMAIndexSubBlock_unittest
gtest
[41/50] incubator-quickstep git commit: Marked SingleNodeQuery for
Insertions.
Posted by zu...@apache.org.
Marked SingleNodeQuery for Insertions.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/0859a17a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/0859a17a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/0859a17a
Branch: refs/heads/quickstep_partition_parser_support
Commit: 0859a17aa4e71ef8d3d261f15e52518b39f617f6
Parents: e50a2b7
Author: Zuyu Zhang <zu...@apache.org>
Authored: Sun Dec 4 14:11:58 2016 -0800
Committer: Zuyu Zhang <zu...@apache.org>
Committed: Sun Dec 4 15:44:24 2016 -0800
----------------------------------------------------------------------
query_execution/ForemanDistributed.cpp | 16 +++++++---
query_execution/PolicyEnforcerDistributed.hpp | 23 +++++++++++---
query_optimizer/ExecutionGenerator.cpp | 3 ++
query_optimizer/QueryHandle.hpp | 37 ++++++++++++++++++++++
4 files changed, 71 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/0859a17a/query_execution/ForemanDistributed.cpp
----------------------------------------------------------------------
diff --git a/query_execution/ForemanDistributed.cpp b/query_execution/ForemanDistributed.cpp
index 61f0603..0dad8b0 100644
--- a/query_execution/ForemanDistributed.cpp
+++ b/query_execution/ForemanDistributed.cpp
@@ -295,14 +295,22 @@ bool ForemanDistributed::isHashJoinRelatedWorkOrder(const S::WorkOrderMessage &p
return true;
}
+namespace {
+constexpr size_t kDefaultShiftbossIndex = 0u;
+} // namespace
+
void ForemanDistributed::dispatchWorkOrderMessages(const vector<unique_ptr<S::WorkOrderMessage>> &messages) {
- const size_t num_shiftbosses = shiftboss_directory_.size();
- size_t shiftboss_index = 0u;
+ static size_t shiftboss_index = kDefaultShiftbossIndex;
+
+ PolicyEnforcerDistributed* policy_enforcer_dist = static_cast<PolicyEnforcerDistributed*>(policy_enforcer_.get());
for (const auto &message : messages) {
DCHECK(message != nullptr);
const S::WorkOrderMessage &proto = *message;
size_t shiftboss_index_for_particular_work_order_type;
- if (isAggregationRelatedWorkOrder(proto, shiftboss_index, &shiftboss_index_for_particular_work_order_type)) {
+ if (policy_enforcer_dist->isSingleNodeQuery(proto.query_id())) {
+ // Always schedule the single-node query to the same Shiftboss.
+ shiftboss_index_for_particular_work_order_type = kDefaultShiftbossIndex;
+ } else if (isAggregationRelatedWorkOrder(proto, shiftboss_index, &shiftboss_index_for_particular_work_order_type)) {
} else if (isHashJoinRelatedWorkOrder(proto, shiftboss_index, &shiftboss_index_for_particular_work_order_type)) {
} else {
// TODO(zuyu): Take data-locality into account for scheduling.
@@ -313,7 +321,7 @@ void ForemanDistributed::dispatchWorkOrderMessages(const vector<unique_ptr<S::Wo
shiftboss_directory_.incrementNumQueuedWorkOrders(shiftboss_index_for_particular_work_order_type);
if (shiftboss_index == shiftboss_index_for_particular_work_order_type) {
- shiftboss_index = (shiftboss_index + 1) % num_shiftbosses;
+ shiftboss_index = (shiftboss_index + 1) % shiftboss_directory_.size();
} else {
// NOTE(zuyu): This is not the exact round-robin scheduling, as in this case,
// <shiftboss_index_for_particular_work_order_type> might be scheduled one
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/0859a17a/query_execution/PolicyEnforcerDistributed.hpp
----------------------------------------------------------------------
diff --git a/query_execution/PolicyEnforcerDistributed.hpp b/query_execution/PolicyEnforcerDistributed.hpp
index e8bc394..2c00a6b 100644
--- a/query_execution/PolicyEnforcerDistributed.hpp
+++ b/query_execution/PolicyEnforcerDistributed.hpp
@@ -17,14 +17,20 @@
#include <cstddef>
#include <memory>
+#include <unordered_map>
+#include <utility>
#include <vector>
#include "query_execution/PolicyEnforcerBase.hpp"
#include "query_execution/QueryContext.hpp"
#include "query_execution/QueryExecutionMessages.pb.h"
+#include "query_execution/QueryManagerBase.hpp"
#include "query_execution/ShiftbossDirectory.hpp"
+#include "query_optimizer/QueryHandle.hpp"
#include "utility/Macros.hpp"
+#include "glog/logging.h"
+
#include "tmb/id_typedefs.h"
namespace tmb {
@@ -35,10 +41,6 @@ class TaggedMessage;
namespace quickstep {
class CatalogDatabaseLite;
-class QueryHandle;
-class QueryManagerBase;
-
-namespace serialization { class WorkOrderMessage; }
/** \addtogroup QueryExecution
* @{
@@ -90,6 +92,19 @@ class PolicyEnforcerDistributed final : public PolicyEnforcerBase {
void processInitiateRebuildResponseMessage(const tmb::TaggedMessage &tagged_message);
/**
+ * @brief Whether the query should be executed on one Shiftboss.
+ *
+ * @param query_id The query id.
+ *
+ * @return Whether the query should be executed on one Shiftboss.
+ **/
+ bool isSingleNodeQuery(const std::size_t query_id) const {
+ const auto cit = admitted_queries_.find(query_id);
+ DCHECK(cit != admitted_queries_.end());
+ return cit->second->query_handle()->is_single_node_query();
+ }
+
+ /**
* @brief Get or set the index of Shiftboss for an Aggregation related
* WorkOrder. If it is the first Aggregation on <aggr_state_index>,
* <shiftboss_index> will be set to <next_shiftboss_index_to_schedule>.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/0859a17a/query_optimizer/ExecutionGenerator.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/ExecutionGenerator.cpp b/query_optimizer/ExecutionGenerator.cpp
index 2e0d8f3..5a2c450 100644
--- a/query_optimizer/ExecutionGenerator.cpp
+++ b/query_optimizer/ExecutionGenerator.cpp
@@ -1096,6 +1096,9 @@ void ExecutionGenerator::convertDropTable(
void ExecutionGenerator::convertInsertTuple(
const P::InsertTuplePtr &physical_plan) {
// InsertTuple is converted to an Insert and a SaveBlocks.
+#ifdef QUICKSTEP_DISTRIBUTED
+ query_handle_->set_is_single_node_query();
+#endif // QUICKSTEP_DISTRIBUTED
const CatalogRelationInfo *input_relation_info =
findRelationInfoOutputByPhysical(physical_plan->input());
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/0859a17a/query_optimizer/QueryHandle.hpp
----------------------------------------------------------------------
diff --git a/query_optimizer/QueryHandle.hpp b/query_optimizer/QueryHandle.hpp
index 1ca6021..cbd1cd9 100644
--- a/query_optimizer/QueryHandle.hpp
+++ b/query_optimizer/QueryHandle.hpp
@@ -26,6 +26,7 @@
#include "catalog/Catalog.pb.h"
#include "query_execution/QueryContext.pb.h"
+#include "query_optimizer/QueryOptimizerConfig.h" // For QUICKSTEP_DISTRIBUTED.
#include "query_optimizer/QueryPlan.hpp"
#include "utility/Macros.hpp"
@@ -134,6 +135,22 @@ class QueryHandle {
query_result_relation_ = relation;
}
+#ifdef QUICKSTEP_DISTRIBUTED
+ /**
+ * @brief Whether the query will be executed in the single node.
+ */
+ bool is_single_node_query() const {
+ return is_single_node_query_;
+ }
+
+ /**
+ * @brief Set the query to be executed in the single node.
+ */
+ void set_is_single_node_query() {
+ is_single_node_query_ = true;
+ }
+#endif // QUICKSTEP_DISTRIBUTED
+
private:
const std::size_t query_id_;
@@ -153,6 +170,26 @@ class QueryHandle {
// and deleted by the Cli shell.
const CatalogRelation *query_result_relation_;
+#ifdef QUICKSTEP_DISTRIBUTED
+ // Indicate whether the query should be executed on the default Shiftboss for
+ // correctness purpose.
+ // An example would be the insert query that might otherwise need block
+ // invalidation among multiple StorageManagers. In this case, an insert query
+ // has scheduled on node 0, and the block is in the buffer pool of node 0.
+ // Another insert query on the same relation might be scheduled on another
+ // node, say node 1, which will pull the block from node 0, and do the
+ // insertion. Thus, two blocks with the same block id in two nodes
+ // have different contents, which is incorrect.
+ // One approach is to evict blocks cached in all other nodes for every
+ // change. It, however, does not scale, and even worse, it will also affect
+ // the performance of each select query.
+ // Alternatively, we choose to mark the query as a single-node query to
+ // modify blocks on the default node only. But if the changed block has also
+ // cached in another node, this approach would still produce inconsistent
+ // query result.
+ bool is_single_node_query_ = false;
+#endif // QUICKSTEP_DISTRIBUTED
+
DISALLOW_COPY_AND_ASSIGN(QueryHandle);
};
[47/50] incubator-quickstep git commit: Fixed the bug in resetting
exactness for InsertSelect.
Posted by zu...@apache.org.
Fixed the bug in resetting exactness for InsertSelect.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/c0d510eb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/c0d510eb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/c0d510eb
Branch: refs/heads/quickstep_partition_parser_support
Commit: c0d510eb0469c095cc216846d2da5df65961507b
Parents: 9fcb0ac
Author: Zuyu Zhang <zu...@apache.org>
Authored: Fri Jan 6 13:45:59 2017 -0800
Committer: Zuyu Zhang <zu...@apache.org>
Committed: Fri Jan 6 13:45:59 2017 -0800
----------------------------------------------------------------------
query_optimizer/ExecutionGenerator.cpp | 2 +-
relational_operators/SaveBlocksOperator.hpp | 6 +++---
2 files changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c0d510eb/query_optimizer/ExecutionGenerator.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/ExecutionGenerator.cpp b/query_optimizer/ExecutionGenerator.cpp
index d24f498..29e67f7 100644
--- a/query_optimizer/ExecutionGenerator.cpp
+++ b/query_optimizer/ExecutionGenerator.cpp
@@ -1243,7 +1243,7 @@ void ExecutionGenerator::convertInsertSelection(
insert_destination_proto->set_relational_op_index(insert_selection_index);
CatalogRelation *mutable_relation =
- catalog_database_->getRelationByIdMutable(selection_relation->getID());
+ catalog_database_->getRelationByIdMutable(destination_relation.getID());
const QueryPlan::DAGNodeIndex save_blocks_index =
execution_plan_->addRelationalOperator(
new SaveBlocksOperator(query_handle_->query_id(), mutable_relation));
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c0d510eb/relational_operators/SaveBlocksOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/SaveBlocksOperator.hpp b/relational_operators/SaveBlocksOperator.hpp
index a8d5327..573d81e 100644
--- a/relational_operators/SaveBlocksOperator.hpp
+++ b/relational_operators/SaveBlocksOperator.hpp
@@ -61,9 +61,9 @@ class SaveBlocksOperator : public RelationalOperator {
* @param force If true, force writing of all blocks to disk, otherwise only
* write dirty blocks.
**/
- explicit SaveBlocksOperator(const std::size_t query_id,
- CatalogRelation *relation,
- const bool force = false)
+ SaveBlocksOperator(const std::size_t query_id,
+ CatalogRelation *relation,
+ const bool force = false)
: RelationalOperator(query_id),
force_(force),
relation_(relation),
[30/50] incubator-quickstep git commit: Adds support for
PartialBulkInserts in StorageBlocks
Posted by zu...@apache.org.
Adds support for PartialBulkInserts in StorageBlocks
- Enables use of PartialBulkInserts in StorageBlocks
- Value accessor changes to allow use of 2 insert destinations
- Enables PartialInserts for SplitRow
- Changes HashJoin operator so that it can take advantage of the
PartialInserts code.
- This also cleans up code from Previous commit.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/2d11ec58
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/2d11ec58
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/2d11ec58
Branch: refs/heads/quickstep_partition_parser_support
Commit: 2d11ec588e7e8d7a7a1a8adfc28fb30e5fe0852e
Parents: 172b51b
Author: navsan <na...@gmail.com>
Authored: Mon Nov 7 14:35:01 2016 -0600
Committer: cramja <ma...@gmail.com>
Committed: Mon Nov 21 14:28:02 2016 -0600
----------------------------------------------------------------------
relational_operators/HashJoinOperator.cpp | 150 ++++++++++++++++---
storage/InsertDestination.cpp | 84 +++++++++++
storage/InsertDestination.hpp | 16 ++
storage/InsertDestinationInterface.hpp | 22 +++
storage/SplitRowStoreTupleStorageSubBlock.hpp | 4 +-
storage/StorageBlock.cpp | 24 +++
storage/StorageBlock.hpp | 44 ++++++
storage/TupleStorageSubBlock.hpp | 50 +++++++
types/containers/ColumnVectorsValueAccessor.hpp | 4 +
9 files changed, 373 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2d11ec58/relational_operators/HashJoinOperator.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/HashJoinOperator.cpp b/relational_operators/HashJoinOperator.cpp
index 4a91f86..2028046 100644
--- a/relational_operators/HashJoinOperator.cpp
+++ b/relational_operators/HashJoinOperator.cpp
@@ -65,10 +65,11 @@ namespace {
// Functor passed to HashTable::getAllFromValueAccessor() to collect matching
// tuples from the inner relation. It stores matching tuple ID pairs
-// in an unordered_map keyed by inner block ID.
-class MapBasedJoinedTupleCollector {
+// in an unordered_map keyed by inner block ID and a vector of
+// pairs of (build-tupleID, probe-tuple-ID).
+class VectorsOfPairsJoinedTuplesCollector {
public:
- MapBasedJoinedTupleCollector() {
+ VectorsOfPairsJoinedTuplesCollector() {
}
template <typename ValueAccessorT>
@@ -95,6 +96,34 @@ class MapBasedJoinedTupleCollector {
std::unordered_map<block_id, std::vector<std::pair<tuple_id, tuple_id>>> joined_tuples_;
};
+// Another collector using an unordered_map keyed on inner block just like above,
+// except that it uses of a pair of (build-tupleIDs-vector, probe-tuple-IDs-vector).
+class PairsOfVectorsJoinedTuplesCollector {
+ public:
+ PairsOfVectorsJoinedTuplesCollector() {
+ }
+
+ template <typename ValueAccessorT>
+ inline void operator()(const ValueAccessorT &accessor,
+ const TupleReference &tref) {
+ joined_tuples_[tref.block].first.push_back(tref.tuple);
+ joined_tuples_[tref.block].second.push_back(accessor.getCurrentPosition());
+ }
+
+ // Get a mutable pointer to the collected map of joined tuple ID pairs. The
+ // key is inner block_id, value is a pair consisting of
+ // inner block tuple IDs (first) and outer block tuple IDs (second).
+ inline std::unordered_map< block_id, std::pair<std::vector<tuple_id>, std::vector<tuple_id>>>*
+ getJoinedTuples() {
+ return &joined_tuples_;
+ }
+
+ private:
+ std::unordered_map<
+ block_id,
+ std::pair<std::vector<tuple_id>, std::vector<tuple_id>>> joined_tuples_;
+};
+
class SemiAntiJoinTupleCollector {
public:
explicit SemiAntiJoinTupleCollector(TupleIdSequence *filter)
@@ -432,7 +461,7 @@ void HashInnerJoinWorkOrder::execute() {
base_accessor->createSharedTupleIdSequenceAdapterVirtual(*existence_map));
}
- MapBasedJoinedTupleCollector collector;
+ PairsOfVectorsJoinedTuplesCollector collector;
if (join_key_attributes_.size() == 1) {
hash_table_.getAllFromValueAccessor(
probe_accessor.get(),
@@ -450,12 +479,14 @@ void HashInnerJoinWorkOrder::execute() {
const relation_id build_relation_id = build_relation_.getID();
const relation_id probe_relation_id = probe_relation_.getID();
- for (std::pair<const block_id, std::vector<std::pair<tuple_id, tuple_id>>>
+ for (std::pair<const block_id, std::pair<std::vector<tuple_id>, std::vector<tuple_id>>>
&build_block_entry : *collector.getJoinedTuples()) {
BlockReference build_block =
storage_manager_->getBlock(build_block_entry.first, build_relation_);
const TupleStorageSubBlock &build_store = build_block->getTupleStorageSubBlock();
std::unique_ptr<ValueAccessor> build_accessor(build_store.createValueAccessor());
+ const std::vector<tuple_id> &build_tids = build_block_entry.second.first;
+ const std::vector<tuple_id> &probe_tids = build_block_entry.second.second;
// Evaluate '*residual_predicate_', if any.
//
@@ -468,17 +499,16 @@ void HashInnerJoinWorkOrder::execute() {
// hash join is below a reasonable threshold so that we don't blow up
// temporary memory requirements to an unreasonable degree.
if (residual_predicate_ != nullptr) {
- std::vector<std::pair<tuple_id, tuple_id>> filtered_matches;
-
- for (const std::pair<tuple_id, tuple_id> &hash_match
- : build_block_entry.second) {
+ std::pair<std::vector<tuple_id>, std::vector<tuple_id>> filtered_matches;
+ for (std::size_t i = 0; i < build_tids.size(); ++i) {
if (residual_predicate_->matchesForJoinedTuples(*build_accessor,
build_relation_id,
- hash_match.first,
+ build_tids[i],
*probe_accessor,
probe_relation_id,
- hash_match.second)) {
- filtered_matches.emplace_back(hash_match);
+ probe_tids[i])) {
+ filtered_matches.first.push_back(build_tids[i]);
+ filtered_matches.second.push_back(probe_tids[i]);
}
}
@@ -501,22 +531,96 @@ void HashInnerJoinWorkOrder::execute() {
// benefit (probably only a real performance win when there are very few
// matching tuples in each individual inner block but very many inner
// blocks with at least one match).
+
+ // We now create ordered value accessors for both build and probe side,
+ // using the joined tuple TIDs. Note that we have to use this Lambda-based
+ // invocation method here because the accessors don't have a virtual
+ // function that creates such an OrderedTupleIdSequenceAdapterValueAccessor.
+ std::unique_ptr<ValueAccessor> ordered_build_accessor, ordered_probe_accessor;
+ InvokeOnValueAccessorNotAdapter(
+ build_accessor.get(),
+ [&](auto *accessor) -> void { // NOLINT(build/c++11)
+ ordered_build_accessor.reset(
+ accessor->createSharedOrderedTupleIdSequenceAdapter(build_tids));
+ });
+
+ if (probe_accessor->isTupleIdSequenceAdapter()) {
+ InvokeOnTupleIdSequenceAdapterValueAccessor(
+ probe_accessor.get(),
+ [&](auto *accessor) -> void { // NOLINT(build/c++11)
+ ordered_probe_accessor.reset(
+ accessor->createSharedOrderedTupleIdSequenceAdapter(probe_tids));
+ });
+ } else {
+ InvokeOnValueAccessorNotAdapter(
+ probe_accessor.get(),
+ [&](auto *accessor) -> void { // NOLINT(build/c++11)
+ ordered_probe_accessor.reset(
+ accessor->createSharedOrderedTupleIdSequenceAdapter(probe_tids));
+ });
+ }
+
+
+ // We also need a temp value accessor to store results of any scalar expressions.
ColumnVectorsValueAccessor temp_result;
- for (vector<unique_ptr<const Scalar>>::const_iterator selection_cit = selection_.begin();
- selection_cit != selection_.end();
- ++selection_cit) {
- temp_result.addColumn((*selection_cit)->getAllValuesForJoin(build_relation_id,
- build_accessor.get(),
- probe_relation_id,
- probe_accessor.get(),
- build_block_entry.second));
+
+ // Create a map of ValueAccessors and what attributes we want to pick from them
+ std::vector<std::pair<ValueAccessor *, std::vector<attribute_id>>> accessor_attribute_map;
+ const std::vector<ValueAccessor *> accessors{
+ ordered_build_accessor.get(), ordered_probe_accessor.get(), &temp_result};
+ const unsigned int build_index = 0, probe_index = 1, temp_index = 2;
+ for (auto &accessor : accessors) {
+ accessor_attribute_map.push_back(std::make_pair(
+ accessor,
+ std::vector<attribute_id>(selection_.size(), kInvalidCatalogId)));
+ }
+
+ attribute_id dest_attr = 0;
+ std::vector<std::pair<tuple_id, tuple_id>> zipped_joined_tuple_ids;
+
+ for (auto &selection_cit : selection_) {
+ // If the Scalar (column) is not an attribute in build/probe blocks, then
+ // insert it into a ColumnVectorsValueAccessor.
+ if (selection_cit->getDataSource() != Scalar::ScalarDataSource::kAttribute) {
+ // Current destination attribute maps to the column we'll create now.
+ accessor_attribute_map[temp_index].second[dest_attr] = temp_result.getNumColumns();
+
+ if (temp_result.getNumColumns() == 0) {
+ // The getAllValuesForJoin function below needs joined tuple IDs as
+ // a vector of pair of (build-tuple-ID, probe-tuple-ID), and we have
+ // a pair of (build-tuple-IDs-vector, probe-tuple-IDs-vector). So
+ // we'll have to zip our two vectors together. We do this inside
+ // the loop because most queries don't exercise this code since
+ // they don't have scalar expressions with attributes from both
+ // build and probe relations (other expressions would have been
+ // pushed down to before the join).
+ zipped_joined_tuple_ids.reserve(build_tids.size());
+ for (std::size_t i = 0; i < build_tids.size(); ++i) {
+ zipped_joined_tuple_ids.push_back(std::make_pair(build_tids[i], probe_tids[i]));
+ }
+ }
+ temp_result.addColumn(
+ selection_cit
+ ->getAllValuesForJoin(build_relation_id, build_accessor.get(),
+ probe_relation_id, probe_accessor.get(),
+ zipped_joined_tuple_ids));
+ } else {
+ auto scalar_attr = static_cast<const ScalarAttribute *>(selection_cit.get());
+ const attribute_id attr_id = scalar_attr->getAttribute().getID();
+ if (scalar_attr->getAttribute().getParent().getID() == build_relation_id) {
+ accessor_attribute_map[build_index].second[dest_attr] = attr_id;
+ } else {
+ accessor_attribute_map[probe_index].second[dest_attr] = attr_id;
+ }
+ }
+ ++dest_attr;
}
// NOTE(chasseur): calling the bulk-insert method of InsertDestination once
// for each pair of joined blocks incurs some extra overhead that could be
// avoided by keeping checked-out MutableBlockReferences across iterations
// of this loop, but that would get messy when combined with partitioning.
- output_destination_->bulkInsertTuples(&temp_result);
+ output_destination_->bulkInsertTuplesFromValueAccessors(accessor_attribute_map);
}
}
@@ -550,7 +654,7 @@ void HashSemiJoinWorkOrder::executeWithResidualPredicate() {
// We collect all the matching probe relation tuples, as there's a residual
// preidcate that needs to be applied after collecting these matches.
- MapBasedJoinedTupleCollector collector;
+ VectorsOfPairsJoinedTuplesCollector collector;
if (join_key_attributes_.size() == 1) {
hash_table_.getAllFromValueAccessor(
probe_accessor.get(),
@@ -759,7 +863,7 @@ void HashAntiJoinWorkOrder::executeWithResidualPredicate() {
base_accessor->createSharedTupleIdSequenceAdapterVirtual(*existence_map));
}
- MapBasedJoinedTupleCollector collector;
+ VectorsOfPairsJoinedTuplesCollector collector;
// We probe the hash table and get all the matches. Unlike
// executeWithoutResidualPredicate(), we have to collect all the matching
// tuples, because after this step we still have to evalute the residual
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2d11ec58/storage/InsertDestination.cpp
----------------------------------------------------------------------
diff --git a/storage/InsertDestination.cpp b/storage/InsertDestination.cpp
index 5e83453..067edf6 100644
--- a/storage/InsertDestination.cpp
+++ b/storage/InsertDestination.cpp
@@ -247,6 +247,90 @@ void InsertDestination::bulkInsertTuplesWithRemappedAttributes(
});
}
+// A common case that we can optimize away is when the attribute_map
+// for an accessor only contains gaps. e.g. This happens for a join when
+// there are no attributes selected from one side.
+void removeGapOnlyAccessors(
+ const std::vector<std::pair<ValueAccessor *, std::vector<attribute_id>>>* accessor_attribute_map,
+ std::vector<std::pair<ValueAccessor *, const std::vector<attribute_id>>>* reduced_accessor_attribute_map) {
+ for (std::size_t i = 0; i < accessor_attribute_map->size(); ++i) {
+ bool all_gaps = true;
+ for (const auto &attr : (*accessor_attribute_map)[i].second)
+ if (attr != kInvalidCatalogId) {
+ all_gaps = false;
+ break;
+ }
+ if (all_gaps)
+ continue;
+ reduced_accessor_attribute_map->push_back((*accessor_attribute_map)[i]);
+ (*accessor_attribute_map)[i].first->beginIterationVirtual();
+ }
+}
+
+void InsertDestination::bulkInsertTuplesFromValueAccessors(
+ const std::vector<std::pair<ValueAccessor *, std::vector<attribute_id>>> &accessor_attribute_map,
+ bool always_mark_full) {
+ // Handle pathological corner case where there are no accessors
+ if (accessor_attribute_map.size() == 0)
+ return;
+
+ std::vector<std::pair<ValueAccessor *, const std::vector<attribute_id>>> reduced_accessor_attribute_map;
+ removeGapOnlyAccessors(&accessor_attribute_map, &reduced_accessor_attribute_map);
+
+ // We assume that all input accessors have the same number of tuples, so
+ // the iterations finish together. Therefore, we can just check the first one.
+ auto first_accessor = reduced_accessor_attribute_map[0].first;
+ while (!first_accessor->iterationFinishedVirtual()) {
+ tuple_id num_tuples_to_insert = kCatalogMaxID;
+ tuple_id num_tuples_inserted = 0;
+ MutableBlockReference output_block = this->getBlockForInsertion();
+
+ // Now iterate through all the accessors and do one round of bulk-insertion
+ // of partial tuples into the selected output_block.
+ // While inserting from the first ValueAccessor, space is reserved for
+ // all the columns including those coming from other ValueAccessors.
+ // Thereafter, in a given round, we only insert the remaining columns of the
+ // same tuples from the other ValueAccessors.
+ for (auto &p : reduced_accessor_attribute_map) {
+ ValueAccessor *accessor = p.first;
+ std::vector<attribute_id> attribute_map = p.second;
+
+
+ InvokeOnAnyValueAccessor(
+ accessor,
+ [&](auto *accessor) -> void { // NOLINT(build/c++11)
+ num_tuples_inserted = output_block->bulkInsertPartialTuples(
+ attribute_map, accessor, num_tuples_to_insert);
+ });
+
+ if (accessor == first_accessor) {
+ // Now we know how many full tuples can be inserted into this
+ // output_block (viz. number of tuples inserted from first ValueAccessor).
+ // We should only insert that many tuples from the remaining
+ // ValueAccessors as well.
+ num_tuples_to_insert = num_tuples_inserted;
+ } else {
+ // Since the bulk insertion of the first ValueAccessor should already
+ // have reserved the space for all the other ValueAccessors' columns,
+ // we must have been able to insert all the tuples we asked to insert.
+ DCHECK(num_tuples_inserted == num_tuples_to_insert);
+ }
+ }
+
+ // After one round of insertions, we have successfully inserted as many
+ // tuples as possible into the output_block. Strictly speaking, it's
+ // possible that there is more space for insertions because the size
+ // estimation of variable length columns is conservative. But we will ignore
+ // that case and proceed assuming that this output_block is full.
+
+ // Update the header for output_block and then return it.
+ output_block->bulkInsertPartialTuplesFinalize(num_tuples_inserted);
+ const bool mark_full = always_mark_full
+ || !first_accessor->iterationFinishedVirtual();
+ this->returnBlock(std::move(output_block), mark_full);
+ }
+}
+
void InsertDestination::insertTuplesFromVector(std::vector<Tuple>::const_iterator begin,
std::vector<Tuple>::const_iterator end) {
if (begin == end) {
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2d11ec58/storage/InsertDestination.hpp
----------------------------------------------------------------------
diff --git a/storage/InsertDestination.hpp b/storage/InsertDestination.hpp
index 408e76b..3487638 100644
--- a/storage/InsertDestination.hpp
+++ b/storage/InsertDestination.hpp
@@ -152,6 +152,10 @@ class InsertDestination : public InsertDestinationInterface {
ValueAccessor *accessor,
bool always_mark_full = false) override;
+ void bulkInsertTuplesFromValueAccessors(
+ const std::vector<std::pair<ValueAccessor *, std::vector<attribute_id>>> &accessor_attribute_map,
+ bool always_mark_full = false) override;
+
void insertTuplesFromVector(std::vector<Tuple>::const_iterator begin,
std::vector<Tuple>::const_iterator end) override;
@@ -313,6 +317,12 @@ class AlwaysCreateBlockInsertDestination : public InsertDestination {
~AlwaysCreateBlockInsertDestination() override {
}
+ void bulkInsertTuplesFromValueAccessors(
+ const std::vector<std::pair<ValueAccessor *, std::vector<attribute_id>>> &accessor_attribute_map,
+ bool always_mark_full = false) override {
+ LOG(FATAL) << "bulkInsertTuplesFromValueAccessors is not implemented for AlwaysCreateBlockInsertDestination";
+ }
+
protected:
MutableBlockReference getBlockForInsertion() override;
@@ -517,6 +527,12 @@ class PartitionAwareInsertDestination : public InsertDestination {
ValueAccessor *accessor,
bool always_mark_full = false) override;
+ void bulkInsertTuplesFromValueAccessors(
+ const std::vector<std::pair<ValueAccessor *, std::vector<attribute_id>>> &accessor_attribute_map,
+ bool always_mark_full = false) override {
+ LOG(FATAL) << "bulkInsertTuplesFromValueAccessors is not implemented for PartitionAwareInsertDestination";
+ }
+
void insertTuplesFromVector(std::vector<Tuple>::const_iterator begin,
std::vector<Tuple>::const_iterator end) override;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2d11ec58/storage/InsertDestinationInterface.hpp
----------------------------------------------------------------------
diff --git a/storage/InsertDestinationInterface.hpp b/storage/InsertDestinationInterface.hpp
index 423dff1..b62d3e5 100644
--- a/storage/InsertDestinationInterface.hpp
+++ b/storage/InsertDestinationInterface.hpp
@@ -20,6 +20,7 @@
#ifndef QUICKSTEP_STORAGE_INSERT_DESTINATION_INTERFACE_HPP_
#define QUICKSTEP_STORAGE_INSERT_DESTINATION_INTERFACE_HPP_
+#include <utility>
#include <vector>
#include "catalog/CatalogTypedefs.hpp"
@@ -122,6 +123,27 @@ class InsertDestinationInterface {
bool always_mark_full = false) = 0;
/**
+ * @brief Bulk-insert tuples from one or more ValueAccessors
+ * into blocks managed by this InsertDestination.
+ *
+ * @warning It is implicitly assumed that all the input ValueAccessors have
+ * the same number of tuples in them.
+ *
+ * @param accessor_attribute_map A vector of pairs of ValueAccessor and
+ * corresponding attribute map
+ * The i-th attribute ID in the attr map for a value accessor is "n"
+ * if the attribute_id "i" in the output relation
+ * is the attribute_id "n" in corresponding input value accessor.
+ * Set the i-th element to kInvalidCatalogId if it doesn't come from
+ * the corresponding value accessor.
+ * @param always_mark_full If \c true, always mark the blocks full after
+ * insertion from ValueAccessor even when partially full.
+ **/
+ virtual void bulkInsertTuplesFromValueAccessors(
+ const std::vector<std::pair<ValueAccessor *, std::vector<attribute_id>>> &accessor_attribute_map,
+ bool always_mark_full = false) = 0;
+
+ /**
* @brief Insert tuples from a range of Tuples in a vector.
* @warning Unlike bulkInsertTuples(), this is not well-optimized and not
* intended for general use. It should only be used by
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2d11ec58/storage/SplitRowStoreTupleStorageSubBlock.hpp
----------------------------------------------------------------------
diff --git a/storage/SplitRowStoreTupleStorageSubBlock.hpp b/storage/SplitRowStoreTupleStorageSubBlock.hpp
index 681001e..89c756d 100644
--- a/storage/SplitRowStoreTupleStorageSubBlock.hpp
+++ b/storage/SplitRowStoreTupleStorageSubBlock.hpp
@@ -304,9 +304,9 @@ class SplitRowStoreTupleStorageSubBlock: public TupleStorageSubBlock {
tuple_id bulkInsertPartialTuples(
const std::vector<attribute_id> &attribute_map,
ValueAccessor *accessor,
- const tuple_id max_num_tuples_to_insert);
+ const tuple_id max_num_tuples_to_insert) override;
- void bulkInsertPartialTuplesFinalize(const tuple_id num_tuples_inserted);
+ void bulkInsertPartialTuplesFinalize(const tuple_id num_tuples_inserted) override;
const void* getAttributeValue(const tuple_id tuple,
const attribute_id attr) const override;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2d11ec58/storage/StorageBlock.cpp
----------------------------------------------------------------------
diff --git a/storage/StorageBlock.cpp b/storage/StorageBlock.cpp
index ea74ee6..6267d6b 100644
--- a/storage/StorageBlock.cpp
+++ b/storage/StorageBlock.cpp
@@ -284,6 +284,30 @@ tuple_id StorageBlock::bulkInsertTuplesWithRemappedAttributes(
return num_inserted;
}
+tuple_id StorageBlock::bulkInsertPartialTuples(
+ const std::vector<attribute_id> &attribute_map,
+ ValueAccessor *accessor,
+ const tuple_id max_num_tuples_to_insert) {
+ const tuple_id num_inserted
+ = tuple_store_->bulkInsertPartialTuples(attribute_map,
+ accessor,
+ max_num_tuples_to_insert);
+ if (num_inserted != 0) {
+ invalidateAllIndexes();
+ dirty_ = true;
+ } else if (tuple_store_->isEmpty()) {
+ if (!accessor->iterationFinishedVirtual()) {
+ throw TupleTooLargeForBlock(0);
+ }
+ }
+ return num_inserted;
+}
+
+void StorageBlock::bulkInsertPartialTuplesFinalize(
+ const tuple_id num_tuples_inserted) {
+ tuple_store_->bulkInsertPartialTuplesFinalize(num_tuples_inserted);
+}
+
void StorageBlock::sample(const bool is_block_sample,
const int percentage,
InsertDestinationInterface *destination) const {
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2d11ec58/storage/StorageBlock.hpp
----------------------------------------------------------------------
diff --git a/storage/StorageBlock.hpp b/storage/StorageBlock.hpp
index 56b3bdc..ed252c5 100644
--- a/storage/StorageBlock.hpp
+++ b/storage/StorageBlock.hpp
@@ -307,6 +307,7 @@ class StorageBlock : public StorageBlockBase {
* iteration will be advanced to the first non-inserted tuple or, if
* all accessible tuples were inserted in this block, to the end
* position.
+ * @param max_tuples_to_insert Insert at most these many tuples
* @return The number of tuples inserted from accessor.
**/
tuple_id bulkInsertTuplesWithRemappedAttributes(
@@ -314,6 +315,49 @@ class StorageBlock : public StorageBlockBase {
ValueAccessor *accessor);
/**
+ * @brief Insert up to max_num_tuples_to_insert tuples from a ValueAccessor
+ * as a single batch, using the attribute_map to project and reorder
+ * columns from the input ValueAccessor. Does not update header.
+ *
+ * @note Typical usage is where you want to bulk-insert columns from two
+ * or more value accessors. Instead of writing out the columns into
+ * one or more column vector value accessors, you can simply use this
+ * function with the appropriate attribute_map for each value
+ * accessor (InsertDestination::bulkInsertTuplesFromValueAccessors
+ * handles all the details) to insert tuples without an extra temp copy.
+ *
+ * @warning Must call bulkInsertPartialTuplesFinalize() to update the header,
+ * until which point, the insertion is not visible to others.
+ * @warning The inserted tuples may be placed in sub-optimal locations in this
+ * TupleStorageSubBlock.
+ *
+ * @param attribute_map A vector which maps the attributes of this
+ * TupleStorageSubBlock's relation (gaps indicated with kInvalidCatalogId)
+ * to the corresponding attributes which should be read from accessor.
+ * @param accessor A ValueAccessor to insert tuples from. The accessor's
+ * iteration will be advanced to the first non-inserted tuple or, if
+ * all accessible tuples were inserted in this sub-block, to the end
+ * position.
+ * @return The number of tuples inserted from accessor.
+ **/
+ tuple_id bulkInsertPartialTuples(
+ const std::vector<attribute_id> &attribute_map,
+ ValueAccessor *accessor,
+ const tuple_id max_num_tuples_to_insert);
+
+ /**
+ * @brief Update header after a bulkInsertPartialTuples.
+ *
+ * @warning Only call this after a bulkInsertPartialTuples, passing in the
+ * number of tuples that were inserted (return value of that function).
+ *
+ * @param num_tuples_inserted Number of tuples inserted (i.e., how much to
+ * advance the header.num_tuples by). Should be equal to the return
+ * value of bulkInsertPartialTuples.
+ **/
+ void bulkInsertPartialTuplesFinalize(tuple_id num_tuples_inserted);
+
+ /**
* @brief Get the IDs of tuples in this StorageBlock which match a given Predicate.
*
* @param predicate The predicate to match.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2d11ec58/storage/TupleStorageSubBlock.hpp
----------------------------------------------------------------------
diff --git a/storage/TupleStorageSubBlock.hpp b/storage/TupleStorageSubBlock.hpp
index aed6eea..26e8027 100644
--- a/storage/TupleStorageSubBlock.hpp
+++ b/storage/TupleStorageSubBlock.hpp
@@ -272,6 +272,56 @@ class TupleStorageSubBlock {
ValueAccessor *accessor) = 0;
/**
+ * @brief Insert up to max_num_tuples_to_insert tuples from a ValueAccessor
+ * as a single batch, using the attribute_map to project and reorder
+ * columns from the input ValueAccessor. Does not update header.
+ *
+ * @note Typical usage is where you want to bulk-insert columns from two
+ * or more value accessors. Instead of writing out the columns into
+ * one or more column vector value accessors, you can simply use this
+ * function with the appropriate attribute_map for each value
+ * accessor (InsertDestination::bulkInsertTuplesFromValueAccessors
+ * handles all the details) to insert tuples without an extra temp copy.
+ *
+ * @warning Must call bulkInsertPartialTuplesFinalize() to update the header,
+ * until which point, the insertion is not visible to others.
+ * @warning The inserted tuples may be placed in a suboptimal position in the
+ * block.
+ *
+ * @param attribute_map A vector which maps the attributes of this
+ * TupleStorageSubBlock's relation (gaps indicated with kInvalidCatalogId)
+ * to the corresponding attributes which should be read from accessor.
+ * @param accessor A ValueAccessor to insert tuples from. The accessor's
+ * iteration will be advanced to the first non-inserted tuple or, if
+ * all accessible tuples were inserted in this sub-block, to the end
+ * position.
+ * @return The number of tuples inserted from accessor.
+ **/
+ virtual tuple_id bulkInsertPartialTuples(
+ const std::vector<attribute_id> &attribute_map,
+ ValueAccessor *accessor,
+ const tuple_id max_num_tuples_to_insert) {
+ LOG(FATAL) << "Partial bulk insert is not supported for this TupleStorageBlock type ("
+ << getTupleStorageSubBlockType() << ").";
+ }
+
+ /**
+ * @brief Update header after a bulkInsertPartialTuples.
+ *
+ * @warning Only call this after a bulkInsertPartialTuples, passing in the
+ * number of tuples that were inserted (return value of that function).
+ *
+ * @param num_tuples_inserted Number of tuples inserted (i.e., how much to
+ * advance the header.num_tuples by). Should be equal to the return
+ * value of bulkInsertPartialTuples.
+ **/
+ virtual void bulkInsertPartialTuplesFinalize(
+ const tuple_id num_tuples_inserted) {
+ LOG(FATAL) << "Partial bulk insert is not supported for this TupleStorageBlock type ("
+ << getTupleStorageSubBlockType() << ").";
+ }
+
+ /**
* @brief Get the (untyped) value of an attribute in a tuple in this buffer.
* @warning This method may not be supported for all implementations of
* TupleStorageSubBlock. supportsUntypedGetAttributeValue() MUST be
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2d11ec58/types/containers/ColumnVectorsValueAccessor.hpp
----------------------------------------------------------------------
diff --git a/types/containers/ColumnVectorsValueAccessor.hpp b/types/containers/ColumnVectorsValueAccessor.hpp
index fe413a0..fbbdc1b 100644
--- a/types/containers/ColumnVectorsValueAccessor.hpp
+++ b/types/containers/ColumnVectorsValueAccessor.hpp
@@ -139,6 +139,10 @@ class ColumnVectorsValueAccessor : public ValueAccessor {
return nullptr;
}
+ inline std::size_t getNumColumns() const {
+ return columns_.size();
+ }
+
template <bool check_null = true>
inline const void* getUntypedValue(const attribute_id attr_id) const {
return getUntypedValueAtAbsolutePosition<check_null>(attr_id, current_position_);
[09/50] incubator-quickstep git commit: Moved some storage-related
constants.
Posted by zu...@apache.org.
Moved some storage-related constants.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/6fa7be6b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/6fa7be6b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/6fa7be6b
Branch: refs/heads/quickstep_partition_parser_support
Commit: 6fa7be6bab6f64fd534b1a7cd56f3a2250f9d330
Parents: 787a325
Author: Zuyu Zhang <zu...@apache.org>
Authored: Mon Nov 14 22:24:55 2016 -0800
Committer: Zuyu Zhang <zu...@apache.org>
Committed: Tue Nov 15 17:55:44 2016 -0800
----------------------------------------------------------------------
CMakeLists.txt | 1 +
cli/QuickstepCli.cpp | 12 +++---------
storage/StorageConstants.hpp | 10 ++++++++++
3 files changed, 14 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/6fa7be6b/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 487aaf9..6191de0 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -769,6 +769,7 @@ target_link_libraries(quickstep_cli_shell
quickstep_queryoptimizer_QueryPlan
quickstep_queryoptimizer_QueryProcessor
quickstep_storage_PreloaderThread
+ quickstep_storage_StorageConstants
quickstep_threading_ThreadIDBasedMap
quickstep_utility_ExecutionDAGVisualizer
quickstep_utility_Macros
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/6fa7be6b/cli/QuickstepCli.cpp
----------------------------------------------------------------------
diff --git a/cli/QuickstepCli.cpp b/cli/QuickstepCli.cpp
index 8269197..31d9834 100644
--- a/cli/QuickstepCli.cpp
+++ b/cli/QuickstepCli.cpp
@@ -29,6 +29,8 @@
#include <vector>
#include <fstream>
+#include "cli/CliConfig.h" // For QUICKSTEP_USE_LINENOISE, QUICKSTEP_ENABLE_GOOGLE_PROFILER, and QUICKSTEP_OS_WINDOWS.
+
// TODO(jmp): If filesystem shows up in C++-17, we can switch to just using that.
#ifdef QUICKSTEP_OS_WINDOWS
#include <filesystem>
@@ -36,7 +38,6 @@
#include <stdlib.h>
#endif
-#include "cli/CliConfig.h" // For QUICKSTEP_USE_LINENOISE, QUICKSTEP_ENABLE_GOOGLE_PROFILER.
#include "cli/CommandExecutor.hpp"
#include "cli/DropRelation.hpp"
@@ -74,6 +75,7 @@ typedef quickstep::LineReaderDumb LineReaderImpl;
#endif
#include "storage/PreloaderThread.hpp"
+#include "storage/StorageConstants.hpp"
#include "threading/ThreadIDBasedMap.hpp"
#include "utility/ExecutionDAGVisualizer.hpp"
#include "utility/Macros.hpp"
@@ -130,14 +132,6 @@ using tmb::client_id;
namespace quickstep {
-#ifdef QUICKSTEP_OS_WINDOWS
-static constexpr char kPathSeparator = '\\';
-static constexpr char kDefaultStoragePath[] = "qsstor\\";
-#else
-static constexpr char kPathSeparator = '/';
-static constexpr char kDefaultStoragePath[] = "qsstor/";
-#endif
-
DEFINE_bool(profile_and_report_workorder_perf, false,
"If true, Quickstep will record the exceution time of all the individual "
"normal work orders and report it at the end of query execution.");
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/6fa7be6b/storage/StorageConstants.hpp
----------------------------------------------------------------------
diff --git a/storage/StorageConstants.hpp b/storage/StorageConstants.hpp
index 90028db..34a714d 100644
--- a/storage/StorageConstants.hpp
+++ b/storage/StorageConstants.hpp
@@ -23,12 +23,22 @@
#include <cstddef>
#include <cstdint>
+#include "cli/CliConfig.h" // For QUICKSTEP_OS_WINDOWS.
+
namespace quickstep {
/** \addtogroup Storage
* @{
*/
+#ifdef QUICKSTEP_OS_WINDOWS
+constexpr char kPathSeparator = '\\';
+constexpr char kDefaultStoragePath[] = "qsstor\\";
+#else
+constexpr char kPathSeparator = '/';
+constexpr char kDefaultStoragePath[] = "qsstor/";
+#endif
+
// Size of a memory slot managed by the StorageManager. This is the smallest
// quantum of allocation for StorageBlocks and StorageBlobs. 2 MB is the large
// page size on x86.
[04/50] incubator-quickstep git commit: Avoid string copy in an
DataExchanger API.
Posted by zu...@apache.org.
Avoid string copy in an DataExchanger API.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/5a3037f2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/5a3037f2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/5a3037f2
Branch: refs/heads/quickstep_partition_parser_support
Commit: 5a3037f2e4d4f732b32374daad81e31a4c8026bc
Parents: 70a3d47
Author: Zuyu Zhang <zu...@apache.org>
Authored: Fri Nov 4 21:29:15 2016 -0700
Committer: Zuyu Zhang <zu...@apache.org>
Committed: Fri Nov 4 21:29:15 2016 -0700
----------------------------------------------------------------------
storage/DataExchangerAsync.hpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5a3037f2/storage/DataExchangerAsync.hpp
----------------------------------------------------------------------
diff --git a/storage/DataExchangerAsync.hpp b/storage/DataExchangerAsync.hpp
index f3b256a..b777108 100644
--- a/storage/DataExchangerAsync.hpp
+++ b/storage/DataExchangerAsync.hpp
@@ -61,7 +61,7 @@ class DataExchangerAsync final : public Thread {
*
* @return Its network address.
**/
- std::string network_address() const {
+ const std::string& network_address() const {
DCHECK(!server_address_.empty());
return server_address_;
}
[37/50] incubator-quickstep git commit: Scheduled WorkOrders w/ the
same aggr_state_index on the same Shiftboss.
Posted by zu...@apache.org.
Scheduled WorkOrders w/ the same aggr_state_index on the same Shiftboss.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/57730e40
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/57730e40
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/57730e40
Branch: refs/heads/quickstep_partition_parser_support
Commit: 57730e406533fa02280568380c853ce1c5dc19ee
Parents: e75c265
Author: Zuyu Zhang <zu...@apache.org>
Authored: Sun Nov 27 17:04:30 2016 -0800
Committer: Zuyu Zhang <zu...@apache.org>
Committed: Sun Nov 27 17:28:30 2016 -0800
----------------------------------------------------------------------
query_execution/ForemanDistributed.cpp | 61 ++++++++++++++++------
query_execution/ForemanDistributed.hpp | 4 ++
query_execution/PolicyEnforcerDistributed.cpp | 12 +++++
query_execution/PolicyEnforcerDistributed.hpp | 18 +++++++
query_execution/QueryManagerDistributed.hpp | 23 ++++++++
5 files changed, 102 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/57730e40/query_execution/ForemanDistributed.cpp
----------------------------------------------------------------------
diff --git a/query_execution/ForemanDistributed.cpp b/query_execution/ForemanDistributed.cpp
index fc9cd3c..61f0603 100644
--- a/query_execution/ForemanDistributed.cpp
+++ b/query_execution/ForemanDistributed.cpp
@@ -243,6 +243,32 @@ bool ForemanDistributed::canCollectNewMessages(const tmb::message_type_id messag
kWorkOrderFeedbackMessage);
}
+bool ForemanDistributed::isAggregationRelatedWorkOrder(const S::WorkOrderMessage &proto,
+ const size_t next_shiftboss_index_to_schedule,
+ size_t *shiftboss_index_for_aggregation) {
+ const S::WorkOrder &work_order_proto = proto.work_order();
+ QueryContext::aggregation_state_id aggr_state_index;
+
+ switch (work_order_proto.work_order_type()) {
+ case S::AGGREGATION:
+ aggr_state_index = work_order_proto.GetExtension(S::AggregationWorkOrder::aggr_state_index);
+ break;
+ case S::FINALIZE_AGGREGATION:
+ aggr_state_index = work_order_proto.GetExtension(S::FinalizeAggregationWorkOrder::aggr_state_index);
+ break;
+ case S::DESTROY_AGGREGATION_STATE:
+ aggr_state_index = work_order_proto.GetExtension(S::DestroyAggregationStateWorkOrder::aggr_state_index);
+ break;
+ default:
+ return false;
+ }
+
+ static_cast<PolicyEnforcerDistributed*>(policy_enforcer_.get())->getShiftbossIndexForAggregation(
+ proto.query_id(), aggr_state_index, next_shiftboss_index_to_schedule, shiftboss_index_for_aggregation);
+
+ return true;
+}
+
bool ForemanDistributed::isHashJoinRelatedWorkOrder(const S::WorkOrderMessage &proto,
const size_t next_shiftboss_index_to_schedule,
size_t *shiftboss_index_for_hash_join) {
@@ -251,13 +277,13 @@ bool ForemanDistributed::isHashJoinRelatedWorkOrder(const S::WorkOrderMessage &p
switch (work_order_proto.work_order_type()) {
case S::BUILD_HASH:
- join_hash_table_index = work_order_proto.GetExtension(serialization::BuildHashWorkOrder::join_hash_table_index);
- break;
- case S::DESTROY_HASH:
- join_hash_table_index = work_order_proto.GetExtension(serialization::DestroyHashWorkOrder::join_hash_table_index);
+ join_hash_table_index = work_order_proto.GetExtension(S::BuildHashWorkOrder::join_hash_table_index);
break;
case S::HASH_JOIN:
- join_hash_table_index = work_order_proto.GetExtension(serialization::HashJoinWorkOrder::join_hash_table_index);
+ join_hash_table_index = work_order_proto.GetExtension(S::HashJoinWorkOrder::join_hash_table_index);
+ break;
+ case S::DESTROY_HASH:
+ join_hash_table_index = work_order_proto.GetExtension(S::DestroyHashWorkOrder::join_hash_table_index);
break;
default:
return false;
@@ -275,20 +301,23 @@ void ForemanDistributed::dispatchWorkOrderMessages(const vector<unique_ptr<S::Wo
for (const auto &message : messages) {
DCHECK(message != nullptr);
const S::WorkOrderMessage &proto = *message;
- size_t shiftboss_index_for_hash_join;
- if (isHashJoinRelatedWorkOrder(proto, shiftboss_index, &shiftboss_index_for_hash_join)) {
- sendWorkOrderMessage(shiftboss_index_for_hash_join, proto);
- shiftboss_directory_.incrementNumQueuedWorkOrders(shiftboss_index_for_hash_join);
-
- if (shiftboss_index == shiftboss_index_for_hash_join) {
- shiftboss_index = (shiftboss_index + 1) % num_shiftbosses;
- }
+ size_t shiftboss_index_for_particular_work_order_type;
+ if (isAggregationRelatedWorkOrder(proto, shiftboss_index, &shiftboss_index_for_particular_work_order_type)) {
+ } else if (isHashJoinRelatedWorkOrder(proto, shiftboss_index, &shiftboss_index_for_particular_work_order_type)) {
} else {
- sendWorkOrderMessage(shiftboss_index, proto);
- shiftboss_directory_.incrementNumQueuedWorkOrders(shiftboss_index);
-
// TODO(zuyu): Take data-locality into account for scheduling.
+ shiftboss_index_for_particular_work_order_type = shiftboss_index;
+ }
+
+ sendWorkOrderMessage(shiftboss_index_for_particular_work_order_type, proto);
+ shiftboss_directory_.incrementNumQueuedWorkOrders(shiftboss_index_for_particular_work_order_type);
+
+ if (shiftboss_index == shiftboss_index_for_particular_work_order_type) {
shiftboss_index = (shiftboss_index + 1) % num_shiftbosses;
+ } else {
+ // NOTE(zuyu): This is not the exact round-robin scheduling, as in this case,
+ // <shiftboss_index_for_particular_work_order_type> might be scheduled one
+ // more WorkOrder for an Aggregation or a HashJoin.
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/57730e40/query_execution/ForemanDistributed.hpp
----------------------------------------------------------------------
diff --git a/query_execution/ForemanDistributed.hpp b/query_execution/ForemanDistributed.hpp
index 0616f30..34bac07 100644
--- a/query_execution/ForemanDistributed.hpp
+++ b/query_execution/ForemanDistributed.hpp
@@ -71,6 +71,10 @@ class ForemanDistributed final : public ForemanBase {
void run() override;
private:
+ bool isAggregationRelatedWorkOrder(const serialization::WorkOrderMessage &proto,
+ const std::size_t next_shiftboss_index_to_schedule,
+ std::size_t *shiftboss_index_for_aggregation);
+
bool isHashJoinRelatedWorkOrder(const serialization::WorkOrderMessage &proto,
const std::size_t next_shiftboss_index_to_schedule,
std::size_t *shiftboss_index_for_hash_join);
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/57730e40/query_execution/PolicyEnforcerDistributed.cpp
----------------------------------------------------------------------
diff --git a/query_execution/PolicyEnforcerDistributed.cpp b/query_execution/PolicyEnforcerDistributed.cpp
index 86b36c8..c5642bc 100644
--- a/query_execution/PolicyEnforcerDistributed.cpp
+++ b/query_execution/PolicyEnforcerDistributed.cpp
@@ -158,6 +158,18 @@ void PolicyEnforcerDistributed::processInitiateRebuildResponseMessage(const tmb:
}
}
+void PolicyEnforcerDistributed::getShiftbossIndexForAggregation(
+ const std::size_t query_id,
+ const QueryContext::aggregation_state_id aggr_state_index,
+ const std::size_t next_shiftboss_index_to_schedule,
+ std::size_t *shiftboss_index) {
+ DCHECK(admitted_queries_.find(query_id) != admitted_queries_.end());
+ QueryManagerDistributed *query_manager = static_cast<QueryManagerDistributed*>(admitted_queries_[query_id].get());
+ query_manager->getShiftbossIndexForAggregation(aggr_state_index,
+ next_shiftboss_index_to_schedule,
+ shiftboss_index);
+}
+
void PolicyEnforcerDistributed::getShiftbossIndexForHashJoin(
const std::size_t query_id,
const QueryContext::join_hash_table_id join_hash_table_index,
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/57730e40/query_execution/PolicyEnforcerDistributed.hpp
----------------------------------------------------------------------
diff --git a/query_execution/PolicyEnforcerDistributed.hpp b/query_execution/PolicyEnforcerDistributed.hpp
index 37326bd..e8bc394 100644
--- a/query_execution/PolicyEnforcerDistributed.hpp
+++ b/query_execution/PolicyEnforcerDistributed.hpp
@@ -90,6 +90,24 @@ class PolicyEnforcerDistributed final : public PolicyEnforcerBase {
void processInitiateRebuildResponseMessage(const tmb::TaggedMessage &tagged_message);
/**
+ * @brief Get or set the index of Shiftboss for an Aggregation related
+ * WorkOrder. If it is the first Aggregation on <aggr_state_index>,
+ * <shiftboss_index> will be set to <next_shiftboss_index_to_schedule>.
+ * Otherwise, <shiftboss_index> will be set to the index of the Shiftboss that
+ * has executed the first Aggregation.
+ *
+ * @param query_id The query id.
+ * @param aggr_state_index The Hash Table for the Aggregation.
+ * @param next_shiftboss_index The index of Shiftboss to schedule a next WorkOrder.
+ * @param shiftboss_index The index of Shiftboss to schedule the WorkOrder.
+ **/
+ void getShiftbossIndexForAggregation(
+ const std::size_t query_id,
+ const QueryContext::aggregation_state_id aggr_state_index,
+ const std::size_t next_shiftboss_index_to_schedule,
+ std::size_t *shiftboss_index);
+
+ /**
* @brief Get or set the index of Shiftboss for a HashJoin related WorkOrder.
* If it is the first BuildHash on <join_hash_table_index>, <shiftboss_index>
* will be set to <next_shiftboss_index_to_schedule>. Otherwise,
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/57730e40/query_execution/QueryManagerDistributed.hpp
----------------------------------------------------------------------
diff --git a/query_execution/QueryManagerDistributed.hpp b/query_execution/QueryManagerDistributed.hpp
index 2b21303..7a07fcb 100644
--- a/query_execution/QueryManagerDistributed.hpp
+++ b/query_execution/QueryManagerDistributed.hpp
@@ -95,6 +95,26 @@ class QueryManagerDistributed final : public QueryManagerBase {
const dag_node_index start_operator_index);
/**
+ * @brief Get the index of Shiftboss for an Aggregation related WorkOrder. If
+ * the Shiftboss index is not found, set using <next_shiftboss_index_to_schedule>.
+ *
+ * @param aggr_state_index The Hash Table for the Aggregation.
+ * @param next_shiftboss_index The index of Shiftboss to schedule a next WorkOrder.
+ * @param shiftboss_index The index of Shiftboss to schedule the WorkOrder.
+ **/
+ void getShiftbossIndexForAggregation(const QueryContext::aggregation_state_id aggr_state_index,
+ const std::size_t next_shiftboss_index_to_schedule,
+ std::size_t *shiftboss_index) {
+ const auto cit = shiftboss_indexes_for_aggrs_.find(aggr_state_index);
+ if (cit != shiftboss_indexes_for_aggrs_.end()) {
+ *shiftboss_index = cit->second;
+ } else {
+ shiftboss_indexes_for_aggrs_.emplace(aggr_state_index, next_shiftboss_index_to_schedule);
+ *shiftboss_index = next_shiftboss_index_to_schedule;
+ }
+ }
+
+ /**
* @brief Get the index of Shiftboss for a HashJoin related WorkOrder. If the
* Shiftboss index is not found, set using <next_shiftboss_index_to_schedule>.
*
@@ -136,6 +156,9 @@ class QueryManagerDistributed final : public QueryManagerBase {
std::unique_ptr<WorkOrderProtosContainer> normal_workorder_protos_container_;
+ // A map from an aggregation id to its scheduled Shiftboss index.
+ std::unordered_map<QueryContext::aggregation_state_id, std::size_t> shiftboss_indexes_for_aggrs_;
+
// A map from a join hash table to its scheduled Shiftboss index.
std::unordered_map<QueryContext::join_hash_table_id, std::size_t> shiftboss_indexes_for_hash_joins_;
[25/50] incubator-quickstep git commit: Fixed the lint issue.
Posted by zu...@apache.org.
Fixed the lint issue.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/212e6cda
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/212e6cda
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/212e6cda
Branch: refs/heads/quickstep_partition_parser_support
Commit: 212e6cdaf89c89bd0cd1ac4b5e27f7cda0785c4a
Parents: c886f7a
Author: Zuyu Zhang <zu...@apache.org>
Authored: Sun Nov 20 20:38:35 2016 -0800
Committer: Zuyu Zhang <zu...@apache.org>
Committed: Sun Nov 20 20:38:35 2016 -0800
----------------------------------------------------------------------
query_optimizer/QueryProcessor.hpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/212e6cda/query_optimizer/QueryProcessor.hpp
----------------------------------------------------------------------
diff --git a/query_optimizer/QueryProcessor.hpp b/query_optimizer/QueryProcessor.hpp
index 3e37c9d..2b39b84 100644
--- a/query_optimizer/QueryProcessor.hpp
+++ b/query_optimizer/QueryProcessor.hpp
@@ -131,7 +131,7 @@ class QueryProcessor {
*
* @param catalog_filename The file to read the serialized catalog from.
**/
- explicit QueryProcessor(std::string &&catalog_filename)
+ explicit QueryProcessor(std::string &&catalog_filename) // NOLINT(whitespace/operators)
: catalog_filename_(std::move(catalog_filename)),
catalog_altered_(false),
query_id_(0) {
[36/50] incubator-quickstep git commit: Scheduled HashJoinWorkOrder
on the same Shiftboss of BuildHashWorkOrder.
Posted by zu...@apache.org.
Scheduled HashJoinWorkOrder on the same Shiftboss of BuildHashWorkOrder.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/e75c265e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/e75c265e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/e75c265e
Branch: refs/heads/quickstep_partition_parser_support
Commit: e75c265eb279256d89f4fa634c542d566ce3cd7a
Parents: c4f4b28
Author: Zuyu Zhang <zu...@apache.org>
Authored: Sun Nov 13 20:20:56 2016 -0800
Committer: Zuyu Zhang <zu...@apache.org>
Committed: Sun Nov 27 16:45:22 2016 -0800
----------------------------------------------------------------------
query_execution/CMakeLists.txt | 3 ++
query_execution/ForemanDistributed.cpp | 47 ++++++++++++++++++++--
query_execution/ForemanDistributed.hpp | 4 ++
query_execution/PolicyEnforcerDistributed.cpp | 12 ++++++
query_execution/PolicyEnforcerDistributed.hpp | 19 +++++++++
query_execution/QueryManagerDistributed.hpp | 25 ++++++++++++
6 files changed, 106 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/e75c265e/query_execution/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_execution/CMakeLists.txt b/query_execution/CMakeLists.txt
index 719d9f3..1f7add8 100644
--- a/query_execution/CMakeLists.txt
+++ b/query_execution/CMakeLists.txt
@@ -108,10 +108,12 @@ if (ENABLE_DISTRIBUTED)
quickstep_queryexecution_ForemanBase
quickstep_queryexecution_PolicyEnforcerBase
quickstep_queryexecution_PolicyEnforcerDistributed
+ quickstep_queryexecution_QueryContext
quickstep_queryexecution_QueryExecutionMessages_proto
quickstep_queryexecution_QueryExecutionTypedefs
quickstep_queryexecution_QueryExecutionUtil
quickstep_queryexecution_ShiftbossDirectory
+ quickstep_relationaloperators_WorkOrder_proto
quickstep_threading_ThreadUtil
quickstep_utility_EqualsAnyConstant
quickstep_utility_Macros
@@ -153,6 +155,7 @@ if (ENABLE_DISTRIBUTED)
quickstep_catalog_CatalogRelation
quickstep_catalog_Catalog_proto
quickstep_queryexecution_PolicyEnforcerBase
+ quickstep_queryexecution_QueryContext
quickstep_queryexecution_QueryContext_proto
quickstep_queryexecution_QueryExecutionMessages_proto
quickstep_queryexecution_QueryExecutionState
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/e75c265e/query_execution/ForemanDistributed.cpp
----------------------------------------------------------------------
diff --git a/query_execution/ForemanDistributed.cpp b/query_execution/ForemanDistributed.cpp
index 7dccce4..fc9cd3c 100644
--- a/query_execution/ForemanDistributed.cpp
+++ b/query_execution/ForemanDistributed.cpp
@@ -30,10 +30,12 @@
#include "query_execution/AdmitRequestMessage.hpp"
#include "query_execution/PolicyEnforcerBase.hpp"
#include "query_execution/PolicyEnforcerDistributed.hpp"
+#include "query_execution/QueryContext.hpp"
#include "query_execution/QueryExecutionMessages.pb.h"
#include "query_execution/QueryExecutionTypedefs.hpp"
#include "query_execution/QueryExecutionUtil.hpp"
#include "query_execution/ShiftbossDirectory.hpp"
+#include "relational_operators/WorkOrder.pb.h"
#include "threading/ThreadUtil.hpp"
#include "utility/EqualsAnyConstant.hpp"
@@ -241,16 +243,53 @@ bool ForemanDistributed::canCollectNewMessages(const tmb::message_type_id messag
kWorkOrderFeedbackMessage);
}
+bool ForemanDistributed::isHashJoinRelatedWorkOrder(const S::WorkOrderMessage &proto,
+ const size_t next_shiftboss_index_to_schedule,
+ size_t *shiftboss_index_for_hash_join) {
+ const S::WorkOrder &work_order_proto = proto.work_order();
+ QueryContext::join_hash_table_id join_hash_table_index;
+
+ switch (work_order_proto.work_order_type()) {
+ case S::BUILD_HASH:
+ join_hash_table_index = work_order_proto.GetExtension(serialization::BuildHashWorkOrder::join_hash_table_index);
+ break;
+ case S::DESTROY_HASH:
+ join_hash_table_index = work_order_proto.GetExtension(serialization::DestroyHashWorkOrder::join_hash_table_index);
+ break;
+ case S::HASH_JOIN:
+ join_hash_table_index = work_order_proto.GetExtension(serialization::HashJoinWorkOrder::join_hash_table_index);
+ break;
+ default:
+ return false;
+ }
+
+ static_cast<PolicyEnforcerDistributed*>(policy_enforcer_.get())->getShiftbossIndexForHashJoin(
+ proto.query_id(), join_hash_table_index, next_shiftboss_index_to_schedule, shiftboss_index_for_hash_join);
+
+ return true;
+}
+
void ForemanDistributed::dispatchWorkOrderMessages(const vector<unique_ptr<S::WorkOrderMessage>> &messages) {
const size_t num_shiftbosses = shiftboss_directory_.size();
size_t shiftboss_index = 0u;
for (const auto &message : messages) {
DCHECK(message != nullptr);
- sendWorkOrderMessage(shiftboss_index, *message);
- shiftboss_directory_.incrementNumQueuedWorkOrders(shiftboss_index);
+ const S::WorkOrderMessage &proto = *message;
+ size_t shiftboss_index_for_hash_join;
+ if (isHashJoinRelatedWorkOrder(proto, shiftboss_index, &shiftboss_index_for_hash_join)) {
+ sendWorkOrderMessage(shiftboss_index_for_hash_join, proto);
+ shiftboss_directory_.incrementNumQueuedWorkOrders(shiftboss_index_for_hash_join);
+
+ if (shiftboss_index == shiftboss_index_for_hash_join) {
+ shiftboss_index = (shiftboss_index + 1) % num_shiftbosses;
+ }
+ } else {
+ sendWorkOrderMessage(shiftboss_index, proto);
+ shiftboss_directory_.incrementNumQueuedWorkOrders(shiftboss_index);
- // TO(zuyu): Take data-locality into account for scheduling.
- shiftboss_index = (shiftboss_index + 1) % num_shiftbosses;
+ // TODO(zuyu): Take data-locality into account for scheduling.
+ shiftboss_index = (shiftboss_index + 1) % num_shiftbosses;
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/e75c265e/query_execution/ForemanDistributed.hpp
----------------------------------------------------------------------
diff --git a/query_execution/ForemanDistributed.hpp b/query_execution/ForemanDistributed.hpp
index ccdd0ae..0616f30 100644
--- a/query_execution/ForemanDistributed.hpp
+++ b/query_execution/ForemanDistributed.hpp
@@ -71,6 +71,10 @@ class ForemanDistributed final : public ForemanBase {
void run() override;
private:
+ bool isHashJoinRelatedWorkOrder(const serialization::WorkOrderMessage &proto,
+ const std::size_t next_shiftboss_index_to_schedule,
+ std::size_t *shiftboss_index_for_hash_join);
+
/**
* @brief Dispatch schedulable WorkOrders, wrapped in WorkOrderMessages to the
* worker threads.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/e75c265e/query_execution/PolicyEnforcerDistributed.cpp
----------------------------------------------------------------------
diff --git a/query_execution/PolicyEnforcerDistributed.cpp b/query_execution/PolicyEnforcerDistributed.cpp
index 6e09ea8..86b36c8 100644
--- a/query_execution/PolicyEnforcerDistributed.cpp
+++ b/query_execution/PolicyEnforcerDistributed.cpp
@@ -158,6 +158,18 @@ void PolicyEnforcerDistributed::processInitiateRebuildResponseMessage(const tmb:
}
}
+void PolicyEnforcerDistributed::getShiftbossIndexForHashJoin(
+ const std::size_t query_id,
+ const QueryContext::join_hash_table_id join_hash_table_index,
+ const std::size_t next_shiftboss_index_to_schedule,
+ std::size_t *shiftboss_index) {
+ DCHECK(admitted_queries_.find(query_id) != admitted_queries_.end());
+ QueryManagerDistributed *query_manager = static_cast<QueryManagerDistributed*>(admitted_queries_[query_id].get());
+ query_manager->getShiftbossIndexForHashJoin(join_hash_table_index,
+ next_shiftboss_index_to_schedule,
+ shiftboss_index);
+}
+
void PolicyEnforcerDistributed::initiateQueryInShiftboss(QueryHandle *query_handle) {
S::QueryInitiateMessage proto;
proto.set_query_id(query_handle->query_id());
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/e75c265e/query_execution/PolicyEnforcerDistributed.hpp
----------------------------------------------------------------------
diff --git a/query_execution/PolicyEnforcerDistributed.hpp b/query_execution/PolicyEnforcerDistributed.hpp
index 146e9af..37326bd 100644
--- a/query_execution/PolicyEnforcerDistributed.hpp
+++ b/query_execution/PolicyEnforcerDistributed.hpp
@@ -20,6 +20,7 @@
#include <vector>
#include "query_execution/PolicyEnforcerBase.hpp"
+#include "query_execution/QueryContext.hpp"
#include "query_execution/QueryExecutionMessages.pb.h"
#include "query_execution/ShiftbossDirectory.hpp"
#include "utility/Macros.hpp"
@@ -88,6 +89,24 @@ class PolicyEnforcerDistributed final : public PolicyEnforcerBase {
**/
void processInitiateRebuildResponseMessage(const tmb::TaggedMessage &tagged_message);
+ /**
+ * @brief Get or set the index of Shiftboss for a HashJoin related WorkOrder.
+ * If it is the first BuildHash on <join_hash_table_index>, <shiftboss_index>
+ * will be set to <next_shiftboss_index_to_schedule>. Otherwise,
+ * <shiftboss_index> will be set to the index of the Shiftboss that has
+ * executed the first BuildHash.
+ *
+ * @param query_id The query id.
+ * @param join_hash_table_index The Hash Table for the Join.
+ * @param next_shiftboss_index The index of Shiftboss to schedule a next WorkOrder.
+ * @param shiftboss_index The index of Shiftboss to schedule the WorkOrder.
+ **/
+ void getShiftbossIndexForHashJoin(
+ const std::size_t query_id,
+ const QueryContext::join_hash_table_id join_hash_table_index,
+ const std::size_t next_shiftboss_index_to_schedule,
+ std::size_t *shiftboss_index);
+
private:
void decrementNumQueuedWorkOrders(const serialization::WorkOrderCompletionMessage &proto) override {
shiftboss_directory_->decrementNumQueuedWorkOrders(proto.shiftboss_index());
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/e75c265e/query_execution/QueryManagerDistributed.hpp
----------------------------------------------------------------------
diff --git a/query_execution/QueryManagerDistributed.hpp b/query_execution/QueryManagerDistributed.hpp
index f8ac53c..2b21303 100644
--- a/query_execution/QueryManagerDistributed.hpp
+++ b/query_execution/QueryManagerDistributed.hpp
@@ -22,7 +22,9 @@
#include <cstddef>
#include <memory>
+#include <unordered_map>
+#include "query_execution/QueryContext.hpp"
#include "query_execution/QueryExecutionState.hpp"
#include "query_execution/QueryManagerBase.hpp"
#include "query_execution/WorkOrderProtosContainer.hpp"
@@ -92,6 +94,26 @@ class QueryManagerDistributed final : public QueryManagerBase {
serialization::WorkOrderMessage* getNextWorkOrderMessage(
const dag_node_index start_operator_index);
+ /**
+ * @brief Get the index of Shiftboss for a HashJoin related WorkOrder. If the
+ * Shiftboss index is not found, set using <next_shiftboss_index_to_schedule>.
+ *
+ * @param join_hash_table_index The Hash Table for the Join.
+ * @param next_shiftboss_index The index of Shiftboss to schedule a next WorkOrder.
+ * @param shiftboss_index The index of Shiftboss to schedule the WorkOrder.
+ **/
+ void getShiftbossIndexForHashJoin(const QueryContext::join_hash_table_id join_hash_table_index,
+ const std::size_t next_shiftboss_index_to_schedule,
+ std::size_t *shiftboss_index) {
+ const auto cit = shiftboss_indexes_for_hash_joins_.find(join_hash_table_index);
+ if (cit != shiftboss_indexes_for_hash_joins_.end()) {
+ *shiftboss_index = cit->second;
+ } else {
+ shiftboss_indexes_for_hash_joins_.emplace(join_hash_table_index, next_shiftboss_index_to_schedule);
+ *shiftboss_index = next_shiftboss_index_to_schedule;
+ }
+ }
+
private:
bool checkNormalExecutionOver(const dag_node_index index) const override {
return (checkAllDependenciesMet(index) &&
@@ -114,6 +136,9 @@ class QueryManagerDistributed final : public QueryManagerBase {
std::unique_ptr<WorkOrderProtosContainer> normal_workorder_protos_container_;
+ // A map from a join hash table to its scheduled Shiftboss index.
+ std::unordered_map<QueryContext::join_hash_table_id, std::size_t> shiftboss_indexes_for_hash_joins_;
+
DISALLOW_COPY_AND_ASSIGN(QueryManagerDistributed);
};
[03/50] incubator-quickstep git commit: QUICKSTEP-61 Fixes varlen
insert bug.
Posted by zu...@apache.org.
QUICKSTEP-61 Fixes varlen insert bug.
This change fixes an incorrect estimate for the maximum size of a
tuple. Previously, it was not accounting for the space used by the
fixed length portion of the tuple.
The unit tests were modified because, now, we insert fewer tuples. It's
actually quite hard to test the correct number of tuples which will be
inserted, so this has been left as a TODO.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/70a3d47c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/70a3d47c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/70a3d47c
Branch: refs/heads/quickstep_partition_parser_support
Commit: 70a3d47cdcf46bbb8f39b1cc1a0fbd2650a6afa0
Parents: fc898b1
Author: cramja <ma...@gmail.com>
Authored: Tue Nov 1 13:42:04 2016 -0500
Committer: cramja <ma...@gmail.com>
Committed: Tue Nov 1 13:42:04 2016 -0500
----------------------------------------------------------------------
storage/SplitRowStoreTupleStorageSubBlock.cpp | 3 ++-
storage/tests/SplitRowStoreTupleStorageSubBlock_unittest.cpp | 8 ++------
2 files changed, 4 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/70a3d47c/storage/SplitRowStoreTupleStorageSubBlock.cpp
----------------------------------------------------------------------
diff --git a/storage/SplitRowStoreTupleStorageSubBlock.cpp b/storage/SplitRowStoreTupleStorageSubBlock.cpp
index 068e975..1e6f7ff 100644
--- a/storage/SplitRowStoreTupleStorageSubBlock.cpp
+++ b/storage/SplitRowStoreTupleStorageSubBlock.cpp
@@ -334,6 +334,7 @@ tuple_id SplitRowStoreTupleStorageSubBlock::bulkInsertPartialTuplesImpl(
varlen_reserve -= relation_.getAttributeById(
copy_groups.varlen_attrs_[vattr_idx].dst_attr_id_)->getType().maximumByteLength();
}
+ DCHECK_GE(relation_.getMaximumVariableByteLength(), varlen_reserve);
}
InvokeOnAnyValueAccessor(
@@ -392,7 +393,7 @@ tuple_id SplitRowStoreTupleStorageSubBlock::bulkInsertPartialTuplesImpl(
(varlen_heap_offset_orig - varlen_heap_offset));
DCHECK_LE(0, remaining_storage_after_inserts);
std::size_t additional_tuples_insert =
- remaining_storage_after_inserts / this->relation_.getMaximumByteLength();
+ remaining_storage_after_inserts / (tuple_slot_bytes_ + this->relation_.getMaximumByteLength());
// We want to avoid a situation where we have several short insert iterations
// near the end of an insertion cycle.
if (additional_tuples_insert > this->getInsertLowerBoundThreshold()) {
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/70a3d47c/storage/tests/SplitRowStoreTupleStorageSubBlock_unittest.cpp
----------------------------------------------------------------------
diff --git a/storage/tests/SplitRowStoreTupleStorageSubBlock_unittest.cpp b/storage/tests/SplitRowStoreTupleStorageSubBlock_unittest.cpp
index b953854..9270d93 100644
--- a/storage/tests/SplitRowStoreTupleStorageSubBlock_unittest.cpp
+++ b/storage/tests/SplitRowStoreTupleStorageSubBlock_unittest.cpp
@@ -656,9 +656,7 @@ TEST_P(SplitRowStoreTupleStorageSubBlockTest, BulkInsertTest) {
// Actually do the bulk-insert.
accessor.beginIteration();
tuple_id num_inserted = tuple_store_->bulkInsertTuples(&accessor);
- if (testVariableLength()) {
- EXPECT_LE(current_tuple_idx - num_inserted, getInsertLowerBoundThreshold());
- } else {
+ if (!testVariableLength()) {
EXPECT_EQ(current_tuple_idx, num_inserted);
ASSERT_TRUE(accessor.iterationFinished());
// Shouldn't be able to insert any more tuples.
@@ -885,9 +883,7 @@ TEST_P(SplitRowStoreTupleStorageSubBlockTest, BulkInsertWithRemappedAttributesTe
// Actually do the bulk-insert.
accessor.beginIteration();
tuple_id num_inserted = tuple_store_->bulkInsertTuplesWithRemappedAttributes(attribute_map, &accessor);
- if (testVariableLength()) {
- EXPECT_LE(current_tuple_idx - num_inserted, getInsertLowerBoundThreshold());
- } else {
+ if (!testVariableLength()) {
EXPECT_EQ(current_tuple_idx, num_inserted);
ASSERT_TRUE(accessor.iterationFinished());
// Shouldn't be able to insert any more tuples.
[10/50] incubator-quickstep git commit: Removed using-declaration in
headers.
Posted by zu...@apache.org.
Removed using-declaration in headers.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/ea5f28bf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/ea5f28bf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/ea5f28bf
Branch: refs/heads/quickstep_partition_parser_support
Commit: ea5f28bf6825071d04fb3b30bc0be0f9c4bf9937
Parents: 6fa7be6
Author: Zuyu Zhang <zu...@apache.org>
Authored: Mon Nov 14 17:49:39 2016 -0800
Committer: Harshad Deshmukh <hb...@apache.org>
Committed: Wed Nov 16 22:15:12 2016 -0600
----------------------------------------------------------------------
cli/CommandExecutor.hpp | 5 -----
cli/LineReader.hpp | 3 ---
2 files changed, 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/ea5f28bf/cli/CommandExecutor.hpp
----------------------------------------------------------------------
diff --git a/cli/CommandExecutor.hpp b/cli/CommandExecutor.hpp
index 25b47cc..a1d9af9 100644
--- a/cli/CommandExecutor.hpp
+++ b/cli/CommandExecutor.hpp
@@ -21,14 +21,9 @@
#define QUICKSTEP_CLI_COMMAND_COMMAND_EXECUTOR_HPP_
#include <cstdio>
-#include <string>
#include "tmb/id_typedefs.h"
-using std::fprintf;
-using std::fputc;
-using std::string;
-
namespace tmb { class MessageBus; }
namespace quickstep {
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/ea5f28bf/cli/LineReader.hpp
----------------------------------------------------------------------
diff --git a/cli/LineReader.hpp b/cli/LineReader.hpp
index 867d676..c7c073f 100644
--- a/cli/LineReader.hpp
+++ b/cli/LineReader.hpp
@@ -24,9 +24,6 @@
#include "utility/Macros.hpp"
-using std::size_t;
-using std::string;
-
namespace quickstep {
/** \addtogroup CLI
[26/50] incubator-quickstep git commit: Moved FLAGS_num_workers into
a file for shared flags.
Posted by zu...@apache.org.
Moved FLAGS_num_workers into a file for shared flags.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/e666a81f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/e666a81f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/e666a81f
Branch: refs/heads/quickstep_partition_parser_support
Commit: e666a81f472e4c8ed81e98995c1914ce655e247a
Parents: 212e6cd
Author: Zuyu Zhang <zu...@apache.org>
Authored: Sun Nov 20 22:42:49 2016 -0800
Committer: Zuyu Zhang <zu...@apache.org>
Committed: Sun Nov 20 22:42:49 2016 -0800
----------------------------------------------------------------------
cli/CMakeLists.txt | 1 +
cli/Flags.cpp | 29 +++++++++++++++++++++++++++++
cli/Flags.hpp | 2 ++
cli/QuickstepCli.cpp | 31 +++++--------------------------
4 files changed, 37 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/e666a81f/cli/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/cli/CMakeLists.txt b/cli/CMakeLists.txt
index 9928f3f..9847787 100644
--- a/cli/CMakeLists.txt
+++ b/cli/CMakeLists.txt
@@ -112,6 +112,7 @@ if(QUICKSTEP_HAVE_LIBNUMA)
${LIBNUMA_LIBRARY})
endif()
target_link_libraries(quickstep_cli_Flags
+ quickstep_cli_DefaultsConfigurator
quickstep_storage_StorageConstants
${GFLAGS_LIB_NAME})
target_link_libraries(quickstep_cli_InputParserUtil
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/e666a81f/cli/Flags.cpp
----------------------------------------------------------------------
diff --git a/cli/Flags.cpp b/cli/Flags.cpp
index 384ef71..87f9f73 100644
--- a/cli/Flags.cpp
+++ b/cli/Flags.cpp
@@ -19,14 +19,43 @@
#include "cli/Flags.hpp"
+#include <cstddef>
+#include <cstdio>
#include <string>
+#include "cli/DefaultsConfigurator.hpp"
#include "storage/StorageConstants.hpp"
#include "gflags/gflags.h"
+using std::fprintf;
+
namespace quickstep {
+static bool ValidateNumWorkers(const char *flagname, int value) {
+ if (value > 0) {
+ return true;
+ }
+
+ // Detect the hardware concurrency level.
+ const std::size_t num_hw_threads =
+ DefaultsConfigurator::GetNumHardwareThreads();
+
+ // Use the command-line value if that was supplied, else use the value
+ // that we computed above, provided it did return a valid value.
+ // TODO(jmp): May need to change this at some point to keep one thread
+ // available for the OS if the hardware concurrency level is high.
+ FLAGS_num_workers = num_hw_threads != 0 ? num_hw_threads : 1;
+
+ return FLAGS_num_workers > 0;
+}
+DEFINE_int32(num_workers, 0, "Number of worker threads. If this value is "
+ "specified and is greater than 0, then this user-supplied value is "
+ "used. Else (i.e. the default case), we examine the reported "
+ "hardware concurrency level, and use that.");
+static const volatile bool num_workers_dummy
+ = gflags::RegisterFlagValidator(&FLAGS_num_workers, &ValidateNumWorkers);
+
static bool ValidateStoragePath(const char *flagname,
const std::string &value) {
if (!value.empty() && value.back() != kPathSeparator) {
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/e666a81f/cli/Flags.hpp
----------------------------------------------------------------------
diff --git a/cli/Flags.hpp b/cli/Flags.hpp
index a623448..b020a3e 100644
--- a/cli/Flags.hpp
+++ b/cli/Flags.hpp
@@ -32,6 +32,8 @@ namespace quickstep {
* @brief A collection of common flags shared by Quickstep CLIs in both the
* single-node and the distributed version.
**/
+DECLARE_int32(num_workers);
+
DECLARE_string(storage_path);
/** @} */
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/e666a81f/cli/QuickstepCli.cpp
----------------------------------------------------------------------
diff --git a/cli/QuickstepCli.cpp b/cli/QuickstepCli.cpp
index 33a9ff4..6a7c500 100644
--- a/cli/QuickstepCli.cpp
+++ b/cli/QuickstepCli.cpp
@@ -109,6 +109,7 @@ using quickstep::AdmitRequestMessage;
using quickstep::CatalogRelation;
using quickstep::DefaultsConfigurator;
using quickstep::DropRelation;
+using quickstep::FLAGS_num_workers;
using quickstep::FLAGS_storage_path;
using quickstep::ForemanSingleNode;
using quickstep::InputParserUtil;
@@ -136,11 +137,6 @@ using tmb::client_id;
namespace quickstep {
-DEFINE_int32(num_workers, 0, "Number of worker threads. If this value is "
- "specified and is greater than 0, then this "
- "user-supplied value is used. Else (i.e. the"
- "default case), we examine the reported "
- "hardware concurrency level, and use that.");
DEFINE_bool(preload_buffer_pool, false,
"If true, pre-load all known blocks into buffer pool before "
"accepting queries (should also set --buffer_pool_slots to be "
@@ -189,25 +185,8 @@ int main(int argc, char* argv[]) {
google::InitGoogleLogging(argv[0]);
gflags::ParseCommandLineFlags(&argc, &argv, true);
- // Detect the hardware concurrency level.
- const std::size_t num_hw_threads =
- DefaultsConfigurator::GetNumHardwareThreads();
-
- // Use the command-line value if that was supplied, else use the value
- // that we computed above, provided it did return a valid value.
- // TODO(jmp): May need to change this at some point to keep one thread
- // available for the OS if the hardware concurrency level is high.
- if (quickstep::FLAGS_num_workers <= 0) {
- LOG(INFO) << "Quickstep expects at least one worker thread, switching to "
- "the default number of worker threads";
- }
- const int real_num_workers = quickstep::FLAGS_num_workers > 0
- ? quickstep::FLAGS_num_workers
- : (num_hw_threads != 0 ? num_hw_threads : 1);
-
- DCHECK_GT(real_num_workers, 0);
- printf("Starting Quickstep with %d worker thread(s) and a %.2f GB buffer pool\n",
- real_num_workers,
+ printf("Starting Quickstep with %d worker thread(s) and a %.2f GB buffer pool.\n",
+ FLAGS_num_workers,
(static_cast<double>(quickstep::FLAGS_buffer_pool_slots) * quickstep::kSlotSizeBytes)/quickstep::kAGigaByte);
#ifdef QUICKSTEP_HAVE_FILE_MANAGER_HDFS
@@ -290,7 +269,7 @@ int main(int argc, char* argv[]) {
// Parse the CPU affinities for workers and the preloader thread, if enabled
// to warm up the buffer pool.
const vector<int> worker_cpu_affinities =
- InputParserUtil::ParseWorkerAffinities(real_num_workers,
+ InputParserUtil::ParseWorkerAffinities(FLAGS_num_workers,
quickstep::FLAGS_worker_affinities);
const std::size_t num_numa_nodes_system = DefaultsConfigurator::GetNumNUMANodes();
@@ -323,7 +302,7 @@ int main(int argc, char* argv[]) {
vector<client_id> worker_client_ids;
// Initialize the worker threads.
- DCHECK_EQ(static_cast<std::size_t>(real_num_workers),
+ DCHECK_EQ(static_cast<std::size_t>(FLAGS_num_workers),
worker_cpu_affinities.size());
for (std::size_t worker_thread_index = 0;
worker_thread_index < worker_cpu_affinities.size();
[50/50] incubator-quickstep git commit: Added optimizer support for
hash partitions.
Posted by zu...@apache.org.
Added optimizer support for hash partitions.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/4890bbe8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/4890bbe8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/4890bbe8
Branch: refs/heads/quickstep_partition_parser_support
Commit: 4890bbe8354d32789d4a0b111ccaedce8ce055bb
Parents: 5fee821
Author: Zuyu Zhang <zu...@apache.org>
Authored: Tue Jan 10 17:00:08 2017 -0800
Committer: Zuyu Zhang <zu...@apache.org>
Committed: Tue Jan 10 17:00:08 2017 -0800
----------------------------------------------------------------------
query_optimizer/ExecutionGenerator.cpp | 4 +
query_optimizer/logical/CMakeLists.txt | 1 +
query_optimizer/logical/CreateTable.hpp | 23 ++++--
query_optimizer/physical/CMakeLists.txt | 1 +
query_optimizer/physical/CreateTable.hpp | 22 ++++--
query_optimizer/resolver/CMakeLists.txt | 3 +
query_optimizer/resolver/Resolver.cpp | 101 ++++++++++++++++++++------
query_optimizer/resolver/Resolver.hpp | 12 +++
query_optimizer/strategy/OneToOne.cpp | 3 +-
9 files changed, 135 insertions(+), 35 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4890bbe8/query_optimizer/ExecutionGenerator.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/ExecutionGenerator.cpp b/query_optimizer/ExecutionGenerator.cpp
index 29e67f7..6ee219e 100644
--- a/query_optimizer/ExecutionGenerator.cpp
+++ b/query_optimizer/ExecutionGenerator.cpp
@@ -1017,6 +1017,10 @@ void ExecutionGenerator::convertCreateTable(
catalog_relation->setDefaultStorageBlockLayout(layout.release());
}
+ if (physical_plan->partition_scheme_header_proto()) {
+ catalog_relation->setPartitionScheme(nullptr);
+ }
+
execution_plan_->addRelationalOperator(
new CreateTableOperator(query_handle_->query_id(),
catalog_relation.release(),
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4890bbe8/query_optimizer/logical/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_optimizer/logical/CMakeLists.txt b/query_optimizer/logical/CMakeLists.txt
index c67f96f..8aca550 100644
--- a/query_optimizer/logical/CMakeLists.txt
+++ b/query_optimizer/logical/CMakeLists.txt
@@ -86,6 +86,7 @@ target_link_libraries(quickstep_queryoptimizer_logical_CreateIndex
quickstep_utility_Macros)
target_link_libraries(quickstep_queryoptimizer_logical_CreateTable
glog
+ quickstep_catalog_Catalog_proto
quickstep_queryoptimizer_OptimizerTree
quickstep_queryoptimizer_expressions_AttributeReference
quickstep_queryoptimizer_logical_Logical
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4890bbe8/query_optimizer/logical/CreateTable.hpp
----------------------------------------------------------------------
diff --git a/query_optimizer/logical/CreateTable.hpp b/query_optimizer/logical/CreateTable.hpp
index cc7c6d7..c01b912 100644
--- a/query_optimizer/logical/CreateTable.hpp
+++ b/query_optimizer/logical/CreateTable.hpp
@@ -24,6 +24,7 @@
#include <string>
#include <vector>
+#include "catalog/Catalog.pb.h"
#include "query_optimizer/OptimizerTree.hpp"
#include "query_optimizer/expressions/AttributeReference.hpp"
#include "query_optimizer/logical/Logical.hpp"
@@ -72,10 +73,17 @@ class CreateTable : public Logical {
return block_properties_;
}
+ /**
+ * @return Shared pointer to the partition scheme.
+ */
+ const std::shared_ptr<const serialization::PartitionSchemeHeader> partition_scheme_header_proto() const {
+ return partition_scheme_header_proto_;
+ }
+
LogicalPtr copyWithNewChildren(
const std::vector<LogicalPtr> &new_children) const override {
DCHECK_EQ(getNumChildren(), new_children.size());
- return Create(relation_name_, attributes_, block_properties_);
+ return Create(relation_name_, attributes_, block_properties_, partition_scheme_header_proto_);
}
std::vector<expressions::AttributeReferencePtr> getOutputAttributes() const override {
@@ -100,8 +108,9 @@ class CreateTable : public Logical {
static CreateTablePtr Create(
const std::string &relation_name,
const std::vector<expressions::AttributeReferencePtr> &attributes,
- const std::shared_ptr<const StorageBlockLayoutDescription> &block_properties) {
- return CreateTablePtr(new CreateTable(relation_name, attributes, block_properties));
+ const std::shared_ptr<const StorageBlockLayoutDescription> &block_properties,
+ const std::shared_ptr<const serialization::PartitionSchemeHeader> &partition_scheme_header_proto) {
+ return CreateTablePtr(new CreateTable(relation_name, attributes, block_properties, partition_scheme_header_proto));
}
protected:
@@ -117,18 +126,20 @@ class CreateTable : public Logical {
CreateTable(
const std::string &relation_name,
const std::vector<expressions::AttributeReferencePtr> &attributes,
- const std::shared_ptr<const StorageBlockLayoutDescription> &block_properties)
+ const std::shared_ptr<const StorageBlockLayoutDescription> &block_properties,
+ const std::shared_ptr<const serialization::PartitionSchemeHeader> &partition_scheme_header_proto)
: relation_name_(relation_name),
attributes_(attributes),
block_properties_(block_properties),
block_properties_representation_(
- getOptimizerRepresentationForProto<OptimizerTreeBaseNodePtr>(block_properties_.get())) {}
+ getOptimizerRepresentationForProto<OptimizerTreeBaseNodePtr>(block_properties_.get())),
+ partition_scheme_header_proto_(partition_scheme_header_proto){}
std::string relation_name_;
std::vector<expressions::AttributeReferencePtr> attributes_;
std::shared_ptr<const StorageBlockLayoutDescription> block_properties_;
std::shared_ptr<const OptimizerProtoRepresentation<OptimizerTreeBaseNodePtr> > block_properties_representation_;
-
+ std::shared_ptr<const serialization::PartitionSchemeHeader> partition_scheme_header_proto_;
DISALLOW_COPY_AND_ASSIGN(CreateTable);
};
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4890bbe8/query_optimizer/physical/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_optimizer/physical/CMakeLists.txt b/query_optimizer/physical/CMakeLists.txt
index 5c2cd0b..7f26943 100644
--- a/query_optimizer/physical/CMakeLists.txt
+++ b/query_optimizer/physical/CMakeLists.txt
@@ -85,6 +85,7 @@ target_link_libraries(quickstep_queryoptimizer_physical_CreateIndex
quickstep_utility_Macros)
target_link_libraries(quickstep_queryoptimizer_physical_CreateTable
glog
+ quickstep_catalog_Catalog_proto
quickstep_queryoptimizer_OptimizerTree
quickstep_queryoptimizer_expressions_AttributeReference
quickstep_queryoptimizer_expressions_ExpressionUtil
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4890bbe8/query_optimizer/physical/CreateTable.hpp
----------------------------------------------------------------------
diff --git a/query_optimizer/physical/CreateTable.hpp b/query_optimizer/physical/CreateTable.hpp
index 8e3bbd4..5fb1c61 100644
--- a/query_optimizer/physical/CreateTable.hpp
+++ b/query_optimizer/physical/CreateTable.hpp
@@ -24,6 +24,7 @@
#include <string>
#include <vector>
+#include "catalog/Catalog.pb.h"
#include "query_optimizer/OptimizerTree.hpp"
#include "query_optimizer/expressions/AttributeReference.hpp"
#include "query_optimizer/expressions/ExpressionUtil.hpp"
@@ -75,10 +76,17 @@ class CreateTable : public Physical {
return block_properties_;
}
+ /**
+ * @return Shared pointer to the partition scheme
+ */
+ const std::shared_ptr<const serialization::PartitionSchemeHeader> partition_scheme_header_proto() const {
+ return partition_scheme_header_proto_;
+ }
+
PhysicalPtr copyWithNewChildren(
const std::vector<PhysicalPtr> &new_children) const override {
DCHECK_EQ(getNumChildren(), new_children.size());
- return Create(relation_name_, attributes_, block_properties_);
+ return Create(relation_name_, attributes_, block_properties_, partition_scheme_header_proto_);
}
std::vector<expressions::AttributeReferencePtr> getOutputAttributes() const override {
@@ -107,8 +115,9 @@ class CreateTable : public Physical {
static CreateTablePtr Create(
const std::string &relation_name,
const std::vector<expressions::AttributeReferencePtr> &attributes,
- const std::shared_ptr<const StorageBlockLayoutDescription> &block_properties) {
- return CreateTablePtr(new CreateTable(relation_name, attributes, block_properties));
+ const std::shared_ptr<const StorageBlockLayoutDescription> &block_properties,
+ const std::shared_ptr<const serialization::PartitionSchemeHeader> &partition_scheme_header_proto) {
+ return CreateTablePtr(new CreateTable(relation_name, attributes, block_properties, partition_scheme_header_proto));
}
protected:
@@ -124,17 +133,20 @@ class CreateTable : public Physical {
CreateTable(
const std::string &relation_name,
const std::vector<expressions::AttributeReferencePtr> &attributes,
- const std::shared_ptr<const StorageBlockLayoutDescription> &block_properties)
+ const std::shared_ptr<const StorageBlockLayoutDescription> &block_properties,
+ const std::shared_ptr<const serialization::PartitionSchemeHeader> &partition_scheme_header_proto)
: relation_name_(relation_name),
attributes_(attributes),
block_properties_(block_properties),
block_properties_representation_(
- getOptimizerRepresentationForProto<OptimizerTreeBaseNodePtr>(block_properties_.get())) {}
+ getOptimizerRepresentationForProto<OptimizerTreeBaseNodePtr>(block_properties_.get())),
+ partition_scheme_header_proto_(partition_scheme_header_proto) {}
std::string relation_name_;
std::vector<expressions::AttributeReferencePtr> attributes_;
std::shared_ptr<const StorageBlockLayoutDescription> block_properties_;
std::shared_ptr<const OptimizerProtoRepresentation<OptimizerTreeBaseNodePtr> > block_properties_representation_;
+ std::shared_ptr<const serialization::PartitionSchemeHeader> partition_scheme_header_proto_;
DISALLOW_COPY_AND_ASSIGN(CreateTable);
};
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4890bbe8/query_optimizer/resolver/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_optimizer/resolver/CMakeLists.txt b/query_optimizer/resolver/CMakeLists.txt
index 5251ccc..a34273e 100644
--- a/query_optimizer/resolver/CMakeLists.txt
+++ b/query_optimizer/resolver/CMakeLists.txt
@@ -34,6 +34,8 @@ target_link_libraries(quickstep_queryoptimizer_resolver_NameResolver
target_link_libraries(quickstep_queryoptimizer_resolver_Resolver
glog
quickstep_catalog_CatalogDatabase
+ quickstep_catalog_CatalogTypedefs
+ quickstep_catalog_Catalog_proto
quickstep_expressions_aggregation_AggregateFunction
quickstep_expressions_aggregation_AggregateFunctionFactory
quickstep_expressions_tablegenerator_GeneratorFunction
@@ -53,6 +55,7 @@ target_link_libraries(quickstep_queryoptimizer_resolver_Resolver
quickstep_parser_ParseLimit
quickstep_parser_ParseLiteralValue
quickstep_parser_ParseOrderBy
+ quickstep_parser_ParsePartitionClause
quickstep_parser_ParsePredicate
quickstep_parser_ParsePredicateExists
quickstep_parser_ParsePredicateInTableQuery
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4890bbe8/query_optimizer/resolver/Resolver.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/resolver/Resolver.cpp b/query_optimizer/resolver/Resolver.cpp
index 2580342..d363a06 100644
--- a/query_optimizer/resolver/Resolver.cpp
+++ b/query_optimizer/resolver/Resolver.cpp
@@ -29,7 +29,9 @@
#include <vector>
#include <utility>
+#include "catalog/Catalog.pb.h"
#include "catalog/CatalogDatabase.hpp"
+#include "catalog/CatalogTypedefs.hpp"
#include "expressions/aggregation/AggregateFunction.hpp"
#include "expressions/aggregation/AggregateFunctionFactory.hpp"
#include "expressions/table_generator/GeneratorFunction.hpp"
@@ -49,6 +51,7 @@
#include "parser/ParseLimit.hpp"
#include "parser/ParseLiteralValue.hpp"
#include "parser/ParseOrderBy.hpp"
+#include "parser/ParsePartitionClause.hpp"
#include "parser/ParsePredicate.hpp"
#include "parser/ParsePredicateExists.hpp"
#include "parser/ParsePredicateInTableQuery.hpp"
@@ -129,12 +132,15 @@
#include "glog/logging.h"
+using std::make_unique;
+
namespace quickstep {
namespace optimizer {
namespace resolver {
namespace E = ::quickstep::optimizer::expressions;
namespace L = ::quickstep::optimizer::logical;
+namespace S = ::quickstep::serialization;
struct Resolver::ExpressionResolutionInfo {
/**
@@ -478,9 +484,32 @@ L::LogicalPtr Resolver::resolveCreateTable(
std::shared_ptr<const StorageBlockLayoutDescription>
block_properties(resolveBlockProperties(create_table_statement));
- return L::CreateTable::Create(relation_name, attributes, block_properties);
+ std::shared_ptr<const S::PartitionSchemeHeader>
+ partition_scheme_header_proto(resolvePartitionClause(create_table_statement));
+
+ return L::CreateTable::Create(relation_name, attributes, block_properties, partition_scheme_header_proto);
+}
+
+namespace {
+
+attribute_id GetAttributeIdFromName(const PtrList<ParseAttributeDefinition>&attribute_definition_list,
+ const std::string &attribute_name) {
+ const std::string lower_attribute_name = ToLower(attribute_name);
+
+ attribute_id attr_id = 0;
+ for (const ParseAttributeDefinition &attribute_definition : attribute_definition_list) {
+ if (lower_attribute_name == ToLower(attribute_definition.name()->value())) {
+ return attr_id;
+ }
+
+ ++attr_id;
+ }
+
+ return kInvalidAttributeID;
}
+} // namespace
+
StorageBlockLayoutDescription* Resolver::resolveBlockProperties(
const ParseStatementCreateTable &create_table_statement) {
const ParseBlockProperties *block_properties
@@ -540,25 +569,6 @@ StorageBlockLayoutDescription* Resolver::resolveBlockProperties(
THROW_SQL_ERROR_AT(type_parse_string) << "Unrecognized storage type.";
}
- // Helper lambda function which will be used in COMPRESS and SORT resolution.
- // Returns the column id from the name of the given attribute. Returns -1 if
- // the attribute is not found.
- auto columnIdFromAttributeName = [&create_table_statement](
- const std::string& attribute_name) -> int {
- const std::string search_name = ToLower(attribute_name);
- int i = 0;
- for (const ParseAttributeDefinition &attribute_definition :
- create_table_statement.attribute_definition_list()) {
- const std::string lower_attribute_name =
- ToLower(attribute_definition.name()->value());
- if (lower_attribute_name.compare(search_name) == 0) {
- return i;
- }
- i++;
- }
- return -1;
- };
-
// Resolve the SORT property.
const ParseString *sort_parse_string = block_properties->getSort();
if (block_requires_sort) {
@@ -566,9 +576,9 @@ StorageBlockLayoutDescription* Resolver::resolveBlockProperties(
THROW_SQL_ERROR_AT(type_parse_string)
<< "The SORT property must be specified as an attribute name.";
} else {
- const std::string &sort_name = sort_parse_string->value();
// Lookup the name and map to a column id.
- int sort_id = columnIdFromAttributeName(sort_name);
+ attribute_id sort_id = GetAttributeIdFromName(create_table_statement.attribute_definition_list(),
+ sort_parse_string->value());
if (sort_id == -1) {
THROW_SQL_ERROR_AT(sort_parse_string)
<< "The SORT property did not match any attribute name.";
@@ -609,7 +619,8 @@ StorageBlockLayoutDescription* Resolver::resolveBlockProperties(
<< "The COMPRESS property must be specified as ALL or a list of attributes.";
}
for (const ParseString &compressed_attribute_name : *compress_parse_strings) {
- int column_id = columnIdFromAttributeName(compressed_attribute_name.value());
+ attribute_id column_id = GetAttributeIdFromName(create_table_statement.attribute_definition_list(),
+ compressed_attribute_name.value());
if (column_id == -1) {
THROW_SQL_ERROR_AT(&compressed_attribute_name)
<< "The given attribute was not found.";
@@ -671,6 +682,50 @@ StorageBlockLayoutDescription* Resolver::resolveBlockProperties(
return storage_block_description.release();
}
+const S::PartitionSchemeHeader* Resolver::resolvePartitionClause(
+ const ParseStatementCreateTable &create_table_statement) {
+ const ParsePartitionClause *partition_clause = create_table_statement.opt_partition_clause();
+ if (partition_clause == nullptr) {
+ return nullptr;
+ }
+
+ const ParseString *partition_type_string = partition_clause->partition_type();
+ if (partition_type_string == nullptr) {
+ THROW_SQL_ERROR_AT(partition_clause)
+ << "Partition type must be specified and be a string.";
+ }
+
+ const PtrList<ParseString> &attribute_name_list = partition_clause->attribute_name_list();
+ if (attribute_name_list.size() != 1) {
+ THROW_SQL_ERROR_AT(partition_clause)
+ << "Partition is supported on only one attribute.";
+ }
+
+ const ParseString &partition_attribute_name = *(attribute_name_list.begin());
+ const attribute_id attr_id = GetAttributeIdFromName(create_table_statement.attribute_definition_list(),
+ partition_attribute_name.value());
+ if (attr_id == kInvalidAttributeID) {
+ THROW_SQL_ERROR_AT(&partition_attribute_name)
+ << "The given attribute was not found.";
+ }
+
+ auto proto = make_unique<S::PartitionSchemeHeader>();
+ proto->set_num_partitions(partition_clause->num_partitions()->long_value());
+ proto->set_partition_attribute_id(attr_id);
+
+ const std::string partition_type = ToLower(partition_type_string->value());
+ if (partition_type == "hash") {
+ proto->set_partition_type(S::PartitionSchemeHeader::HASH);
+ } else if (partition_type == "range") {
+ THROW_SQL_ERROR_AT(partition_clause)
+ << "Range partition is not supported.";
+ } else {
+ THROW_SQL_ERROR_AT(partition_type_string) << "Unrecognized partition type.";
+ }
+
+ return proto.release();
+}
+
L::LogicalPtr Resolver::resolveCreateIndex(
const ParseStatementCreateIndex &create_index_statement) {
// Resolve relation reference.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4890bbe8/query_optimizer/resolver/Resolver.hpp
----------------------------------------------------------------------
diff --git a/query_optimizer/resolver/Resolver.hpp b/query_optimizer/resolver/Resolver.hpp
index 855e6ba..e4351e0 100644
--- a/query_optimizer/resolver/Resolver.hpp
+++ b/query_optimizer/resolver/Resolver.hpp
@@ -74,6 +74,8 @@ class PtrList;
class StorageBlockLayoutDescription;
class Type;
+namespace serialization { class PartitionSchemeHeader; }
+
} // namespace quickstep
namespace quickstep {
@@ -213,6 +215,16 @@ class Resolver {
const ParseStatementCreateTable &create_table_statement);
/**
+ * @brief Resolves the PARTITION clause of a CREATE TABLE statement to a
+ * the serialized PartitionSchemeHeader describing the user input.
+ *
+ * @param create_table_statement The create table statement.
+ * @return A pointer to a user-owned serialized PartitionSchemeHeader.
+ */
+ const serialization::PartitionSchemeHeader* resolvePartitionClause(
+ const ParseStatementCreateTable &create_table_statement);
+
+ /**
* @brief Resolves a DELETE query and returns a logical plan.
*
* @param delete_statement The DELETE parse tree.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4890bbe8/query_optimizer/strategy/OneToOne.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/strategy/OneToOne.cpp b/query_optimizer/strategy/OneToOne.cpp
index 78003f4..7d0c4cb 100644
--- a/query_optimizer/strategy/OneToOne.cpp
+++ b/query_optimizer/strategy/OneToOne.cpp
@@ -121,7 +121,8 @@ bool OneToOne::generatePlan(const L::LogicalPtr &logical_input,
std::static_pointer_cast<const L::CreateTable>(logical_input);
*physical_output = P::CreateTable::Create(create_table->relation_name(),
create_table->attributes(),
- create_table->block_properties());
+ create_table->block_properties(),
+ create_table->partition_scheme_header_proto());
return true;
}
case L::LogicalType::kDeleteTuples: {
[45/50] incubator-quickstep git commit: Fixed PlanVisualizer crashes
Posted by zu...@apache.org.
Fixed PlanVisualizer crashes
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/1b0328bd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/1b0328bd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/1b0328bd
Branch: refs/heads/quickstep_partition_parser_support
Commit: 1b0328bd29a38e259d89f5bcf8142b1f5aa1ba50
Parents: 6ae0cdd
Author: Jianqiao Zhu <ji...@cs.wisc.edu>
Authored: Wed Dec 14 18:41:38 2016 -0600
Committer: Jianqiao Zhu <ji...@cs.wisc.edu>
Committed: Wed Dec 14 20:48:21 2016 -0600
----------------------------------------------------------------------
query_optimizer/cost_model/CostModel.hpp | 28 ++++++++++++++++++++
query_optimizer/cost_model/SimpleCostModel.cpp | 3 ++-
.../cost_model/StarSchemaSimpleCostModel.cpp | 3 ++-
utility/PlanVisualizer.cpp | 18 ++++++++++---
4 files changed, 46 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1b0328bd/query_optimizer/cost_model/CostModel.hpp
----------------------------------------------------------------------
diff --git a/query_optimizer/cost_model/CostModel.hpp b/query_optimizer/cost_model/CostModel.hpp
index 20bc208..21f1856 100644
--- a/query_optimizer/cost_model/CostModel.hpp
+++ b/query_optimizer/cost_model/CostModel.hpp
@@ -21,6 +21,8 @@
#define QUERY_OPTIMIZER_COST_MODEL_COST_MODEL_HPP_
#include <cstddef>
+#include <exception>
+#include <string>
#include "query_optimizer/physical/Aggregate.hpp"
#include "query_optimizer/physical/Physical.hpp"
@@ -35,6 +37,32 @@ namespace cost {
*/
/**
+ * @brief Exception thrown for unsupported physical plan.
+ **/
+class UnsupportedPhysicalPlan : public std::exception {
+ public:
+ /**
+ * @brief Constructor.
+ *
+ * @param physical_plan The physical plan that is not supported by the cost
+ * model.
+ **/
+ explicit UnsupportedPhysicalPlan(const physical::PhysicalPtr &physical_plan)
+ : message_("UnsupportedPhysicalPlan: \n" + physical_plan->toString()) {
+ }
+
+ ~UnsupportedPhysicalPlan() throw() {
+ }
+
+ virtual const char* what() const throw() {
+ return message_.c_str();
+ }
+
+ private:
+ std::string message_;
+};
+
+/**
* @brief Interface to a cost model of physical plans.
*/
class CostModel {
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1b0328bd/query_optimizer/cost_model/SimpleCostModel.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/cost_model/SimpleCostModel.cpp b/query_optimizer/cost_model/SimpleCostModel.cpp
index a803c67..bf6da6a 100644
--- a/query_optimizer/cost_model/SimpleCostModel.cpp
+++ b/query_optimizer/cost_model/SimpleCostModel.cpp
@@ -24,6 +24,7 @@
#include "catalog/CatalogRelation.hpp"
#include "catalog/CatalogRelationStatistics.hpp"
+#include "query_optimizer/cost_model/CostModel.hpp"
#include "query_optimizer/physical/Aggregate.hpp"
#include "query_optimizer/physical/NestedLoopsJoin.hpp"
#include "query_optimizer/physical/HashJoin.hpp"
@@ -82,7 +83,7 @@ std::size_t SimpleCostModel::estimateCardinality(
return estimateCardinalityForWindowAggregate(
std::static_pointer_cast<const P::WindowAggregate>(physical_plan));
default:
- LOG(FATAL) << "Unsupported physical plan:" << physical_plan->toString();
+ throw UnsupportedPhysicalPlan(physical_plan);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1b0328bd/query_optimizer/cost_model/StarSchemaSimpleCostModel.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/cost_model/StarSchemaSimpleCostModel.cpp b/query_optimizer/cost_model/StarSchemaSimpleCostModel.cpp
index 1075739..dfafa7d 100644
--- a/query_optimizer/cost_model/StarSchemaSimpleCostModel.cpp
+++ b/query_optimizer/cost_model/StarSchemaSimpleCostModel.cpp
@@ -25,6 +25,7 @@
#include <vector>
#include "catalog/CatalogRelation.hpp"
+#include "query_optimizer/cost_model/CostModel.hpp"
#include "query_optimizer/expressions/AttributeReference.hpp"
#include "query_optimizer/expressions/ComparisonExpression.hpp"
#include "query_optimizer/expressions/ExprId.hpp"
@@ -93,7 +94,7 @@ std::size_t StarSchemaSimpleCostModel::estimateCardinality(
return estimateCardinalityForWindowAggregate(
std::static_pointer_cast<const P::WindowAggregate>(physical_plan));
default:
- LOG(FATAL) << "Unsupported physical plan:" << physical_plan->toString();
+ throw UnsupportedPhysicalPlan(physical_plan);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1b0328bd/utility/PlanVisualizer.cpp
----------------------------------------------------------------------
diff --git a/utility/PlanVisualizer.cpp b/utility/PlanVisualizer.cpp
index 5d70c86..df7a20c 100644
--- a/utility/PlanVisualizer.cpp
+++ b/utility/PlanVisualizer.cpp
@@ -20,6 +20,7 @@
#include "utility/PlanVisualizer.hpp"
#include <cstddef>
+#include <exception>
#include <memory>
#include <set>
#include <sstream>
@@ -189,10 +190,19 @@ void PlanVisualizer::visit(const P::PhysicalPtr &input) {
}
}
- node_info.labels.emplace_back(
- "est. # = " + std::to_string(cost_model_->estimateCardinality(input)));
- node_info.labels.emplace_back(
- "est. Selectivity = " + std::to_string(cost_model_->estimateSelectivity(input)));
+ try {
+ const std::size_t estimated_cardinality = cost_model_->estimateCardinality(input);
+ const double estimated_selectivity = cost_model_->estimateSelectivity(input);
+
+ node_info.labels.emplace_back(
+ "est. # = " + std::to_string(estimated_cardinality));
+ node_info.labels.emplace_back(
+ "est. Selectivity = " + std::to_string(estimated_selectivity));
+ } catch (const std::exception &e) {
+ // NOTE(jianqiao): CostModel::estimateCardinality() may throw UnsupportedPhysicalPlan
+ // exception for some type of physical nodes such as CreateTable.
+ // In this case, we omit the node's cardinality/selectivity information.
+ }
}
} // namespace quickstep
[18/50] incubator-quickstep git commit: Refactored
PolicyEnforcerBase::decrementNumQueuedWorkOrders.
Posted by zu...@apache.org.
Refactored PolicyEnforcerBase::decrementNumQueuedWorkOrders.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/76e37cab
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/76e37cab
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/76e37cab
Branch: refs/heads/quickstep_partition_parser_support
Commit: 76e37cabcf24ff4fec7e50d5bd2b590fb9fe93d2
Parents: 37a78cb
Author: Zuyu Zhang <zu...@apache.org>
Authored: Fri Nov 18 12:20:12 2016 -0800
Committer: Zuyu Zhang <zu...@apache.org>
Committed: Sun Nov 20 19:29:11 2016 -0800
----------------------------------------------------------------------
query_execution/CMakeLists.txt | 1 +
query_execution/PolicyEnforcerBase.cpp | 4 ++--
query_execution/PolicyEnforcerBase.hpp | 6 ++++--
query_execution/PolicyEnforcerDistributed.hpp | 5 +++--
query_execution/PolicyEnforcerSingleNode.cpp | 4 ----
query_execution/PolicyEnforcerSingleNode.hpp | 7 +++++--
query_execution/QueryExecutionMessages.proto | 3 +++
query_execution/QueryExecutionTypedefs.hpp | 3 ++-
query_execution/Shiftboss.cpp | 8 +++++++-
query_execution/Shiftboss.hpp | 1 +
query_execution/Worker.cpp | 14 ++++++++++++++
query_execution/Worker.hpp | 9 +++++++++
12 files changed, 51 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/76e37cab/query_execution/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_execution/CMakeLists.txt b/query_execution/CMakeLists.txt
index b5e07df..0f6c282 100644
--- a/query_execution/CMakeLists.txt
+++ b/query_execution/CMakeLists.txt
@@ -160,6 +160,7 @@ target_link_libraries(quickstep_queryexecution_PolicyEnforcerSingleNode
glog
quickstep_catalog_CatalogTypedefs
quickstep_queryexecution_PolicyEnforcerBase
+ quickstep_queryexecution_QueryExecutionMessages_proto
quickstep_queryexecution_QueryExecutionState
quickstep_queryexecution_QueryManagerBase
quickstep_queryexecution_QueryManagerSingleNode
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/76e37cab/query_execution/PolicyEnforcerBase.cpp
----------------------------------------------------------------------
diff --git a/query_execution/PolicyEnforcerBase.cpp b/query_execution/PolicyEnforcerBase.cpp
index 4e8c782..6e922a8 100644
--- a/query_execution/PolicyEnforcerBase.cpp
+++ b/query_execution/PolicyEnforcerBase.cpp
@@ -50,7 +50,7 @@ void PolicyEnforcerBase::processMessage(const TaggedMessage &tagged_message) {
// WorkOrder. It can be accessed in this scope.
CHECK(proto.ParseFromArray(tagged_message.message(),
tagged_message.message_bytes()));
- decrementNumQueuedWorkOrders(proto.worker_thread_index());
+ decrementNumQueuedWorkOrders(proto);
if (profile_individual_workorders_) {
recordTimeForWorkOrder(proto);
@@ -69,7 +69,7 @@ void PolicyEnforcerBase::processMessage(const TaggedMessage &tagged_message) {
// rebuild WorkOrder. It can be accessed in this scope.
CHECK(proto.ParseFromArray(tagged_message.message(),
tagged_message.message_bytes()));
- decrementNumQueuedWorkOrders(proto.worker_thread_index());
+ decrementNumQueuedWorkOrders(proto);
query_id = proto.query_id();
DCHECK(admitted_queries_.find(query_id) != admitted_queries_.end());
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/76e37cab/query_execution/PolicyEnforcerBase.hpp
----------------------------------------------------------------------
diff --git a/query_execution/PolicyEnforcerBase.hpp b/query_execution/PolicyEnforcerBase.hpp
index 7009a0a..414367e 100644
--- a/query_execution/PolicyEnforcerBase.hpp
+++ b/query_execution/PolicyEnforcerBase.hpp
@@ -196,9 +196,11 @@ class PolicyEnforcerBase {
/**
* @brief Decrement the number of queued workorders for the given worker by 1.
*
- * @param worker_index The logical ID of the given worker.
+ * @param proto The completion message proto received after the WorkOrder
+ * execution.
**/
- virtual void decrementNumQueuedWorkOrders(const std::size_t worker_index) = 0;
+ virtual void decrementNumQueuedWorkOrders(
+ const serialization::WorkOrderCompletionMessage &proto) = 0;
DISALLOW_COPY_AND_ASSIGN(PolicyEnforcerBase);
};
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/76e37cab/query_execution/PolicyEnforcerDistributed.hpp
----------------------------------------------------------------------
diff --git a/query_execution/PolicyEnforcerDistributed.hpp b/query_execution/PolicyEnforcerDistributed.hpp
index bce3e0c..0bf249c 100644
--- a/query_execution/PolicyEnforcerDistributed.hpp
+++ b/query_execution/PolicyEnforcerDistributed.hpp
@@ -20,6 +20,7 @@
#include <vector>
#include "query_execution/PolicyEnforcerBase.hpp"
+#include "query_execution/QueryExecutionMessages.pb.h"
#include "query_execution/ShiftbossDirectory.hpp"
#include "utility/Macros.hpp"
@@ -89,8 +90,8 @@ class PolicyEnforcerDistributed final : public PolicyEnforcerBase {
void processInitiateRebuildResponseMessage(const tmb::TaggedMessage &tagged_message);
private:
- void decrementNumQueuedWorkOrders(const std::size_t shiftboss_index) override {
- shiftboss_directory_->decrementNumQueuedWorkOrders(shiftboss_index);
+ void decrementNumQueuedWorkOrders(const serialization::WorkOrderCompletionMessage &proto) override {
+ shiftboss_directory_->decrementNumQueuedWorkOrders(proto.shiftboss_index());
}
void onQueryCompletion(QueryManagerBase *query_manager) override;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/76e37cab/query_execution/PolicyEnforcerSingleNode.cpp
----------------------------------------------------------------------
diff --git a/query_execution/PolicyEnforcerSingleNode.cpp b/query_execution/PolicyEnforcerSingleNode.cpp
index 0bdb9b1..0aa2ca8 100644
--- a/query_execution/PolicyEnforcerSingleNode.cpp
+++ b/query_execution/PolicyEnforcerSingleNode.cpp
@@ -108,8 +108,4 @@ bool PolicyEnforcerSingleNode::admitQuery(QueryHandle *query_handle) {
}
}
-void PolicyEnforcerSingleNode::decrementNumQueuedWorkOrders(const std::size_t worker_index) {
- worker_directory_->decrementNumQueuedWorkOrders(worker_index);
-}
-
} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/76e37cab/query_execution/PolicyEnforcerSingleNode.hpp
----------------------------------------------------------------------
diff --git a/query_execution/PolicyEnforcerSingleNode.hpp b/query_execution/PolicyEnforcerSingleNode.hpp
index 870df95..16c7a0d 100644
--- a/query_execution/PolicyEnforcerSingleNode.hpp
+++ b/query_execution/PolicyEnforcerSingleNode.hpp
@@ -25,6 +25,8 @@
#include <vector>
#include "query_execution/PolicyEnforcerBase.hpp"
+#include "query_execution/QueryExecutionMessages.pb.h"
+#include "query_execution/WorkerDirectory.hpp"
#include "utility/Macros.hpp"
#include "tmb/id_typedefs.h"
@@ -36,7 +38,6 @@ namespace quickstep {
class CatalogDatabaseLite;
class QueryHandle;
class StorageManager;
-class WorkerDirectory;
class WorkerMessage;
/** \addtogroup QueryExecution
@@ -89,7 +90,9 @@ class PolicyEnforcerSingleNode final : public PolicyEnforcerBase {
std::vector<std::unique_ptr<WorkerMessage>> *worker_messages);
private:
- void decrementNumQueuedWorkOrders(const std::size_t worker_index) override;
+ void decrementNumQueuedWorkOrders(const serialization::WorkOrderCompletionMessage &proto) override {
+ worker_directory_->decrementNumQueuedWorkOrders(proto.worker_thread_index());
+ }
const tmb::client_id foreman_client_id_;
const std::size_t num_numa_nodes_;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/76e37cab/query_execution/QueryExecutionMessages.proto
----------------------------------------------------------------------
diff --git a/query_execution/QueryExecutionMessages.proto b/query_execution/QueryExecutionMessages.proto
index 165a194..e6d741a 100644
--- a/query_execution/QueryExecutionMessages.proto
+++ b/query_execution/QueryExecutionMessages.proto
@@ -46,6 +46,9 @@ message WorkOrderCompletionMessage {
// Epoch time in microseconds.
optional uint64 execution_start_time = 5;
optional uint64 execution_end_time = 6;
+
+ // Required in the distributed version.
+ optional uint64 shiftboss_index = 7;
}
message CatalogRelationNewBlockMessage {
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/76e37cab/query_execution/QueryExecutionTypedefs.hpp
----------------------------------------------------------------------
diff --git a/query_execution/QueryExecutionTypedefs.hpp b/query_execution/QueryExecutionTypedefs.hpp
index 22c0ae1..fb9a9d6 100644
--- a/query_execution/QueryExecutionTypedefs.hpp
+++ b/query_execution/QueryExecutionTypedefs.hpp
@@ -79,7 +79,8 @@ enum QueryExecutionMessageType : message_type_id {
#ifdef QUICKSTEP_DISTRIBUTED
kShiftbossRegistrationMessage, // From Shiftboss to Foreman.
- kShiftbossRegistrationResponseMessage, // From Foreman to Shiftboss.
+ kShiftbossRegistrationResponseMessage, // From Foreman to Shiftboss, or from
+ // Shiftboss to Worker.
kQueryInitiateMessage, // From Foreman to Shiftboss.
kQueryInitiateResponseMessage, // From Shiftboss to Foreman.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/76e37cab/query_execution/Shiftboss.cpp
----------------------------------------------------------------------
diff --git a/query_execution/Shiftboss.cpp b/query_execution/Shiftboss.cpp
index 30b2ae2..ed4bade 100644
--- a/query_execution/Shiftboss.cpp
+++ b/query_execution/Shiftboss.cpp
@@ -273,7 +273,7 @@ void Shiftboss::registerWithForeman() {
}
void Shiftboss::processShiftbossRegistrationResponseMessage() {
- const AnnotatedMessage annotated_message(bus_->Receive(shiftboss_client_id_, 0, true));
+ AnnotatedMessage annotated_message(bus_->Receive(shiftboss_client_id_, 0, true));
const TaggedMessage &tagged_message = annotated_message.tagged_message;
DCHECK_EQ(kShiftbossRegistrationResponseMessage, tagged_message.message_type());
@@ -286,6 +286,12 @@ void Shiftboss::processShiftbossRegistrationResponseMessage() {
CHECK(proto.ParseFromArray(tagged_message.message(), tagged_message.message_bytes()));
shiftboss_index_ = proto.shiftboss_index();
+
+ // Forward this message to Workers regarding <shiftboss_index_>.
+ QueryExecutionUtil::BroadcastMessage(shiftboss_client_id_,
+ worker_addresses_,
+ move(annotated_message.tagged_message),
+ bus_);
}
void Shiftboss::processQueryInitiateMessage(
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/76e37cab/query_execution/Shiftboss.hpp
----------------------------------------------------------------------
diff --git a/query_execution/Shiftboss.hpp b/query_execution/Shiftboss.hpp
index 442e61e..6538d48 100644
--- a/query_execution/Shiftboss.hpp
+++ b/query_execution/Shiftboss.hpp
@@ -102,6 +102,7 @@ class Shiftboss : public Thread {
bus_->RegisterClientAsSender(shiftboss_client_id_, kSaveQueryResultResponseMessage);
// Message sent to Worker.
+ bus_->RegisterClientAsSender(shiftboss_client_id_, kShiftbossRegistrationResponseMessage);
bus_->RegisterClientAsSender(shiftboss_client_id_, kRebuildWorkOrderMessage);
// Forward the following message types from Foreman to Workers.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/76e37cab/query_execution/Worker.cpp
----------------------------------------------------------------------
diff --git a/query_execution/Worker.cpp b/query_execution/Worker.cpp
index 0db17b4..31eb964 100644
--- a/query_execution/Worker.cpp
+++ b/query_execution/Worker.cpp
@@ -29,6 +29,7 @@
#include "query_execution/QueryExecutionTypedefs.hpp"
#include "query_execution/QueryExecutionUtil.hpp"
#include "query_execution/WorkerMessage.hpp"
+#include "query_optimizer/QueryOptimizerConfig.h" // For QUICKSTEP_DISTRIBUTED.
#include "relational_operators/WorkOrder.hpp"
#include "threading/ThreadIDBasedMap.hpp"
#include "threading/ThreadUtil.hpp"
@@ -62,6 +63,15 @@ void Worker::run() {
bus_->Receive(worker_client_id_, 0, true);
const TaggedMessage &tagged_message = annotated_msg.tagged_message;
switch (tagged_message.message_type()) {
+#ifdef QUICKSTEP_DISTRIBUTED
+ case kShiftbossRegistrationResponseMessage: {
+ serialization::ShiftbossRegistrationResponseMessage proto;
+ CHECK(proto.ParseFromArray(tagged_message.message(), tagged_message.message_bytes()));
+
+ shiftboss_index_ = proto.shiftboss_index();
+ break;
+ }
+#endif // QUICKSTEP_DISTRIBUTED
case kWorkOrderMessage: {
WorkOrderCompletionMessage proto;
executeWorkOrderHelper(tagged_message, &proto);
@@ -136,6 +146,10 @@ void Worker::executeWorkOrderHelper(const TaggedMessage &tagged_message,
proto->set_worker_thread_index(worker_thread_index_);
proto->set_execution_start_time(execution_start_time);
proto->set_execution_end_time(execution_end_time);
+
+#ifdef QUICKSTEP_DISTRIBUTED
+ proto->set_shiftboss_index(shiftboss_index_);
+#endif // QUICKSTEP_DISTRIBUTED
}
} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/76e37cab/query_execution/Worker.hpp
----------------------------------------------------------------------
diff --git a/query_execution/Worker.hpp b/query_execution/Worker.hpp
index fec2242..feb7c7d 100644
--- a/query_execution/Worker.hpp
+++ b/query_execution/Worker.hpp
@@ -24,6 +24,7 @@
#include <cstdint>
#include "query_execution/QueryExecutionTypedefs.hpp"
+#include "query_optimizer/QueryOptimizerConfig.h" // For QUICKSTEP_DISTRIBUTED.
#include "threading/Thread.hpp"
#include "utility/Macros.hpp"
@@ -75,6 +76,10 @@ class Worker : public Thread {
bus_->RegisterClientAsReceiver(worker_client_id_, kWorkOrderMessage);
bus_->RegisterClientAsReceiver(worker_client_id_, kRebuildWorkOrderMessage);
bus_->RegisterClientAsReceiver(worker_client_id_, kPoisonMessage);
+
+#ifdef QUICKSTEP_DISTRIBUTED
+ bus_->RegisterClientAsReceiver(worker_client_id_, kShiftbossRegistrationResponseMessage);
+#endif // QUICKSTEP_DISTRIBUTED
}
~Worker() override {}
@@ -132,6 +137,10 @@ class Worker : public Thread {
const int cpu_id_;
client_id worker_client_id_;
+#ifdef QUICKSTEP_DISTRIBUTED
+ std::size_t shiftboss_index_;
+#endif // QUICKSTEP_DISTRIBUTED
+
DISALLOW_COPY_AND_ASSIGN(Worker);
};
[14/50] incubator-quickstep git commit: Unified
WorkOrderCompletionMessage.
Posted by zu...@apache.org.
Unified WorkOrderCompletionMessage.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/178ed4b3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/178ed4b3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/178ed4b3
Branch: refs/heads/quickstep_partition_parser_support
Commit: 178ed4b3241d5c16736459f9e9073922a03d99b2
Parents: 7095987
Author: Zuyu Zhang <zu...@apache.org>
Authored: Sat Nov 12 16:44:39 2016 -0800
Committer: Zuyu Zhang <zu...@apache.org>
Committed: Fri Nov 18 13:21:38 2016 -0800
----------------------------------------------------------------------
query_execution/PolicyEnforcerBase.cpp | 6 ++---
query_execution/PolicyEnforcerBase.hpp | 4 +--
query_execution/QueryExecutionMessages.proto | 32 ++++++++---------------
query_execution/Worker.cpp | 28 +++++++++-----------
query_execution/Worker.hpp | 13 +++++----
5 files changed, 35 insertions(+), 48 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/178ed4b3/query_execution/PolicyEnforcerBase.cpp
----------------------------------------------------------------------
diff --git a/query_execution/PolicyEnforcerBase.cpp b/query_execution/PolicyEnforcerBase.cpp
index 745ded6..4e8c782 100644
--- a/query_execution/PolicyEnforcerBase.cpp
+++ b/query_execution/PolicyEnforcerBase.cpp
@@ -45,7 +45,7 @@ void PolicyEnforcerBase::processMessage(const TaggedMessage &tagged_message) {
switch (tagged_message.message_type()) {
case kWorkOrderCompleteMessage: {
- serialization::NormalWorkOrderCompletionMessage proto;
+ serialization::WorkOrderCompletionMessage proto;
// Note: This proto message contains the time it took to execute the
// WorkOrder. It can be accessed in this scope.
CHECK(proto.ParseFromArray(tagged_message.message(),
@@ -64,7 +64,7 @@ void PolicyEnforcerBase::processMessage(const TaggedMessage &tagged_message) {
break;
}
case kRebuildWorkOrderCompleteMessage: {
- serialization::RebuildWorkOrderCompletionMessage proto;
+ serialization::WorkOrderCompletionMessage proto;
// Note: This proto message contains the time it took to execute the
// rebuild WorkOrder. It can be accessed in this scope.
CHECK(proto.ParseFromArray(tagged_message.message(),
@@ -157,7 +157,7 @@ bool PolicyEnforcerBase::admitQueries(
}
void PolicyEnforcerBase::recordTimeForWorkOrder(
- const serialization::NormalWorkOrderCompletionMessage &proto) {
+ const serialization::WorkOrderCompletionMessage &proto) {
const std::size_t query_id = proto.query_id();
std::vector<WorkOrderTimeEntry> &workorder_time_entries
= workorder_time_recorder_[query_id];
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/178ed4b3/query_execution/PolicyEnforcerBase.hpp
----------------------------------------------------------------------
diff --git a/query_execution/PolicyEnforcerBase.hpp b/query_execution/PolicyEnforcerBase.hpp
index ea2c06f..7009a0a 100644
--- a/query_execution/PolicyEnforcerBase.hpp
+++ b/query_execution/PolicyEnforcerBase.hpp
@@ -38,7 +38,7 @@ namespace quickstep {
class CatalogDatabaseLite;
class QueryHandle;
-namespace serialization { class NormalWorkOrderCompletionMessage; }
+namespace serialization { class WorkOrderCompletionMessage; }
/** \addtogroup QueryExecution
* @{
@@ -165,7 +165,7 @@ class PolicyEnforcerBase {
* execution.
**/
void recordTimeForWorkOrder(
- const serialization::NormalWorkOrderCompletionMessage &proto);
+ const serialization::WorkOrderCompletionMessage &proto);
CatalogDatabaseLite *catalog_database_;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/178ed4b3/query_execution/QueryExecutionMessages.proto
----------------------------------------------------------------------
diff --git a/query_execution/QueryExecutionMessages.proto b/query_execution/QueryExecutionMessages.proto
index 1a2cb78..165a194 100644
--- a/query_execution/QueryExecutionMessages.proto
+++ b/query_execution/QueryExecutionMessages.proto
@@ -31,31 +31,21 @@ import "relational_operators/WorkOrder.proto";
// order completion message, we may be interested in adding the compression
// ratio or dictionary size of the rebuilt block.
-// TODO(harshad) : If there are different fields in the two message types below,
-// create a base message class called WorkOrderCompletionMessage and make the
-// two classes below extend the base class. All the common fields in both the
-// classes can be moved to the base class.
+message WorkOrderCompletionMessage {
+ enum WorkOrderType {
+ NORMAL = 0;
+ REBUILD = 1;
+ }
-// A message sent upon completion of a normal (not rebuild) WorkOrder execution.
-message NormalWorkOrderCompletionMessage {
- required uint64 operator_index = 1;
- required uint64 worker_thread_index = 2;
- required uint64 query_id = 3;
-
- // Epoch time in microseconds.
- optional uint64 execution_start_time = 4;
- optional uint64 execution_end_time = 5;
-}
+ required WorkOrderType work_order_type = 1;
-// A message sent upon completion of a rebuild WorkOrder execution.
-message RebuildWorkOrderCompletionMessage {
- required uint64 operator_index = 1;
- required uint64 worker_thread_index = 2;
- required uint64 query_id = 3;
+ required uint64 operator_index = 2;
+ required uint64 worker_thread_index = 3;
+ required uint64 query_id = 4;
// Epoch time in microseconds.
- optional uint64 execution_start_time = 4;
- optional uint64 execution_end_time = 5;
+ optional uint64 execution_start_time = 5;
+ optional uint64 execution_end_time = 6;
}
message CatalogRelationNewBlockMessage {
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/178ed4b3/query_execution/Worker.cpp
----------------------------------------------------------------------
diff --git a/query_execution/Worker.cpp b/query_execution/Worker.cpp
index 0b1efba..0db17b4 100644
--- a/query_execution/Worker.cpp
+++ b/query_execution/Worker.cpp
@@ -47,6 +47,8 @@ using tmb::TaggedMessage;
namespace quickstep {
+using serialization::WorkOrderCompletionMessage;
+
void Worker::run() {
if (cpu_id_ >= 0) {
ThreadUtil::BindToCPU(cpu_id_);
@@ -61,21 +63,16 @@ void Worker::run() {
const TaggedMessage &tagged_message = annotated_msg.tagged_message;
switch (tagged_message.message_type()) {
case kWorkOrderMessage: {
- serialization::NormalWorkOrderCompletionMessage proto;
- executeWorkOrderHelper<serialization::NormalWorkOrderCompletionMessage>(
- tagged_message, &proto);
- sendWorkOrderCompleteMessage<
- serialization::NormalWorkOrderCompletionMessage>(
+ WorkOrderCompletionMessage proto;
+ executeWorkOrderHelper(tagged_message, &proto);
+ sendWorkOrderCompleteMessage(
annotated_msg.sender, proto, kWorkOrderCompleteMessage);
break;
}
case kRebuildWorkOrderMessage: {
- serialization::RebuildWorkOrderCompletionMessage proto;
- executeWorkOrderHelper<
- serialization::RebuildWorkOrderCompletionMessage>(tagged_message,
- &proto);
- sendWorkOrderCompleteMessage<
- serialization::RebuildWorkOrderCompletionMessage>(
+ WorkOrderCompletionMessage proto;
+ executeWorkOrderHelper(tagged_message, &proto, true /* is_rebuild */);
+ sendWorkOrderCompleteMessage(
annotated_msg.sender, proto, kRebuildWorkOrderCompleteMessage);
break;
}
@@ -88,9 +85,8 @@ void Worker::run() {
}
}
-template <typename CompletionMessageProtoT>
void Worker::sendWorkOrderCompleteMessage(const tmb::client_id receiver,
- const CompletionMessageProtoT &proto,
+ const WorkOrderCompletionMessage &proto,
const message_type_id message_type) {
// NOTE(zuyu): Using the heap memory to serialize proto as a c-like string.
const size_t proto_length = proto.ByteSize();
@@ -109,9 +105,9 @@ void Worker::sendWorkOrderCompleteMessage(const tmb::client_id receiver,
CHECK(send_status == tmb::MessageBus::SendStatus::kOK);
}
-template <typename CompletionMessageProtoT>
void Worker::executeWorkOrderHelper(const TaggedMessage &tagged_message,
- CompletionMessageProtoT *proto) {
+ WorkOrderCompletionMessage *proto,
+ const bool is_rebuild_work_order) {
std::chrono::time_point<std::chrono::steady_clock> start, end;
WorkerMessage worker_message(
*static_cast<const WorkerMessage *>(tagged_message.message()));
@@ -133,6 +129,8 @@ void Worker::executeWorkOrderHelper(const TaggedMessage &tagged_message,
end.time_since_epoch()).count();
// Construct the proto message.
+ proto->set_work_order_type(is_rebuild_work_order ? WorkOrderCompletionMessage::REBUILD
+ : WorkOrderCompletionMessage::NORMAL);
proto->set_operator_index(worker_message.getRelationalOpIndex());
proto->set_query_id(query_id_for_workorder);
proto->set_worker_thread_index(worker_thread_index_);
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/178ed4b3/query_execution/Worker.hpp
----------------------------------------------------------------------
diff --git a/query_execution/Worker.hpp b/query_execution/Worker.hpp
index aa39bb3..fec2242 100644
--- a/query_execution/Worker.hpp
+++ b/query_execution/Worker.hpp
@@ -34,6 +34,8 @@ namespace tmb { class TaggedMessge; }
namespace quickstep {
+namespace serialization { class WorkOrderCompletionMessage; }
+
/** \addtogroup QueryExecution
* @{
*/
@@ -100,7 +102,6 @@ class Worker : public Thread {
* @brief A helper method to execute the WorkOrder and construct a
* completion message.
*
- * @note CompletionMessageProtoT is the type of the completion message.
* @note Right now a single helper method works for all message types.
* If different message types need to collect different statistics for
* the WorkOrder execution, we need to create different helper methods,
@@ -108,23 +109,21 @@ class Worker : public Thread {
*
* @param tagged_message The TaggedMessage which consists of the WorkOrder.
* @param proto The proto message to be sent.
+ * @param is_rebuild_work_order Whether it is used for a RebuildWorkOrder.
**/
- template <typename CompletionMessageProtoT>
void executeWorkOrderHelper(const TaggedMessage &tagged_message,
- CompletionMessageProtoT *proto);
+ serialization::WorkOrderCompletionMessage *proto,
+ const bool is_rebuild_work_order = false);
/**
* @brief A helper method to send the WorkOrder completion message.
*
- * @note CompletionMessageProtoT is the type of the completion message.
- *
* @param receiver The TMB client ID of the receiver.
* @param proto The proto message to be sent.
* @param message_type The ID of the type of the message being sent.
**/
- template <typename CompletionMessageProtoT>
void sendWorkOrderCompleteMessage(const tmb::client_id receiver,
- const CompletionMessageProtoT &proto,
+ const serialization::WorkOrderCompletionMessage &proto,
const message_type_id message_type);
const std::size_t worker_thread_index_;
[19/50] incubator-quickstep git commit: Refactored
printWorkOrderProfilingResults in Foreman.
Posted by zu...@apache.org.
Refactored printWorkOrderProfilingResults in Foreman.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/7fb96f3d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/7fb96f3d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/7fb96f3d
Branch: refs/heads/quickstep_partition_parser_support
Commit: 7fb96f3da140f4631060132b58ded2fd38191167
Parents: 76e37ca
Author: Zuyu Zhang <zu...@apache.org>
Authored: Fri Nov 18 22:07:27 2016 -0800
Committer: Zuyu Zhang <zu...@apache.org>
Committed: Sun Nov 20 19:41:16 2016 -0800
----------------------------------------------------------------------
query_execution/CMakeLists.txt | 3 +++
query_execution/ForemanBase.hpp | 33 +++++++++++++++++++++++++++++
query_execution/ForemanDistributed.cpp | 11 ++++++----
query_execution/ForemanDistributed.hpp | 14 +-----------
query_execution/ForemanSingleNode.cpp | 17 ++++++++-------
query_execution/ForemanSingleNode.hpp | 29 +------------------------
query_execution/PolicyEnforcerBase.hpp | 18 ++++++++--------
7 files changed, 63 insertions(+), 62 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/7fb96f3d/query_execution/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_execution/CMakeLists.txt b/query_execution/CMakeLists.txt
index 0f6c282..cf9d5b0 100644
--- a/query_execution/CMakeLists.txt
+++ b/query_execution/CMakeLists.txt
@@ -86,6 +86,7 @@ if (ENABLE_DISTRIBUTED)
endif(ENABLE_DISTRIBUTED)
target_link_libraries(quickstep_queryexecution_ForemanBase
glog
+ quickstep_queryexecution_PolicyEnforcerBase
quickstep_threading_Thread
quickstep_utility_Macros
tmb)
@@ -98,6 +99,7 @@ if (ENABLE_DISTRIBUTED)
quickstep_catalog_Catalog_proto
quickstep_queryexecution_AdmitRequestMessage
quickstep_queryexecution_ForemanBase
+ quickstep_queryexecution_PolicyEnforcerBase
quickstep_queryexecution_PolicyEnforcerDistributed
quickstep_queryexecution_QueryExecutionMessages_proto
quickstep_queryexecution_QueryExecutionTypedefs
@@ -113,6 +115,7 @@ target_link_libraries(quickstep_queryexecution_ForemanSingleNode
glog
quickstep_queryexecution_AdmitRequestMessage
quickstep_queryexecution_ForemanBase
+ quickstep_queryexecution_PolicyEnforcerBase
quickstep_queryexecution_PolicyEnforcerSingleNode
quickstep_queryexecution_QueryExecutionTypedefs
quickstep_queryexecution_QueryExecutionUtil
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/7fb96f3d/query_execution/ForemanBase.hpp
----------------------------------------------------------------------
diff --git a/query_execution/ForemanBase.hpp b/query_execution/ForemanBase.hpp
index 951f34d..ee6c7ce 100644
--- a/query_execution/ForemanBase.hpp
+++ b/query_execution/ForemanBase.hpp
@@ -20,6 +20,11 @@
#ifndef QUICKSTEP_QUERY_EXECUTION_FOREMAN_BASE_HPP_
#define QUICKSTEP_QUERY_EXECUTION_FOREMAN_BASE_HPP_
+#include <cstdio>
+#include <memory>
+#include <vector>
+
+#include "query_execution/PolicyEnforcerBase.hpp"
#include "threading/Thread.hpp"
#include "utility/Macros.hpp"
@@ -30,6 +35,8 @@
namespace quickstep {
+struct WorkOrderTimeEntry;
+
/** \addtogroup QueryExecution
* @{
*/
@@ -59,6 +66,30 @@ class ForemanBase : public Thread {
~ForemanBase() override {}
/**
+ * @brief Print the results of profiling individual work orders for a given
+ * query.
+ *
+ * TODO(harshad) - Add the name of the operator to the output.
+ *
+ * @param query_id The ID of the query for which the results are to be printed.
+ * @param out The file stream.
+ **/
+ virtual void printWorkOrderProfilingResults(const std::size_t query_id,
+ std::FILE *out) const = 0;
+
+ /**
+ * @brief Get the results of profiling individual work orders for a given
+ * query.
+ *
+ * @param query_id The ID of the query for which the results are to be printed.
+ * @return A vector of records, each being a single profiling entry.
+ **/
+ const std::vector<WorkOrderTimeEntry>& getWorkOrderProfilingResults(
+ const std::size_t query_id) const {
+ return policy_enforcer_->getProfilingResults(query_id);
+ }
+
+ /**
* @brief Get the TMB client ID of Foreman thread.
*
* @return TMB client ID of foreman thread.
@@ -77,6 +108,8 @@ class ForemanBase : public Thread {
// The ID of the CPU that the Foreman thread can optionally be pinned to.
const int cpu_id_;
+ std::unique_ptr<PolicyEnforcerBase> policy_enforcer_;
+
private:
DISALLOW_COPY_AND_ASSIGN(ForemanBase);
};
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/7fb96f3d/query_execution/ForemanDistributed.cpp
----------------------------------------------------------------------
diff --git a/query_execution/ForemanDistributed.cpp b/query_execution/ForemanDistributed.cpp
index 56b319b..aa4db17 100644
--- a/query_execution/ForemanDistributed.cpp
+++ b/query_execution/ForemanDistributed.cpp
@@ -28,6 +28,7 @@
#include "catalog/CatalogRelation.hpp"
#include "catalog/CatalogTypedefs.hpp"
#include "query_execution/AdmitRequestMessage.hpp"
+#include "query_execution/PolicyEnforcerBase.hpp"
#include "query_execution/PolicyEnforcerDistributed.hpp"
#include "query_execution/QueryExecutionMessages.pb.h"
#include "query_execution/QueryExecutionTypedefs.hpp"
@@ -98,12 +99,12 @@ ForemanDistributed::ForemanDistributed(
bus_->RegisterClientAsReceiver(foreman_client_id_, message_type);
}
- policy_enforcer_.reset(new PolicyEnforcerDistributed(
+ policy_enforcer_ = std::make_unique<PolicyEnforcerDistributed>(
foreman_client_id_,
catalog_database_,
&shiftboss_directory_,
bus_,
- profile_individual_workorders));
+ profile_individual_workorders);
}
void ForemanDistributed::run() {
@@ -180,7 +181,8 @@ void ForemanDistributed::run() {
}
case kInitiateRebuildResponseMessage: {
// A unique case in the distributed version.
- policy_enforcer_->processInitiateRebuildResponseMessage(tagged_message);
+ static_cast<PolicyEnforcerDistributed*>(policy_enforcer_.get())->
+ processInitiateRebuildResponseMessage(tagged_message);
break;
}
case kSaveQueryResultResponseMessage: {
@@ -228,7 +230,8 @@ void ForemanDistributed::run() {
if (canCollectNewMessages(message_type)) {
vector<unique_ptr<S::WorkOrderMessage>> new_messages;
- policy_enforcer_->getWorkOrderProtoMessages(&new_messages);
+ static_cast<PolicyEnforcerDistributed*>(policy_enforcer_.get())->
+ getWorkOrderProtoMessages(&new_messages);
dispatchWorkOrderMessages(new_messages);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/7fb96f3d/query_execution/ForemanDistributed.hpp
----------------------------------------------------------------------
diff --git a/query_execution/ForemanDistributed.hpp b/query_execution/ForemanDistributed.hpp
index b42795c..7f2d2f9 100644
--- a/query_execution/ForemanDistributed.hpp
+++ b/query_execution/ForemanDistributed.hpp
@@ -24,7 +24,6 @@
#include "catalog/CatalogTypedefs.hpp"
#include "query_execution/ForemanBase.hpp"
-#include "query_execution/PolicyEnforcerDistributed.hpp"
#include "query_execution/ShiftbossDirectory.hpp"
#include "utility/Macros.hpp"
@@ -68,17 +67,8 @@ class ForemanDistributed final : public ForemanBase {
~ForemanDistributed() override {}
- /**
- * @brief Print the results of profiling individual work orders for a given
- * query.
- *
- * TODO(harshad) - Add the name of the operator to the output.
- *
- * @param query_id The ID of the query for which the results are to be printed.
- * @param out The file stream.
- **/
void printWorkOrderProfilingResults(const std::size_t query_id,
- std::FILE *out) const;
+ std::FILE *out) const override;
protected:
void run() override;
@@ -120,8 +110,6 @@ class ForemanDistributed final : public ForemanBase {
CatalogDatabaseLite *catalog_database_;
- std::unique_ptr<PolicyEnforcerDistributed> policy_enforcer_;
-
// From a query id to a set of Shiftbosses that save query result.
std::unordered_map<std::size_t, std::unordered_set<std::size_t>> query_result_saved_shiftbosses_;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/7fb96f3d/query_execution/ForemanSingleNode.cpp
----------------------------------------------------------------------
diff --git a/query_execution/ForemanSingleNode.cpp b/query_execution/ForemanSingleNode.cpp
index 4661c37..dfdfb71 100644
--- a/query_execution/ForemanSingleNode.cpp
+++ b/query_execution/ForemanSingleNode.cpp
@@ -26,6 +26,7 @@
#include <vector>
#include "query_execution/AdmitRequestMessage.hpp"
+#include "query_execution/PolicyEnforcerBase.hpp"
#include "query_execution/PolicyEnforcerSingleNode.hpp"
#include "query_execution/QueryExecutionTypedefs.hpp"
#include "query_execution/QueryExecutionUtil.hpp"
@@ -92,14 +93,14 @@ ForemanSingleNode::ForemanSingleNode(
bus_->RegisterClientAsReceiver(foreman_client_id_, message_type);
}
- policy_enforcer_.reset(new PolicyEnforcerSingleNode(
+ policy_enforcer_ = std::make_unique<PolicyEnforcerSingleNode>(
foreman_client_id_,
num_numa_nodes,
catalog_database_,
storage_manager_,
worker_directory_,
bus_,
- profile_individual_workorders));
+ profile_individual_workorders);
}
void ForemanSingleNode::run() {
@@ -157,7 +158,8 @@ void ForemanSingleNode::run() {
if (canCollectNewMessages(message_type)) {
vector<unique_ptr<WorkerMessage>> new_messages;
- policy_enforcer_->getWorkerMessages(&new_messages);
+ static_cast<PolicyEnforcerSingleNode*>(policy_enforcer_.get())->
+ getWorkerMessages(&new_messages);
dispatchWorkerMessages(new_messages);
}
@@ -233,13 +235,12 @@ void ForemanSingleNode::sendWorkerMessage(const size_t worker_thread_index,
CHECK(send_status == tmb::MessageBus::SendStatus::kOK);
}
-const std::vector<WorkOrderTimeEntry>& ForemanSingleNode
- ::getWorkOrderProfilingResults(const std::size_t query_id) const {
- return policy_enforcer_->getProfilingResults(query_id);
-}
-
void ForemanSingleNode::printWorkOrderProfilingResults(const std::size_t query_id,
std::FILE *out) const {
+ // TODO(harshad) - Add the CPU core ID of the operator to the output. This
+ // will require modifying the WorkerDirectory to remember worker affinities.
+ // Until then, the users can refer to the worker_affinities provided to the
+ // cli to infer the CPU core ID where a given worker is pinned.
const std::vector<WorkOrderTimeEntry> &recorded_times =
policy_enforcer_->getProfilingResults(query_id);
fputs("Query ID,Worker ID,NUMA Socket,Operator ID,Time (microseconds)\n", out);
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/7fb96f3d/query_execution/ForemanSingleNode.hpp
----------------------------------------------------------------------
diff --git a/query_execution/ForemanSingleNode.hpp b/query_execution/ForemanSingleNode.hpp
index 5a368aa..4cc7a63 100644
--- a/query_execution/ForemanSingleNode.hpp
+++ b/query_execution/ForemanSingleNode.hpp
@@ -26,7 +26,6 @@
#include <vector>
#include "query_execution/ForemanBase.hpp"
-#include "query_execution/PolicyEnforcerSingleNode.hpp"
#include "utility/Macros.hpp"
#include "tmb/id_typedefs.h"
@@ -78,32 +77,8 @@ class ForemanSingleNode final : public ForemanBase {
~ForemanSingleNode() override {}
-
- /**
- * @brief Get the results of profiling individual work orders for a given
- * query.
- *
- * @param query_id The ID of the query for which the results are to be printed.
- * @return A vector of records, each being a single profiling entry.
- **/
- const std::vector<WorkOrderTimeEntry>& getWorkOrderProfilingResults(
- const std::size_t query_id) const;
-
- /**
- * @brief Print the results of profiling individual work orders for a given
- * query.
- *
- * TODO(harshad) - Add the name of the operator to the output.
- * TODO(harshad) - Add the CPU core ID of the operator to the output. This
- * will require modifying the WorkerDirectory to remember worker affinities.
- * Until then, the users can refer to the worker_affinities provided to the
- * cli to infer the CPU core ID where a given worker is pinned.
- *
- * @param query_id The ID of the query for which the results are to be printed.
- * @param out The file stream.
- **/
void printWorkOrderProfilingResults(const std::size_t query_id,
- std::FILE *out) const;
+ std::FILE *out) const override;
protected:
void run() override;
@@ -142,8 +117,6 @@ class ForemanSingleNode final : public ForemanBase {
CatalogDatabaseLite *catalog_database_;
StorageManager *storage_manager_;
- std::unique_ptr<PolicyEnforcerSingleNode> policy_enforcer_;
-
DISALLOW_COPY_AND_ASSIGN(ForemanSingleNode);
};
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/7fb96f3d/query_execution/PolicyEnforcerBase.hpp
----------------------------------------------------------------------
diff --git a/query_execution/PolicyEnforcerBase.hpp b/query_execution/PolicyEnforcerBase.hpp
index 414367e..aa070b8 100644
--- a/query_execution/PolicyEnforcerBase.hpp
+++ b/query_execution/PolicyEnforcerBase.hpp
@@ -146,6 +146,15 @@ class PolicyEnforcerBase {
return workorder_time_recorder_.at(query_id);
}
+ /**
+ * @brief Admit a query to the system.
+ *
+ * @param query_handle The QueryHandle for the new query.
+ *
+ * @return Whether the query was admitted to the system.
+ **/
+ virtual bool admitQuery(QueryHandle *query_handle) = 0;
+
protected:
static constexpr std::size_t kMaxConcurrentQueries = 1;
@@ -185,15 +194,6 @@ class PolicyEnforcerBase {
private:
/**
- * @brief Admit a query to the system.
- *
- * @param query_handle The QueryHandle for the new query.
- *
- * @return Whether the query was admitted to the system.
- **/
- virtual bool admitQuery(QueryHandle *query_handle) = 0;
-
- /**
* @brief Decrement the number of queued workorders for the given worker by 1.
*
* @param proto The completion message proto received after the WorkOrder
[24/50] incubator-quickstep git commit: Used the network util in
DataExchanger.
Posted by zu...@apache.org.
Used the network util in DataExchanger.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/c886f7af
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/c886f7af
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/c886f7af
Branch: refs/heads/quickstep_partition_parser_support
Commit: c886f7af5835a8577f516b2c7a39692d18d5b9b2
Parents: 8f1b0e5
Author: Zuyu Zhang <zu...@apache.org>
Authored: Sun Nov 13 15:55:31 2016 -0800
Committer: Zuyu Zhang <zu...@apache.org>
Committed: Sun Nov 20 20:08:27 2016 -0800
----------------------------------------------------------------------
storage/CMakeLists.txt | 1 +
storage/DataExchangerAsync.cpp | 9 +++++----
storage/DataExchangerAsync.hpp | 2 --
3 files changed, 6 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c886f7af/storage/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/storage/CMakeLists.txt b/storage/CMakeLists.txt
index 559d86d..61a8a99 100644
--- a/storage/CMakeLists.txt
+++ b/storage/CMakeLists.txt
@@ -625,6 +625,7 @@ if (ENABLE_DISTRIBUTED)
quickstep_storage_StorageManager
quickstep_threading_Thread
quickstep_utility_Macros
+ quickstep_utility_NetworkUtil
${GRPCPLUSPLUS_LIBRARIES})
endif()
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c886f7af/storage/DataExchangerAsync.cpp
----------------------------------------------------------------------
diff --git a/storage/DataExchangerAsync.cpp b/storage/DataExchangerAsync.cpp
index 1d2f7db..d349430 100644
--- a/storage/DataExchangerAsync.cpp
+++ b/storage/DataExchangerAsync.cpp
@@ -28,6 +28,7 @@
#include "storage/DataExchange.grpc.pb.h"
#include "storage/DataExchange.pb.h"
#include "storage/StorageManager.hpp"
+#include "utility/NetworkUtil.hpp"
#include "glog/logging.h"
@@ -127,18 +128,18 @@ void CallContext::Proceed() {
} // namespace
-const char *DataExchangerAsync::kLocalNetworkAddress = "0.0.0.0:";
-
DataExchangerAsync::DataExchangerAsync() {
+ const std::string ipv4_address(GetIpv4Address() + ':');
+
grpc::ServerBuilder builder;
- builder.AddListeningPort(kLocalNetworkAddress, grpc::InsecureServerCredentials(), &port_);
+ builder.AddListeningPort(ipv4_address, grpc::InsecureServerCredentials(), &port_);
builder.RegisterService(&service_);
queue_ = builder.AddCompletionQueue();
server_ = builder.BuildAndStart();
DCHECK_GT(port_, 0);
- server_address_ = kLocalNetworkAddress + std::to_string(port_);
+ server_address_ = ipv4_address + std::to_string(port_);
LOG(INFO) << "DataExchangerAsync Service listening on " << server_address_;
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c886f7af/storage/DataExchangerAsync.hpp
----------------------------------------------------------------------
diff --git a/storage/DataExchangerAsync.hpp b/storage/DataExchangerAsync.hpp
index b777108..4504c08 100644
--- a/storage/DataExchangerAsync.hpp
+++ b/storage/DataExchangerAsync.hpp
@@ -79,8 +79,6 @@ class DataExchangerAsync final : public Thread {
void run() override;
private:
- static const char *kLocalNetworkAddress;
-
DataExchange::AsyncService service_;
int port_ = -1;
[16/50] incubator-quickstep git commit: Used the move semantics in
constructing QueryProcessor.
Posted by zu...@apache.org.
Used the move semantics in constructing QueryProcessor.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/24367d7d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/24367d7d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/24367d7d
Branch: refs/heads/quickstep_partition_parser_support
Commit: 24367d7d878e066ce131a6e13032b2ba0ecef8f7
Parents: dec8723
Author: Zuyu Zhang <zu...@apache.org>
Authored: Fri Nov 18 23:28:34 2016 -0800
Committer: Zuyu Zhang <zu...@apache.org>
Committed: Sun Nov 20 19:25:55 2016 -0800
----------------------------------------------------------------------
cli/QuickstepCli.cpp | 2 +-
query_optimizer/QueryProcessor.hpp | 7 ++++---
2 files changed, 5 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/24367d7d/cli/QuickstepCli.cpp
----------------------------------------------------------------------
diff --git a/cli/QuickstepCli.cpp b/cli/QuickstepCli.cpp
index 31215f6..f17e18b 100644
--- a/cli/QuickstepCli.cpp
+++ b/cli/QuickstepCli.cpp
@@ -285,7 +285,7 @@ int main(int argc, char* argv[]) {
// Setup QueryProcessor, including CatalogDatabase.
std::unique_ptr<QueryProcessor> query_processor;
try {
- query_processor = std::make_unique<QueryProcessor>(catalog_path);
+ query_processor = std::make_unique<QueryProcessor>(std::move(catalog_path));
} catch (const std::exception &e) {
LOG(FATAL) << "FATAL ERROR DURING STARTUP: "
<< e.what()
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/24367d7d/query_optimizer/QueryProcessor.hpp
----------------------------------------------------------------------
diff --git a/query_optimizer/QueryProcessor.hpp b/query_optimizer/QueryProcessor.hpp
index 5e5c115..3e37c9d 100644
--- a/query_optimizer/QueryProcessor.hpp
+++ b/query_optimizer/QueryProcessor.hpp
@@ -24,6 +24,7 @@
#include <exception>
#include <memory>
#include <string>
+#include <utility>
#include "catalog/Catalog.hpp"
#include "query_optimizer/Optimizer.hpp"
@@ -130,8 +131,8 @@ class QueryProcessor {
*
* @param catalog_filename The file to read the serialized catalog from.
**/
- explicit QueryProcessor(const std::string &catalog_filename)
- : catalog_filename_(catalog_filename),
+ explicit QueryProcessor(std::string &&catalog_filename)
+ : catalog_filename_(std::move(catalog_filename)),
catalog_altered_(false),
query_id_(0) {
loadCatalog();
@@ -185,7 +186,7 @@ class QueryProcessor {
optimizer::Optimizer optimizer_;
- std::string catalog_filename_;
+ const std::string catalog_filename_;
std::unique_ptr<Catalog> catalog_;
[22/50] incubator-quickstep git commit: Added an util to get block
domain from BlockLocator.
Posted by zu...@apache.org.
Added an util to get block domain from BlockLocator.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/56555117
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/56555117
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/56555117
Branch: refs/heads/quickstep_partition_parser_support
Commit: 565551173e797c65651eef0408c74b55e27a5796
Parents: f7d1543
Author: Zuyu Zhang <zu...@apache.org>
Authored: Sun Nov 20 15:17:46 2016 -0800
Committer: Zuyu Zhang <zu...@apache.org>
Committed: Sun Nov 20 19:58:14 2016 -0800
----------------------------------------------------------------------
query_execution/BlockLocatorUtil.cpp | 93 ++++++++++++++++++++
query_execution/BlockLocatorUtil.hpp | 59 +++++++++++++
query_execution/CMakeLists.txt | 9 ++
query_execution/tests/BlockLocator_unittest.cpp | 42 +--------
query_optimizer/tests/CMakeLists.txt | 4 +-
.../DistributedExecutionGeneratorTestRunner.cpp | 54 +++---------
.../DistributedExecutionGeneratorTestRunner.hpp | 4 +-
storage/CMakeLists.txt | 4 +-
storage/tests/DataExchange_unittest.cpp | 50 ++---------
9 files changed, 187 insertions(+), 132 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/56555117/query_execution/BlockLocatorUtil.cpp
----------------------------------------------------------------------
diff --git a/query_execution/BlockLocatorUtil.cpp b/query_execution/BlockLocatorUtil.cpp
new file mode 100644
index 0000000..d2d1e96
--- /dev/null
+++ b/query_execution/BlockLocatorUtil.cpp
@@ -0,0 +1,93 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ **/
+
+#include "query_execution/BlockLocatorUtil.hpp"
+
+#include <cstdlib>
+#include <string>
+#include <utility>
+
+#include "query_execution/QueryExecutionMessages.pb.h"
+#include "query_execution/QueryExecutionTypedefs.hpp"
+#include "storage/StorageBlockInfo.hpp"
+
+#include "glog/logging.h"
+
+#include "tmb/address.h"
+#include "tmb/id_typedefs.h"
+#include "tmb/message_bus.h"
+#include "tmb/message_style.h"
+#include "tmb/tagged_message.h"
+
+using tmb::TaggedMessage;
+using tmb::MessageBus;
+using tmb::client_id;
+
+namespace quickstep {
+namespace block_locator {
+
+namespace S = ::quickstep::serialization;
+
+block_id_domain getBlockDomain(const std::string &network_address,
+ const client_id cli_id,
+ client_id *locator_client_id,
+ MessageBus *bus) {
+ tmb::Address address;
+ address.All(true);
+ // NOTE(zuyu): The singleton BlockLocator would need only one copy of the message.
+ tmb::MessageStyle style;
+
+ S::BlockDomainRegistrationMessage proto;
+ proto.set_domain_network_address(network_address);
+
+ const int proto_length = proto.ByteSize();
+ char *proto_bytes = static_cast<char*>(std::malloc(proto_length));
+ CHECK(proto.SerializeToArray(proto_bytes, proto_length));
+
+ TaggedMessage message(static_cast<const void*>(proto_bytes),
+ proto_length,
+ kBlockDomainRegistrationMessage);
+ std::free(proto_bytes);
+
+ DLOG(INFO) << "Client (id '" << cli_id
+ << "') broadcasts BlockDomainRegistrationMessage (typed '" << kBlockDomainRegistrationMessage
+ << "') to BlockLocator.";
+
+ CHECK(MessageBus::SendStatus::kOK ==
+ bus->Send(cli_id, address, style, std::move(message)));
+
+ const tmb::AnnotatedMessage annotated_message(bus->Receive(cli_id, 0, true));
+ const TaggedMessage &tagged_message = annotated_message.tagged_message;
+ CHECK_EQ(kBlockDomainRegistrationResponseMessage, tagged_message.message_type());
+
+ *locator_client_id = annotated_message.sender;
+
+ DLOG(INFO) << "Client (id '" << cli_id
+ << "') received BlockDomainRegistrationResponseMessage (typed '"
+ << kBlockDomainRegistrationResponseMessage
+ << "') from BlockLocator (id '" << *locator_client_id << "').";
+
+ S::BlockDomainMessage response_proto;
+ CHECK(response_proto.ParseFromArray(tagged_message.message(), tagged_message.message_bytes()));
+
+ return static_cast<block_id_domain>(response_proto.block_domain());
+}
+
+} // namespace block_locator
+} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/56555117/query_execution/BlockLocatorUtil.hpp
----------------------------------------------------------------------
diff --git a/query_execution/BlockLocatorUtil.hpp b/query_execution/BlockLocatorUtil.hpp
new file mode 100644
index 0000000..74f65e4
--- /dev/null
+++ b/query_execution/BlockLocatorUtil.hpp
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ **/
+
+#ifndef QUICKSTEP_QUERY_EXECUTION_BLOCK_LOCATOR_UTIL_HPP_
+#define QUICKSTEP_QUERY_EXECUTION_BLOCK_LOCATOR_UTIL_HPP_
+
+#include <string>
+
+#include "storage/StorageBlockInfo.hpp"
+
+#include "tmb/id_typedefs.h"
+
+namespace tmb { class MessageBus; }
+
+namespace quickstep {
+namespace block_locator {
+
+/** \addtogroup QueryExecution
+ * @{
+ */
+
+/**
+ * @brief Broadcast to find BlockLocator to get a block domain for
+ * StorageManager with the given network address.
+ *
+ * @param network_address The network address of the StorageManager.
+ * @param cli_id The client ID of the block domain requester.
+ * @param locator_client_id The client ID of BlockLocator to set.
+ * @param bus A pointer to the TMB.
+ *
+ * @return The requested block domain.
+ **/
+block_id_domain getBlockDomain(const std::string &network_address,
+ const tmb::client_id cli_id,
+ tmb::client_id *locator_client_id,
+ tmb::MessageBus *bus);
+
+/** @} */
+
+} // namespace block_locator
+} // namespace quickstep
+
+#endif // QUICKSTEP_QUERY_EXECUTION_BLOCK_LOCATOR_UTIL_HPP_
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/56555117/query_execution/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_execution/CMakeLists.txt b/query_execution/CMakeLists.txt
index eec0029..719d9f3 100644
--- a/query_execution/CMakeLists.txt
+++ b/query_execution/CMakeLists.txt
@@ -31,6 +31,7 @@ endif()
add_library(quickstep_queryexecution_AdmitRequestMessage ../empty_src.cpp AdmitRequestMessage.hpp)
if (ENABLE_DISTRIBUTED)
add_library(quickstep_queryexecution_BlockLocator BlockLocator.cpp BlockLocator.hpp)
+ add_library(quickstep_queryexecution_BlockLocatorUtil BlockLocatorUtil.cpp BlockLocatorUtil.hpp)
endif(ENABLE_DISTRIBUTED)
add_library(quickstep_queryexecution_ForemanBase ../empty_src.cpp ForemanBase.hpp)
if (ENABLE_DISTRIBUTED)
@@ -83,6 +84,12 @@ if (ENABLE_DISTRIBUTED)
quickstep_threading_ThreadUtil
quickstep_utility_Macros
tmb)
+ target_link_libraries(quickstep_queryexecution_BlockLocatorUtil
+ glog
+ quickstep_queryexecution_QueryExecutionMessages_proto
+ quickstep_queryexecution_QueryExecutionTypedefs
+ quickstep_storage_StorageBlockInfo
+ tmb)
endif(ENABLE_DISTRIBUTED)
target_link_libraries(quickstep_queryexecution_ForemanBase
glog
@@ -345,6 +352,7 @@ target_link_libraries(quickstep_queryexecution
if (ENABLE_DISTRIBUTED)
target_link_libraries(quickstep_queryexecution
quickstep_queryexecution_BlockLocator
+ quickstep_queryexecution_BlockLocatorUtil
quickstep_queryexecution_ForemanDistributed
quickstep_queryexecution_PolicyEnforcerDistributed
quickstep_queryexecution_QueryManagerDistributed
@@ -363,6 +371,7 @@ if (ENABLE_DISTRIBUTED)
quickstep_catalog_CatalogAttribute
quickstep_catalog_CatalogRelation
quickstep_queryexecution_BlockLocator
+ quickstep_queryexecution_BlockLocatorUtil
quickstep_queryexecution_QueryExecutionMessages_proto
quickstep_queryexecution_QueryExecutionTypedefs
quickstep_queryexecution_QueryExecutionUtil
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/56555117/query_execution/tests/BlockLocator_unittest.cpp
----------------------------------------------------------------------
diff --git a/query_execution/tests/BlockLocator_unittest.cpp b/query_execution/tests/BlockLocator_unittest.cpp
index 465f2a3..32437c3 100644
--- a/query_execution/tests/BlockLocator_unittest.cpp
+++ b/query_execution/tests/BlockLocator_unittest.cpp
@@ -26,6 +26,7 @@
#include "catalog/CatalogAttribute.hpp"
#include "catalog/CatalogRelation.hpp"
#include "query_execution/BlockLocator.hpp"
+#include "query_execution/BlockLocatorUtil.hpp"
#include "query_execution/QueryExecutionMessages.pb.h"
#include "query_execution/QueryExecutionTypedefs.hpp"
#include "query_execution/QueryExecutionUtil.hpp"
@@ -71,7 +72,6 @@ class BlockLocatorTest : public ::testing::Test {
bus_.Initialize();
locator_.reset(new BlockLocator(&bus_));
- locator_client_id_ = locator_->getBusClientID();
locator_->start();
worker_client_id_ = bus_.Connect();
@@ -84,7 +84,9 @@ class BlockLocatorTest : public ::testing::Test {
bus_.RegisterClientAsSender(worker_client_id_, kPoisonMessage);
- block_domain_ = getBlockDomain(kDomainNetworkAddress);
+ block_domain_ =
+ block_locator::getBlockDomain(kDomainNetworkAddress, worker_client_id_, &locator_client_id_, &bus_);
+ DCHECK_EQ(locator_->getBusClientID(), locator_client_id_);
storage_manager_.reset(
new StorageManager(kStoragePath, block_domain_, locator_client_id_, &bus_));
@@ -168,42 +170,6 @@ class BlockLocatorTest : public ::testing::Test {
unique_ptr<StorageManager> storage_manager_;
private:
- block_id_domain getBlockDomain(const string &network_address) {
- serialization::BlockDomainRegistrationMessage proto;
- proto.set_domain_network_address(network_address);
-
- const int proto_length = proto.ByteSize();
- char *proto_bytes = static_cast<char*>(malloc(proto_length));
- CHECK(proto.SerializeToArray(proto_bytes, proto_length));
-
- TaggedMessage message(static_cast<const void*>(proto_bytes),
- proto_length,
- kBlockDomainRegistrationMessage);
- free(proto_bytes);
-
- LOG(INFO) << "Worker (id '" << worker_client_id_
- << "') sent BlockDomainRegistrationMessage (typed '" << kBlockDomainRegistrationMessage
- << "') to BlockLocator (id '" << locator_client_id_ << "')";
-
- CHECK(MessageBus::SendStatus::kOK ==
- QueryExecutionUtil::SendTMBMessage(&bus_,
- worker_client_id_,
- locator_client_id_,
- move(message)));
-
- const AnnotatedMessage annotated_message(bus_.Receive(worker_client_id_, 0, true));
- const TaggedMessage &tagged_message = annotated_message.tagged_message;
- EXPECT_EQ(locator_client_id_, annotated_message.sender);
- EXPECT_EQ(kBlockDomainRegistrationResponseMessage, tagged_message.message_type());
- LOG(INFO) << "Worker (id '" << worker_client_id_
- << "') received BlockDomainRegistrationResponseMessage from BlockLocator";
-
- serialization::BlockDomainMessage response_proto;
- CHECK(response_proto.ParseFromArray(tagged_message.message(), tagged_message.message_bytes()));
-
- return static_cast<block_id_domain>(response_proto.block_domain());
- }
-
MessageBusImpl bus_;
unique_ptr<BlockLocator> locator_;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/56555117/query_optimizer/tests/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_optimizer/tests/CMakeLists.txt b/query_optimizer/tests/CMakeLists.txt
index 9c764e4..b987322 100644
--- a/query_optimizer/tests/CMakeLists.txt
+++ b/query_optimizer/tests/CMakeLists.txt
@@ -115,8 +115,8 @@ if (ENABLE_DISTRIBUTED)
quickstep_parser_ParseStatement
quickstep_parser_SqlParserWrapper
quickstep_queryexecution_BlockLocator
+ quickstep_queryexecution_BlockLocatorUtil
quickstep_queryexecution_ForemanDistributed
- quickstep_queryexecution_QueryExecutionMessages_proto
quickstep_queryexecution_QueryExecutionTypedefs
quickstep_queryexecution_QueryExecutionUtil
quickstep_queryexecution_Shiftboss
@@ -127,7 +127,7 @@ if (ENABLE_DISTRIBUTED)
quickstep_queryoptimizer_QueryHandle
quickstep_queryoptimizer_tests_TestDatabaseLoader
quickstep_storage_DataExchangerAsync
- quickstep_storage_StorageBlockInfo
+ quickstep_storage_StorageManager
quickstep_utility_Macros
quickstep_utility_MemStream
quickstep_utility_SqlError
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/56555117/query_optimizer/tests/DistributedExecutionGeneratorTestRunner.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/tests/DistributedExecutionGeneratorTestRunner.cpp b/query_optimizer/tests/DistributedExecutionGeneratorTestRunner.cpp
index 0403e77..2351dcd 100644
--- a/query_optimizer/tests/DistributedExecutionGeneratorTestRunner.cpp
+++ b/query_optimizer/tests/DistributedExecutionGeneratorTestRunner.cpp
@@ -32,14 +32,14 @@
#include "cli/PrintToScreen.hpp"
#include "parser/ParseStatement.hpp"
#include "query_execution/BlockLocator.hpp"
+#include "query_execution/BlockLocatorUtil.hpp"
#include "query_execution/ForemanDistributed.hpp"
-#include "query_execution/QueryExecutionMessages.pb.h"
#include "query_execution/QueryExecutionTypedefs.hpp"
#include "query_execution/QueryExecutionUtil.hpp"
#include "query_optimizer/OptimizerContext.hpp"
#include "query_optimizer/QueryHandle.hpp"
#include "storage/DataExchangerAsync.hpp"
-#include "storage/StorageBlockInfo.hpp"
+#include "storage/StorageManager.hpp"
#include "utility/MemStream.hpp"
#include "utility/SqlError.hpp"
@@ -81,14 +81,15 @@ DistributedExecutionGeneratorTestRunner::DistributedExecutionGeneratorTestRunner
bus_.RegisterClientAsReceiver(cli_id_, kBlockDomainRegistrationResponseMessage);
block_locator_ = make_unique<BlockLocator>(&bus_);
- locator_client_id_ = block_locator_->getBusClientID();
block_locator_->start();
test_database_loader_ = make_unique<TestDatabaseLoader>(
storage_path,
- getBlockDomain(test_database_loader_data_exchanger_.network_address()),
+ block_locator::getBlockDomain(
+ test_database_loader_data_exchanger_.network_address(), cli_id_, &locator_client_id_, &bus_),
locator_client_id_,
&bus_);
+ DCHECK_EQ(block_locator_->getBusClientID(), locator_client_id_);
test_database_loader_data_exchanger_.set_storage_manager(test_database_loader_->storage_manager());
test_database_loader_data_exchanger_.start();
@@ -111,7 +112,11 @@ DistributedExecutionGeneratorTestRunner::DistributedExecutionGeneratorTestRunner
make_unique<WorkerDirectory>(worker_client_ids.size(), worker_client_ids, numa_nodes));
auto storage_manager = make_unique<StorageManager>(
- storage_path, getBlockDomain(data_exchangers_[i].network_address()), locator_client_id_, &bus_);
+ storage_path,
+ block_locator::getBlockDomain(
+ data_exchangers_[i].network_address(), cli_id_, &locator_client_id_, &bus_),
+ locator_client_id_, &bus_);
+ DCHECK_EQ(block_locator_->getBusClientID(), locator_client_id_);
data_exchangers_[i].set_storage_manager(storage_manager.get());
shiftbosses_.push_back(
@@ -193,44 +198,5 @@ void DistributedExecutionGeneratorTestRunner::runTestCase(
}
}
-block_id_domain DistributedExecutionGeneratorTestRunner::getBlockDomain(
- const string &network_address) {
- serialization::BlockDomainRegistrationMessage proto;
- proto.set_domain_network_address(network_address);
-
- const int proto_length = proto.ByteSize();
- char *proto_bytes = static_cast<char*>(malloc(proto_length));
- CHECK(proto.SerializeToArray(proto_bytes, proto_length));
-
- TaggedMessage message(static_cast<const void*>(proto_bytes),
- proto_length,
- kBlockDomainRegistrationMessage);
- free(proto_bytes);
-
- DLOG(INFO) << "Client (id '" << cli_id_
- << "') sent BlockDomainRegistrationMessage (typed '" << kBlockDomainRegistrationMessage
- << "') to BlockLocator (id '" << locator_client_id_ << "')";
-
- CHECK(MessageBus::SendStatus::kOK ==
- QueryExecutionUtil::SendTMBMessage(&bus_,
- cli_id_,
- locator_client_id_,
- move(message)));
-
- const tmb::AnnotatedMessage annotated_message(bus_.Receive(cli_id_, 0, true));
- const TaggedMessage &tagged_message = annotated_message.tagged_message;
- CHECK_EQ(locator_client_id_, annotated_message.sender);
- CHECK_EQ(kBlockDomainRegistrationResponseMessage, tagged_message.message_type());
- DLOG(INFO) << "Client (id '" << cli_id_
- << "') received BlockDomainRegistrationResponseMessage (typed '"
- << kBlockDomainRegistrationResponseMessage
- << "') from BlockLocator";
-
- serialization::BlockDomainMessage response_proto;
- CHECK(response_proto.ParseFromArray(tagged_message.message(), tagged_message.message_bytes()));
-
- return static_cast<block_id_domain>(response_proto.block_domain());
-}
-
} // namespace optimizer
} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/56555117/query_optimizer/tests/DistributedExecutionGeneratorTestRunner.hpp
----------------------------------------------------------------------
diff --git a/query_optimizer/tests/DistributedExecutionGeneratorTestRunner.hpp b/query_optimizer/tests/DistributedExecutionGeneratorTestRunner.hpp
index d2b13e4..63e320d 100644
--- a/query_optimizer/tests/DistributedExecutionGeneratorTestRunner.hpp
+++ b/query_optimizer/tests/DistributedExecutionGeneratorTestRunner.hpp
@@ -38,7 +38,7 @@
#include "query_optimizer/Optimizer.hpp"
#include "query_optimizer/tests/TestDatabaseLoader.hpp"
#include "storage/DataExchangerAsync.hpp"
-#include "storage/StorageBlockInfo.hpp"
+#include "storage/StorageManager.hpp"
#include "utility/Macros.hpp"
#include "utility/textbased_test/TextBasedTestRunner.hpp"
@@ -115,8 +115,6 @@ class DistributedExecutionGeneratorTestRunner : public TextBasedTestRunner {
std::string *output) override;
private:
- block_id_domain getBlockDomain(const std::string &network_address);
-
std::size_t query_id_;
SqlParserWrapper sql_parser_;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/56555117/storage/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/storage/CMakeLists.txt b/storage/CMakeLists.txt
index be60662..559d86d 100644
--- a/storage/CMakeLists.txt
+++ b/storage/CMakeLists.txt
@@ -862,7 +862,7 @@ target_link_libraries(quickstep_storage_PartitionedHashTablePool
quickstep_storage_FastHashTableFactory
quickstep_storage_HashTableBase
quickstep_utility_Macros
- quickstep_utility_StringUtil)
+ quickstep_utility_StringUtil)
target_link_libraries(quickstep_storage_PreloaderThread
glog
quickstep_catalog_CatalogDatabase
@@ -1502,7 +1502,7 @@ if (ENABLE_DISTRIBUTED)
quickstep_catalog_CatalogRelation
quickstep_catalog_CatalogTypedefs
quickstep_queryexecution_BlockLocator
- quickstep_queryexecution_QueryExecutionMessages_proto
+ quickstep_queryexecution_BlockLocatorUtil
quickstep_queryexecution_QueryExecutionTypedefs
quickstep_queryexecution_QueryExecutionUtil
quickstep_storage_CountedReference
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/56555117/storage/tests/DataExchange_unittest.cpp
----------------------------------------------------------------------
diff --git a/storage/tests/DataExchange_unittest.cpp b/storage/tests/DataExchange_unittest.cpp
index 9c75150..ac39728 100644
--- a/storage/tests/DataExchange_unittest.cpp
+++ b/storage/tests/DataExchange_unittest.cpp
@@ -27,7 +27,7 @@
#include "catalog/CatalogRelation.hpp"
#include "catalog/CatalogTypedefs.hpp"
#include "query_execution/BlockLocator.hpp"
-#include "query_execution/QueryExecutionMessages.pb.h"
+#include "query_execution/BlockLocatorUtil.hpp"
#include "query_execution/QueryExecutionTypedefs.hpp"
#include "query_execution/QueryExecutionUtil.hpp"
#include "storage/CountedReference.hpp"
@@ -77,7 +77,6 @@ class DataExchangeTest : public ::testing::Test {
bus_.Initialize();
locator_.reset(new BlockLocator(&bus_));
- locator_client_id_ = locator_->getBusClientID();
locator_->start();
worker_client_id_ = bus_.Connect();
@@ -88,18 +87,22 @@ class DataExchangeTest : public ::testing::Test {
storage_manager_expected_.reset(new StorageManager(
kStoragePath,
- getBlockDomain(data_exchanger_expected_.network_address()),
+ block_locator::getBlockDomain(
+ data_exchanger_expected_.network_address(), worker_client_id_, &locator_client_id_, &bus_),
locator_client_id_,
&bus_));
+ DCHECK_EQ(locator_->getBusClientID(), locator_client_id_);
data_exchanger_expected_.set_storage_manager(storage_manager_expected_.get());
data_exchanger_expected_.start();
storage_manager_checked_.reset(new StorageManager(
kStoragePath,
- getBlockDomain(kCheckedDomainNetworkAddress),
+ block_locator::getBlockDomain(
+ kCheckedDomainNetworkAddress, worker_client_id_, &locator_client_id_, &bus_),
locator_client_id_,
&bus_));
+ DCHECK_EQ(locator_->getBusClientID(), locator_client_id_);
}
virtual void TearDown() {
@@ -123,45 +126,6 @@ class DataExchangeTest : public ::testing::Test {
unique_ptr<StorageManager> storage_manager_expected_, storage_manager_checked_;
private:
- block_id_domain getBlockDomain(const string &network_address) {
- serialization::BlockDomainRegistrationMessage proto;
- proto.set_domain_network_address(network_address);
-
- const int proto_length = proto.ByteSize();
- char *proto_bytes = static_cast<char*>(malloc(proto_length));
- CHECK(proto.SerializeToArray(proto_bytes, proto_length));
-
- TaggedMessage message(static_cast<const void*>(proto_bytes),
- proto_length,
- kBlockDomainRegistrationMessage);
- free(proto_bytes);
-
- LOG(INFO) << "Worker (id '" << worker_client_id_
- << "') sent BlockDomainRegistrationMessage (typed '" << kBlockDomainRegistrationMessage
- << "') to BlockLocator (id '" << locator_client_id_ << "')";
-
- CHECK(MessageBus::SendStatus::kOK ==
- QueryExecutionUtil::SendTMBMessage(&bus_,
- worker_client_id_,
- locator_client_id_,
- move(message)));
-
- const tmb::AnnotatedMessage annotated_message(bus_.Receive(worker_client_id_, 0, true));
- const TaggedMessage &tagged_message = annotated_message.tagged_message;
- EXPECT_EQ(locator_client_id_, annotated_message.sender);
- EXPECT_EQ(kBlockDomainRegistrationResponseMessage, tagged_message.message_type());
-
- LOG(INFO) << "Worker (id '" << worker_client_id_
- << "') received BlockDomainRegistrationResponseMessage (typed '"
- << kBlockDomainRegistrationResponseMessage
- << "') from BlockLocator";
-
- serialization::BlockDomainMessage response_proto;
- CHECK(response_proto.ParseFromArray(tagged_message.message(), tagged_message.message_bytes()));
-
- return static_cast<block_id_domain>(response_proto.block_domain());
- }
-
MessageBusImpl bus_;
unique_ptr<BlockLocator> locator_;
[06/50] incubator-quickstep git commit: Refine estimates for
estimateCardinality for TableReference
Posted by zu...@apache.org.
Refine estimates for estimateCardinality for TableReference
- Use exact CatalogRelation statistics in SimpleCostModel's
estimateCardinality method, whenever stats on that relation are available.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/c45d68e1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/c45d68e1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/c45d68e1
Branch: refs/heads/quickstep_partition_parser_support
Commit: c45d68e19c53ffb5a6e017aa62c7577860204d0b
Parents: 1340fcb
Author: Harshad Deshmukh <hb...@apache.org>
Authored: Sat Nov 5 10:04:07 2016 -0500
Committer: Harshad Deshmukh <hb...@apache.org>
Committed: Sun Nov 6 13:09:03 2016 -0600
----------------------------------------------------------------------
query_optimizer/cost_model/CMakeLists.txt | 1 +
query_optimizer/cost_model/SimpleCostModel.cpp | 9 ++++++++-
2 files changed, 9 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c45d68e1/query_optimizer/cost_model/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_optimizer/cost_model/CMakeLists.txt b/query_optimizer/cost_model/CMakeLists.txt
index d616696..032e34c 100644
--- a/query_optimizer/cost_model/CMakeLists.txt
+++ b/query_optimizer/cost_model/CMakeLists.txt
@@ -30,6 +30,7 @@ target_link_libraries(quickstep_queryoptimizer_costmodel_CostModel
target_link_libraries(quickstep_queryoptimizer_costmodel_SimpleCostModel
glog
quickstep_catalog_CatalogRelation
+ quickstep_catalog_CatalogRelationStatistics
quickstep_queryoptimizer_costmodel_CostModel
quickstep_queryoptimizer_physical_Aggregate
quickstep_queryoptimizer_physical_HashJoin
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c45d68e1/query_optimizer/cost_model/SimpleCostModel.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/cost_model/SimpleCostModel.cpp b/query_optimizer/cost_model/SimpleCostModel.cpp
index 74b62d6..a803c67 100644
--- a/query_optimizer/cost_model/SimpleCostModel.cpp
+++ b/query_optimizer/cost_model/SimpleCostModel.cpp
@@ -23,6 +23,7 @@
#include <memory>
#include "catalog/CatalogRelation.hpp"
+#include "catalog/CatalogRelationStatistics.hpp"
#include "query_optimizer/physical/Aggregate.hpp"
#include "query_optimizer/physical/NestedLoopsJoin.hpp"
#include "query_optimizer/physical/HashJoin.hpp"
@@ -92,7 +93,13 @@ std::size_t SimpleCostModel::estimateCardinalityForTopLevelPlan(
std::size_t SimpleCostModel::estimateCardinalityForTableReference(
const P::TableReferencePtr &physical_plan) {
- return physical_plan->relation()->estimateTupleCardinality();
+ const std::size_t num_tuples_in_relation =
+ physical_plan->relation()->getStatistics().getNumTuples();
+ if (num_tuples_in_relation == 0) {
+ return physical_plan->relation()->estimateTupleCardinality();
+ } else {
+ return num_tuples_in_relation;
+ }
}
std::size_t SimpleCostModel::estimateCardinalityForSelection(
[43/50] incubator-quickstep git commit: Added DistributedCli
executable.
Posted by zu...@apache.org.
Added DistributedCli executable.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/b949c504
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/b949c504
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/b949c504
Branch: refs/heads/quickstep_partition_parser_support
Commit: b949c50450625f16c40a581db3ed811be2b9ccf2
Parents: bc4086b
Author: Zuyu Zhang <zu...@apache.org>
Authored: Sun Nov 27 22:32:24 2016 -0800
Committer: Zuyu Zhang <zu...@apache.org>
Committed: Mon Dec 5 00:15:16 2016 -0800
----------------------------------------------------------------------
CMakeLists.txt | 17 ++
cli/CMakeLists.txt | 3 +
cli/distributed/CMakeLists.txt | 93 +++++++++
cli/distributed/Cli.cpp | 239 ++++++++++++++++++++++
cli/distributed/Cli.hpp | 71 +++++++
cli/distributed/CliDistributedModule.hpp | 23 +++
cli/distributed/Conductor.cpp | 180 ++++++++++++++++
cli/distributed/Conductor.hpp | 80 ++++++++
cli/distributed/Executor.cpp | 87 ++++++++
cli/distributed/Executor.hpp | 83 ++++++++
cli/distributed/QuickstepDistributedCli.cpp | 81 ++++++++
cli/distributed/Role.cpp | 51 +++++
cli/distributed/Role.hpp | 69 +++++++
query_execution/QueryExecutionMessages.proto | 8 +
query_execution/QueryExecutionTypedefs.hpp | 6 +-
validate_cmakelists.py | 4 +-
16 files changed, 1093 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/b949c504/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 126b47b..391cb26 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -782,3 +782,20 @@ endif()
# Link against other required system and third-party libraries.
target_link_libraries(quickstep_cli_shell ${LIBS})
+
+if (ENABLE_DISTRIBUTED)
+ # Build the quickstep_distributed_cli_shell executable.
+ add_executable (quickstep_distributed_cli_shell cli/distributed/QuickstepDistributedCli.cpp)
+ # Link against direct deps (will transitively pull in everything needed).
+ # NOTE(zuyu): Link quickstep_cli_LineReader on behalf of quickstep_cli_distributed_Cli,
+ # as a workaround for bypassing conditionally built target checks in validate_cmakelists.py.
+ target_link_libraries(quickstep_distributed_cli_shell
+ glog
+ quickstep_cli_LineReader
+ quickstep_cli_distributed_Cli
+ quickstep_cli_distributed_Conductor
+ quickstep_cli_distributed_Executor
+ quickstep_utility_StringUtil
+ ${GFLAGS_LIB_NAME}
+ ${GRPCPLUSPLUS_LIBRARIES})
+endif(ENABLE_DISTRIBUTED)
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/b949c504/cli/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/cli/CMakeLists.txt b/cli/CMakeLists.txt
index 9b62af9..be13c82 100644
--- a/cli/CMakeLists.txt
+++ b/cli/CMakeLists.txt
@@ -16,6 +16,9 @@
# under the License.
include_directories(${CMAKE_CURRENT_BINARY_DIR})
+if (ENABLE_DISTRIBUTED)
+ add_subdirectory(distributed)
+endif(ENABLE_DISTRIBUTED)
add_subdirectory(tests)
if (WIN32)
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/b949c504/cli/distributed/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/cli/distributed/CMakeLists.txt b/cli/distributed/CMakeLists.txt
new file mode 100644
index 0000000..e16d8af
--- /dev/null
+++ b/cli/distributed/CMakeLists.txt
@@ -0,0 +1,93 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+if (BUILD_SHARED_LIBS)
+ set(GFLAGS_LIB_NAME gflags_nothreads-shared)
+else()
+ set(GFLAGS_LIB_NAME gflags_nothreads-static)
+endif()
+
+# Declare micro-libs and link dependencies:
+add_library(quickstep_cli_distributed_Cli Cli.cpp Cli.hpp)
+add_library(quickstep_cli_distributed_Conductor Conductor.cpp Conductor.hpp)
+add_library(quickstep_cli_distributed_Executor Executor.cpp Executor.hpp)
+add_library(quickstep_cli_distributed_Role Role.cpp Role.hpp)
+
+# Link dependencies:
+target_link_libraries(quickstep_cli_distributed_Cli
+ glog
+ quickstep_catalog_CatalogRelation
+ quickstep_cli_Flags
+ quickstep_cli_PrintToScreen
+ quickstep_cli_distributed_Role
+ quickstep_parser_ParseStatement
+ quickstep_parser_SqlParserWrapper
+ quickstep_queryexecution_BlockLocatorUtil
+ quickstep_queryexecution_QueryExecutionMessages_proto
+ quickstep_queryexecution_QueryExecutionTypedefs
+ quickstep_queryexecution_QueryExecutionUtil
+ quickstep_storage_DataExchangerAsync
+ quickstep_storage_StorageBlockInfo
+ quickstep_storage_StorageManager
+ quickstep_utility_Macros
+ quickstep_utility_StringUtil
+ tmb)
+target_link_libraries(quickstep_cli_distributed_Conductor
+ glog
+ quickstep_cli_DefaultsConfigurator
+ quickstep_cli_Flags
+ quickstep_cli_distributed_Role
+ quickstep_parser_ParseStatement
+ quickstep_parser_SqlParserWrapper
+ quickstep_queryexecution_BlockLocator
+ quickstep_queryexecution_ForemanDistributed
+ quickstep_queryexecution_QueryExecutionMessages_proto
+ quickstep_queryexecution_QueryExecutionTypedefs
+ quickstep_queryexecution_QueryExecutionUtil
+ quickstep_queryoptimizer_QueryHandle
+ quickstep_queryoptimizer_QueryProcessor
+ quickstep_storage_StorageConstants
+ quickstep_utility_Macros
+ quickstep_utility_SqlError
+ tmb)
+target_link_libraries(quickstep_cli_distributed_Executor
+ glog
+ quickstep_catalog_CatalogTypedefs
+ quickstep_cli_Flags
+ quickstep_cli_distributed_Role
+ quickstep_queryexecution_BlockLocatorUtil
+ quickstep_queryexecution_QueryExecutionTypedefs
+ quickstep_queryexecution_Shiftboss
+ quickstep_queryexecution_Worker
+ quickstep_queryexecution_WorkerDirectory
+ quickstep_storage_DataExchangerAsync
+ quickstep_storage_StorageManager
+ quickstep_utility_Macros
+ tmb)
+target_link_libraries(quickstep_cli_distributed_Role
+ quickstep_utility_Macros
+ tmb
+ ${GFLAGS_LIB_NAME})
+
+# Module all-in-one library:
+add_library(quickstep_cli_distributed ../../empty_src.cpp CliDistributedModule.hpp)
+
+target_link_libraries(quickstep_cli_distributed
+ quickstep_cli_distributed_Cli
+ quickstep_cli_distributed_Conductor
+ quickstep_cli_distributed_Executor
+ quickstep_cli_distributed_Role)
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/b949c504/cli/distributed/Cli.cpp
----------------------------------------------------------------------
diff --git a/cli/distributed/Cli.cpp b/cli/distributed/Cli.cpp
new file mode 100644
index 0000000..01f824d
--- /dev/null
+++ b/cli/distributed/Cli.cpp
@@ -0,0 +1,239 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ **/
+
+#include "cli/distributed/Cli.hpp"
+
+#include <chrono>
+#include <cstddef>
+#include <cstdio>
+#include <cstdlib>
+#include <memory>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "catalog/CatalogRelation.hpp"
+#include "cli/CliConfig.h" // For QUICKSTEP_USE_LINENOISE.
+#include "cli/Flags.hpp"
+
+#ifdef QUICKSTEP_USE_LINENOISE
+#include "cli/LineReaderLineNoise.hpp"
+typedef quickstep::LineReaderLineNoise LineReaderImpl;
+#else
+#include "cli/LineReaderDumb.hpp"
+typedef quickstep::LineReaderDumb LineReaderImpl;
+#endif
+
+#include "cli/PrintToScreen.hpp"
+#include "parser/ParseStatement.hpp"
+#include "parser/SqlParserWrapper.hpp"
+#include "query_execution/BlockLocatorUtil.hpp"
+#include "query_execution/QueryExecutionMessages.pb.h"
+#include "query_execution/QueryExecutionTypedefs.hpp"
+#include "query_execution/QueryExecutionUtil.hpp"
+#include "storage/DataExchangerAsync.hpp"
+#include "utility/StringUtil.hpp"
+#include "storage/StorageBlockInfo.hpp"
+
+#include "tmb/address.h"
+#include "tmb/id_typedefs.h"
+#include "tmb/message_bus.h"
+#include "tmb/message_style.h"
+#include "tmb/native_net_client_message_bus.h"
+#include "tmb/tagged_message.h"
+
+#include "glog/logging.h"
+
+using std::fprintf;
+using std::free;
+using std::make_unique;
+using std::malloc;
+using std::move;
+using std::printf;
+using std::size_t;
+using std::string;
+using std::vector;
+
+using tmb::AnnotatedMessage;
+using tmb::TaggedMessage;
+using tmb::client_id;
+
+namespace quickstep {
+
+namespace S = ::quickstep::serialization;
+
+void Cli::init() {
+ cli_id_ = bus_.Connect();
+ DLOG(INFO) << "DistributedCli TMB Client ID: " << cli_id_;
+
+ bus_.RegisterClientAsSender(cli_id_, kDistributedCliRegistrationMessage);
+ bus_.RegisterClientAsReceiver(cli_id_, kDistributedCliRegistrationResponseMessage);
+
+ DLOG(INFO) << "DistributedCli sent DistributedCliRegistrationMessage (typed '"
+ << kDistributedCliRegistrationMessage
+ << "') to all";
+
+ tmb::Address all_addresses;
+ all_addresses.All(true);
+ // NOTE(zuyu): The singleton Conductor would need one copy of the message.
+ tmb::MessageStyle style;
+
+ TaggedMessage cli_reg_message(kDistributedCliRegistrationMessage);
+ DCHECK(tmb::MessageBus::SendStatus::kOK ==
+ bus_.Send(cli_id_, all_addresses, style, move(cli_reg_message)));
+
+ // Wait for Conductor to response.
+ const AnnotatedMessage cli_reg_response_message(bus_.Receive(cli_id_, 0, true));
+ DCHECK_EQ(kDistributedCliRegistrationResponseMessage,
+ cli_reg_response_message.tagged_message.message_type());
+ conductor_client_id_ = cli_reg_response_message.sender;
+
+ DLOG(INFO) << "DistributedCli received typed '" << kDistributedCliRegistrationResponseMessage
+ << "' message from Conductor (id'" << conductor_client_id_ << "').";
+
+ // Setup StorageManager.
+ bus_.RegisterClientAsSender(cli_id_, kBlockDomainRegistrationMessage);
+ bus_.RegisterClientAsReceiver(cli_id_, kBlockDomainRegistrationResponseMessage);
+
+ client_id locator_client_id;
+ storage_manager_ = make_unique<StorageManager>(
+ FLAGS_storage_path,
+ block_locator::getBlockDomain(data_exchanger_.network_address(), cli_id_, &locator_client_id, &bus_),
+ locator_client_id, &bus_);
+
+ data_exchanger_.set_storage_manager(storage_manager_.get());
+ data_exchanger_.start();
+
+ // Prepare for submitting a query.
+ bus_.RegisterClientAsSender(cli_id_, kSqlQueryMessage);
+ bus_.RegisterClientAsReceiver(cli_id_, kQueryExecutionSuccessMessage);
+ bus_.RegisterClientAsReceiver(cli_id_, kQueryExecutionErrorMessage);
+}
+
+void Cli::run() {
+ LineReaderImpl line_reader("distributed_quickstep> ",
+ " ...> ");
+ auto parser_wrapper = make_unique<SqlParserWrapper>();
+ std::chrono::time_point<std::chrono::steady_clock> start, end;
+
+ for (;;) {
+ string *command_string = new string();
+ *command_string = line_reader.getNextCommand();
+ if (command_string->size() == 0) {
+ delete command_string;
+ break;
+ }
+
+ parser_wrapper->feedNextBuffer(command_string);
+
+ bool quitting = false;
+ // A parse error should reset the parser. This is because the thrown quickstep
+ // SqlError does not do the proper reset work of the YYABORT macro.
+ bool reset_parser = false;
+ for (;;) {
+ ParseResult result = parser_wrapper->getNextStatement();
+ const ParseStatement &statement = *result.parsed_statement;
+ if (result.condition == ParseResult::kSuccess) {
+ if (statement.getStatementType() == ParseStatement::kQuit) {
+ quitting = true;
+ break;
+ }
+
+ CHECK_NE(statement.getStatementType(), ParseStatement::kCommand)
+ << "TODO(quickstep-team)";
+
+ DLOG(INFO) << "DistributedCli sent SqlQueryMessage (typed '" << kSqlQueryMessage
+ << "') to Conductor";
+ S::SqlQueryMessage proto;
+ proto.set_sql_query(*command_string);
+
+ const size_t proto_length = proto.ByteSize();
+ char *proto_bytes = static_cast<char*>(malloc(proto_length));
+ CHECK(proto.SerializeToArray(proto_bytes, proto_length));
+
+ TaggedMessage sql_query_message(static_cast<const void*>(proto_bytes),
+ proto_length,
+ kSqlQueryMessage);
+ free(proto_bytes);
+
+ QueryExecutionUtil::SendTMBMessage(&bus_,
+ cli_id_,
+ conductor_client_id_,
+ move(sql_query_message));
+
+ start = std::chrono::steady_clock::now();
+
+ const AnnotatedMessage annotated_message(bus_.Receive(cli_id_, 0, true));
+ const TaggedMessage &tagged_message = annotated_message.tagged_message;
+ DLOG(INFO) << "DistributedCli received typed '" << tagged_message.message_type()
+ << "' message from client " << annotated_message.sender;
+ switch (tagged_message.message_type()) {
+ case kQueryExecutionSuccessMessage: {
+ end = std::chrono::steady_clock::now();
+
+ S::QueryExecutionSuccessMessage proto;
+ CHECK(proto.ParseFromArray(tagged_message.message(), tagged_message.message_bytes()));
+
+ if (proto.has_result_relation()) {
+ CatalogRelation result_relation(proto.result_relation());
+
+ PrintToScreen::PrintRelation(result_relation, storage_manager_.get(), stdout);
+
+ const vector<block_id> blocks(result_relation.getBlocksSnapshot());
+ for (const block_id block : blocks) {
+ storage_manager_->deleteBlockOrBlobFile(block);
+ }
+ }
+
+ std::chrono::duration<double, std::milli> time_in_ms = end - start;
+ printf("Time: %s ms\n", DoubleToStringWithSignificantDigits(time_in_ms.count(), 3).c_str());
+ break;
+ }
+ case kQueryExecutionErrorMessage: {
+ S::QueryExecutionErrorMessage proto;
+ CHECK(proto.ParseFromArray(tagged_message.message(), tagged_message.message_bytes()));
+
+ fprintf(stderr, "%s", proto.error_message().c_str());
+ break;
+ }
+ default: {
+ LOG(ERROR) << "Unknown TMB message type";
+ }
+ }
+ } else {
+ if (result.condition == ParseResult::kError) {
+ fprintf(stderr, "%s", result.error_message.c_str());
+ }
+ reset_parser = true;
+ break;
+ }
+ }
+
+ if (quitting) {
+ break;
+ } else if (reset_parser) {
+ parser_wrapper = make_unique<SqlParserWrapper>();
+ reset_parser = false;
+ }
+ }
+
+ bus_.Disconnect(cli_id_);
+}
+
+} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/b949c504/cli/distributed/Cli.hpp
----------------------------------------------------------------------
diff --git a/cli/distributed/Cli.hpp b/cli/distributed/Cli.hpp
new file mode 100644
index 0000000..32c178f
--- /dev/null
+++ b/cli/distributed/Cli.hpp
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ **/
+
+#ifndef QUICKSTEP_CLI_DISTRIBUTED_CLI_HPP_
+#define QUICKSTEP_CLI_DISTRIBUTED_CLI_HPP_
+
+#include <memory>
+
+#include "cli/distributed/Role.hpp"
+#include "storage/DataExchangerAsync.hpp"
+#include "storage/StorageManager.hpp"
+#include "utility/Macros.hpp"
+
+#include "tmb/id_typedefs.h"
+
+namespace quickstep {
+
+/** \addtogroup CliDistributed
+ * @{
+ */
+
+/**
+ * @brief A class for the Cli component in the distributed version.
+ **/
+class Cli final : public Role {
+ public:
+ /**
+ * @brief Constructor.
+ **/
+ Cli() = default;
+
+ ~Cli() override {
+ data_exchanger_.shutdown();
+ storage_manager_.reset();
+ data_exchanger_.join();
+ }
+
+ void init() override;
+
+ void run() override;
+
+ private:
+ tmb::client_id cli_id_, conductor_client_id_;
+
+ DataExchangerAsync data_exchanger_;
+ std::unique_ptr<StorageManager> storage_manager_;
+
+ DISALLOW_COPY_AND_ASSIGN(Cli);
+};
+
+/** @} */
+
+} // namespace quickstep
+
+#endif // QUICKSTEP_CLI_DISTRIBUTED_CLI_HPP_
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/b949c504/cli/distributed/CliDistributedModule.hpp
----------------------------------------------------------------------
diff --git a/cli/distributed/CliDistributedModule.hpp b/cli/distributed/CliDistributedModule.hpp
new file mode 100644
index 0000000..cfa1e1b
--- /dev/null
+++ b/cli/distributed/CliDistributedModule.hpp
@@ -0,0 +1,23 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ **/
+
+/** @defgroup CliDistributed
+ *
+ * The distributed QuickStep command-line interface.
+ **/
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/b949c504/cli/distributed/Conductor.cpp
----------------------------------------------------------------------
diff --git a/cli/distributed/Conductor.cpp b/cli/distributed/Conductor.cpp
new file mode 100644
index 0000000..c4a2721
--- /dev/null
+++ b/cli/distributed/Conductor.cpp
@@ -0,0 +1,180 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ **/
+
+#include "cli/distributed/Conductor.hpp"
+
+#include <cstddef>
+#include <cstdlib>
+#include <exception>
+#include <memory>
+#include <sstream>
+#include <string>
+#include <utility>
+
+#include "cli/DefaultsConfigurator.hpp"
+#include "cli/Flags.hpp"
+#include "parser/ParseStatement.hpp"
+#include "parser/SqlParserWrapper.hpp"
+#include "query_execution/BlockLocator.hpp"
+#include "query_execution/ForemanDistributed.hpp"
+#include "query_execution/QueryExecutionMessages.pb.h"
+#include "query_execution/QueryExecutionTypedefs.hpp"
+#include "query_execution/QueryExecutionUtil.hpp"
+#include "query_optimizer/QueryHandle.hpp"
+#include "query_optimizer/QueryProcessor.hpp"
+#include "storage/StorageConstants.hpp"
+#include "utility/SqlError.hpp"
+
+#include "tmb/id_typedefs.h"
+#include "tmb/native_net_client_message_bus.h"
+#include "tmb/tagged_message.h"
+
+#include "glog/logging.h"
+
+using std::free;
+using std::make_unique;
+using std::malloc;
+using std::move;
+using std::size_t;
+using std::string;
+
+using tmb::AnnotatedMessage;
+using tmb::MessageBus;
+using tmb::TaggedMessage;
+using tmb::client_id;
+
+namespace quickstep {
+
+namespace S = ::quickstep::serialization;
+
+void Conductor::init() {
+ try {
+ string catalog_path = FLAGS_storage_path + kCatalogFilename;
+
+ if (quickstep::FLAGS_initialize_db) { // Initialize the database
+ DefaultsConfigurator::InitializeDefaultDatabase(FLAGS_storage_path, catalog_path);
+ }
+
+ query_processor_ = make_unique<QueryProcessor>(move(catalog_path));
+ } catch (const std::exception &e) {
+ LOG(FATAL) << "FATAL ERROR DURING STARTUP: " << e.what()
+ << "\nIf you intended to create a new database, "
+ << "please use the \"-initialize_db=true\" command line option.";
+ } catch (...) {
+ LOG(FATAL) << "NON-STANDARD EXCEPTION DURING STARTUP";
+ }
+
+ bus_.ResetBus();
+
+ conductor_client_id_ = bus_.Connect();
+ DLOG(INFO) << "Conductor TMB Client ID: " << conductor_client_id_;
+
+ bus_.RegisterClientAsReceiver(conductor_client_id_, kDistributedCliRegistrationMessage);
+ bus_.RegisterClientAsSender(conductor_client_id_, kDistributedCliRegistrationResponseMessage);
+
+ bus_.RegisterClientAsReceiver(conductor_client_id_, kSqlQueryMessage);
+ bus_.RegisterClientAsSender(conductor_client_id_, kQueryExecutionErrorMessage);
+ bus_.RegisterClientAsSender(conductor_client_id_, kAdmitRequestMessage);
+
+ block_locator_ = make_unique<BlockLocator>(&bus_);
+ block_locator_->start();
+
+ foreman_ = make_unique<ForemanDistributed>(*block_locator_, &bus_, query_processor_->getDefaultDatabase());
+ foreman_->start();
+}
+
+void Conductor::run() {
+ for (;;) {
+ AnnotatedMessage annotated_message(bus_.Receive(conductor_client_id_, 0, true));
+ const TaggedMessage &tagged_message = annotated_message.tagged_message;
+ const client_id sender = annotated_message.sender;
+
+ DLOG(INFO) << "Conductor received typed '" << tagged_message.message_type()
+ << "' message from client " << sender;
+ switch (tagged_message.message_type()) {
+ case kDistributedCliRegistrationMessage: {
+ TaggedMessage message(kDistributedCliRegistrationResponseMessage);
+
+ DLOG(INFO) << "Conductor sent DistributedCliRegistrationResponseMessage (typed '"
+ << kDistributedCliRegistrationResponseMessage
+ << "') to Distributed CLI " << sender;
+ CHECK(MessageBus::SendStatus::kOK ==
+ QueryExecutionUtil::SendTMBMessage(&bus_, conductor_client_id_, sender, move(message)));
+ break;
+ }
+ case kSqlQueryMessage: {
+ S::SqlQueryMessage proto;
+ CHECK(proto.ParseFromArray(tagged_message.message(), tagged_message.message_bytes()));
+ DLOG(INFO) << "Conductor received the following SQL query: " << proto.sql_query();
+
+ processSqlQueryMessage(sender, new string(move(proto.sql_query())));
+ break;
+ }
+ default:
+ LOG(FATAL) << "Unknown TMB message type";
+ }
+ }
+}
+
+void Conductor::processSqlQueryMessage(const tmb::client_id sender, string *command_string) {
+ parser_wrapper_.feedNextBuffer(command_string);
+ ParseResult parse_result = parser_wrapper_.getNextStatement();
+
+ CHECK(parse_result.condition == ParseResult::kSuccess)
+ << "Any SQL syntax error should be addressed in the DistributedCli.";
+
+ const ParseStatement &statement = *parse_result.parsed_statement;
+ CHECK(statement.getStatementType() != ParseStatement::kCommand)
+ << "TODO(quickstep-team)";
+
+ try {
+ auto query_handle = make_unique<QueryHandle>(query_processor_->query_id(),
+ sender,
+ statement.getPriority());
+ query_processor_->generateQueryHandle(statement, query_handle.get());
+ DCHECK(query_handle->getQueryPlanMutable() != nullptr);
+
+ QueryExecutionUtil::ConstructAndSendAdmitRequestMessage(
+ conductor_client_id_,
+ foreman_->getBusClientID(),
+ query_handle.release(),
+ &bus_);
+ } catch (const SqlError &sql_error) {
+ // Set the query execution status along with the error message.
+ S::QueryExecutionErrorMessage proto;
+ proto.set_error_message(sql_error.formatMessage(*command_string));
+
+ const size_t proto_length = proto.ByteSize();
+ char *proto_bytes = static_cast<char*>(malloc(proto_length));
+ CHECK(proto.SerializeToArray(proto_bytes, proto_length));
+
+ TaggedMessage message(static_cast<const void*>(proto_bytes),
+ proto_length,
+ kQueryExecutionErrorMessage);
+ free(proto_bytes);
+
+ DLOG(INFO) << "Conductor (on behalf of Optimizer) sent QueryExecutionErrorMessage (typed '"
+ << kQueryExecutionErrorMessage
+ << "') to Distributed CLI " << sender;
+ CHECK(MessageBus::SendStatus::kOK ==
+ QueryExecutionUtil::SendTMBMessage(&bus_, conductor_client_id_, sender, move(message)));
+ }
+}
+
+} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/b949c504/cli/distributed/Conductor.hpp
----------------------------------------------------------------------
diff --git a/cli/distributed/Conductor.hpp b/cli/distributed/Conductor.hpp
new file mode 100644
index 0000000..e8c9582
--- /dev/null
+++ b/cli/distributed/Conductor.hpp
@@ -0,0 +1,80 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ **/
+
+#ifndef QUICKSTEP_CLI_DISTRIBUTED_CONDUCTOR_HPP_
+#define QUICKSTEP_CLI_DISTRIBUTED_CONDUCTOR_HPP_
+
+#include <memory>
+#include <string>
+
+#include "cli/distributed/Role.hpp"
+#include "parser/SqlParserWrapper.hpp"
+#include "query_execution/BlockLocator.hpp"
+#include "query_execution/ForemanDistributed.hpp"
+#include "query_optimizer/QueryProcessor.hpp"
+#include "utility/Macros.hpp"
+
+#include "tmb/id_typedefs.h"
+
+namespace quickstep {
+
+/** \addtogroup CliDistributed
+ * @{
+ */
+
+/**
+ * @brief A class for the Conductor component in the distributed version.
+ **/
+class Conductor final : public Role {
+ public:
+ /**
+ * @brief Constructor.
+ **/
+ Conductor() = default;
+
+ ~Conductor() override {
+ foreman_->join();
+ block_locator_->join();
+ }
+
+ void init() override;
+
+ void run() override;
+
+ private:
+ void processSqlQueryMessage(const tmb::client_id sender, std::string *command_string);
+
+ SqlParserWrapper parser_wrapper_;
+
+ std::unique_ptr<QueryProcessor> query_processor_;
+
+ tmb::client_id conductor_client_id_;
+
+ std::unique_ptr<BlockLocator> block_locator_;
+
+ std::unique_ptr<ForemanDistributed> foreman_;
+
+ DISALLOW_COPY_AND_ASSIGN(Conductor);
+};
+
+/** @} */
+
+} // namespace quickstep
+
+#endif // QUICKSTEP_CLI_DISTRIBUTED_CONDUCTOR_HPP_
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/b949c504/cli/distributed/Executor.cpp
----------------------------------------------------------------------
diff --git a/cli/distributed/Executor.cpp b/cli/distributed/Executor.cpp
new file mode 100644
index 0000000..1d03579
--- /dev/null
+++ b/cli/distributed/Executor.cpp
@@ -0,0 +1,87 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ **/
+
+#include "cli/distributed/Executor.hpp"
+
+#include <cstddef>
+#include <memory>
+#include <vector>
+
+#include "catalog/CatalogTypedefs.hpp"
+#include "cli/Flags.hpp"
+#include "query_execution/BlockLocatorUtil.hpp"
+#include "query_execution/QueryExecutionTypedefs.hpp"
+#include "query_execution/Shiftboss.hpp"
+#include "query_execution/Worker.hpp"
+#include "query_execution/WorkerDirectory.hpp"
+#include "storage/DataExchangerAsync.hpp"
+#include "storage/StorageManager.hpp"
+
+#include "tmb/id_typedefs.h"
+#include "tmb/native_net_client_message_bus.h"
+
+#include "glog/logging.h"
+
+using std::make_unique;
+using std::size_t;
+using std::vector;
+
+using tmb::client_id;
+
+namespace quickstep {
+
+void Executor::init() {
+ executor_client_id_ = bus_.Connect();
+ DLOG(INFO) << "Executor TMB Client ID: " << executor_client_id_;
+
+ bus_.RegisterClientAsSender(executor_client_id_, kBlockDomainRegistrationMessage);
+ bus_.RegisterClientAsReceiver(executor_client_id_, kBlockDomainRegistrationResponseMessage);
+
+ vector<client_id> worker_client_ids;
+ vector<numa_node_id> worker_numa_nodes(FLAGS_num_workers, kAnyNUMANodeID);
+
+ for (std::size_t worker_thread_index = 0;
+ worker_thread_index < FLAGS_num_workers;
+ ++worker_thread_index) {
+ workers_.push_back(make_unique<Worker>(worker_thread_index, &bus_));
+ worker_client_ids.push_back(workers_.back()->getBusClientID());
+ }
+
+ worker_directory_ =
+ make_unique<WorkerDirectory>(worker_client_ids.size(), worker_client_ids, worker_numa_nodes);
+
+ client_id locator_client_id;
+ storage_manager_ = make_unique<StorageManager>(
+ FLAGS_storage_path,
+ block_locator::getBlockDomain(data_exchanger_.network_address(), executor_client_id_, &locator_client_id, &bus_),
+ locator_client_id, &bus_);
+
+ data_exchanger_.set_storage_manager(storage_manager_.get());
+ data_exchanger_.start();
+
+ shiftboss_ =
+ make_unique<Shiftboss>(&bus_, storage_manager_.get(), worker_directory_.get());
+ shiftboss_->start();
+
+ for (const auto &worker : workers_) {
+ worker->start();
+ }
+}
+
+} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/b949c504/cli/distributed/Executor.hpp
----------------------------------------------------------------------
diff --git a/cli/distributed/Executor.hpp b/cli/distributed/Executor.hpp
new file mode 100644
index 0000000..6ffa756
--- /dev/null
+++ b/cli/distributed/Executor.hpp
@@ -0,0 +1,83 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ **/
+
+#ifndef QUICKSTEP_CLI_DISTRIBUTED_EXECUTOR_HPP_
+#define QUICKSTEP_CLI_DISTRIBUTED_EXECUTOR_HPP_
+
+#include <memory>
+#include <vector>
+
+#include "cli/distributed/Role.hpp"
+#include "query_execution/Shiftboss.hpp"
+#include "query_execution/Worker.hpp"
+#include "query_execution/WorkerDirectory.hpp"
+#include "storage/DataExchangerAsync.hpp"
+#include "storage/StorageManager.hpp"
+#include "utility/Macros.hpp"
+
+#include "tmb/id_typedefs.h"
+
+namespace quickstep {
+
+/** \addtogroup CliDistributed
+ * @{
+ */
+
+/**
+ * @brief A class for the Executor component in the distributed version.
+ **/
+class Executor final : public Role {
+ public:
+ /**
+ * @brief Constructor.
+ **/
+ Executor() = default;
+
+ ~Executor() override {
+ for (const auto &worker : workers_) {
+ worker->join();
+ }
+ shiftboss_->join();
+
+ data_exchanger_.shutdown();
+ storage_manager_.reset();
+ data_exchanger_.join();
+ }
+
+ void init() override;
+
+ void run() override {}
+
+ private:
+ tmb::client_id executor_client_id_;
+
+ std::vector<std::unique_ptr<Worker>> workers_;
+ std::unique_ptr<WorkerDirectory> worker_directory_;
+ DataExchangerAsync data_exchanger_;
+ std::unique_ptr<StorageManager> storage_manager_;
+ std::unique_ptr<Shiftboss> shiftboss_;
+
+ DISALLOW_COPY_AND_ASSIGN(Executor);
+};
+
+/** @} */
+
+} // namespace quickstep
+
+#endif // QUICKSTEP_CLI_DISTRIBUTED_EXECUTOR_HPP_
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/b949c504/cli/distributed/QuickstepDistributedCli.cpp
----------------------------------------------------------------------
diff --git a/cli/distributed/QuickstepDistributedCli.cpp b/cli/distributed/QuickstepDistributedCli.cpp
new file mode 100644
index 0000000..f01cd13
--- /dev/null
+++ b/cli/distributed/QuickstepDistributedCli.cpp
@@ -0,0 +1,81 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ **/
+
+/* A standalone command-line interface to Distributed QuickStep */
+
+#include <memory>
+#include <string>
+
+#include "cli/distributed/Cli.hpp"
+#include "cli/distributed/Conductor.hpp"
+#include "cli/distributed/Executor.hpp"
+#include "cli/distributed/Role.hpp"
+#include "utility/StringUtil.hpp"
+
+#include "gflags/gflags.h"
+#include "glog/logging.h"
+#include "grpc/grpc.h"
+
+using std::make_unique;
+
+namespace quickstep {
+
+constexpr char kCliRole[] = "cli";
+constexpr char kConductorRole[] = "conductor";
+constexpr char kExecutorRole[] = "executor";
+
+DEFINE_string(role, "",
+ "The role in the distributed Quickstep: Conductor, Executor, or Cli.");
+static bool ValidateRole(const char *flagname,
+ const std::string &value) {
+ if (value.empty()) {
+ return false;
+ }
+
+ FLAGS_role = ToLower(value);
+ return FLAGS_role == kCliRole ||
+ FLAGS_role == kConductorRole ||
+ FLAGS_role == kExecutorRole;
+}
+static const volatile bool role_dummy
+ = gflags::RegisterFlagValidator(&FLAGS_role, &ValidateRole);
+
+} // namespace quickstep
+
+using quickstep::FLAGS_role;
+
+int main(int argc, char *argv[]) {
+ google::InitGoogleLogging(argv[0]);
+ gflags::ParseCommandLineFlags(&argc, &argv, true);
+ grpc_init();
+
+ std::unique_ptr<quickstep::Role> role;
+ if (FLAGS_role == quickstep::kCliRole) {
+ role = make_unique<quickstep::Cli>();
+ } else if (FLAGS_role == quickstep::kConductorRole) {
+ role = make_unique<quickstep::Conductor>();
+ } else if (FLAGS_role == quickstep::kExecutorRole) {
+ role = make_unique<quickstep::Executor>();
+ }
+
+ role->init();
+ role->run();
+
+ return 0;
+}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/b949c504/cli/distributed/Role.cpp
----------------------------------------------------------------------
diff --git a/cli/distributed/Role.cpp b/cli/distributed/Role.cpp
new file mode 100644
index 0000000..d56ef09
--- /dev/null
+++ b/cli/distributed/Role.cpp
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ **/
+
+#include "cli/distributed/Role.hpp"
+
+#include <cstdio>
+#include <cstdint>
+
+#include "gflags/gflags.h"
+
+#include "tmb/native_net_client_message_bus.h"
+
+namespace quickstep {
+
+DEFINE_string(tmb_server_ip, "127.0.0.1", "IP Address of the TMB Server.");
+
+static bool ValidateTmbServerPort(const char *flagname,
+ std::int32_t value) {
+ if (value > 0 && value < 65536) {
+ return true;
+ } else {
+ std::fprintf(stderr, "--%s must be between 1 and 65535 (inclusive)\n", flagname);
+ return false;
+ }
+}
+DEFINE_int32(tmb_server_port, 4575, "Port of the TMB Server.");
+static const bool tmb_server_port_dummy
+ = gflags::RegisterFlagValidator(&FLAGS_tmb_server_port, &ValidateTmbServerPort);
+
+Role::Role() {
+ bus_.AddServer(FLAGS_tmb_server_ip, FLAGS_tmb_server_port);
+ bus_.Initialize();
+}
+
+} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/b949c504/cli/distributed/Role.hpp
----------------------------------------------------------------------
diff --git a/cli/distributed/Role.hpp b/cli/distributed/Role.hpp
new file mode 100644
index 0000000..b802543
--- /dev/null
+++ b/cli/distributed/Role.hpp
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ **/
+
+#ifndef QUICKSTEP_CLI_DISTRIBUTED_ROLE_HPP_
+#define QUICKSTEP_CLI_DISTRIBUTED_ROLE_HPP_
+
+#include "utility/Macros.hpp"
+
+#include "tmb/native_net_client_message_bus.h"
+
+namespace quickstep {
+
+/** \addtogroup CliDistributed
+ * @{
+ */
+
+/**
+ * @brief A base class for all components in the distributed version.
+ **/
+class Role {
+ public:
+ /**
+ * @brief Constructor.
+ **/
+ Role();
+
+ /**
+ * @brief Virtual destructor.
+ **/
+ virtual ~Role() {}
+
+ /**
+ * @brief Initialize the component.
+ **/
+ virtual void init() = 0;
+
+ /**
+ * @brief Start the component.
+ **/
+ virtual void run() = 0;
+
+ protected:
+ tmb::NativeNetClientMessageBus bus_;
+
+ private:
+ DISALLOW_COPY_AND_ASSIGN(Role);
+};
+
+/** @} */
+
+} // namespace quickstep
+
+#endif // QUICKSTEP_CLI_DISTRIBUTED_ROLE_HPP_
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/b949c504/query_execution/QueryExecutionMessages.proto
----------------------------------------------------------------------
diff --git a/query_execution/QueryExecutionMessages.proto b/query_execution/QueryExecutionMessages.proto
index 93e458c..28b5ebd 100644
--- a/query_execution/QueryExecutionMessages.proto
+++ b/query_execution/QueryExecutionMessages.proto
@@ -78,6 +78,10 @@ message ShiftbossRegistrationResponseMessage {
required uint64 shiftboss_index = 1;
}
+message SqlQueryMessage {
+ required string sql_query = 1;
+}
+
message QueryInitiateMessage {
required uint64 query_id = 1;
required CatalogDatabase catalog_database_cache = 2;
@@ -131,6 +135,10 @@ message QueryExecutionSuccessMessage {
optional CatalogRelationSchema result_relation = 1;
}
+message QueryExecutionErrorMessage {
+ required string error_message = 1;
+}
+
// BlockLocator related messages.
message BlockDomainRegistrationMessage {
// Format IP:Port, i.e., "0.0.0.0:0".
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/b949c504/query_execution/QueryExecutionTypedefs.hpp
----------------------------------------------------------------------
diff --git a/query_execution/QueryExecutionTypedefs.hpp b/query_execution/QueryExecutionTypedefs.hpp
index 919e45b..faf2132 100644
--- a/query_execution/QueryExecutionTypedefs.hpp
+++ b/query_execution/QueryExecutionTypedefs.hpp
@@ -81,6 +81,9 @@ enum QueryExecutionMessageType : message_type_id {
kShiftbossRegistrationMessage, // From Shiftboss to Foreman.
kShiftbossRegistrationResponseMessage, // From Foreman to Shiftboss, or from
// Shiftboss to Worker.
+ kDistributedCliRegistrationMessage, // From CLI to Conductor.
+ kDistributedCliRegistrationResponseMessage, // From Conductor to CLI.
+ kSqlQueryMessage, // From CLI to Conductor.
kQueryInitiateMessage, // From Foreman to Shiftboss.
kQueryInitiateResponseMessage, // From Shiftboss to Foreman.
@@ -92,8 +95,9 @@ enum QueryExecutionMessageType : message_type_id {
kSaveQueryResultMessage, // From Foreman to Shiftboss.
kSaveQueryResultResponseMessage, // From Shiftboss to Foreman.
- // From Foreman to CLI.
+ // From Foreman / Conductor to CLI.
kQueryExecutionSuccessMessage,
+ kQueryExecutionErrorMessage,
// BlockLocator related messages, sorted in a life cycle of StorageManager
// with a unique block domain.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/b949c504/validate_cmakelists.py
----------------------------------------------------------------------
diff --git a/validate_cmakelists.py b/validate_cmakelists.py
index f691d1f..9d1f530 100755
--- a/validate_cmakelists.py
+++ b/validate_cmakelists.py
@@ -46,7 +46,9 @@ EXCLUDED_TOP_LEVEL_DIRS = ["build", "third_party"]
# Explicitly ignored dependencies (special headers with no other quickstep
# dependencies).
IGNORED_DEPENDENCIES = frozenset(
- ["quickstep_storage_DataExchange.grpc_proto",
+ ["quickstep_cli_LineReaderDumb",
+ "quickstep_cli_LineReaderLineNoise",
+ "quickstep_storage_DataExchange.grpc_proto",
"quickstep_threading_WinThreadsAPI",
"quickstep_utility_textbasedtest_TextBasedTest",
"quickstep_utility_textbasedtest_TextBasedTestDriver",
[32/50] incubator-quickstep git commit: Moved the init-db flag and
fixed a bug regarding db-init.
Posted by zu...@apache.org.
Moved the init-db flag and fixed a bug regarding db-init.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/3047d89a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/3047d89a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/3047d89a
Branch: refs/heads/quickstep_partition_parser_support
Commit: 3047d89ab0e02bf0a30e914f55e6b49435c07e6a
Parents: 1b27888
Author: Zuyu Zhang <zu...@apache.org>
Authored: Mon Nov 21 23:44:13 2016 -0800
Committer: Zuyu Zhang <zu...@apache.org>
Committed: Mon Nov 21 23:44:13 2016 -0800
----------------------------------------------------------------------
cli/DefaultsConfigurator.cpp | 2 +-
cli/Flags.cpp | 2 ++
cli/Flags.hpp | 3 +++
cli/QuickstepCli.cpp | 1 -
4 files changed, 6 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3047d89a/cli/DefaultsConfigurator.cpp
----------------------------------------------------------------------
diff --git a/cli/DefaultsConfigurator.cpp b/cli/DefaultsConfigurator.cpp
index 94280a7..21b0af6 100644
--- a/cli/DefaultsConfigurator.cpp
+++ b/cli/DefaultsConfigurator.cpp
@@ -55,7 +55,7 @@ void DefaultsConfigurator::InitializeDefaultDatabase(const string &storage_path,
#else
{
const string path_name = "mkdir " + storage_path;
- CHECK(std::system(path_name.c_str()))
+ CHECK(!std::system(path_name.c_str()))
<< "Failed when attempting to create the directory: " << storage_path;
}
#endif // QUICKSTEP_OS_WINDOWS
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3047d89a/cli/Flags.cpp
----------------------------------------------------------------------
diff --git a/cli/Flags.cpp b/cli/Flags.cpp
index 87f9f73..1e3ce1d 100644
--- a/cli/Flags.cpp
+++ b/cli/Flags.cpp
@@ -32,6 +32,8 @@ using std::fprintf;
namespace quickstep {
+DEFINE_bool(initialize_db, false, "If true, initialize a database.");
+
static bool ValidateNumWorkers(const char *flagname, int value) {
if (value > 0) {
return true;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3047d89a/cli/Flags.hpp
----------------------------------------------------------------------
diff --git a/cli/Flags.hpp b/cli/Flags.hpp
index b020a3e..70aee98 100644
--- a/cli/Flags.hpp
+++ b/cli/Flags.hpp
@@ -32,6 +32,9 @@ namespace quickstep {
* @brief A collection of common flags shared by Quickstep CLIs in both the
* single-node and the distributed version.
**/
+
+DECLARE_bool(initialize_db);
+
DECLARE_int32(num_workers);
DECLARE_string(storage_path);
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3047d89a/cli/QuickstepCli.cpp
----------------------------------------------------------------------
diff --git a/cli/QuickstepCli.cpp b/cli/QuickstepCli.cpp
index 656786a..f4816a8 100644
--- a/cli/QuickstepCli.cpp
+++ b/cli/QuickstepCli.cpp
@@ -128,7 +128,6 @@ DEFINE_string(worker_affinities, "",
"the affinity mask of the Quickstep process, which typically "
"means that they will all be runable on any CPU according to "
"the kernel's own scheduling policy).");
-DEFINE_bool(initialize_db, false, "If true, initialize a database.");
DEFINE_bool(print_query, false,
"Print each input query statement. This is useful when running a "
"large number of queries in a batch.");
[07/50] incubator-quickstep git commit: Allows filters to be
evaluated.
Posted by zu...@apache.org.
Allows filters to be evaluated.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/3093e744
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/3093e744
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/3093e744
Branch: refs/heads/quickstep_partition_parser_support
Commit: 3093e74484b620f828e435f434d674390f81a1c9
Parents: c45d68e
Author: cramja <ma...@gmail.com>
Authored: Tue Nov 8 17:08:22 2016 -0600
Committer: cramja <ma...@gmail.com>
Committed: Wed Nov 9 11:26:20 2016 -0600
----------------------------------------------------------------------
storage/SMAIndexSubBlock.cpp | 26 +++++++++++++-------------
1 file changed, 13 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3093e744/storage/SMAIndexSubBlock.cpp
----------------------------------------------------------------------
diff --git a/storage/SMAIndexSubBlock.cpp b/storage/SMAIndexSubBlock.cpp
index 621bafd..3b3b879 100644
--- a/storage/SMAIndexSubBlock.cpp
+++ b/storage/SMAIndexSubBlock.cpp
@@ -672,25 +672,25 @@ predicate_cost_t SMAIndexSubBlock::estimatePredicateEvaluationCost(
TupleIdSequence* SMAIndexSubBlock::getMatchesForPredicate(
const ComparisonPredicate &predicate,
const TupleIdSequence *filter) const {
- if (filter != nullptr) {
- LOG(FATAL) << "SMAIndex cannot evaluate filters.";
- }
-
Selectivity selectivity = getSelectivityForPredicate(predicate);
if (selectivity == Selectivity::kAll) {
- TupleIdSequence* tidseq = new TupleIdSequence(tuple_store_.numTuples());
-
- // Set all existing tuples to true, selected.
- if (tuple_store_.isPacked()) {
- tidseq->setRange(0, tuple_store_.numTuples(), true);
+ if (filter != nullptr) {
+ return new TupleIdSequence(filter->length(), filter->getInternalBitVector());
} else {
- for (tuple_id tid = 0; tid <= tuple_store_.getMaxTupleID(); ++tid) {
- if (tuple_store_.hasTupleWithID(tid)) {
- tidseq->set(tid, true);
+ TupleIdSequence* tidseq = new TupleIdSequence(tuple_store_.numTuples());
+
+ // Set all existing tuples to true, selected.
+ if (tuple_store_.isPacked()) {
+ tidseq->setRange(0, tuple_store_.numTuples(), true);
+ } else {
+ for (tuple_id tid = 0; tid <= tuple_store_.getMaxTupleID(); ++tid) {
+ if (tuple_store_.hasTupleWithID(tid)) {
+ tidseq->set(tid, true);
+ }
}
}
+ return tidseq;
}
- return tidseq;
} else if (selectivity == Selectivity::kNone) {
// A new tuple ID sequence is initialized to false for all values.
return new TupleIdSequence(tuple_store_.numTuples());
[33/50] incubator-quickstep git commit: Added back the debug build
with llvm.
Posted by zu...@apache.org.
Added back the debug build with llvm.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/a31dde4e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/a31dde4e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/a31dde4e
Branch: refs/heads/quickstep_partition_parser_support
Commit: a31dde4eaab6122768e21812b958a05d3048e50f
Parents: 3047d89
Author: Zuyu Zhang <zu...@apache.org>
Authored: Fri Nov 25 21:52:43 2016 -0800
Committer: Zuyu Zhang <zu...@apache.org>
Committed: Sat Nov 26 23:49:57 2016 -0800
----------------------------------------------------------------------
.travis.yml | 11 ++++++++++-
1 file changed, 10 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/a31dde4e/.travis.yml
----------------------------------------------------------------------
diff --git a/.travis.yml b/.travis.yml
index 9b43b16..8915eeb 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,4 +1,4 @@
-# NOTE(quickstep-team): In Travis-CI, jobs timeout if they take more than 120
+# NOTE(quickstep-team): In Travis-CI, jobs timeout if they take more than 50
# mins or if there is no log output for more than 10 mins. Hence, we use -O0 to
# speed up compilation in release build. Also, jobs can only use upto 20GB of
# disk space. Hence, we minimize the amount of debug symbol using -g0 (DEBUG
@@ -15,9 +15,18 @@ compiler:
- clang
env:
+ - BUILD_TYPE=Debug VECTOR_COPY_ELISION_LEVEL=joinwithbinaryexpressions
+ - BUILD_TYPE=Debug VECTOR_COPY_ELISION_LEVEL=selection
- BUILD_TYPE=Release VECTOR_COPY_ELISION_LEVEL=joinwithbinaryexpressions
- BUILD_TYPE=Release VECTOR_COPY_ELISION_LEVEL=selection
+matrix:
+ exclude: # Due to time-out.
+ - compiler: gcc
+ env: BUILD_TYPE=Debug VECTOR_COPY_ELISION_LEVEL=joinwithbinaryexpressions
+ - compiler: gcc
+ env: BUILD_TYPE=Debug VECTOR_COPY_ELISION_LEVEL=selection
+
install:
- export TEST_JOBS=2;
- if [ "$CC" = "gcc" ]; then
[21/50] incubator-quickstep git commit: Moved a CLI flag into a new
file.
Posted by zu...@apache.org.
Moved a CLI flag into a new file.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/f7d15436
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/f7d15436
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/f7d15436
Branch: refs/heads/quickstep_partition_parser_support
Commit: f7d1543671a3d3b3e52d2fe0c8170b64796aa9d6
Parents: e7c664b
Author: Zuyu Zhang <zu...@apache.org>
Authored: Sat Nov 19 00:18:55 2016 -0800
Committer: Zuyu Zhang <zu...@apache.org>
Committed: Sun Nov 20 19:56:10 2016 -0800
----------------------------------------------------------------------
CMakeLists.txt | 2 ++
cli/CMakeLists.txt | 5 +++++
cli/Flags.cpp | 43 +++++++++++++++++++++++++++++++++++++++++++
cli/Flags.hpp | 41 +++++++++++++++++++++++++++++++++++++++++
cli/QuickstepCli.cpp | 23 +++++++++--------------
5 files changed, 100 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f7d15436/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 6191de0..cd53967 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -752,6 +752,7 @@ target_link_libraries(quickstep_cli_shell
quickstep_cli_CommandExecutor
quickstep_cli_DefaultsConfigurator
quickstep_cli_DropRelation
+ quickstep_cli_Flags
quickstep_cli_InputParserUtil
quickstep_cli_LineReader
quickstep_cli_PrintToScreen
@@ -770,6 +771,7 @@ target_link_libraries(quickstep_cli_shell
quickstep_queryoptimizer_QueryProcessor
quickstep_storage_PreloaderThread
quickstep_storage_StorageConstants
+ quickstep_storage_StorageManager
quickstep_threading_ThreadIDBasedMap
quickstep_utility_ExecutionDAGVisualizer
quickstep_utility_Macros
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f7d15436/cli/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/cli/CMakeLists.txt b/cli/CMakeLists.txt
index b86821a..9928f3f 100644
--- a/cli/CMakeLists.txt
+++ b/cli/CMakeLists.txt
@@ -54,6 +54,7 @@ target_link_libraries(quickstep_cli_DropRelation
quickstep_storage_StorageBlockInfo
quickstep_storage_StorageManager
quickstep_utility_Macros)
+add_library(quickstep_cli_Flags Flags.cpp Flags.hpp)
if(USE_LINENOISE)
add_library(quickstep_cli_LineReader
@@ -110,6 +111,9 @@ if(QUICKSTEP_HAVE_LIBNUMA)
target_link_libraries(quickstep_cli_DefaultsConfigurator
${LIBNUMA_LIBRARY})
endif()
+target_link_libraries(quickstep_cli_Flags
+ quickstep_storage_StorageConstants
+ ${GFLAGS_LIB_NAME})
target_link_libraries(quickstep_cli_InputParserUtil
glog
quickstep_utility_Macros
@@ -139,6 +143,7 @@ target_link_libraries(quickstep_cli
quickstep_cli_CommandExecutor
quickstep_cli_DefaultsConfigurator
quickstep_cli_DropRelation
+ quickstep_cli_Flags
quickstep_cli_InputParserUtil
quickstep_cli_LineReader
quickstep_cli_PrintToScreen)
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f7d15436/cli/Flags.cpp
----------------------------------------------------------------------
diff --git a/cli/Flags.cpp b/cli/Flags.cpp
new file mode 100644
index 0000000..384ef71
--- /dev/null
+++ b/cli/Flags.cpp
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ **/
+
+#include "cli/Flags.hpp"
+
+#include <string>
+
+#include "storage/StorageConstants.hpp"
+
+#include "gflags/gflags.h"
+
+namespace quickstep {
+
+static bool ValidateStoragePath(const char *flagname,
+ const std::string &value) {
+ if (!value.empty() && value.back() != kPathSeparator) {
+ FLAGS_storage_path.push_back(kPathSeparator);
+ }
+
+ return true;
+}
+DEFINE_string(storage_path, kDefaultStoragePath,
+ "Filesystem path to store the Quickstep database.");
+static const volatile bool storage_path_dummy
+ = gflags::RegisterFlagValidator(&FLAGS_storage_path, &ValidateStoragePath);
+
+} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f7d15436/cli/Flags.hpp
----------------------------------------------------------------------
diff --git a/cli/Flags.hpp b/cli/Flags.hpp
new file mode 100644
index 0000000..a623448
--- /dev/null
+++ b/cli/Flags.hpp
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ **/
+
+#ifndef QUICKSTEP_CLI_FLAGS_HPP_
+#define QUICKSTEP_CLI_FLAGS_HPP_
+
+#include "gflags/gflags_declare.h"
+
+namespace quickstep {
+
+/** \addtogroup CLI
+ * @{
+ */
+
+/**
+ * @brief A collection of common flags shared by Quickstep CLIs in both the
+ * single-node and the distributed version.
+ **/
+DECLARE_string(storage_path);
+
+/** @} */
+
+} // namespace quickstep
+
+#endif // QUICKSTEP_CLI_FLAGS_HPP_
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f7d15436/cli/QuickstepCli.cpp
----------------------------------------------------------------------
diff --git a/cli/QuickstepCli.cpp b/cli/QuickstepCli.cpp
index a4e55d7..33a9ff4 100644
--- a/cli/QuickstepCli.cpp
+++ b/cli/QuickstepCli.cpp
@@ -54,6 +54,7 @@ typedef quickstep::LineReaderDumb LineReaderImpl;
#endif
#include "cli/DefaultsConfigurator.hpp"
+#include "cli/Flags.hpp"
#include "cli/InputParserUtil.hpp"
#include "cli/PrintToScreen.hpp"
#include "parser/ParseStatement.hpp"
@@ -76,6 +77,7 @@ typedef quickstep::LineReaderDumb LineReaderImpl;
#include "storage/PreloaderThread.hpp"
#include "storage/StorageConstants.hpp"
+#include "storage/StorageManager.hpp"
#include "threading/ThreadIDBasedMap.hpp"
#include "utility/ExecutionDAGVisualizer.hpp"
#include "utility/Macros.hpp"
@@ -107,6 +109,7 @@ using quickstep::AdmitRequestMessage;
using quickstep::CatalogRelation;
using quickstep::DefaultsConfigurator;
using quickstep::DropRelation;
+using quickstep::FLAGS_storage_path;
using quickstep::ForemanSingleNode;
using quickstep::InputParserUtil;
using quickstep::MessageBusImpl;
@@ -142,8 +145,6 @@ DEFINE_bool(preload_buffer_pool, false,
"If true, pre-load all known blocks into buffer pool before "
"accepting queries (should also set --buffer_pool_slots to be "
"large enough to accomodate the entire database).");
-DEFINE_string(storage_path, kDefaultStoragePath,
- "Filesystem path to store the Quickstep database.");
DEFINE_string(worker_affinities, "",
"A comma-separated list of CPU IDs to pin worker threads to "
"(leaving this empty will cause all worker threads to inherit "
@@ -231,15 +232,9 @@ int main(int argc, char* argv[]) {
bus.RegisterClientAsSender(main_thread_client_id, kPoisonMessage);
bus.RegisterClientAsReceiver(main_thread_client_id, kWorkloadCompletionMessage);
- // Setup the paths used by StorageManager.
- string fixed_storage_path(quickstep::FLAGS_storage_path);
- if (!fixed_storage_path.empty()
- && (fixed_storage_path.back() != quickstep::kPathSeparator)) {
- fixed_storage_path.push_back(quickstep::kPathSeparator);
- }
- quickstep::StorageManager storage_manager(fixed_storage_path);
+ quickstep::StorageManager storage_manager(FLAGS_storage_path);
- string catalog_path(fixed_storage_path);
+ string catalog_path(FLAGS_storage_path);
catalog_path.append(kCatalogFilename);
if (quickstep::FLAGS_initialize_db) { // Initialize the database
// TODO(jmp): Refactor the code in this file!
@@ -249,14 +244,14 @@ int main(int argc, char* argv[]) {
// TODO(jmp): At some point, likely in C++-17, we will just have the
// filesystem path, and we can clean this up
#ifdef QUICKSTEP_OS_WINDOWS
- std::filesystem::create_directories(fixed_storage_path);
- LOG(FATAL) << "Failed when attempting to create the directory: " << fixed_storage_path << "\n";
+ std::filesystem::create_directories(FLAGS_storage_path);
+ LOG(FATAL) << "Failed when attempting to create the directory: " << FLAGS_storage_path << "\n";
LOG(FATAL) << "Check if the directory already exists. If so, delete it or move it before initializing \n";
#else
{
- string path_name = "mkdir " + fixed_storage_path;
+ string path_name = "mkdir " + FLAGS_storage_path;
if (std::system(path_name.c_str())) {
- LOG(FATAL) << "Failed when attempting to create the directory: " << fixed_storage_path << "\n";
+ LOG(FATAL) << "Failed when attempting to create the directory: " << FLAGS_storage_path << "\n";
}
}
#endif
[15/50] incubator-quickstep git commit: Build only the release jobs.
Posted by zu...@apache.org.
Build only the release jobs.
- Use only one thread to avoid OOM.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/dec87234
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/dec87234
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/dec87234
Branch: refs/heads/quickstep_partition_parser_support
Commit: dec87234c53db8bd27bbe6a22b28b7e461cae9c6
Parents: 178ed4b
Author: Zuyu Zhang <zu...@apache.org>
Authored: Mon Nov 14 21:57:19 2016 -0800
Committer: Zuyu Zhang <zu...@apache.org>
Committed: Sun Nov 20 19:22:51 2016 -0800
----------------------------------------------------------------------
.travis.yml | 9 +--------
1 file changed, 1 insertion(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/dec87234/.travis.yml
----------------------------------------------------------------------
diff --git a/.travis.yml b/.travis.yml
index 784a46f..9b43b16 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -15,17 +15,10 @@ compiler:
- clang
env:
- - BUILD_TYPE=Debug VECTOR_COPY_ELISION_LEVEL=joinwithbinaryexpressions
- BUILD_TYPE=Release VECTOR_COPY_ELISION_LEVEL=joinwithbinaryexpressions
- - BUILD_TYPE=Debug VECTOR_COPY_ELISION_LEVEL=selection
- BUILD_TYPE=Release VECTOR_COPY_ELISION_LEVEL=selection
install:
- - if [ "$CC" = "gcc" ] || [[ "$BUILD_TYPE" = "Release" ]]; then
- export MAKE_JOBS=1;
- else
- export MAKE_JOBS=2;
- fi
- export TEST_JOBS=2;
- if [ "$CC" = "gcc" ]; then
export CC="gcc-5";
@@ -61,7 +54,7 @@ script:
- ./third_party/cpplint/lint_everything.py
- ./validate_cmakelists.py
- ./cyclic_dependency.py
- - (cd build && make -j$MAKE_JOBS)
+ - (cd build && make)
- (cd build && ctest --output-on-failure -j$TEST_JOBS)
after_failure:
[35/50] incubator-quickstep git commit: Used multiple Shiftbosses in
the distributed unit tests.
Posted by zu...@apache.org.
Used multiple Shiftbosses in the distributed unit tests.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/c4f4b285
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/c4f4b285
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/c4f4b285
Branch: refs/heads/quickstep_partition_parser_support
Commit: c4f4b285b63e8a3d6bb8fd312f6d00cc8f1cbd45
Parents: 4286d75
Author: Zuyu Zhang <zu...@apache.org>
Authored: Sat Nov 12 22:31:42 2016 -0800
Committer: Zuyu Zhang <zu...@apache.org>
Committed: Sun Nov 27 11:46:23 2016 -0800
----------------------------------------------------------------------
query_execution/ForemanDistributed.cpp | 14 +++---
query_execution/PolicyEnforcerDistributed.cpp | 6 ++-
query_execution/QueryExecutionState.hpp | 53 ++++++++++++++++++++++
query_execution/QueryManagerDistributed.cpp | 35 ++++++++------
query_execution/QueryManagerDistributed.hpp | 4 +-
5 files changed, 88 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c4f4b285/query_execution/ForemanDistributed.cpp
----------------------------------------------------------------------
diff --git a/query_execution/ForemanDistributed.cpp b/query_execution/ForemanDistributed.cpp
index d619657..7dccce4 100644
--- a/query_execution/ForemanDistributed.cpp
+++ b/query_execution/ForemanDistributed.cpp
@@ -238,17 +238,19 @@ void ForemanDistributed::run() {
bool ForemanDistributed::canCollectNewMessages(const tmb::message_type_id message_type) {
return !QUICKSTEP_EQUALS_ANY_CONSTANT(message_type,
kCatalogRelationNewBlockMessage,
- kWorkOrderFeedbackMessage) &&
- // TODO(zuyu): Multiple Shiftbosses support.
- !shiftboss_directory_.hasReachedCapacity(0);
+ kWorkOrderFeedbackMessage);
}
void ForemanDistributed::dispatchWorkOrderMessages(const vector<unique_ptr<S::WorkOrderMessage>> &messages) {
+ const size_t num_shiftbosses = shiftboss_directory_.size();
+ size_t shiftboss_index = 0u;
for (const auto &message : messages) {
DCHECK(message != nullptr);
- // TODO(zuyu): Multiple Shiftbosses support.
- sendWorkOrderMessage(0, *message);
- shiftboss_directory_.incrementNumQueuedWorkOrders(0);
+ sendWorkOrderMessage(shiftboss_index, *message);
+ shiftboss_directory_.incrementNumQueuedWorkOrders(shiftboss_index);
+
+ // TO(zuyu): Take data-locality into account for scheduling.
+ shiftboss_index = (shiftboss_index + 1) % num_shiftbosses;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c4f4b285/query_execution/PolicyEnforcerDistributed.cpp
----------------------------------------------------------------------
diff --git a/query_execution/PolicyEnforcerDistributed.cpp b/query_execution/PolicyEnforcerDistributed.cpp
index c06fd86..6e09ea8 100644
--- a/query_execution/PolicyEnforcerDistributed.cpp
+++ b/query_execution/PolicyEnforcerDistributed.cpp
@@ -140,8 +140,10 @@ void PolicyEnforcerDistributed::processInitiateRebuildResponseMessage(const tmb:
QueryManagerDistributed *query_manager = static_cast<QueryManagerDistributed*>(admitted_queries_[query_id].get());
const std::size_t num_rebuild_work_orders = proto.num_rebuild_work_orders();
- query_manager->processInitiateRebuildResponseMessage(proto.operator_index(), num_rebuild_work_orders);
- shiftboss_directory_->addNumQueuedWorkOrders(proto.shiftboss_index(), num_rebuild_work_orders);
+ const size_t shiftboss_index = proto.shiftboss_index();
+ query_manager->processInitiateRebuildResponseMessage(
+ proto.operator_index(), num_rebuild_work_orders, shiftboss_index);
+ shiftboss_directory_->addNumQueuedWorkOrders(shiftboss_index, num_rebuild_work_orders);
if (query_manager->getQueryExecutionState().hasQueryExecutionFinished()) {
onQueryCompletion(query_manager);
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c4f4b285/query_execution/QueryExecutionState.hpp
----------------------------------------------------------------------
diff --git a/query_execution/QueryExecutionState.hpp b/query_execution/QueryExecutionState.hpp
index f5281d5..a4273dc 100644
--- a/query_execution/QueryExecutionState.hpp
+++ b/query_execution/QueryExecutionState.hpp
@@ -22,6 +22,12 @@
#include <cstddef>
#include <unordered_map>
+
+#include "query_optimizer/QueryOptimizerConfig.h" // For QUICKSTEP_DISTRIBUTED.
+#ifdef QUICKSTEP_DISTRIBUTED
+#include <unordered_set>
+#endif // QUICKSTEP_DISTRIBUTED
+
#include <utility>
#include <vector>
@@ -103,6 +109,49 @@ class QueryExecutionState {
}
}
+#ifdef QUICKSTEP_DISTRIBUTED
+ /**
+ * @brief Update the rebuild status of the given operator the number of
+ * pending rebuild work orders, after the rebuild has been initiated.
+ *
+ * @param operator_index The index of the given operator.
+ * @param num_rebuild_workorders The number of rebuild workorders of the given
+ * operator.
+ * @param shiftboss_index The index of the Shiftboss that rebuilt.
+ **/
+ void updateRebuildStatus(const std::size_t operator_index,
+ const std::size_t num_rebuild_workorders,
+ const std::size_t shiftboss_index) {
+ DCHECK_LT(operator_index, num_operators_);
+ auto search_res = rebuild_status_.find(operator_index);
+ DCHECK(search_res != rebuild_status_.end() && search_res->second.has_initiated);
+ search_res->second.num_pending_workorders += num_rebuild_workorders;
+ search_res->second.rebuilt_shiftboss_indexes.insert(shiftboss_index);
+ }
+
+ /**
+ * @brief Check if the rebuild has been finished for the given operator.
+ *
+ * @param operator_index The index of the given operator.
+ * @param num_shiftbosses The number of the Shiftbosses for rebuilt.
+ *
+ * @return True if the rebuild has been finished, false otherwise.
+ **/
+ inline bool hasRebuildFinished(const std::size_t operator_index,
+ const std::size_t num_shiftbosses) const {
+ DCHECK_LT(operator_index, num_operators_);
+ const auto search_res = rebuild_status_.find(operator_index);
+ DCHECK(search_res != rebuild_status_.end());
+
+ const auto &rebuild_status = search_res->second;
+ DCHECK(rebuild_status.has_initiated);
+
+ return rebuild_status.rebuilt_shiftboss_indexes.size() == num_shiftbosses &&
+ rebuild_status.num_pending_workorders == 0u;
+ }
+
+#endif // QUICKSTEP_DISTRIBUTED
+
/**
* @brief Check if the rebuild has been initiated for the given operator.
*
@@ -314,6 +363,10 @@ class QueryExecutionState {
// The number of pending rebuild workorders for the operator.
// Valid if and only if 'has_initiated' is true.
std::size_t num_pending_workorders;
+
+#ifdef QUICKSTEP_DISTRIBUTED
+ std::unordered_set<std::size_t> rebuilt_shiftboss_indexes;
+#endif // QUICKSTEP_DISTRIBUTED
};
// Key is dag_node_index for which rebuild is required.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c4f4b285/query_execution/QueryManagerDistributed.cpp
----------------------------------------------------------------------
diff --git a/query_execution/QueryManagerDistributed.cpp b/query_execution/QueryManagerDistributed.cpp
index 7d45933..20650d0 100644
--- a/query_execution/QueryManagerDistributed.cpp
+++ b/query_execution/QueryManagerDistributed.cpp
@@ -36,7 +36,9 @@
#include "glog/logging.h"
+#include "tmb/address.h"
#include "tmb/id_typedefs.h"
+#include "tmb/tagged_message.h"
using std::free;
using std::malloc;
@@ -125,16 +127,16 @@ bool QueryManagerDistributed::fetchNormalWorkOrders(const dag_node_index index)
}
void QueryManagerDistributed::processInitiateRebuildResponseMessage(const dag_node_index op_index,
- const std::size_t num_rebuild_work_orders) {
- // TODO(zuyu): Multiple Shiftbosses support.
- query_exec_state_->setRebuildStatus(op_index, num_rebuild_work_orders, true);
+ const std::size_t num_rebuild_work_orders,
+ const std::size_t shiftboss_index) {
+ query_exec_state_->updateRebuildStatus(op_index, num_rebuild_work_orders, shiftboss_index);
- if (num_rebuild_work_orders != 0u) {
+ if (!query_exec_state_->hasRebuildFinished(op_index, shiftboss_directory_->size())) {
// Wait for the rebuild work orders to finish.
return;
}
- // No needs for rebuilds.
+ // No needs for rebuilds, or the rebuild has finished.
markOperatorFinished(op_index);
for (const std::pair<dag_node_index, bool> &dependent_link :
@@ -168,17 +170,20 @@ bool QueryManagerDistributed::initiateRebuild(const dag_node_index index) {
kInitiateRebuildMessage);
free(proto_bytes);
+ // TODO(quickstep-team): Dynamically scale-up/down Shiftbosses.
+ tmb::Address shiftboss_addresses;
+ for (std::size_t i = 0; i < shiftboss_directory_->size(); ++i) {
+ shiftboss_addresses.AddRecipient(shiftboss_directory_->getClientId(i));
+ }
+
LOG(INFO) << "ForemanDistributed sent InitiateRebuildMessage (typed '" << kInitiateRebuildMessage
- << "') to Shiftboss";
- // TODO(zuyu): Multiple workers support.
- QueryExecutionUtil::SendTMBMessage(bus_,
- foreman_client_id_,
- shiftboss_directory_->getClientId(0),
- move(tagged_msg));
-
- // The negative value indicates that the number of rebuild work orders is to be
- // determined.
- query_exec_state_->setRebuildStatus(index, -1, true);
+ << "') to all Shiftbosses";
+ QueryExecutionUtil::BroadcastMessage(foreman_client_id_,
+ shiftboss_addresses,
+ move(tagged_msg),
+ bus_);
+
+ query_exec_state_->setRebuildStatus(index, 0, true);
// Wait for Shiftbosses to report the number of rebuild work orders.
return false;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c4f4b285/query_execution/QueryManagerDistributed.hpp
----------------------------------------------------------------------
diff --git a/query_execution/QueryManagerDistributed.hpp b/query_execution/QueryManagerDistributed.hpp
index e609ab8..f8ac53c 100644
--- a/query_execution/QueryManagerDistributed.hpp
+++ b/query_execution/QueryManagerDistributed.hpp
@@ -73,9 +73,11 @@ class QueryManagerDistributed final : public QueryManagerBase {
* for initiating the rebuild work order.
* @param num_rebuild_work_orders The number of the rebuild work orders
* generated for the operator indexed by 'op_index'.
+ * @param shiftboss_index The index of the Shiftboss that sends the message.
**/
void processInitiateRebuildResponseMessage(const dag_node_index op_index,
- const std::size_t num_rebuild_work_orders);
+ const std::size_t num_rebuild_work_orders,
+ const std::size_t shiftboss_index);
/**
* @brief Get the next normal workorder to be excuted, wrapped in a
[42/50] incubator-quickstep git commit: Scheduling based on data
locality info.
Posted by zu...@apache.org.
Scheduling based on data locality info.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/bc4086be
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/bc4086be
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/bc4086be
Branch: refs/heads/quickstep_partition_parser_support
Commit: bc4086be44bb400f29ae9cada17b951241040bee
Parents: 0859a17
Author: Zuyu Zhang <zu...@apache.org>
Authored: Mon Dec 5 00:13:59 2016 -0800
Committer: Zuyu Zhang <zu...@apache.org>
Committed: Mon Dec 5 00:13:59 2016 -0800
----------------------------------------------------------------------
query_execution/BlockLocator.cpp | 53 +++++++++++++++++---
query_execution/BlockLocator.hpp | 53 ++++++++++++++++++++
query_execution/CMakeLists.txt | 3 ++
query_execution/ForemanDistributed.cpp | 53 +++++++++++++++++++-
query_execution/ForemanDistributed.hpp | 13 +++--
query_execution/QueryExecutionMessages.proto | 6 +++
query_execution/QueryExecutionTypedefs.hpp | 1 +
query_execution/Shiftboss.cpp | 1 +
.../DistributedExecutionGeneratorTestRunner.cpp | 2 +-
storage/StorageManager.cpp | 29 +++++++++++
storage/StorageManager.hpp | 9 ++++
11 files changed, 210 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bc4086be/query_execution/BlockLocator.cpp
----------------------------------------------------------------------
diff --git a/query_execution/BlockLocator.cpp b/query_execution/BlockLocator.cpp
index 5de6a54..fa6db51 100644
--- a/query_execution/BlockLocator.cpp
+++ b/query_execution/BlockLocator.cpp
@@ -27,6 +27,7 @@
#include "query_execution/QueryExecutionTypedefs.hpp"
#include "query_execution/QueryExecutionUtil.hpp"
#include "storage/StorageBlockInfo.hpp"
+#include "threading/SpinSharedMutex.hpp"
#include "threading/ThreadUtil.hpp"
#include "glog/logging.h"
@@ -65,6 +66,18 @@ void BlockLocator::run() {
processBlockDomainRegistrationMessage(sender, proto.domain_network_address());
break;
}
+ case kBlockDomainToShiftbossIndexMessage: {
+ serialization::BlockDomainToShiftbossIndexMessage proto;
+ CHECK(proto.ParseFromArray(tagged_message.message(), tagged_message.message_bytes()));
+
+ {
+ // Lock 'block_domain_to_shiftboss_index_shared_mutex_' as briefly as
+ // possible to insert an entry for the new Shiftboss index.
+ SpinSharedMutexExclusiveLock<false> write_lock(block_domain_to_shiftboss_index_shared_mutex_);
+ block_domain_to_shiftboss_index_.emplace(proto.block_domain(), proto.shiftboss_index());
+ }
+ break;
+ }
case kAddBlockLocationMessage: {
serialization::BlockLocationMessage proto;
CHECK(proto.ParseFromArray(tagged_message.message(), tagged_message.message_bytes()));
@@ -72,9 +85,15 @@ void BlockLocator::run() {
const block_id block = proto.block_id();
const block_id_domain domain = proto.block_domain();
- const auto result_block_locations = block_locations_[block].insert(domain);
const auto result_domain_blocks = domain_blocks_[domain].insert(block);
- DCHECK_EQ(result_block_locations.second, result_domain_blocks.second);
+
+ {
+ // Lock 'block_locations_shared_mutex_' as briefly as possible to
+ // insert an entry for the new block location.
+ SpinSharedMutexExclusiveLock<false> write_lock(block_locations_shared_mutex_);
+ const auto result_block_locations = block_locations_[block].insert(domain);
+ DCHECK_EQ(result_block_locations.second, result_domain_blocks.second);
+ }
if (result_domain_blocks.second) {
DLOG(INFO) << "Block " << BlockIdUtil::ToString(block) << " loaded in Domain " << domain;
@@ -90,11 +109,20 @@ void BlockLocator::run() {
const block_id block = proto.block_id();
const block_id_domain domain = proto.block_domain();
- const auto cit = block_locations_[block].find(domain);
- if (cit != block_locations_[block].end()) {
- block_locations_[block].erase(domain);
- domain_blocks_[domain].erase(block);
+ bool block_found = false;
+ {
+ // Lock 'block_locations_shared_mutex_' as briefly as possible to
+ // delete an entry for the block location.
+ SpinSharedMutexExclusiveLock<false> write_lock(block_locations_shared_mutex_);
+ const auto cit = block_locations_[block].find(domain);
+ if (cit != block_locations_[block].end()) {
+ block_locations_[block].erase(domain);
+ block_found = true;
+ }
+ }
+ if (block_found) {
+ domain_blocks_[domain].erase(block);
DLOG(INFO) << "Block " << BlockIdUtil::ToString(block) << " evicted in Domain " << domain;
} else {
DLOG(INFO) << "Block " << BlockIdUtil::ToString(block) << " not found in Domain " << domain;
@@ -123,8 +151,13 @@ void BlockLocator::run() {
domain_network_addresses_.erase(domain);
- for (const block_id block : domain_blocks_[domain]) {
- block_locations_[block].erase(domain);
+ {
+ // Lock 'block_locations_shared_mutex_' as briefly as possible to
+ // delete all entry for the block domain.
+ SpinSharedMutexExclusiveLock<false> write_lock(block_locations_shared_mutex_);
+ for (const block_id block : domain_blocks_[domain]) {
+ block_locations_[block].erase(domain);
+ }
}
domain_blocks_.erase(domain);
@@ -172,6 +205,8 @@ void BlockLocator::processLocateBlockMessage(const client_id receiver,
const block_id block) {
serialization::LocateBlockResponseMessage proto;
+ // NOTE(zuyu): We don't need to protect here, as all the writers are in the
+ // single thread.
for (const block_id_domain domain : block_locations_[block]) {
proto.add_block_domains(domain);
}
@@ -199,6 +234,8 @@ void BlockLocator::processGetPeerDomainNetworkAddressesMessage(const client_id r
const block_id block) {
serialization::GetPeerDomainNetworkAddressesResponseMessage proto;
+ // NOTE(zuyu): We don't need to protect here, as all the writers are in the
+ // single thread.
for (const block_id_domain domain : block_locations_[block]) {
proto.add_domain_network_addresses(domain_network_addresses_[domain]);
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bc4086be/query_execution/BlockLocator.hpp
----------------------------------------------------------------------
diff --git a/query_execution/BlockLocator.hpp b/query_execution/BlockLocator.hpp
index a83a394..4690369 100644
--- a/query_execution/BlockLocator.hpp
+++ b/query_execution/BlockLocator.hpp
@@ -21,6 +21,7 @@
#define QUICKSTEP_QUERY_EXECUTION_BLOCK_LOCATOR_HPP_
#include <atomic>
+#include <cstddef>
#include <string>
#include <unordered_map>
#include <unordered_set>
@@ -28,6 +29,7 @@
#include "query_execution/QueryExecutionTypedefs.hpp"
#include "storage/StorageBlockInfo.hpp"
#include "storage/StorageConstants.hpp"
+#include "threading/SpinSharedMutex.hpp"
#include "threading/Thread.hpp"
#include "utility/Macros.hpp"
@@ -67,6 +69,8 @@ class BlockLocator : public Thread {
bus_->RegisterClientAsReceiver(locator_client_id_, kBlockDomainRegistrationMessage);
bus_->RegisterClientAsSender(locator_client_id_, kBlockDomainRegistrationResponseMessage);
+ bus_->RegisterClientAsReceiver(locator_client_id_, kBlockDomainToShiftbossIndexMessage);
+
bus_->RegisterClientAsReceiver(locator_client_id_, kAddBlockLocationMessage);
bus_->RegisterClientAsReceiver(locator_client_id_, kDeleteBlockLocationMessage);
@@ -91,6 +95,46 @@ class BlockLocator : public Thread {
return locator_client_id_;
}
+ /**
+ * @brief Get the block locality info for scheduling in ForemanDistributed.
+ *
+ * @param block The given block.
+ * @param shiftboss_index_for_block The index of Shiftboss that has loaded the
+ * block in the buffer pool.
+ *
+ * @return Whether the block locality info has found.
+ **/
+ bool getBlockLocalityInfo(const block_id block, std::size_t *shiftboss_index_for_block) const {
+ std::unordered_set<block_id_domain> block_domains;
+ {
+ // Lock 'block_locations_shared_mutex_' as briefly as possible as a
+ // reader.
+ SpinSharedMutexSharedLock<false> read_lock(block_locations_shared_mutex_);
+ const auto cit = block_locations_.find(block);
+ if (cit != block_locations_.end()) {
+ block_domains = cit->second;
+ } else {
+ return false;
+ }
+ }
+
+ {
+ // NOTE(zuyu): This lock is held for the rest duration of this call, as the
+ // exclusive case is rare.
+ SpinSharedMutexSharedLock<false> read_lock(block_domain_to_shiftboss_index_shared_mutex_);
+ for (const block_id_domain block_domain : block_domains) {
+ // TODO(quickstep-team): choose the best node, instead of the first.
+ const auto cit = block_domain_to_shiftboss_index_.find(block_domain);
+ if (cit != block_domain_to_shiftboss_index_.end()) {
+ *shiftboss_index_for_block = cit->second;
+ return true;
+ }
+ }
+ }
+
+ return false;
+ }
+
protected:
void run() override;
@@ -110,8 +154,17 @@ class BlockLocator : public Thread {
// "0.0.0.0:0".
std::unordered_map<block_id_domain, const std::string> domain_network_addresses_;
+ // From a block domain to its Shiftboss index, used by ForemanDistributed
+ // to schedule based on the data-locality info.
+ // Note that not every 'block_id_domain' has a Shiftboss index. For example,
+ // DistributedCli has StorageManager with a 'block_id_domain', which is not
+ // a part of Shiftboss.
+ std::unordered_map<block_id_domain, std::size_t> block_domain_to_shiftboss_index_;
+ alignas(kCacheLineBytes) mutable SpinSharedMutex<false> block_domain_to_shiftboss_index_shared_mutex_;
+
// From a block to its domains.
std::unordered_map<block_id, std::unordered_set<block_id_domain>> block_locations_;
+ alignas(kCacheLineBytes) mutable SpinSharedMutex<false> block_locations_shared_mutex_;
// From a block domain to all blocks loaded in its buffer pool.
std::unordered_map<block_id_domain, std::unordered_set<block_id>> domain_blocks_;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bc4086be/query_execution/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_execution/CMakeLists.txt b/query_execution/CMakeLists.txt
index 1f7add8..0f74384 100644
--- a/query_execution/CMakeLists.txt
+++ b/query_execution/CMakeLists.txt
@@ -80,6 +80,7 @@ if (ENABLE_DISTRIBUTED)
quickstep_queryexecution_QueryExecutionUtil
quickstep_storage_StorageBlockInfo
quickstep_storage_StorageConstants
+ quickstep_threading_SpinSharedMutex
quickstep_threading_Thread
quickstep_threading_ThreadUtil
quickstep_utility_Macros
@@ -105,6 +106,7 @@ if (ENABLE_DISTRIBUTED)
quickstep_catalog_CatalogTypedefs
quickstep_catalog_Catalog_proto
quickstep_queryexecution_AdmitRequestMessage
+ quickstep_queryexecution_BlockLocator
quickstep_queryexecution_ForemanBase
quickstep_queryexecution_PolicyEnforcerBase
quickstep_queryexecution_PolicyEnforcerDistributed
@@ -114,6 +116,7 @@ if (ENABLE_DISTRIBUTED)
quickstep_queryexecution_QueryExecutionUtil
quickstep_queryexecution_ShiftbossDirectory
quickstep_relationaloperators_WorkOrder_proto
+ quickstep_storage_StorageBlockInfo
quickstep_threading_ThreadUtil
quickstep_utility_EqualsAnyConstant
quickstep_utility_Macros
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bc4086be/query_execution/ForemanDistributed.cpp
----------------------------------------------------------------------
diff --git a/query_execution/ForemanDistributed.cpp b/query_execution/ForemanDistributed.cpp
index 0dad8b0..0fa701d 100644
--- a/query_execution/ForemanDistributed.cpp
+++ b/query_execution/ForemanDistributed.cpp
@@ -28,6 +28,7 @@
#include "catalog/CatalogRelation.hpp"
#include "catalog/CatalogTypedefs.hpp"
#include "query_execution/AdmitRequestMessage.hpp"
+#include "query_execution/BlockLocator.hpp"
#include "query_execution/PolicyEnforcerBase.hpp"
#include "query_execution/PolicyEnforcerDistributed.hpp"
#include "query_execution/QueryContext.hpp"
@@ -36,6 +37,7 @@
#include "query_execution/QueryExecutionUtil.hpp"
#include "query_execution/ShiftbossDirectory.hpp"
#include "relational_operators/WorkOrder.pb.h"
+#include "storage/StorageBlockInfo.hpp"
#include "threading/ThreadUtil.hpp"
#include "utility/EqualsAnyConstant.hpp"
@@ -64,10 +66,12 @@ namespace S = serialization;
class QueryHandle;
ForemanDistributed::ForemanDistributed(
+ const BlockLocator &block_locator,
MessageBus *bus,
CatalogDatabaseLite *catalog_database,
const int cpu_id)
: ForemanBase(bus, cpu_id),
+ block_locator_(block_locator),
catalog_database_(DCHECK_NOTNULL(catalog_database)) {
const std::vector<QueryExecutionMessageType> sender_message_types{
kShiftbossRegistrationResponseMessage,
@@ -296,7 +300,50 @@ bool ForemanDistributed::isHashJoinRelatedWorkOrder(const S::WorkOrderMessage &p
}
namespace {
+
constexpr size_t kDefaultShiftbossIndex = 0u;
+
+bool isNestedLoopsJoinWorkOrder(const serialization::WorkOrder &work_order_proto,
+ const BlockLocator &block_locator,
+ std::size_t *shiftboss_index_for_join) {
+ if (work_order_proto.work_order_type() != S::NESTED_LOOP_JOIN) {
+ return false;
+ }
+
+ const block_id left_block = work_order_proto.GetExtension(S::NestedLoopsJoinWorkOrder::left_block_id);
+ if (block_locator.getBlockLocalityInfo(left_block, shiftboss_index_for_join)) {
+ return true;
+ }
+
+ const block_id right_block = work_order_proto.GetExtension(S::NestedLoopsJoinWorkOrder::right_block_id);
+ return block_locator.getBlockLocalityInfo(right_block, shiftboss_index_for_join);
+}
+
+bool hasBlockLocalityInfo(const serialization::WorkOrder &work_order_proto,
+ const BlockLocator &block_locator,
+ std::size_t *shiftboss_index_for_block) {
+ block_id block = kInvalidBlockId;
+ switch (work_order_proto.work_order_type()) {
+ case S::SAVE_BLOCKS: {
+ block = work_order_proto.GetExtension(S::SaveBlocksWorkOrder::block_id);
+ break;
+ }
+ case S::SELECT: {
+ block = work_order_proto.GetExtension(S::SelectWorkOrder::block_id);
+ break;
+ }
+ case S::SORT_RUN_GENERATION: {
+ block = work_order_proto.GetExtension(S::SortRunGenerationWorkOrder::block_id);
+ break;
+ }
+ default:
+ return false;
+ }
+
+ DCHECK_NE(block, kInvalidBlockId);
+ return block_locator.getBlockLocalityInfo(block, shiftboss_index_for_block);
+}
+
} // namespace
void ForemanDistributed::dispatchWorkOrderMessages(const vector<unique_ptr<S::WorkOrderMessage>> &messages) {
@@ -306,14 +353,18 @@ void ForemanDistributed::dispatchWorkOrderMessages(const vector<unique_ptr<S::Wo
for (const auto &message : messages) {
DCHECK(message != nullptr);
const S::WorkOrderMessage &proto = *message;
+ const S::WorkOrder &work_order_proto = proto.work_order();
size_t shiftboss_index_for_particular_work_order_type;
if (policy_enforcer_dist->isSingleNodeQuery(proto.query_id())) {
// Always schedule the single-node query to the same Shiftboss.
shiftboss_index_for_particular_work_order_type = kDefaultShiftbossIndex;
} else if (isAggregationRelatedWorkOrder(proto, shiftboss_index, &shiftboss_index_for_particular_work_order_type)) {
} else if (isHashJoinRelatedWorkOrder(proto, shiftboss_index, &shiftboss_index_for_particular_work_order_type)) {
+ } else if (hasBlockLocalityInfo(work_order_proto, block_locator_,
+ &shiftboss_index_for_particular_work_order_type)) {
+ } else if (isNestedLoopsJoinWorkOrder(work_order_proto, block_locator_,
+ &shiftboss_index_for_particular_work_order_type)) {
} else {
- // TODO(zuyu): Take data-locality into account for scheduling.
shiftboss_index_for_particular_work_order_type = shiftboss_index;
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bc4086be/query_execution/ForemanDistributed.hpp
----------------------------------------------------------------------
diff --git a/query_execution/ForemanDistributed.hpp b/query_execution/ForemanDistributed.hpp
index 34bac07..ed09fda 100644
--- a/query_execution/ForemanDistributed.hpp
+++ b/query_execution/ForemanDistributed.hpp
@@ -33,6 +33,7 @@ namespace tmb { class MessageBus; }
namespace quickstep {
+class BlockLocator;
class CatalogDatabaseLite;
namespace serialization { class WorkOrderMessage; }
@@ -51,6 +52,7 @@ class ForemanDistributed final : public ForemanBase {
/**
* @brief Constructor.
*
+ * @param block_locator The block locator that manages block location info.
* @param bus A pointer to the TMB.
* @param catalog_database The catalog database where this query is executed.
* @param cpu_id The ID of the CPU to which the Foreman thread can be pinned.
@@ -58,9 +60,11 @@ class ForemanDistributed final : public ForemanBase {
* @note If cpu_id is not specified, Foreman thread can be possibly moved
* around on different CPUs by the OS.
**/
- ForemanDistributed(tmb::MessageBus *bus,
- CatalogDatabaseLite *catalog_database,
- const int cpu_id = -1);
+ ForemanDistributed(
+ const BlockLocator &block_locator,
+ tmb::MessageBus *bus,
+ CatalogDatabaseLite *catalog_database,
+ const int cpu_id = -1);
~ForemanDistributed() override {}
@@ -111,6 +115,9 @@ class ForemanDistributed final : public ForemanBase {
**/
bool canCollectNewMessages(const tmb::message_type_id message_type);
+ // To get block locality info for scheduling.
+ const BlockLocator &block_locator_;
+
ShiftbossDirectory shiftboss_directory_;
CatalogDatabaseLite *catalog_database_;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bc4086be/query_execution/QueryExecutionMessages.proto
----------------------------------------------------------------------
diff --git a/query_execution/QueryExecutionMessages.proto b/query_execution/QueryExecutionMessages.proto
index e6d741a..93e458c 100644
--- a/query_execution/QueryExecutionMessages.proto
+++ b/query_execution/QueryExecutionMessages.proto
@@ -142,6 +142,12 @@ message BlockDomainMessage {
required uint32 block_domain = 1;
}
+// Used for the block locality based scheduling in ForemanDistributed.
+message BlockDomainToShiftbossIndexMessage {
+ required uint32 block_domain = 1;
+ required uint64 shiftboss_index = 2;
+}
+
// Used when StorageManager loads or evicts a block or a blob from its buffer
// pool.
message BlockLocationMessage {
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bc4086be/query_execution/QueryExecutionTypedefs.hpp
----------------------------------------------------------------------
diff --git a/query_execution/QueryExecutionTypedefs.hpp b/query_execution/QueryExecutionTypedefs.hpp
index fb9a9d6..919e45b 100644
--- a/query_execution/QueryExecutionTypedefs.hpp
+++ b/query_execution/QueryExecutionTypedefs.hpp
@@ -99,6 +99,7 @@ enum QueryExecutionMessageType : message_type_id {
// with a unique block domain.
kBlockDomainRegistrationMessage, // From Worker to BlockLocator.
kBlockDomainRegistrationResponseMessage, // From BlockLocator to Worker.
+ kBlockDomainToShiftbossIndexMessage, // From StorageManager to BlockLocator.
kAddBlockLocationMessage, // From StorageManager to BlockLocator.
kDeleteBlockLocationMessage, // From StorageManager to BlockLocator.
kLocateBlockMessage, // From StorageManager to BlockLocator.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bc4086be/query_execution/Shiftboss.cpp
----------------------------------------------------------------------
diff --git a/query_execution/Shiftboss.cpp b/query_execution/Shiftboss.cpp
index ed4bade..2ed42d0 100644
--- a/query_execution/Shiftboss.cpp
+++ b/query_execution/Shiftboss.cpp
@@ -286,6 +286,7 @@ void Shiftboss::processShiftbossRegistrationResponseMessage() {
CHECK(proto.ParseFromArray(tagged_message.message(), tagged_message.message_bytes()));
shiftboss_index_ = proto.shiftboss_index();
+ storage_manager_->sendBlockDomainToShiftbossIndexMessage(shiftboss_index_);
// Forward this message to Workers regarding <shiftboss_index_>.
QueryExecutionUtil::BroadcastMessage(shiftboss_client_id_,
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bc4086be/query_optimizer/tests/DistributedExecutionGeneratorTestRunner.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/tests/DistributedExecutionGeneratorTestRunner.cpp b/query_optimizer/tests/DistributedExecutionGeneratorTestRunner.cpp
index 5100651..45d4fdf 100644
--- a/query_optimizer/tests/DistributedExecutionGeneratorTestRunner.cpp
+++ b/query_optimizer/tests/DistributedExecutionGeneratorTestRunner.cpp
@@ -98,7 +98,7 @@ DistributedExecutionGeneratorTestRunner::DistributedExecutionGeneratorTestRunner
// NOTE(zuyu): Foreman should initialize before Shiftboss so that the former
// could receive a registration message from the latter.
- foreman_ = make_unique<ForemanDistributed>(&bus_, test_database_loader_->catalog_database());
+ foreman_ = make_unique<ForemanDistributed>(*block_locator_, &bus_, test_database_loader_->catalog_database());
// We don't use the NUMA aware version of worker code.
const vector<numa_node_id> numa_nodes(1 /* Number of worker threads per instance */,
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bc4086be/storage/StorageManager.cpp
----------------------------------------------------------------------
diff --git a/storage/StorageManager.cpp b/storage/StorageManager.cpp
index a202f71..6299cda 100644
--- a/storage/StorageManager.cpp
+++ b/storage/StorageManager.cpp
@@ -230,6 +230,8 @@ StorageManager::StorageManager(
bus_->RegisterClientAsSender(storage_manager_client_id_, kGetPeerDomainNetworkAddressesMessage);
bus_->RegisterClientAsReceiver(storage_manager_client_id_, kGetPeerDomainNetworkAddressesResponseMessage);
+ bus_->RegisterClientAsSender(storage_manager_client_id_, kBlockDomainToShiftbossIndexMessage);
+
bus_->RegisterClientAsSender(storage_manager_client_id_, kAddBlockLocationMessage);
bus_->RegisterClientAsSender(storage_manager_client_id_, kDeleteBlockLocationMessage);
bus_->RegisterClientAsSender(storage_manager_client_id_, kBlockDomainUnregistrationMessage);
@@ -470,6 +472,33 @@ block_id StorageManager::allocateNewBlockOrBlob(const std::size_t num_slots,
}
#ifdef QUICKSTEP_DISTRIBUTED
+void StorageManager::sendBlockDomainToShiftbossIndexMessage(const std::size_t shiftboss_index) {
+ serialization::BlockDomainToShiftbossIndexMessage proto;
+ proto.set_block_domain(block_domain_);
+ proto.set_shiftboss_index(shiftboss_index);
+
+ const int proto_length = proto.ByteSize();
+ char *proto_bytes = static_cast<char*>(malloc(proto_length));
+ CHECK(proto.SerializeToArray(proto_bytes, proto_length));
+
+ TaggedMessage message(static_cast<const void*>(proto_bytes),
+ proto_length,
+ kBlockDomainToShiftbossIndexMessage);
+ free(proto_bytes);
+
+ DLOG(INFO) << "StorageManager (id '" << storage_manager_client_id_
+ << "') sent BlockDomainToShiftbossIndexMessage (typed '" << kBlockDomainToShiftbossIndexMessage
+ << "') to BlockLocator";
+
+ DCHECK_NE(block_locator_client_id_, tmb::kClientIdNone);
+ DCHECK(bus_ != nullptr);
+ CHECK(MessageBus::SendStatus::kOK ==
+ QueryExecutionUtil::SendTMBMessage(bus_,
+ storage_manager_client_id_,
+ block_locator_client_id_,
+ move(message)));
+}
+
void StorageManager::pullBlockOrBlob(const block_id block,
PullResponse *response) const {
SpinSharedMutexSharedLock<false> read_lock(blocks_shared_mutex_);
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bc4086be/storage/StorageManager.hpp
----------------------------------------------------------------------
diff --git a/storage/StorageManager.hpp b/storage/StorageManager.hpp
index 066953b..b61f10a 100644
--- a/storage/StorageManager.hpp
+++ b/storage/StorageManager.hpp
@@ -382,6 +382,15 @@ class StorageManager {
#ifdef QUICKSTEP_DISTRIBUTED
/**
+ * @brief Send BlockDomainToShiftbossIndexMessage to BlockLocator so that
+ * ForemanDistributed could take advantages of block locality info
+ * for a better scheduling policy.
+ *
+ * @param shiftboss_index The Shiftboss index.
+ **/
+ void sendBlockDomainToShiftbossIndexMessage(const std::size_t shiftboss_index);
+
+ /**
* @brief Pull a block or a blob. Used by DataExchangerAsync.
*
* @param block The id of the block or blob.
[31/50] incubator-quickstep git commit: Set Default block type to
only be the splitrow store.
Posted by zu...@apache.org.
Set Default block type to only be the splitrow store.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/1b27888c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/1b27888c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/1b27888c
Branch: refs/heads/quickstep_partition_parser_support
Commit: 1b27888c3ced7ff6b104893cc6fe1a02d5cb33b2
Parents: 2d11ec5
Author: cramja <ma...@gmail.com>
Authored: Thu Nov 17 16:12:55 2016 -0600
Committer: cramja <ma...@gmail.com>
Committed: Mon Nov 21 14:28:02 2016 -0600
----------------------------------------------------------------------
storage/StorageBlockLayout.cpp | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1b27888c/storage/StorageBlockLayout.cpp
----------------------------------------------------------------------
diff --git a/storage/StorageBlockLayout.cpp b/storage/StorageBlockLayout.cpp
index bdeb9cb..3b1bcc0 100644
--- a/storage/StorageBlockLayout.cpp
+++ b/storage/StorageBlockLayout.cpp
@@ -148,17 +148,17 @@ void StorageBlockLayout::copyHeaderTo(void *dest) const {
StorageBlockLayout* StorageBlockLayout::GenerateDefaultLayout(const CatalogRelationSchema &relation,
const bool relation_variable_length) {
+ // TODO(marc): In the future we may use relation_variable_length (columnstores as intermediate
+ // blocks), but for now, all we do is add (void) so that the compiler will not complain
+ // about an unused variable.
+ (void) relation_variable_length;
StorageBlockLayout *layout = new StorageBlockLayout(relation);
StorageBlockLayoutDescription *description = layout->getDescriptionMutable();
description->set_num_slots(1);
TupleStorageSubBlockDescription *tuple_store_description = description->mutable_tuple_store_description();
- if (relation_variable_length) {
- tuple_store_description->set_sub_block_type(TupleStorageSubBlockDescription::SPLIT_ROW_STORE);
- } else {
- tuple_store_description->set_sub_block_type(TupleStorageSubBlockDescription::PACKED_ROW_STORE);
- }
+ tuple_store_description->set_sub_block_type(TupleStorageSubBlockDescription::SPLIT_ROW_STORE);
layout->finalize();
return layout;
[17/50] incubator-quickstep git commit: Maintained the catalog
filename as a constant.
Posted by zu...@apache.org.
Maintained the catalog filename as a constant.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/37a78cb1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/37a78cb1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/37a78cb1
Branch: refs/heads/quickstep_partition_parser_support
Commit: 37a78cb112da2df57973fcef054dba77bd9d9ee1
Parents: 24367d7
Author: Zuyu Zhang <zu...@apache.org>
Authored: Sat Nov 19 11:59:13 2016 -0800
Committer: Zuyu Zhang <zu...@apache.org>
Committed: Sun Nov 20 19:27:54 2016 -0800
----------------------------------------------------------------------
BUILDING.md | 8 ++------
cli/QuickstepCli.cpp | 7 ++++---
storage/StorageConstants.hpp | 2 ++
3 files changed, 8 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/37a78cb1/BUILDING.md
----------------------------------------------------------------------
diff --git a/BUILDING.md b/BUILDING.md
index 581fbdd..97552c6 100644
--- a/BUILDING.md
+++ b/BUILDING.md
@@ -190,16 +190,12 @@ number of jobs).
Running Quickstep
-----------------
-To use quickstep, just run `quickstep_cli_shell` in the build directory.
+To use quickstep, just run `quickstep_cli_shell` in the build directory. For the
+first time user, run once with `-initialize_db=true` to set up an empty catalog.
Quickstep has number of command-line flags that control its behavior. Run
`quickstep_cli_shell --help` to see a listing of the options and how to use
them.
-The provided build directory has a skeleton qsstor directory with a fresh
-catalog.pb.bin file which is needed to run `quickstep_cli_shell`. If you want
-to wipe out your database and start fresh, delete all the files in qsstor and
-copy `catalog.pb.bin.orig` to `qsstor/catalog.pb.bin`.
-
Running Tests
-------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/37a78cb1/cli/QuickstepCli.cpp
----------------------------------------------------------------------
diff --git a/cli/QuickstepCli.cpp b/cli/QuickstepCli.cpp
index f17e18b..0e8468e 100644
--- a/cli/QuickstepCli.cpp
+++ b/cli/QuickstepCli.cpp
@@ -125,6 +125,7 @@ using quickstep::Worker;
using quickstep::WorkerDirectory;
using quickstep::WorkerMessage;
using quickstep::kAdmitRequestMessage;
+using quickstep::kCatalogFilename;
using quickstep::kPoisonMessage;
using quickstep::kWorkloadCompletionMessage;
@@ -243,7 +244,7 @@ int main(int argc, char* argv[]) {
quickstep::StorageManager storage_manager(fixed_storage_path);
string catalog_path(fixed_storage_path);
- catalog_path.append("catalog.pb.bin");
+ catalog_path.append(kCatalogFilename);
if (quickstep::FLAGS_initialize_db) { // Initialize the database
// TODO(jmp): Refactor the code in this file!
LOG(INFO) << "Initializing the database, creating a new catalog file and storage directory\n";
@@ -267,14 +268,14 @@ int main(int argc, char* argv[]) {
// Create the default catalog file.
std::ofstream catalog_file(catalog_path);
if (!catalog_file.good()) {
- LOG(FATAL) << "ERROR: Unable to open catalog.pb.bin for writing.\n";
+ LOG(FATAL) << "ERROR: Unable to open " << kCatalogFilename << " for writing.\n";
}
quickstep::Catalog catalog;
catalog.addDatabase(new quickstep::CatalogDatabase(nullptr, "default"));
if (!catalog.getProto().SerializeToOstream(&catalog_file)) {
- LOG(FATAL) << "ERROR: Unable to serialize catalog proto to file catalog.pb.bin\n";
+ LOG(FATAL) << "ERROR: Unable to serialize catalog proto to file " << kCatalogFilename;
return 1;
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/37a78cb1/storage/StorageConstants.hpp
----------------------------------------------------------------------
diff --git a/storage/StorageConstants.hpp b/storage/StorageConstants.hpp
index 34a714d..037a8a9 100644
--- a/storage/StorageConstants.hpp
+++ b/storage/StorageConstants.hpp
@@ -39,6 +39,8 @@ constexpr char kPathSeparator = '/';
constexpr char kDefaultStoragePath[] = "qsstor/";
#endif
+constexpr char kCatalogFilename[] = "catalog.pb.bin";
+
// Size of a memory slot managed by the StorageManager. This is the smallest
// quantum of allocation for StorageBlocks and StorageBlobs. 2 MB is the large
// page size on x86.
[13/50] incubator-quickstep git commit: Refactor WorkOrderFactory for
a better debug info.
Posted by zu...@apache.org.
Refactor WorkOrderFactory for a better debug info.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/70959876
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/70959876
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/70959876
Branch: refs/heads/quickstep_partition_parser_support
Commit: 70959876590fb65b8f195a77d7af62e33b166dd4
Parents: 365fff6
Author: Zuyu Zhang <zu...@apache.org>
Authored: Thu Nov 17 15:09:17 2016 -0800
Committer: Zuyu Zhang <zu...@apache.org>
Committed: Fri Nov 18 11:30:44 2016 -0800
----------------------------------------------------------------------
query_execution/Shiftboss.cpp | 1 +
relational_operators/WorkOrderFactory.cpp | 48 ++++++++++++++------------
relational_operators/WorkOrderFactory.hpp | 4 +++
3 files changed, 30 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/70959876/query_execution/Shiftboss.cpp
----------------------------------------------------------------------
diff --git a/query_execution/Shiftboss.cpp b/query_execution/Shiftboss.cpp
index 09d7846..30b2ae2 100644
--- a/query_execution/Shiftboss.cpp
+++ b/query_execution/Shiftboss.cpp
@@ -99,6 +99,7 @@ void Shiftboss::run() {
DCHECK_EQ(1u, query_contexts_.count(query_id));
WorkOrder *work_order = WorkOrderFactory::ReconstructFromProto(proto.work_order(),
+ shiftboss_index_,
&database_cache_,
query_contexts_[query_id].get(),
storage_manager_,
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/70959876/relational_operators/WorkOrderFactory.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/WorkOrderFactory.cpp b/relational_operators/WorkOrderFactory.cpp
index 6943a52..871f74d 100644
--- a/relational_operators/WorkOrderFactory.cpp
+++ b/relational_operators/WorkOrderFactory.cpp
@@ -19,6 +19,7 @@
#include "relational_operators/WorkOrderFactory.hpp"
+#include <cstddef>
#include <memory>
#include <utility>
#include <vector>
@@ -67,6 +68,7 @@ class Predicate;
class Scalar;
WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder &proto,
+ const std::size_t shiftboss_index,
CatalogDatabaseLite *catalog_database,
QueryContext *query_context,
StorageManager *storage_manager,
@@ -79,7 +81,7 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder
switch (proto.work_order_type()) {
case serialization::AGGREGATION: {
- LOG(INFO) << "Creating AggregationWorkOrder";
+ LOG(INFO) << "Creating AggregationWorkOrder in Shiftboss " << shiftboss_index;
return new AggregationWorkOrder(
proto.query_id(),
proto.GetExtension(serialization::AggregationWorkOrder::block_id),
@@ -89,7 +91,7 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder
proto.GetExtension(serialization::AggregationWorkOrder::lip_deployment_index), query_context));
}
case serialization::BUILD_HASH: {
- LOG(INFO) << "Creating BuildHashWorkOrder";
+ LOG(INFO) << "Creating BuildHashWorkOrder in Shiftboss " << shiftboss_index;
vector<attribute_id> join_key_attributes;
for (int i = 0; i < proto.ExtensionSize(serialization::BuildHashWorkOrder::join_key_attributes); ++i) {
join_key_attributes.push_back(
@@ -110,7 +112,7 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder
proto.GetExtension(serialization::BuildHashWorkOrder::lip_deployment_index), query_context));
}
case serialization::DELETE: {
- LOG(INFO) << "Creating DeleteWorkOrder";
+ LOG(INFO) << "Creating DeleteWorkOrder in Shiftboss " << shiftboss_index;
return new DeleteWorkOrder(
proto.query_id(),
catalog_database->getRelationSchemaById(
@@ -124,7 +126,7 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder
bus);
}
case serialization::DESTROY_AGGREGATION_STATE: {
- LOG(INFO) << "Creating DestroyAggregationStateWorkOrder";
+ LOG(INFO) << "Creating DestroyAggregationStateWorkOrder in Shiftboss " << shiftboss_index;
return new DestroyAggregationStateWorkOrder(
proto.query_id(),
proto.GetExtension(
@@ -132,7 +134,7 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder
query_context);
}
case serialization::DESTROY_HASH: {
- LOG(INFO) << "Creating DestroyHashWorkOrder";
+ LOG(INFO) << "Creating DestroyHashWorkOrder in Shiftboss " << shiftboss_index;
return new DestroyHashWorkOrder(
proto.query_id(),
proto.GetExtension(
@@ -140,7 +142,7 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder
query_context);
}
case serialization::DROP_TABLE: {
- LOG(INFO) << "Creating DropTableWorkOrder";
+ LOG(INFO) << "Creating DropTableWorkOrder in Shiftboss " << shiftboss_index;
vector<block_id> blocks;
for (int i = 0; i < proto.ExtensionSize(serialization::DropTableWorkOrder::block_ids); ++i) {
blocks.push_back(
@@ -157,7 +159,7 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder
catalog_database);
}
case serialization::FINALIZE_AGGREGATION: {
- LOG(INFO) << "Creating FinalizeAggregationWorkOrder";
+ LOG(INFO) << "Creating FinalizeAggregationWorkOrder in Shiftboss " << shiftboss_index;
return new FinalizeAggregationWorkOrder(
proto.query_id(),
query_context->getAggregationState(proto.GetExtension(
@@ -212,7 +214,7 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder
switch (hash_join_work_order_type) {
case serialization::HashJoinWorkOrder::HASH_ANTI_JOIN: {
- LOG(INFO) << "Creating HashAntiJoinWorkOrder";
+ LOG(INFO) << "Creating HashAntiJoinWorkOrder in Shiftboss " << shiftboss_index;
return new HashAntiJoinWorkOrder(
proto.query_id(),
build_relation,
@@ -228,7 +230,7 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder
lip_filter_adaptive_prober);
}
case serialization::HashJoinWorkOrder::HASH_INNER_JOIN: {
- LOG(INFO) << "Creating HashInnerJoinWorkOrder";
+ LOG(INFO) << "Creating HashInnerJoinWorkOrder in Shiftboss " << shiftboss_index;
return new HashInnerJoinWorkOrder(
proto.query_id(),
build_relation,
@@ -252,7 +254,7 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder
proto.GetExtension(serialization::HashJoinWorkOrder::is_selection_on_build, i));
}
- LOG(INFO) << "Creating HashOuterJoinWorkOrder";
+ LOG(INFO) << "Creating HashOuterJoinWorkOrder in Shiftboss " << shiftboss_index;
return new HashOuterJoinWorkOrder(
proto.query_id(),
build_relation,
@@ -268,7 +270,7 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder
lip_filter_adaptive_prober);
}
case serialization::HashJoinWorkOrder::HASH_SEMI_JOIN: {
- LOG(INFO) << "Creating HashSemiJoinWorkOrder";
+ LOG(INFO) << "Creating HashSemiJoinWorkOrder in Shiftboss " << shiftboss_index;
return new HashSemiJoinWorkOrder(
proto.query_id(),
build_relation,
@@ -288,7 +290,7 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder
}
}
case serialization::INSERT: {
- LOG(INFO) << "Creating InsertWorkOrder";
+ LOG(INFO) << "Creating InsertWorkOrder in Shiftboss " << shiftboss_index;
return new InsertWorkOrder(
proto.query_id(),
query_context->getInsertDestination(
@@ -297,7 +299,7 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder
proto.GetExtension(serialization::InsertWorkOrder::tuple_index)));
}
case serialization::NESTED_LOOP_JOIN: {
- LOG(INFO) << "Creating NestedLoopsJoinWorkOrder";
+ LOG(INFO) << "Creating NestedLoopsJoinWorkOrder in Shiftboss " << shiftboss_index;
return new NestedLoopsJoinWorkOrder(
proto.query_id(),
catalog_database->getRelationSchemaById(
@@ -315,7 +317,7 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder
storage_manager);
}
case serialization::SAMPLE: {
- LOG(INFO) << "Creating SampleWorkOrder";
+ LOG(INFO) << "Creating SampleWorkOrder in Shiftboss " << shiftboss_index;
return new SampleWorkOrder(
proto.query_id(),
catalog_database->getRelationSchemaById(
@@ -328,7 +330,7 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder
storage_manager);
}
case serialization::SAVE_BLOCKS: {
- LOG(INFO) << "Creating SaveBlocksWorkOrder";
+ LOG(INFO) << "Creating SaveBlocksWorkOrder in Shiftboss " << shiftboss_index;
return new SaveBlocksWorkOrder(
proto.query_id(),
proto.GetExtension(serialization::SaveBlocksWorkOrder::block_id),
@@ -336,7 +338,7 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder
storage_manager);
}
case serialization::SELECT: {
- LOG(INFO) << "Creating SelectWorkOrder";
+ LOG(INFO) << "Creating SelectWorkOrder in Shiftboss " << shiftboss_index;
const bool simple_projection =
proto.GetExtension(serialization::SelectWorkOrder::simple_projection);
vector<attribute_id> simple_selection;
@@ -364,7 +366,7 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder
proto.GetExtension(serialization::SelectWorkOrder::lip_deployment_index), query_context));
}
case serialization::SORT_MERGE_RUN: {
- LOG(INFO) << "Creating SortMergeRunWorkOrder";
+ LOG(INFO) << "Creating SortMergeRunWorkOrder in Shiftboss " << shiftboss_index;
vector<merge_run_operator::Run> runs;
for (int i = 0; i < proto.ExtensionSize(serialization::SortMergeRunWorkOrder::runs); ++i) {
merge_run_operator::Run run;
@@ -393,7 +395,7 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder
bus);
}
case serialization::SORT_RUN_GENERATION: {
- LOG(INFO) << "Creating SortRunGenerationWorkOrder";
+ LOG(INFO) << "Creating SortRunGenerationWorkOrder in Shiftboss " << shiftboss_index;
return new SortRunGenerationWorkOrder(
proto.query_id(),
catalog_database->getRelationSchemaById(
@@ -406,7 +408,7 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder
storage_manager);
}
case serialization::TABLE_GENERATOR: {
- LOG(INFO) << "Creating SortRunGenerationWorkOrder";
+ LOG(INFO) << "Creating SortRunGenerationWorkOrder in Shiftboss " << shiftboss_index;
return new TableGeneratorWorkOrder(
proto.query_id(),
query_context->getGeneratorFunctionHandle(
@@ -415,7 +417,7 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder
proto.GetExtension(serialization::TableGeneratorWorkOrder::insert_destination_index)));
}
case serialization::TEXT_SCAN: {
- LOG(INFO) << "Creating TextScanWorkOrder";
+ LOG(INFO) << "Creating TextScanWorkOrder in Shiftboss " << shiftboss_index;
return new TextScanWorkOrder(
proto.query_id(),
proto.GetExtension(serialization::TextScanWorkOrder::filename),
@@ -427,7 +429,7 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder
proto.GetExtension(serialization::TextScanWorkOrder::insert_destination_index)));
}
case serialization::UPDATE: {
- LOG(INFO) << "Creating UpdateWorkOrder";
+ LOG(INFO) << "Creating UpdateWorkOrder in Shiftboss " << shiftboss_index;
return new UpdateWorkOrder(
proto.query_id(),
catalog_database->getRelationSchemaById(
@@ -445,7 +447,7 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder
bus);
}
case serialization::WINDOW_AGGREGATION: {
- LOG(INFO) << "Creating WindowAggregationWorkOrder";
+ LOG(INFO) << "Creating WindowAggregationWorkOrder in Shiftboss " << shiftboss_index;
vector<block_id> blocks;
for (int i = 0; i < proto.ExtensionSize(serialization::WindowAggregationWorkOrder::block_ids); ++i) {
blocks.push_back(
@@ -461,7 +463,7 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder
proto.GetExtension(serialization::WindowAggregationWorkOrder::insert_destination_index)));
}
default:
- LOG(FATAL) << "Unknown WorkOrder Type in WorkOrderFactory::ReconstructFromProto";
+ LOG(FATAL) << "Unknown WorkOrder Type in WorkOrderFactory::ReconstructFromProto in Shiftboss" << shiftboss_index;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/70959876/relational_operators/WorkOrderFactory.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/WorkOrderFactory.hpp b/relational_operators/WorkOrderFactory.hpp
index 45e2cbc..acf3855 100644
--- a/relational_operators/WorkOrderFactory.hpp
+++ b/relational_operators/WorkOrderFactory.hpp
@@ -20,6 +20,8 @@
#ifndef QUICKSTEP_RELATIONAL_OPERATORS_WORK_ORDER_FACTORY_HPP_
#define QUICKSTEP_RELATIONAL_OPERATORS_WORK_ORDER_FACTORY_HPP_
+#include <cstddef>
+
#include "utility/Macros.hpp"
#include "tmb/id_typedefs.h"
@@ -50,6 +52,7 @@ class WorkOrderFactory {
*
* @param proto The Protocol Buffer representation of a WorkOrder object,
* originally generated by RelationalOperator::getAllWorkOrders.
+ * @param shiftboss_index The index of Shiftboss that is using.
* @param catalog_database The database to resolve relation and attribute
* references in.
* @param query_context A pointer to QueryContext.
@@ -60,6 +63,7 @@ class WorkOrderFactory {
* @return A new WorkOrder reconstructed from the supplied Protocol Buffer.
**/
static WorkOrder* ReconstructFromProto(const serialization::WorkOrder &proto,
+ const std::size_t shiftboss_index,
CatalogDatabaseLite *catalog_database,
QueryContext *query_context,
StorageManager *storage_manager,
[05/50] incubator-quickstep git commit: Auto pin workers to CPU cores
Posted by zu...@apache.org.
Auto pin workers to CPU cores
- Automatically pin worker threads to CPU cores.
- Use auto pinning when either there are no affinities provided, or the
provided affinities are incorrect.
- Try to balance CPU cores across multiple sockets, when maximum
paralellism is not used.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/1340fcb7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/1340fcb7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/1340fcb7
Branch: refs/heads/quickstep_partition_parser_support
Commit: 1340fcb7f47f8be0f5f8f397fdc9c9f08db120b3
Parents: 5a3037f
Author: Harshad Deshmukh <hb...@apache.org>
Authored: Thu Nov 3 23:34:37 2016 -0500
Committer: Hakan Memisoglu <ha...@gmail.com>
Committed: Sat Nov 5 14:59:53 2016 -0500
----------------------------------------------------------------------
cli/InputParserUtil.cpp | 68 ++++++++++++++++++++++++++++++++++++++------
1 file changed, 59 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1340fcb7/cli/InputParserUtil.cpp
----------------------------------------------------------------------
diff --git a/cli/InputParserUtil.cpp b/cli/InputParserUtil.cpp
index 0538afc..e45605c 100644
--- a/cli/InputParserUtil.cpp
+++ b/cli/InputParserUtil.cpp
@@ -21,6 +21,7 @@
#include <cstddef>
#include <iostream>
+#include <memory>
#include <ostream>
#include <string>
#include <utility>
@@ -44,23 +45,72 @@ std::vector<int> InputParserUtil::ParseWorkerAffinities(
const int num_workers,
const string &affinity_string) {
std::vector<int> affinities;
+ bool switch_to_default_affinities = false;
if (affinity_string.empty()) {
- affinities.resize(num_workers, -1);
- return affinities;
- }
-
- if (!ParseIntString(affinity_string, ',', &affinities)) {
- LOG(FATAL) << "--worker_affinities must be a comma-separated list of "
- << "integer CPU ids.\n";
+ switch_to_default_affinities = true;
+ LOG(INFO) << "Empty worker affinities provided, switching to default "
+ "worker affinities";
+ } else if (!ParseIntString(affinity_string, ',', &affinities)) {
+ switch_to_default_affinities = true;
+ LOG(INFO) << "Invalid worker affinities provided, switching to default "
+ "affinities";
}
for (const int affinity : affinities) {
if (affinity < -1) {
- LOG(FATAL) << "CPU affinities specified by --worker_affinities must be "
- << "non-negative, or -1 to specify no affinity.\n";
+ switch_to_default_affinities = true;
+ LOG(INFO) << "CPU affinities specified by --worker_affinities must be "
+ "non-negative, or -1 to specify no affinity. Switching to "
+ "default worker affinities";
+ break;
}
}
+ if (switch_to_default_affinities) {
+ // Set default affinities.
+ // If the number of worker threads is less than the maximum parallelism on
+ // the box, we try to balance workers on all sockets. The intention is to
+ // balance the memory bandwidth usage across all sockets. This may however
+ // hurt the performance (due to poor data locality) when the machine has
+ // many sockets and data is not partitioned.
+#ifdef QUICKSTEP_HAVE_LIBNUMA
+ // This code is inspired from the print_node_cpus() function of numactl.
+ // WARNING - If some NUMA sockets are disabled, we can't detect it.
+ const int num_sockets = numa_num_configured_nodes();
+ CHECK_GT(num_sockets, 0);
+ // A vector V where V[i] denotes a vector of CPU cores that belong to the
+ // socket i.
+ std::vector<std::vector<int>> cpus_from_sockets;
+ cpus_from_sockets.resize(num_sockets);
+ for (int curr_socket = 0; curr_socket < num_sockets; ++curr_socket) {
+ std::unique_ptr<struct bitmask> cpus(numa_allocate_cpumask());
+ const int err = numa_node_to_cpus(curr_socket, cpus.get());
+ if (err >= 0) {
+ for (int i = 0; i < static_cast<int>(cpus->size); i++) {
+ if (numa_bitmask_isbitset(cpus.get(), i)) {
+ // The current CPU belongs to curr_socket.
+ cpus_from_sockets[curr_socket].push_back(i);
+ }
+ }
+ }
+ }
+ // Now assign affinity to each worker, picking one CPU from each socket in a
+ // round robin manner.
+ int curr_socket = 0;
+ std::size_t iteration = 0;
+ for (int curr_worker = 0; curr_worker < num_workers; ++curr_worker) {
+ if (iteration < cpus_from_sockets[curr_socket].size()) {
+ const int curr_worker_affinity =
+ cpus_from_sockets[curr_socket][iteration];
+ affinities.push_back(curr_worker_affinity);
+ }
+ // Increase iteration number only when we are at the last socket.
+ iteration = iteration + ((curr_socket + 1) / num_sockets);
+ curr_socket = (curr_socket + 1) % num_sockets;
+ }
+#endif
+ }
+
if (affinities.size() < static_cast<std::size_t>(num_workers)) {
std::cout << "--num_workers is " << num_workers << ", but only "
<< "specified " << affinities.size() << " CPU affinities "
[38/50] incubator-quickstep git commit: Transfered the ownership of
QueryHandle to QueryManager.
Posted by zu...@apache.org.
Transfered the ownership of QueryHandle to QueryManager.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/c608e99e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/c608e99e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/c608e99e
Branch: refs/heads/quickstep_partition_parser_support
Commit: c608e99eff883afacafa2f0fefce99a0513f9963
Parents: 57730e4
Author: Zuyu Zhang <zu...@apache.org>
Authored: Sun Nov 20 12:04:24 2016 -0800
Committer: Zuyu Zhang <zu...@apache.org>
Committed: Sun Nov 27 18:00:34 2016 -0800
----------------------------------------------------------------------
cli/CommandExecutor.cpp | 30 +++++++------
cli/QuickstepCli.cpp | 46 ++++++++++----------
cli/tests/CommandExecutorTestRunner.cpp | 31 +++++++------
query_execution/QueryManagerBase.hpp | 7 ++-
.../tests/QueryManagerSingleNode_unittest.cpp | 8 ++--
.../DistributedExecutionGeneratorTestRunner.cpp | 34 ++++++++-------
.../tests/ExecutionGeneratorTestRunner.cpp | 35 ++++++++-------
7 files changed, 99 insertions(+), 92 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c608e99e/cli/CommandExecutor.cpp
----------------------------------------------------------------------
diff --git a/cli/CommandExecutor.cpp b/cli/CommandExecutor.cpp
index 4ab32de..3c510e7 100644
--- a/cli/CommandExecutor.cpp
+++ b/cli/CommandExecutor.cpp
@@ -213,25 +213,27 @@ inline std::vector<TypedValue> executeQueryForSingleRow(
DCHECK(result.condition == ParseResult::kSuccess);
const ParseStatement &statement = *result.parsed_statement;
+ const CatalogRelation *query_result_relation = nullptr;
- // Generate the query plan.
- std::unique_ptr<QueryHandle> query_handle(
- std::make_unique<QueryHandle>(query_processor->query_id(),
- main_thread_client_id,
- statement.getPriority()));
- query_processor->generateQueryHandle(statement, query_handle.get());
- DCHECK(query_handle->getQueryPlanMutable() != nullptr);
-
- // Use foreman to execute the query plan.
- QueryExecutionUtil::ConstructAndSendAdmitRequestMessage(
- main_thread_client_id, foreman_client_id, query_handle.get(), bus);
+ {
+ // Generate the query plan.
+ auto query_handle =
+ std::make_unique<QueryHandle>(query_processor->query_id(),
+ main_thread_client_id,
+ statement.getPriority());
+ query_processor->generateQueryHandle(statement, query_handle.get());
+ DCHECK(query_handle->getQueryPlanMutable() != nullptr);
+ query_result_relation = query_handle->getQueryResultRelation();
+ DCHECK(query_result_relation != nullptr);
+
+ // Use foreman to execute the query plan.
+ QueryExecutionUtil::ConstructAndSendAdmitRequestMessage(
+ main_thread_client_id, foreman_client_id, query_handle.release(), bus);
+ }
QueryExecutionUtil::ReceiveQueryCompletionMessage(main_thread_client_id, bus);
// Retrieve the scalar result from the result relation.
- const CatalogRelation *query_result_relation = query_handle->getQueryResultRelation();
- DCHECK(query_result_relation != nullptr);
-
std::vector<TypedValue> values;
{
std::vector<block_id> blocks = query_result_relation->getBlocksSnapshot();
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c608e99e/cli/QuickstepCli.cpp
----------------------------------------------------------------------
diff --git a/cli/QuickstepCli.cpp b/cli/QuickstepCli.cpp
index f4816a8..9db7577 100644
--- a/cli/QuickstepCli.cpp
+++ b/cli/QuickstepCli.cpp
@@ -341,38 +341,41 @@ int main(int argc, char* argv[]) {
continue;
}
- std::unique_ptr<QueryHandle> query_handle(
- std::make_unique<QueryHandle>(query_processor->query_id(),
- main_thread_client_id,
- statement.getPriority()));
+ const std::size_t query_id = query_processor->query_id();
+ const CatalogRelation *query_result_relation = nullptr;
+ std::unique_ptr<quickstep::ExecutionDAGVisualizer> dag_visualizer;
+
try {
+ auto query_handle = std::make_unique<QueryHandle>(query_id,
+ main_thread_client_id,
+ statement.getPriority());
query_processor->generateQueryHandle(statement, query_handle.get());
+ DCHECK(query_handle->getQueryPlanMutable() != nullptr);
+
+ if (quickstep::FLAGS_visualize_execution_dag) {
+ dag_visualizer =
+ std::make_unique<quickstep::ExecutionDAGVisualizer>(*query_handle->getQueryPlanMutable());
+ }
+
+ query_result_relation = query_handle->getQueryResultRelation();
+
+ start = std::chrono::steady_clock::now();
+ QueryExecutionUtil::ConstructAndSendAdmitRequestMessage(
+ main_thread_client_id,
+ foreman.getBusClientID(),
+ query_handle.release(),
+ &bus);
} catch (const quickstep::SqlError &sql_error) {
fprintf(stderr, "%s", sql_error.formatMessage(*command_string).c_str());
reset_parser = true;
break;
}
- DCHECK(query_handle->getQueryPlanMutable() != nullptr);
- std::unique_ptr<quickstep::ExecutionDAGVisualizer> dag_visualizer;
- if (quickstep::FLAGS_visualize_execution_dag) {
- dag_visualizer.reset(
- new quickstep::ExecutionDAGVisualizer(*query_handle->getQueryPlanMutable()));
- }
-
- start = std::chrono::steady_clock::now();
- QueryExecutionUtil::ConstructAndSendAdmitRequestMessage(
- main_thread_client_id,
- foreman.getBusClientID(),
- query_handle.get(),
- &bus);
-
try {
QueryExecutionUtil::ReceiveQueryCompletionMessage(
main_thread_client_id, &bus);
end = std::chrono::steady_clock::now();
- const CatalogRelation *query_result_relation = query_handle->getQueryResultRelation();
if (query_result_relation) {
PrintToScreen::PrintRelation(*query_result_relation,
&storage_manager,
@@ -394,12 +397,11 @@ int main(int argc, char* argv[]) {
time_ms.count(), 3).c_str());
if (quickstep::FLAGS_profile_and_report_workorder_perf) {
// TODO(harshad) - Allow user specified file instead of stdout.
- foreman.printWorkOrderProfilingResults(query_handle->query_id(),
- stdout);
+ foreman.printWorkOrderProfilingResults(query_id, stdout);
}
if (quickstep::FLAGS_visualize_execution_dag) {
const auto &profiling_stats =
- foreman.getWorkOrderProfilingResults(query_handle->query_id());
+ foreman.getWorkOrderProfilingResults(query_id);
dag_visualizer->bindProfilingStats(profiling_stats);
std::cerr << "\n" << dag_visualizer->toDOT() << "\n";
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c608e99e/cli/tests/CommandExecutorTestRunner.cpp
----------------------------------------------------------------------
diff --git a/cli/tests/CommandExecutorTestRunner.cpp b/cli/tests/CommandExecutorTestRunner.cpp
index 41cc9da..e352b24 100644
--- a/cli/tests/CommandExecutorTestRunner.cpp
+++ b/cli/tests/CommandExecutorTestRunner.cpp
@@ -20,6 +20,7 @@
#include "cli/tests/CommandExecutorTestRunner.hpp"
#include <cstdio>
+#include <memory>
#include <set>
#include <string>
#include <utility>
@@ -88,28 +89,26 @@ void CommandExecutorTestRunner::runTestCase(
nullptr,
output_stream.file());
} else {
- QueryHandle query_handle(0 /* query_id */, main_thread_client_id_);
- O::OptimizerContext optimizer_context;
-
- optimizer_.generateQueryHandle(parse_statement,
- test_database_loader_.catalog_database(),
- &optimizer_context,
- &query_handle);
-
- AdmitRequestMessage request_message(&query_handle);
- TaggedMessage admit_tagged_message(
- &request_message, sizeof(request_message), kAdmitRequestMessage);
- QueryExecutionUtil::SendTMBMessage(&bus_,
- main_thread_client_id_,
- foreman_->getBusClientID(),
- std::move(admit_tagged_message));
+ const CatalogRelation *query_result_relation = nullptr;
+ {
+ auto query_handle = std::make_unique<QueryHandle>(0 /* query_id */, main_thread_client_id_);
+ O::OptimizerContext optimizer_context;
+
+ optimizer_.generateQueryHandle(parse_statement,
+ test_database_loader_.catalog_database(),
+ &optimizer_context,
+ query_handle.get());
+ query_result_relation = query_handle->getQueryResultRelation();
+
+ QueryExecutionUtil::ConstructAndSendAdmitRequestMessage(
+ main_thread_client_id_, foreman_->getBusClientID(), query_handle.release(), &bus_);
+ }
// Receive workload completion message from Foreman.
const AnnotatedMessage annotated_msg =
bus_.Receive(main_thread_client_id_, 0, true);
const TaggedMessage &tagged_message = annotated_msg.tagged_message;
DCHECK_EQ(kWorkloadCompletionMessage, tagged_message.message_type());
- const CatalogRelation *query_result_relation = query_handle.getQueryResultRelation();
if (query_result_relation) {
PrintToScreen::PrintRelation(*query_result_relation,
test_database_loader_.storage_manager(),
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c608e99e/query_execution/QueryManagerBase.hpp
----------------------------------------------------------------------
diff --git a/query_execution/QueryManagerBase.hpp b/query_execution/QueryManagerBase.hpp
index a274742..ddb76d5 100644
--- a/query_execution/QueryManagerBase.hpp
+++ b/query_execution/QueryManagerBase.hpp
@@ -26,6 +26,7 @@
#include "catalog/CatalogTypedefs.hpp"
#include "query_execution/QueryExecutionState.hpp"
+#include "query_optimizer/QueryHandle.hpp"
#include "relational_operators/RelationalOperator.hpp"
#include "relational_operators/WorkOrder.hpp"
#include "storage/StorageBlockInfo.hpp"
@@ -34,8 +35,6 @@
namespace quickstep {
-class QueryHandle;
-
/** \addtogroup QueryExecution
* @{
*/
@@ -77,7 +76,7 @@ class QueryManagerBase {
* @brief Get the query handle.
**/
const QueryHandle* query_handle() const {
- return query_handle_;
+ return query_handle_.get();
}
/**
@@ -259,7 +258,7 @@ class QueryManagerBase {
return query_exec_state_->hasRebuildInitiated(index);
}
- const QueryHandle *query_handle_;
+ std::unique_ptr<QueryHandle> query_handle_;
const std::size_t query_id_;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c608e99e/query_execution/tests/QueryManagerSingleNode_unittest.cpp
----------------------------------------------------------------------
diff --git a/query_execution/tests/QueryManagerSingleNode_unittest.cpp b/query_execution/tests/QueryManagerSingleNode_unittest.cpp
index f65ec53..6ec6521 100644
--- a/query_execution/tests/QueryManagerSingleNode_unittest.cpp
+++ b/query_execution/tests/QueryManagerSingleNode_unittest.cpp
@@ -234,14 +234,14 @@ class QueryManagerTest : public ::testing::Test {
db_.reset(new CatalogDatabase(nullptr /* catalog */, "database"));
storage_manager_.reset(new StorageManager("./"));
bus_.Initialize();
- query_handle_.reset(new QueryHandle(0 /* dummy query ID */, tmb::kClientIdNone /* cli_id */));
+ query_handle_ = new QueryHandle(0 /* dummy query ID */, tmb::kClientIdNone /* cli_id */);
query_plan_ = query_handle_->getQueryPlanMutable();
query_handle_->getQueryContextProtoMutable()->set_query_id(query_handle_->query_id());
}
inline void constructQueryManager() {
query_manager_.reset(new QueryManagerSingleNode(
- 0, 1, query_handle_.get(), db_.get(), storage_manager_.get(), &bus_));
+ 0, 1, query_handle_, db_.get(), storage_manager_.get(), &bus_));
}
inline const int getNumWorkOrdersInExecution(const QueryPlan::DAGNodeIndex index) const {
@@ -291,8 +291,8 @@ class QueryManagerTest : public ::testing::Test {
unique_ptr<CatalogDatabase> db_;
unique_ptr<StorageManager> storage_manager_;
- QueryPlan *query_plan_;
- unique_ptr<QueryHandle> query_handle_;
+ QueryPlan *query_plan_; // Owned by 'query_handle_'.
+ QueryHandle* query_handle_; // Owned by 'query_manager_'.
unique_ptr<QueryManagerSingleNode> query_manager_;
MessageBusImpl bus_;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c608e99e/query_optimizer/tests/DistributedExecutionGeneratorTestRunner.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/tests/DistributedExecutionGeneratorTestRunner.cpp b/query_optimizer/tests/DistributedExecutionGeneratorTestRunner.cpp
index 2351dcd..5100651 100644
--- a/query_optimizer/tests/DistributedExecutionGeneratorTestRunner.cpp
+++ b/query_optimizer/tests/DistributedExecutionGeneratorTestRunner.cpp
@@ -160,37 +160,39 @@ void DistributedExecutionGeneratorTestRunner::runTestCase(
const ParseStatement &parse_statement = *result.parsed_statement;
std::printf("%s\n", parse_statement.toString().c_str());
+
+ const CatalogRelation *query_result_relation = nullptr;
try {
OptimizerContext optimizer_context;
- QueryHandle query_handle(query_id_++, cli_id_);
+ auto query_handle = std::make_unique<QueryHandle>(query_id_++, cli_id_);
optimizer_.generateQueryHandle(parse_statement,
test_database_loader_->catalog_database(),
&optimizer_context,
- &query_handle);
+ query_handle.get());
+ query_result_relation = query_handle->getQueryResultRelation();
QueryExecutionUtil::ConstructAndSendAdmitRequestMessage(
cli_id_,
foreman_->getBusClientID(),
- &query_handle,
+ query_handle.release(),
&bus_);
-
- const tmb::AnnotatedMessage annotated_message = bus_.Receive(cli_id_, 0, true);
- DCHECK_EQ(kQueryExecutionSuccessMessage, annotated_message.tagged_message.message_type());
-
- const CatalogRelation *query_result_relation = query_handle.getQueryResultRelation();
- if (query_result_relation) {
- PrintToScreen::PrintRelation(*query_result_relation,
- test_database_loader_->storage_manager(),
- output_stream.file());
- DropRelation::Drop(*query_result_relation,
- test_database_loader_->catalog_database(),
- test_database_loader_->storage_manager());
- }
} catch (const SqlError &error) {
*output = error.formatMessage(input);
break;
}
+
+ const tmb::AnnotatedMessage annotated_message = bus_.Receive(cli_id_, 0, true);
+ DCHECK_EQ(kQueryExecutionSuccessMessage, annotated_message.tagged_message.message_type());
+
+ if (query_result_relation) {
+ PrintToScreen::PrintRelation(*query_result_relation,
+ test_database_loader_->storage_manager(),
+ output_stream.file());
+ DropRelation::Drop(*query_result_relation,
+ test_database_loader_->catalog_database(),
+ test_database_loader_->storage_manager());
+ }
}
if (output->empty()) {
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c608e99e/query_optimizer/tests/ExecutionGeneratorTestRunner.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/tests/ExecutionGeneratorTestRunner.cpp b/query_optimizer/tests/ExecutionGeneratorTestRunner.cpp
index 06397d4..ee9bee7 100644
--- a/query_optimizer/tests/ExecutionGeneratorTestRunner.cpp
+++ b/query_optimizer/tests/ExecutionGeneratorTestRunner.cpp
@@ -20,6 +20,7 @@
#include "query_optimizer/tests/ExecutionGeneratorTestRunner.hpp"
#include <cstdio>
+#include <memory>
#include <set>
#include <string>
@@ -71,37 +72,39 @@ void ExecutionGeneratorTestRunner::runTestCase(
} else {
const ParseStatement &parse_statement = *result.parsed_statement;
std::printf("%s\n", parse_statement.toString().c_str());
+
+ const CatalogRelation *query_result_relation = nullptr;
try {
- QueryHandle query_handle(0 /* query_id */, main_thread_client_id_);
OptimizerContext optimizer_context;
+ auto query_handle = std::make_unique<QueryHandle>(0 /* query_id */, main_thread_client_id_);
optimizer_.generateQueryHandle(parse_statement,
test_database_loader_.catalog_database(),
&optimizer_context,
- &query_handle);
+ query_handle.get());
+ query_result_relation = query_handle->getQueryResultRelation();
QueryExecutionUtil::ConstructAndSendAdmitRequestMessage(
main_thread_client_id_,
foreman_->getBusClientID(),
- &query_handle,
+ query_handle.release(),
&bus_);
-
- QueryExecutionUtil::ReceiveQueryCompletionMessage(
- main_thread_client_id_, &bus_);
-
- const CatalogRelation *query_result_relation = query_handle.getQueryResultRelation();
- if (query_result_relation) {
- PrintToScreen::PrintRelation(*query_result_relation,
- test_database_loader_.storage_manager(),
- output_stream.file());
- DropRelation::Drop(*query_result_relation,
- test_database_loader_.catalog_database(),
- test_database_loader_.storage_manager());
- }
} catch (const SqlError &error) {
*output = error.formatMessage(input);
break;
}
+
+ QueryExecutionUtil::ReceiveQueryCompletionMessage(
+ main_thread_client_id_, &bus_);
+
+ if (query_result_relation) {
+ PrintToScreen::PrintRelation(*query_result_relation,
+ test_database_loader_.storage_manager(),
+ output_stream.file());
+ DropRelation::Drop(*query_result_relation,
+ test_database_loader_.catalog_database(),
+ test_database_loader_.storage_manager());
+ }
}
}
[12/50] incubator-quickstep git commit: Refactor Shiftboss for better
debug info.
Posted by zu...@apache.org.
Refactor Shiftboss for better debug info.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/365fff6a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/365fff6a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/365fff6a
Branch: refs/heads/quickstep_partition_parser_support
Commit: 365fff6a3371f6516488a808cc79e929ff789b4a
Parents: 9a005f3
Author: Zuyu Zhang <zu...@apache.org>
Authored: Thu Nov 17 15:02:17 2016 -0800
Committer: Zuyu Zhang <zu...@apache.org>
Committed: Thu Nov 17 15:02:17 2016 -0800
----------------------------------------------------------------------
query_execution/Shiftboss.cpp | 56 ++++++++++++++++++++------------------
query_execution/Shiftboss.hpp | 2 ++
2 files changed, 32 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/365fff6a/query_execution/Shiftboss.cpp
----------------------------------------------------------------------
diff --git a/query_execution/Shiftboss.cpp b/query_execution/Shiftboss.cpp
index a434527..09d7846 100644
--- a/query_execution/Shiftboss.cpp
+++ b/query_execution/Shiftboss.cpp
@@ -56,6 +56,7 @@ using std::string;
using std::unique_ptr;
using std::vector;
+using tmb::AnnotatedMessage;
using tmb::MessageBus;
using tmb::TaggedMessage;
@@ -69,25 +70,16 @@ void Shiftboss::run() {
ThreadUtil::BindToCPU(cpu_id_);
}
+ processShiftbossRegistrationResponseMessage();
+
for (;;) {
// Receive() is a blocking call, causing this thread to sleep until next
// message is received.
AnnotatedMessage annotated_message(bus_->Receive(shiftboss_client_id_, 0, true));
- DLOG(INFO) << "Shiftboss (id '" << shiftboss_client_id_
+ DLOG(INFO) << "Shiftboss " << shiftboss_index_ << " (id '" << shiftboss_client_id_
<< "') received the typed '" << annotated_message.tagged_message.message_type()
<< "' message from client " << annotated_message.sender;
switch (annotated_message.tagged_message.message_type()) {
- case kShiftbossRegistrationResponseMessage: {
- foreman_client_id_ = annotated_message.sender;
-
- const TaggedMessage &tagged_message = annotated_message.tagged_message;
-
- serialization::ShiftbossRegistrationResponseMessage proto;
- CHECK(proto.ParseFromArray(tagged_message.message(), tagged_message.message_bytes()));
-
- shiftboss_index_ = proto.shiftboss_index();
- break;
- }
case kQueryInitiateMessage: {
const TaggedMessage &tagged_message = annotated_message.tagged_message;
@@ -121,7 +113,7 @@ void Shiftboss::run() {
kWorkOrderMessage);
const size_t worker_index = getSchedulableWorker();
- DLOG(INFO) << "Shiftboss (id '" << shiftboss_client_id_
+ DLOG(INFO) << "Shiftboss " << shiftboss_index_ << " (id '" << shiftboss_client_id_
<< "') forwarded WorkOrderMessage (typed '" << kWorkOrderMessage
<< "') from Foreman to worker " << worker_index;
@@ -152,7 +144,7 @@ void Shiftboss::run() {
case kWorkOrderFeedbackMessage:
case kWorkOrderCompleteMessage:
case kRebuildWorkOrderCompleteMessage: {
- DLOG(INFO) << "Shiftboss (id '" << shiftboss_client_id_
+ DLOG(INFO) << "Shiftboss " << shiftboss_index_ << " (id '" << shiftboss_client_id_
<< "') forwarded typed '" << annotated_message.tagged_message.message_type()
<< "' message from Worker with TMB client ID '" << annotated_message.sender
<< "' to Foreman with TMB client ID " << foreman_client_id_;
@@ -203,7 +195,7 @@ void Shiftboss::run() {
kSaveQueryResultResponseMessage);
free(proto_response_bytes);
- DLOG(INFO) << "Shiftboss (id '" << shiftboss_client_id_
+ DLOG(INFO) << "Shiftboss " << shiftboss_index_ << " (id '" << shiftboss_client_id_
<< "') sent SaveQueryResultResponseMessage (typed '" << kSaveQueryResultResponseMessage
<< "') to Foreman with TMB client ID " << foreman_client_id_;
const MessageBus::SendStatus send_status =
@@ -215,7 +207,7 @@ void Shiftboss::run() {
break;
}
case kPoisonMessage: {
- DLOG(INFO) << "Shiftboss (id '" << shiftboss_client_id_
+ DLOG(INFO) << "Shiftboss " << shiftboss_index_ << " (id '" << shiftboss_client_id_
<< "') forwarded PoisonMessage (typed '" << kPoisonMessage
<< "') from Foreman to all workers";
@@ -271,7 +263,7 @@ void Shiftboss::registerWithForeman() {
kShiftbossRegistrationMessage);
free(proto_bytes);
- DLOG(INFO) << "Shiftboss (id '" << shiftboss_client_id_
+ DLOG(INFO) << "Shiftboss " << shiftboss_index_ << " (id '" << shiftboss_client_id_
<< "') sent ShiftbossRegistrationMessage (typed '" << kShiftbossRegistrationMessage
<< "') to all";
tmb::MessageBus::SendStatus send_status =
@@ -279,18 +271,30 @@ void Shiftboss::registerWithForeman() {
DCHECK(send_status == tmb::MessageBus::SendStatus::kOK);
}
+void Shiftboss::processShiftbossRegistrationResponseMessage() {
+ const AnnotatedMessage annotated_message(bus_->Receive(shiftboss_client_id_, 0, true));
+ const TaggedMessage &tagged_message = annotated_message.tagged_message;
+ DCHECK_EQ(kShiftbossRegistrationResponseMessage, tagged_message.message_type());
+
+ foreman_client_id_ = annotated_message.sender;
+ DLOG(INFO) << "Shiftboss (id '" << shiftboss_client_id_
+ << "') received the typed '" << kShiftbossRegistrationResponseMessage
+ << "' message from ForemanDistributed with client " << foreman_client_id_;
+
+ serialization::ShiftbossRegistrationResponseMessage proto;
+ CHECK(proto.ParseFromArray(tagged_message.message(), tagged_message.message_bytes()));
+
+ shiftboss_index_ = proto.shiftboss_index();
+}
+
void Shiftboss::processQueryInitiateMessage(
const std::size_t query_id,
const serialization::CatalogDatabase &catalog_database_cache_proto,
const serialization::QueryContext &query_context_proto) {
database_cache_.update(catalog_database_cache_proto);
- unique_ptr<QueryContext> query_context(
- new QueryContext(query_context_proto,
- database_cache_,
- storage_manager_,
- shiftboss_client_id_,
- bus_));
+ auto query_context = std::make_unique<QueryContext>(
+ query_context_proto, database_cache_, storage_manager_, shiftboss_client_id_, bus_);
query_contexts_.emplace(query_id, move(query_context));
serialization::QueryInitiateResponseMessage proto;
@@ -305,7 +309,7 @@ void Shiftboss::processQueryInitiateMessage(
kQueryInitiateResponseMessage);
free(proto_bytes);
- DLOG(INFO) << "Shiftboss (id '" << shiftboss_client_id_
+ DLOG(INFO) << "Shiftboss " << shiftboss_index_ << " (id '" << shiftboss_client_id_
<< "') sent QueryInitiateResponseMessage (typed '" << kQueryInitiateResponseMessage
<< "') to Foreman with TMB client ID " << foreman_client_id_;
const MessageBus::SendStatus send_status =
@@ -344,7 +348,7 @@ void Shiftboss::processInitiateRebuildMessage(const std::size_t query_id,
kInitiateRebuildResponseMessage);
free(proto_bytes);
- DLOG(INFO) << "Shiftboss (id '" << shiftboss_client_id_
+ DLOG(INFO) << "Shiftboss " << shiftboss_index_ << " (id '" << shiftboss_client_id_
<< "') sent InitiateRebuildResponseMessage (typed '" << kInitiateRebuildResponseMessage
<< "') to Foreman with TMB client ID " << foreman_client_id_;
const MessageBus::SendStatus send_status =
@@ -373,7 +377,7 @@ void Shiftboss::processInitiateRebuildMessage(const std::size_t query_id,
kRebuildWorkOrderMessage);
const size_t worker_index = getSchedulableWorker();
- DLOG(INFO) << "Shiftboss (id '" << shiftboss_client_id_
+ DLOG(INFO) << "Shiftboss " << shiftboss_index_ << " (id '" << shiftboss_client_id_
<< "') sent RebuildWorkOrderMessage (typed '" << kRebuildWorkOrderMessage
<< "') to worker " << worker_index;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/365fff6a/query_execution/Shiftboss.hpp
----------------------------------------------------------------------
diff --git a/query_execution/Shiftboss.hpp b/query_execution/Shiftboss.hpp
index 94b10a2..442e61e 100644
--- a/query_execution/Shiftboss.hpp
+++ b/query_execution/Shiftboss.hpp
@@ -191,6 +191,8 @@ class Shiftboss : public Thread {
private:
void registerWithForeman();
+ void processShiftbossRegistrationResponseMessage();
+
/**
* @brief Process the Shiftboss initiate message and ack back.
*
[39/50] incubator-quickstep git commit: Logged only in the debug mode.
Posted by zu...@apache.org.
Logged only in the debug mode.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/5ff89dd5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/5ff89dd5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/5ff89dd5
Branch: refs/heads/quickstep_partition_parser_support
Commit: 5ff89dd5427bf904e1406111bc2877836f586196
Parents: c608e99
Author: Zuyu Zhang <zu...@apache.org>
Authored: Wed Nov 30 23:07:25 2016 -0800
Committer: Zuyu Zhang <zu...@apache.org>
Committed: Wed Nov 30 23:07:25 2016 -0800
----------------------------------------------------------------------
storage/StorageManager.cpp | 26 +++++++++++++-------------
1 file changed, 13 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5ff89dd5/storage/StorageManager.cpp
----------------------------------------------------------------------
diff --git a/storage/StorageManager.cpp b/storage/StorageManager.cpp
index 56ca323..a202f71 100644
--- a/storage/StorageManager.cpp
+++ b/storage/StorageManager.cpp
@@ -557,9 +557,9 @@ vector<string> StorageManager::getPeerDomainNetworkAddresses(const block_id bloc
kGetPeerDomainNetworkAddressesMessage);
free(proto_bytes);
- LOG(INFO) << "StorageManager (id '" << storage_manager_client_id_
- << "') sent GetPeerDomainNetworkAddressesMessage (typed '" << kGetPeerDomainNetworkAddressesMessage
- << "') to BlockLocator";
+ DLOG(INFO) << "StorageManager (id '" << storage_manager_client_id_
+ << "') sent GetPeerDomainNetworkAddressesMessage (typed '" << kGetPeerDomainNetworkAddressesMessage
+ << "') to BlockLocator";
DCHECK_NE(block_locator_client_id_, tmb::kClientIdNone);
DCHECK(bus_ != nullptr);
@@ -573,8 +573,8 @@ vector<string> StorageManager::getPeerDomainNetworkAddresses(const block_id bloc
const TaggedMessage &tagged_message = annotated_message.tagged_message;
CHECK_EQ(block_locator_client_id_, annotated_message.sender);
CHECK_EQ(kGetPeerDomainNetworkAddressesResponseMessage, tagged_message.message_type());
- LOG(INFO) << "StorageManager (id '" << storage_manager_client_id_
- << "') received GetPeerDomainNetworkAddressesResponseMessage from BlockLocator";
+ DLOG(INFO) << "StorageManager (id '" << storage_manager_client_id_
+ << "') received GetPeerDomainNetworkAddressesResponseMessage from BlockLocator";
serialization::GetPeerDomainNetworkAddressesResponseMessage response_proto;
CHECK(response_proto.ParseFromArray(tagged_message.message(), tagged_message.message_bytes()));
@@ -591,10 +591,10 @@ void StorageManager::sendBlockLocationMessage(const block_id block,
const tmb::message_type_id message_type) {
switch (message_type) {
case kAddBlockLocationMessage:
- LOG(INFO) << "Loaded Block " << BlockIdUtil::ToString(block) << " in Domain " << block_domain_;
+ DLOG(INFO) << "Loaded Block " << BlockIdUtil::ToString(block) << " in Domain " << block_domain_;
break;
case kDeleteBlockLocationMessage:
- LOG(INFO) << "Evicted Block " << BlockIdUtil::ToString(block) << " in Domain " << block_domain_;
+ DLOG(INFO) << "Evicted Block " << BlockIdUtil::ToString(block) << " in Domain " << block_domain_;
break;
default:
LOG(FATAL) << "Unknown message type " << message_type;
@@ -613,9 +613,9 @@ void StorageManager::sendBlockLocationMessage(const block_id block,
message_type);
free(proto_bytes);
- LOG(INFO) << "StorageManager (id '" << storage_manager_client_id_
- << "') sent BlockLocationMessage (typed '" << message_type
- << "') to BlockLocator";
+ DLOG(INFO) << "StorageManager (id '" << storage_manager_client_id_
+ << "') sent BlockLocationMessage (typed '" << message_type
+ << "') to BlockLocator";
CHECK(MessageBus::SendStatus::kOK ==
QueryExecutionUtil::SendTMBMessage(bus_,
storage_manager_client_id_,
@@ -635,7 +635,7 @@ StorageManager::BlockHandle StorageManager::loadBlockOrBlob(
// TODO(quickstep-team): Use a cost model to determine whether to load from
// a remote peer or the disk.
if (BlockIdUtil::Domain(block) != block_domain_) {
- LOG(INFO) << "Pulling Block " << BlockIdUtil::ToString(block) << " from a remote peer";
+ DLOG(INFO) << "Pulling Block " << BlockIdUtil::ToString(block) << " from a remote peer";
const vector<string> peer_domain_network_addresses = getPeerDomainNetworkAddresses(block);
for (const string &peer_domain_network_address : peer_domain_network_addresses) {
DataExchangerClientAsync client(
@@ -648,8 +648,8 @@ StorageManager::BlockHandle StorageManager::loadBlockOrBlob(
}
}
- LOG(INFO) << "Failed to pull Block " << BlockIdUtil::ToString(block)
- << " from remote peers, so try to load from disk.";
+ DLOG(INFO) << "Failed to pull Block " << BlockIdUtil::ToString(block)
+ << " from remote peers, so try to load from disk.";
}
#endif
[08/50] incubator-quickstep git commit: Use BlockLocator and
DataExchangerAsync in the distributed tests.
Posted by zu...@apache.org.
Use BlockLocator and DataExchangerAsync in the distributed tests.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/787a3251
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/787a3251
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/787a3251
Branch: refs/heads/quickstep_partition_parser_support
Commit: 787a3251019162610ebe13efbd341b3f9ac7a268
Parents: 3093e74
Author: Zuyu Zhang <zu...@apache.org>
Authored: Fri Nov 4 23:12:09 2016 -0700
Committer: Zuyu Zhang <zu...@apache.org>
Committed: Mon Nov 14 20:38:04 2016 -0800
----------------------------------------------------------------------
query_optimizer/tests/CMakeLists.txt | 8 ++
.../DistributedExecutionGeneratorTestRunner.cpp | 104 ++++++++++++++++---
.../DistributedExecutionGeneratorTestRunner.hpp | 32 +++++-
query_optimizer/tests/TestDatabaseLoader.hpp | 59 +++++++++--
storage/DataExchangerAsync.cpp | 4 +-
5 files changed, 178 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/787a3251/query_optimizer/tests/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_optimizer/tests/CMakeLists.txt b/query_optimizer/tests/CMakeLists.txt
index ac4548a..9c764e4 100644
--- a/query_optimizer/tests/CMakeLists.txt
+++ b/query_optimizer/tests/CMakeLists.txt
@@ -79,6 +79,10 @@ target_link_libraries(quickstep_queryoptimizer_tests_TestDatabaseLoader
quickstep_types_containers_Tuple
quickstep_utility_Macros
tmb)
+if (ENABLE_DISTRIBUTED)
+ target_link_libraries(quickstep_queryoptimizer_tests_TestDatabaseLoader
+ quickstep_storage_StorageBlockInfo)
+endif(ENABLE_DISTRIBUTED)
if (ENABLE_DISTRIBUTED)
add_executable(quickstep_queryoptimizer_tests_DistributedExecutionGeneratorTest
@@ -110,7 +114,9 @@ if (ENABLE_DISTRIBUTED)
quickstep_cli_PrintToScreen
quickstep_parser_ParseStatement
quickstep_parser_SqlParserWrapper
+ quickstep_queryexecution_BlockLocator
quickstep_queryexecution_ForemanDistributed
+ quickstep_queryexecution_QueryExecutionMessages_proto
quickstep_queryexecution_QueryExecutionTypedefs
quickstep_queryexecution_QueryExecutionUtil
quickstep_queryexecution_Shiftboss
@@ -120,6 +126,8 @@ if (ENABLE_DISTRIBUTED)
quickstep_queryoptimizer_OptimizerContext
quickstep_queryoptimizer_QueryHandle
quickstep_queryoptimizer_tests_TestDatabaseLoader
+ quickstep_storage_DataExchangerAsync
+ quickstep_storage_StorageBlockInfo
quickstep_utility_Macros
quickstep_utility_MemStream
quickstep_utility_SqlError
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/787a3251/query_optimizer/tests/DistributedExecutionGeneratorTestRunner.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/tests/DistributedExecutionGeneratorTestRunner.cpp b/query_optimizer/tests/DistributedExecutionGeneratorTestRunner.cpp
index 5cccc21..0403e77 100644
--- a/query_optimizer/tests/DistributedExecutionGeneratorTestRunner.cpp
+++ b/query_optimizer/tests/DistributedExecutionGeneratorTestRunner.cpp
@@ -20,18 +20,26 @@
#include "query_optimizer/tests/DistributedExecutionGeneratorTestRunner.hpp"
#include <cstdio>
+#include <cstdlib>
+#include <memory>
#include <set>
#include <string>
+#include <utility>
#include <vector>
#include "catalog/CatalogTypedefs.hpp"
#include "cli/DropRelation.hpp"
#include "cli/PrintToScreen.hpp"
#include "parser/ParseStatement.hpp"
+#include "query_execution/BlockLocator.hpp"
#include "query_execution/ForemanDistributed.hpp"
+#include "query_execution/QueryExecutionMessages.pb.h"
#include "query_execution/QueryExecutionTypedefs.hpp"
+#include "query_execution/QueryExecutionUtil.hpp"
#include "query_optimizer/OptimizerContext.hpp"
#include "query_optimizer/QueryHandle.hpp"
+#include "storage/DataExchangerAsync.hpp"
+#include "storage/StorageBlockInfo.hpp"
#include "utility/MemStream.hpp"
#include "utility/SqlError.hpp"
@@ -41,10 +49,15 @@
#include "tmb/message_bus.h"
#include "tmb/tagged_message.h"
-using std::string;
+using std::free;
using std::make_unique;
+using std::malloc;
+using std::move;
+using std::string;
using std::vector;
+using tmb::TaggedMessage;
+
namespace quickstep {
class CatalogRelation;
@@ -56,10 +69,7 @@ const char *DistributedExecutionGeneratorTestRunner::kResetOption =
DistributedExecutionGeneratorTestRunner::DistributedExecutionGeneratorTestRunner(const string &storage_path)
: query_id_(0),
- test_database_loader_(storage_path) {
- test_database_loader_.createTestRelation(false /* allow_vchar */);
- test_database_loader_.loadTestRelation();
-
+ data_exchangers_(kNumInstances) {
bus_.Initialize();
cli_id_ = bus_.Connect();
@@ -67,9 +77,27 @@ DistributedExecutionGeneratorTestRunner::DistributedExecutionGeneratorTestRunner
bus_.RegisterClientAsSender(cli_id_, kPoisonMessage);
bus_.RegisterClientAsReceiver(cli_id_, kQueryExecutionSuccessMessage);
+ bus_.RegisterClientAsSender(cli_id_, kBlockDomainRegistrationMessage);
+ bus_.RegisterClientAsReceiver(cli_id_, kBlockDomainRegistrationResponseMessage);
+
+ block_locator_ = make_unique<BlockLocator>(&bus_);
+ locator_client_id_ = block_locator_->getBusClientID();
+ block_locator_->start();
+
+ test_database_loader_ = make_unique<TestDatabaseLoader>(
+ storage_path,
+ getBlockDomain(test_database_loader_data_exchanger_.network_address()),
+ locator_client_id_,
+ &bus_);
+ test_database_loader_data_exchanger_.set_storage_manager(test_database_loader_->storage_manager());
+ test_database_loader_data_exchanger_.start();
+
+ test_database_loader_->createTestRelation(false /* allow_vchar */);
+ test_database_loader_->loadTestRelation();
+
// NOTE(zuyu): Foreman should initialize before Shiftboss so that the former
// could receive a registration message from the latter.
- foreman_ = make_unique<ForemanDistributed>(&bus_, test_database_loader_.catalog_database());
+ foreman_ = make_unique<ForemanDistributed>(&bus_, test_database_loader_->catalog_database());
// We don't use the NUMA aware version of worker code.
const vector<numa_node_id> numa_nodes(1 /* Number of worker threads per instance */,
@@ -78,17 +106,24 @@ DistributedExecutionGeneratorTestRunner::DistributedExecutionGeneratorTestRunner
for (int i = 0; i < kNumInstances; ++i) {
workers_.push_back(make_unique<Worker>(0 /* worker_thread_index */, &bus_));
- const vector<tmb::client_id> worker_client_ids(1, workers_[i]->getBusClientID());
+ const vector<tmb::client_id> worker_client_ids(1, workers_.back()->getBusClientID());
worker_directories_.push_back(
make_unique<WorkerDirectory>(worker_client_ids.size(), worker_client_ids, numa_nodes));
+ auto storage_manager = make_unique<StorageManager>(
+ storage_path, getBlockDomain(data_exchangers_[i].network_address()), locator_client_id_, &bus_);
+
+ data_exchangers_[i].set_storage_manager(storage_manager.get());
shiftbosses_.push_back(
- make_unique<Shiftboss>(&bus_, test_database_loader_.storage_manager(), worker_directories_[i].get()));
+ make_unique<Shiftboss>(&bus_, storage_manager.get(), worker_directories_.back().get()));
+
+ storage_managers_.push_back(move(storage_manager));
}
foreman_->start();
for (int i = 0; i < kNumInstances; ++i) {
+ data_exchangers_[i].start();
shiftbosses_[i]->start();
workers_[i]->start();
}
@@ -101,9 +136,9 @@ void DistributedExecutionGeneratorTestRunner::runTestCase(
VLOG(4) << "Test SQL(s): " << input;
if (options.find(kResetOption) != options.end()) {
- test_database_loader_.clear();
- test_database_loader_.createTestRelation(false /* allow_vchar */);
- test_database_loader_.loadTestRelation();
+ test_database_loader_->clear();
+ test_database_loader_->createTestRelation(false /* allow_vchar */);
+ test_database_loader_->loadTestRelation();
}
MemStream output_stream;
@@ -125,7 +160,7 @@ void DistributedExecutionGeneratorTestRunner::runTestCase(
QueryHandle query_handle(query_id_++, cli_id_);
optimizer_.generateQueryHandle(parse_statement,
- test_database_loader_.catalog_database(),
+ test_database_loader_->catalog_database(),
&optimizer_context,
&query_handle);
@@ -141,11 +176,11 @@ void DistributedExecutionGeneratorTestRunner::runTestCase(
const CatalogRelation *query_result_relation = query_handle.getQueryResultRelation();
if (query_result_relation) {
PrintToScreen::PrintRelation(*query_result_relation,
- test_database_loader_.storage_manager(),
+ test_database_loader_->storage_manager(),
output_stream.file());
DropRelation::Drop(*query_result_relation,
- test_database_loader_.catalog_database(),
- test_database_loader_.storage_manager());
+ test_database_loader_->catalog_database(),
+ test_database_loader_->storage_manager());
}
} catch (const SqlError &error) {
*output = error.formatMessage(input);
@@ -158,5 +193,44 @@ void DistributedExecutionGeneratorTestRunner::runTestCase(
}
}
+block_id_domain DistributedExecutionGeneratorTestRunner::getBlockDomain(
+ const string &network_address) {
+ serialization::BlockDomainRegistrationMessage proto;
+ proto.set_domain_network_address(network_address);
+
+ const int proto_length = proto.ByteSize();
+ char *proto_bytes = static_cast<char*>(malloc(proto_length));
+ CHECK(proto.SerializeToArray(proto_bytes, proto_length));
+
+ TaggedMessage message(static_cast<const void*>(proto_bytes),
+ proto_length,
+ kBlockDomainRegistrationMessage);
+ free(proto_bytes);
+
+ DLOG(INFO) << "Client (id '" << cli_id_
+ << "') sent BlockDomainRegistrationMessage (typed '" << kBlockDomainRegistrationMessage
+ << "') to BlockLocator (id '" << locator_client_id_ << "')";
+
+ CHECK(MessageBus::SendStatus::kOK ==
+ QueryExecutionUtil::SendTMBMessage(&bus_,
+ cli_id_,
+ locator_client_id_,
+ move(message)));
+
+ const tmb::AnnotatedMessage annotated_message(bus_.Receive(cli_id_, 0, true));
+ const TaggedMessage &tagged_message = annotated_message.tagged_message;
+ CHECK_EQ(locator_client_id_, annotated_message.sender);
+ CHECK_EQ(kBlockDomainRegistrationResponseMessage, tagged_message.message_type());
+ DLOG(INFO) << "Client (id '" << cli_id_
+ << "') received BlockDomainRegistrationResponseMessage (typed '"
+ << kBlockDomainRegistrationResponseMessage
+ << "') from BlockLocator";
+
+ serialization::BlockDomainMessage response_proto;
+ CHECK(response_proto.ParseFromArray(tagged_message.message(), tagged_message.message_bytes()));
+
+ return static_cast<block_id_domain>(response_proto.block_domain());
+}
+
} // namespace optimizer
} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/787a3251/query_optimizer/tests/DistributedExecutionGeneratorTestRunner.hpp
----------------------------------------------------------------------
diff --git a/query_optimizer/tests/DistributedExecutionGeneratorTestRunner.hpp b/query_optimizer/tests/DistributedExecutionGeneratorTestRunner.hpp
index ab10841..d2b13e4 100644
--- a/query_optimizer/tests/DistributedExecutionGeneratorTestRunner.hpp
+++ b/query_optimizer/tests/DistributedExecutionGeneratorTestRunner.hpp
@@ -28,6 +28,7 @@
#include <vector>
#include "parser/SqlParserWrapper.hpp"
+#include "query_execution/BlockLocator.hpp"
#include "query_execution/ForemanDistributed.hpp"
#include "query_execution/QueryExecutionTypedefs.hpp"
#include "query_execution/QueryExecutionUtil.hpp"
@@ -36,6 +37,8 @@
#include "query_execution/WorkerDirectory.hpp"
#include "query_optimizer/Optimizer.hpp"
#include "query_optimizer/tests/TestDatabaseLoader.hpp"
+#include "storage/DataExchangerAsync.hpp"
+#include "storage/StorageBlockInfo.hpp"
#include "utility/Macros.hpp"
#include "utility/textbased_test/TextBasedTestRunner.hpp"
@@ -86,6 +89,25 @@ class DistributedExecutionGeneratorTestRunner : public TextBasedTestRunner {
}
foreman_->join();
+
+ test_database_loader_data_exchanger_.shutdown();
+ test_database_loader_.reset();
+ for (int i = 0; i < kNumInstances; ++i) {
+ data_exchangers_[i].shutdown();
+ storage_managers_[i].reset();
+ }
+
+ CHECK(MessageBus::SendStatus::kOK ==
+ QueryExecutionUtil::SendTMBMessage(&bus_,
+ cli_id_,
+ locator_client_id_,
+ tmb::TaggedMessage(quickstep::kPoisonMessage)));
+
+ test_database_loader_data_exchanger_.join();
+ for (int i = 0; i < kNumInstances; ++i) {
+ data_exchangers_[i].join();
+ }
+ block_locator_->join();
}
void runTestCase(const std::string &input,
@@ -93,20 +115,26 @@ class DistributedExecutionGeneratorTestRunner : public TextBasedTestRunner {
std::string *output) override;
private:
+ block_id_domain getBlockDomain(const std::string &network_address);
+
std::size_t query_id_;
SqlParserWrapper sql_parser_;
- TestDatabaseLoader test_database_loader_;
+ std::unique_ptr<TestDatabaseLoader> test_database_loader_;
+ DataExchangerAsync test_database_loader_data_exchanger_;
Optimizer optimizer_;
MessageBusImpl bus_;
+ tmb::client_id cli_id_, locator_client_id_;
- tmb::client_id cli_id_;
+ std::unique_ptr<BlockLocator> block_locator_;
std::unique_ptr<ForemanDistributed> foreman_;
std::vector<std::unique_ptr<Worker>> workers_;
std::vector<std::unique_ptr<WorkerDirectory>> worker_directories_;
+ std::vector<DataExchangerAsync> data_exchangers_;
+ std::vector<std::unique_ptr<StorageManager>> storage_managers_;
std::vector<std::unique_ptr<Shiftboss>> shiftbosses_;
DISALLOW_COPY_AND_ASSIGN(DistributedExecutionGeneratorTestRunner);
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/787a3251/query_optimizer/tests/TestDatabaseLoader.hpp
----------------------------------------------------------------------
diff --git a/query_optimizer/tests/TestDatabaseLoader.hpp b/query_optimizer/tests/TestDatabaseLoader.hpp
index d49719d..87c19c6 100644
--- a/query_optimizer/tests/TestDatabaseLoader.hpp
+++ b/query_optimizer/tests/TestDatabaseLoader.hpp
@@ -24,12 +24,21 @@
#include "catalog/CatalogDatabase.hpp"
#include "query_execution/QueryExecutionTypedefs.hpp"
+
+#ifdef QUICKSTEP_DISTRIBUTED
+#include "storage/StorageBlockInfo.hpp"
+#endif // QUICKSTEP_DISTRIBUTED
+
#include "storage/StorageManager.hpp"
#include "threading/ThreadIDBasedMap.hpp"
#include "utility/Macros.hpp"
#include "tmb/id_typedefs.h"
+#ifdef QUICKSTEP_DISTRIBUTED
+namespace tmb { class MessageBus; }
+#endif // QUICKSTEP_DISTRIBUTED
+
namespace quickstep {
class CatalogRelation;
@@ -60,18 +69,34 @@ class TestDatabaseLoader {
0 /* id */),
storage_manager_(storage_path),
test_relation_(nullptr) {
- bus_.Initialize();
-
- const tmb::client_id worker_thread_client_id = bus_.Connect();
- bus_.RegisterClientAsSender(worker_thread_client_id, kCatalogRelationNewBlockMessage);
-
- // Refer to InsertDestination::sendBlockFilledMessage for the rationale
- // behind using ClientIDMap.
- thread_id_map_->addValue(worker_thread_client_id);
+ init();
+ }
- scheduler_client_id_ = bus_.Connect();
- bus_.RegisterClientAsReceiver(scheduler_client_id_, kCatalogRelationNewBlockMessage);
+#ifdef QUICKSTEP_DISTRIBUTED
+ /**
+ * @brief Constructor for the distributed version.
+ *
+ * @param storage_path A filesystem directory where the blocks may be
+ * evicted to during the execution of a test query.
+ * Can be empty if the test query is not executed
+ * in the query engine.
+ * @param block_domain The block_domain for StorageManager.
+ * @param locator_client_id The client id of BlockLocator for StorageManager.
+ * @param bus_global The Bus for StorageManager.
+ */
+ TestDatabaseLoader(const std::string &storage_path,
+ const block_id_domain block_domain,
+ const tmb::client_id locator_client_id,
+ tmb::MessageBus *bus_global)
+ : thread_id_map_(ClientIDMap::Instance()),
+ catalog_database_(nullptr /* parent */,
+ "TestDatabase" /* name */,
+ 0 /* id */),
+ storage_manager_(storage_path, block_domain, locator_client_id, bus_global),
+ test_relation_(nullptr) {
+ init();
}
+#endif // QUICKSTEP_DISTRIBUTED
~TestDatabaseLoader() {
clear();
@@ -139,6 +164,20 @@ class TestDatabaseLoader {
void clear();
private:
+ void init() {
+ bus_.Initialize();
+
+ const tmb::client_id worker_thread_client_id = bus_.Connect();
+ bus_.RegisterClientAsSender(worker_thread_client_id, kCatalogRelationNewBlockMessage);
+
+ // Refer to InsertDestination::sendBlockFilledMessage for the rationale
+ // behind using ClientIDMap.
+ thread_id_map_->addValue(worker_thread_client_id);
+
+ scheduler_client_id_ = bus_.Connect();
+ bus_.RegisterClientAsReceiver(scheduler_client_id_, kCatalogRelationNewBlockMessage);
+ }
+
/**
* @brief Simulate Foreman to add all new blocks to the relation.
*/
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/787a3251/storage/DataExchangerAsync.cpp
----------------------------------------------------------------------
diff --git a/storage/DataExchangerAsync.cpp b/storage/DataExchangerAsync.cpp
index 59f5ebf..1d2f7db 100644
--- a/storage/DataExchangerAsync.cpp
+++ b/storage/DataExchangerAsync.cpp
@@ -155,11 +155,11 @@ void DataExchangerAsync::run() {
if (ok) {
call_context->Proceed();
} else {
- LOG(WARNING) << "Not ok\n";
+ LOG(WARNING) << "DataExchangerAsync " << server_address_ << " is not ok";
delete call_context;
}
} else {
- LOG(INFO) << "Shutdown\n";
+ LOG(INFO) << "DataExchangerAsync " << server_address_ << " shuts down";
return;
}
}
[34/50] incubator-quickstep git commit: A better debug log for Worker.
Posted by zu...@apache.org.
A better debug log for Worker.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/4286d75e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/4286d75e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/4286d75e
Branch: refs/heads/quickstep_partition_parser_support
Commit: 4286d75ec622ac6e60813c6892b131660ce3a931
Parents: a31dde4
Author: Zuyu Zhang <zu...@apache.org>
Authored: Sun Nov 27 11:45:28 2016 -0800
Committer: Zuyu Zhang <zu...@apache.org>
Committed: Sun Nov 27 11:45:28 2016 -0800
----------------------------------------------------------------------
query_execution/Worker.cpp | 9 ++++++++-
1 file changed, 8 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4286d75e/query_execution/Worker.cpp
----------------------------------------------------------------------
diff --git a/query_execution/Worker.cpp b/query_execution/Worker.cpp
index 31eb964..9a548fd 100644
--- a/query_execution/Worker.cpp
+++ b/query_execution/Worker.cpp
@@ -107,7 +107,14 @@ void Worker::sendWorkOrderCompleteMessage(const tmb::client_id receiver,
static_cast<const void *>(proto_bytes), proto_length, message_type);
std::free(proto_bytes);
- DLOG(INFO) << "Worker sent WorkOrderCompleteMessage (typed '" << message_type
+ DLOG(INFO) << "Worker " << worker_thread_index_
+#ifdef QUICKSTEP_DISTRIBUTED
+ << " in Shiftboss " << shiftboss_index_
+#endif // QUICKSTEP_DISTRIBUTED
+ << " sent "
+ << (message_type == kWorkOrderCompleteMessage ? "WorkOrderCompleteMessage"
+ : "RebuildWorkOrderCompleteMessage")
+ << " (typed '" << message_type
<< "') to Scheduler with TMB client ID " << receiver;
const tmb::MessageBus::SendStatus send_status =
QueryExecutionUtil::SendTMBMessage(
[44/50] incubator-quickstep git commit: QUICKSTEP-65 Fix Quickstep
build failure on Mac OSX 10.12 by turning off deprecation errors
Posted by zu...@apache.org.
QUICKSTEP-65 Fix Quickstep build failure on Mac OSX 10.12 by turning off deprecation errors
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/6ae0cdd3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/6ae0cdd3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/6ae0cdd3
Branch: refs/heads/quickstep_partition_parser_support
Commit: 6ae0cdd3d5db40febc98be00ce94baeed31ba680
Parents: b949c50
Author: Saket Saurabh <ss...@cs.wisc.edu>
Authored: Tue Dec 6 09:53:57 2016 -0600
Committer: Saket Saurabh <ss...@cs.wisc.edu>
Committed: Tue Dec 6 10:05:42 2016 -0600
----------------------------------------------------------------------
CMakeLists.txt | 19 +++++++++++++++++++
1 file changed, 19 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/6ae0cdd3/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 391cb26..4dcc56a 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -291,6 +291,25 @@ else()
endif()
endif()
+ # OSX 10.12 has deprecated certain system-level APIs which causes protobuf & glog
+ # builds to fail. As a short-term workaround for now, we turn off deprecated
+ # warnings so that they do not cause build failures anymore.
+ # TODO: Remove this workaround by fixing the protobuf_cmake and glog_cmake.
+ if (${CMAKE_SYSTEM} MATCHES "Darwin-16.1.0")
+ if (${CMAKE_CXX_COMPILER_ID} MATCHES "Clang")
+ CHECK_CXX_COMPILER_FLAG("-Wno-error=deprecated-declarations" COMPILER_HAS_WNO_DEPRECATED)
+ if (COMPILER_HAS_WNO_DEPRECATED)
+ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error=deprecated-declarations")
+ endif()
+ endif()
+ if(CMAKE_COMPILER_IS_GNUCXX)
+ CHECK_CXX_COMPILER_FLAG("-Wno-deprecated-declarations" COMPILER_HAS_WNO_DEPRECATED)
+ if (COMPILER_HAS_WNO_DEPRECATED)
+ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-deprecated-declarations")
+ endif()
+ endif()
+ endif()
+
# One of the protobuf headers includes a nested anonymous union within
# another anonymous type. Ordinarily we work around this by compiling the
# protobuf libraries themselves with "-Wno-nested-anon-types" and including
[46/50] incubator-quickstep git commit: Add min/max statistics and
the exactness flag into CatalogRelationStatistics.
Posted by zu...@apache.org.
Add min/max statistics and the exactness flag into CatalogRelationStatistics.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/9fcb0aca
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/9fcb0aca
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/9fcb0aca
Branch: refs/heads/quickstep_partition_parser_support
Commit: 9fcb0aca86cf0e63b04e307534f65644877ee327
Parents: 1b0328b
Author: Jianqiao Zhu <ji...@cs.wisc.edu>
Authored: Thu Oct 27 14:16:32 2016 -0500
Committer: jianqiao <ji...@node-2.jianqiao.quickstep-pg0.wisc.cloudlab.us>
Committed: Tue Dec 20 09:19:50 2016 -0600
----------------------------------------------------------------------
catalog/CMakeLists.txt | 3 +
catalog/Catalog.proto | 11 +-
catalog/CatalogRelationStatistics.cpp | 60 +++++--
catalog/CatalogRelationStatistics.hpp | 174 ++++++++++++++++---
cli/CMakeLists.txt | 3 +
cli/CommandExecutor.cpp | 105 ++++++++---
query_optimizer/ExecutionGenerator.cpp | 36 ++--
query_optimizer/cost_model/CMakeLists.txt | 1 +
query_optimizer/cost_model/SimpleCostModel.cpp | 13 +-
.../cost_model/StarSchemaSimpleCostModel.cpp | 16 +-
relational_operators/CMakeLists.txt | 1 +
relational_operators/SaveBlocksOperator.cpp | 11 ++
relational_operators/SaveBlocksOperator.hpp | 7 +
13 files changed, 353 insertions(+), 88 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/9fcb0aca/catalog/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/catalog/CMakeLists.txt b/catalog/CMakeLists.txt
index dd4ef99..143d12f 100644
--- a/catalog/CMakeLists.txt
+++ b/catalog/CMakeLists.txt
@@ -120,6 +120,9 @@ target_link_libraries(quickstep_catalog_CatalogRelation
target_link_libraries(quickstep_catalog_CatalogRelationStatistics
quickstep_catalog_CatalogTypedefs
quickstep_catalog_Catalog_proto
+ quickstep_types_LongType
+ quickstep_types_NullType
+ quickstep_types_TypedValue
quickstep_utility_Macros)
target_link_libraries(quickstep_catalog_IndexScheme
glog
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/9fcb0aca/catalog/Catalog.proto
----------------------------------------------------------------------
diff --git a/catalog/Catalog.proto b/catalog/Catalog.proto
index 90ce37e..a54862f 100644
--- a/catalog/Catalog.proto
+++ b/catalog/Catalog.proto
@@ -81,13 +81,16 @@ message IndexScheme {
}
message CatalogRelationStatistics {
- optional fixed64 num_tuples = 1;
+ optional bool is_exact = 1;
+ optional TypedValue num_tuples = 2;
- message NumDistinctValuesEntry {
+ message ColumnStats {
required int32 attr_id = 1;
- required fixed64 num_distinct_values = 2;
+ optional TypedValue num_distinct_values = 2;
+ optional TypedValue min_value = 3;
+ optional TypedValue max_value = 4;
}
- repeated NumDistinctValuesEntry num_distinct_values_map = 2;
+ repeated ColumnStats column_stats = 3;
}
message CatalogRelationSchema {
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/9fcb0aca/catalog/CatalogRelationStatistics.cpp
----------------------------------------------------------------------
diff --git a/catalog/CatalogRelationStatistics.cpp b/catalog/CatalogRelationStatistics.cpp
index 6a51570..f3f224d 100644
--- a/catalog/CatalogRelationStatistics.cpp
+++ b/catalog/CatalogRelationStatistics.cpp
@@ -23,28 +23,68 @@
namespace quickstep {
+const TypedValue CatalogRelationStatistics::kNullValue =
+ NullType::InstanceNullable().makeNullValue();
+
CatalogRelationStatistics::CatalogRelationStatistics(
const serialization::CatalogRelationStatistics &proto) {
+ if (proto.has_is_exact()) {
+ is_exact_ = proto.is_exact();
+ } else {
+ is_exact_ = false;
+ }
+
if (proto.has_num_tuples()) {
- num_tuples_ = proto.num_tuples();
+ num_tuples_ = TypedValue::ReconstructFromProto(proto.num_tuples());
+ } else {
+ num_tuples_ = kNullValue;
}
- for (int i = 0; i < proto.num_distinct_values_map_size(); ++i) {
- const auto &entry = proto.num_distinct_values_map(i);
- num_distinct_values_map_.emplace(entry.attr_id(),
- entry.num_distinct_values());
+
+ for (int i = 0; i < proto.column_stats_size(); ++i) {
+ const auto &stat_proto = proto.column_stats(i);
+ auto &stat = column_stats_[stat_proto.attr_id()];
+ if (stat_proto.has_num_distinct_values()) {
+ stat.num_distinct_values =
+ TypedValue::ReconstructFromProto(stat_proto.num_distinct_values());
+ }
+ if (stat_proto.has_min_value()) {
+ stat.min_value =
+ TypedValue::ReconstructFromProto(stat_proto.min_value());
+ }
+ if (stat_proto.has_max_value()) {
+ stat.max_value =
+ TypedValue::ReconstructFromProto(stat_proto.max_value());
+ }
}
}
serialization::CatalogRelationStatistics CatalogRelationStatistics::getProto() const {
serialization::CatalogRelationStatistics proto;
- if (num_tuples_ != 0) {
- proto.set_num_tuples(num_tuples_);
+
+ proto.set_is_exact(is_exact_);
+
+ if (!num_tuples_.isNull()) {
+ proto.mutable_num_tuples()->CopyFrom(num_tuples_.getProto());
}
- for (const auto &pair : num_distinct_values_map_) {
- auto entry = proto.add_num_distinct_values_map();
+
+ for (const auto &pair : column_stats_) {
+ auto entry = proto.add_column_stats();
entry->set_attr_id(pair.first);
- entry->set_num_distinct_values(pair.second);
+ const auto &stat = pair.second;
+ if (!stat.num_distinct_values.isNull()) {
+ entry->mutable_num_distinct_values()->CopyFrom(
+ stat.num_distinct_values.getProto());
+ }
+ if (!stat.min_value.isNull()) {
+ entry->mutable_min_value()->CopyFrom(
+ stat.min_value.getProto());
+ }
+ if (!stat.max_value.isNull()) {
+ entry->mutable_max_value()->CopyFrom(
+ stat.max_value.getProto());
+ }
}
+
return proto;
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/9fcb0aca/catalog/CatalogRelationStatistics.hpp
----------------------------------------------------------------------
diff --git a/catalog/CatalogRelationStatistics.hpp b/catalog/CatalogRelationStatistics.hpp
index f2056f3..df95231 100644
--- a/catalog/CatalogRelationStatistics.hpp
+++ b/catalog/CatalogRelationStatistics.hpp
@@ -26,6 +26,9 @@
#include "catalog/Catalog.pb.h"
#include "catalog/CatalogTypedefs.hpp"
+#include "types/LongType.hpp"
+#include "types/NullType.hpp"
+#include "types/TypedValue.hpp"
#include "utility/Macros.hpp"
namespace quickstep {
@@ -44,7 +47,8 @@ class CatalogRelationStatistics {
* @brief Constructor.
**/
CatalogRelationStatistics()
- : num_tuples_(0) {}
+ : is_exact_(false),
+ num_tuples_(kNullValue) {}
/**
* @brief Reconstruct a CatalogRelationStatistics object from its serialized
@@ -64,12 +68,31 @@ class CatalogRelationStatistics {
serialization::CatalogRelationStatistics getProto() const;
/**
- * @brief Set the number of tuples statistic.
+ * @brief Check whether the statistics are exact for the relation.
*
- * @param num_tuples The number of tuples statistic.
+ * return True if the statistics are exact for the relation, false otherwise.
*/
- void setNumTuples(std::size_t num_tuples) {
- num_tuples_ = num_tuples;
+ bool isExact() const {
+ return is_exact_;
+ }
+
+ /**
+ * @brief Set the boolean flag that indicates whether the statistics are exact
+ * for the relation.
+ *
+ * @param Exactness of the statistics.
+ */
+ void setExactness(const bool is_exact) {
+ is_exact_ = is_exact;
+ }
+
+ /**
+ * @brief Check whether the number of tuples statistic is available.
+ *
+ * @return True if the number of tuples statistic is available, false otherwise.
+ */
+ bool hasNumTuples() const {
+ return !num_tuples_.isNull();
}
/**
@@ -78,41 +101,148 @@ class CatalogRelationStatistics {
* @return The number of tuples. Returns 0 if the statistic is not set.
*/
std::size_t getNumTuples() const {
- return num_tuples_;
+ DCHECK(hasNumTuples());
+ return num_tuples_.getLiteral<std::int64_t>();
+ }
+
+ /**
+ * @brief Set the number of tuples statistic.
+ *
+ * @param num_tuples The number of tuples statistic.
+ */
+ void setNumTuples(const std::size_t num_tuples) {
+ num_tuples_ = LongType::InstanceNonNullable().makeValue(&num_tuples);
}
/**
- * @brief Set the number of distinct values statistic for a column (catalog attribute).
+ * @brief Check whether the number of distinct values statistic is available
+ * for a column.
+ *
+ * @param The id of the column.
+ * @return True if the number of distinct values statistic is available,
+ * false otherwise.
+ */
+ bool hasNumDistinctValues(const attribute_id attr_id) const {
+ const ColumnStats *stats = getColumnStats(attr_id);
+ return (stats != nullptr && !stats->num_distinct_values.isNull());
+ }
+
+ /**
+ * @brief Get the number of distinct values statistic for a column.
+ *
+ * @param The id of the column.
+ * @return The number of distinct values statistic for the column.
+ */
+ std::size_t getNumDistinctValues(const attribute_id attr_id) const {
+ DCHECK(hasNumDistinctValues(attr_id));
+ return column_stats_.at(attr_id).num_distinct_values.getLiteral<std::int64_t>();
+ }
+
+ /**
+ * @brief Set the number of distinct values statistic for a column.
*
* @param attr_id The id of the column.
* @param num_distinct_values The number of distinct values statistic.
*/
- void setNumDistinctValues(attribute_id attr_id, std::size_t num_distinct_values) {
- num_distinct_values_map_[attr_id] = num_distinct_values;
+ void setNumDistinctValues(const attribute_id attr_id,
+ const std::size_t num_distinct_values) {
+ column_stats_[attr_id].num_distinct_values =
+ LongType::InstanceNonNullable().makeValue(&num_distinct_values);
}
/**
- * @brief Get the number of distinct values statistic for a column (catalog attribute).
+ * @brief Check whether the minimum value statistic is available for a column.
*
* @param The id of the column.
- * @return The number of distinct values statistic for the column. Returns 0
- * if the statistic is not set.
+ * @return True if the minimum value statistic is available, false otherwise.
*/
- std::size_t getNumDistinctValues(attribute_id attr_id) const {
- const auto it = num_distinct_values_map_.find(attr_id);
- if (it == num_distinct_values_map_.end()) {
- return static_cast<std::size_t>(0);
- } else {
- return it->second;
- }
+ bool hasMinValue(const attribute_id attr_id) const {
+ const ColumnStats *stats = getColumnStats(attr_id);
+ return (stats != nullptr && !stats->min_value.isNull());
+ }
+
+ /**
+ * @brief Get the minimum value statistic for a column.
+ *
+ * @param The id of the column.
+ * @return The minimum value statistic for the column.
+ */
+ const TypedValue& getMinValue(const attribute_id attr_id) const {
+ DCHECK(hasMinValue(attr_id));
+ return column_stats_.at(attr_id).min_value;
+ }
+
+ /**
+ * @brief Set the minimum value statistic for a column.
+ *
+ * @param The id of the column.
+ * @return The minimum value statistic for the column.
+ */
+ void setMinValue(const attribute_id attr_id,
+ const TypedValue &min_value) {
+ column_stats_[attr_id].min_value = min_value;
+ }
+
+ /**
+ * @brief Check whether the maximum value statistic is available for a column.
+ *
+ * @param The id of the column.
+ * @return True if the maximum value statistic is available, false otherwise.
+ */
+ bool hasMaxValue(const attribute_id attr_id) const {
+ const ColumnStats *stats = getColumnStats(attr_id);
+ return (stats != nullptr && !stats->max_value.isNull());
+ }
+
+ /**
+ * @brief Get the maximum value statistic for a column.
+ *
+ * @param The id of the column.
+ * @return The maximum value statistic for the column.
+ */
+ const TypedValue& getMaxValue(const attribute_id attr_id) const {
+ DCHECK(hasMaxValue(attr_id));
+ return column_stats_.at(attr_id).max_value;
+ }
+
+ /**
+ * @brief Set the maximum value statistic for a column.
+ *
+ * @param The id of the column.
+ * @return The maximum value statistic for the column.
+ */
+ void setMaxValue(const attribute_id attr_id,
+ const TypedValue &max_value) {
+ column_stats_[attr_id].max_value = max_value;
}
private:
+ struct ColumnStats {
+ ColumnStats()
+ : num_distinct_values(kNullValue),
+ min_value(kNullValue),
+ max_value(kNullValue) {
+ }
+ TypedValue num_distinct_values;
+ TypedValue min_value;
+ TypedValue max_value;
+ };
+
+ inline const ColumnStats* getColumnStats(const attribute_id attr_id) const {
+ const auto it = column_stats_.find(attr_id);
+ return (it == column_stats_.end() ? nullptr : &it->second);
+ }
+
+ static const TypedValue kNullValue;
+
+ // Whether the statistics are exact (i.e. up-to-date) for the relation.
+ bool is_exact_;
+
// Total number of tuples in the relation.
- std::size_t num_tuples_;
+ TypedValue num_tuples_;
- // Number of distinct values for each column.
- std::unordered_map<attribute_id, std::size_t> num_distinct_values_map_;
+ // Statistics for each column.
+ std::unordered_map<attribute_id, ColumnStats> column_stats_;
DISALLOW_COPY_AND_ASSIGN(CatalogRelationStatistics);
};
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/9fcb0aca/cli/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/cli/CMakeLists.txt b/cli/CMakeLists.txt
index be13c82..8c7fe2d 100644
--- a/cli/CMakeLists.txt
+++ b/cli/CMakeLists.txt
@@ -89,8 +89,11 @@ target_link_libraries(quickstep_cli_CommandExecutor
quickstep_catalog_CatalogDatabase
quickstep_catalog_CatalogRelation
quickstep_catalog_CatalogRelationSchema
+ quickstep_catalog_CatalogRelationStatistics
quickstep_cli_DropRelation
quickstep_cli_PrintToScreen
+ quickstep_expressions_aggregation_AggregateFunctionMax
+ quickstep_expressions_aggregation_AggregateFunctionMin
quickstep_parser_ParseStatement
quickstep_parser_SqlParserWrapper
quickstep_queryoptimizer_QueryHandle
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/9fcb0aca/cli/CommandExecutor.cpp
----------------------------------------------------------------------
diff --git a/cli/CommandExecutor.cpp b/cli/CommandExecutor.cpp
index 3c510e7..78bff98 100644
--- a/cli/CommandExecutor.cpp
+++ b/cli/CommandExecutor.cpp
@@ -31,8 +31,11 @@
#include "catalog/CatalogDatabase.hpp"
#include "catalog/CatalogRelation.hpp"
#include "catalog/CatalogRelationSchema.hpp"
+#include "catalog/CatalogRelationStatistics.hpp"
#include "cli/DropRelation.hpp"
#include "cli/PrintToScreen.hpp"
+#include "expressions/aggregation/AggregateFunctionMax.hpp"
+#include "expressions/aggregation/AggregateFunctionMin.hpp"
#include "parser/ParseStatement.hpp"
#include "parser/ParseString.hpp"
#include "parser/SqlParserWrapper.hpp"
@@ -69,7 +72,7 @@ namespace {
namespace C = ::quickstep::cli;
-void executeDescribeDatabase(
+void ExecuteDescribeDatabase(
const PtrVector<ParseString> *arguments,
const CatalogDatabase &catalog_database,
StorageManager *storage_manager,
@@ -131,7 +134,7 @@ void executeDescribeDatabase(
}
}
-void executeDescribeTable(
+void ExecuteDescribeTable(
const PtrVector<ParseString> *arguments,
const CatalogDatabase &catalog_database, FILE *out) {
const ParseString &table_name = arguments->front();
@@ -199,7 +202,7 @@ void executeDescribeTable(
/**
* @brief A helper function that executes a SQL query to obtain a row of results.
*/
-inline std::vector<TypedValue> executeQueryForSingleRow(
+inline std::vector<TypedValue> ExecuteQueryForSingleRow(
const tmb::client_id main_thread_client_id,
const tmb::client_id foreman_client_id,
const std::string &query_string,
@@ -270,7 +273,7 @@ inline std::vector<TypedValue> executeQueryForSingleRow(
/**
* @brief A helper function that executes a SQL query to obtain a scalar result.
*/
-inline TypedValue executeQueryForSingleResult(
+inline TypedValue ExecuteQueryForSingleResult(
const tmb::client_id main_thread_client_id,
const tmb::client_id foreman_client_id,
const std::string &query_string,
@@ -279,7 +282,7 @@ inline TypedValue executeQueryForSingleResult(
QueryProcessor *query_processor,
SqlParserWrapper *parser_wrapper) {
std::vector<TypedValue> results =
- executeQueryForSingleRow(main_thread_client_id,
+ ExecuteQueryForSingleRow(main_thread_client_id,
foreman_client_id,
query_string,
bus,
@@ -290,7 +293,21 @@ inline TypedValue executeQueryForSingleResult(
return results[0];
}
-void executeAnalyze(const PtrVector<ParseString> *arguments,
+/**
+ * @brief A helper function for escaping quotes (i.e. ' or ").
+ */
+std::string EscapeQuotes(const std::string &str, const char quote) {
+ std::string ret;
+ for (const char c : str) {
+ ret.push_back(c);
+ if (c == quote) {
+ ret.push_back(c);
+ }
+ }
+ return ret;
+}
+
+void ExecuteAnalyze(const PtrVector<ParseString> *arguments,
const tmb::client_id main_thread_client_id,
const tmb::client_id foreman_client_id,
MessageBus *bus,
@@ -321,17 +338,42 @@ void executeAnalyze(const PtrVector<ParseString> *arguments,
CatalogRelation *mutable_relation =
query_processor->getDefaultDatabase()->getRelationByIdMutable(relation.getID());
+ CatalogRelationStatistics *mutable_stat =
+ mutable_relation->getStatisticsMutable();
+
+ const std::string rel_name = EscapeQuotes(relation.getName(), '"');
// Get the number of distinct values for each column.
for (const CatalogAttribute &attribute : relation) {
- std::string query_string = "SELECT COUNT(DISTINCT ";
- query_string.append(attribute.getName());
- query_string.append(") FROM ");
- query_string.append(relation.getName());
- query_string.append(";");
+ const std::string attr_name = EscapeQuotes(attribute.getName(), '"');
+ const Type &attr_type = attribute.getType();
+ bool is_min_applicable =
+ AggregateFunctionMin::Instance().canApplyToTypes({&attr_type});
+ bool is_max_applicable =
+ AggregateFunctionMax::Instance().canApplyToTypes({&attr_type});
+
+ // NOTE(jianqiao): Note that the relation name and the attribute names may
+ // contain non-letter characters, e.g. CREATE TABLE "with space"("1" int).
+ // So here we need to format the names with double quotes (").
+ std::string query_string = "SELECT COUNT(DISTINCT \"";
+ query_string.append(attr_name);
+ query_string.append("\")");
+ if (is_min_applicable) {
+ query_string.append(", MIN(\"");
+ query_string.append(attr_name);
+ query_string.append("\")");
+ }
+ if (is_max_applicable) {
+ query_string.append(", MAX(\"");
+ query_string.append(attr_name);
+ query_string.append("\")");
+ }
+ query_string.append(" FROM \"");
+ query_string.append(rel_name);
+ query_string.append("\";");
std::vector<TypedValue> results =
- executeQueryForSingleRow(main_thread_client_id,
+ ExecuteQueryForSingleRow(main_thread_client_id,
foreman_client_id,
query_string,
bus,
@@ -339,21 +381,29 @@ void executeAnalyze(const PtrVector<ParseString> *arguments,
query_processor,
parser_wrapper.get());
- auto *stat = mutable_relation->getStatisticsMutable();
- const attribute_id attr_id = attribute.getID();
+ auto results_it = results.begin();
+ DCHECK(results_it->getTypeID() == TypeID::kLong);
- DCHECK(results[0].getTypeID() == TypeID::kLong);
- stat->setNumDistinctValues(attr_id,
- results[0].getLiteral<std::int64_t>());
+ const attribute_id attr_id = attribute.getID();
+ mutable_stat->setNumDistinctValues(attr_id,
+ results_it->getLiteral<std::int64_t>());
+ if (is_min_applicable) {
+ ++results_it;
+ mutable_stat->setMinValue(attr_id, *results_it);
+ }
+ if (is_max_applicable) {
+ ++results_it;
+ mutable_stat->setMaxValue(attr_id, *results_it);
+ }
}
// Get the number of tuples for the relation.
- std::string query_string = "SELECT COUNT(*) FROM ";
- query_string.append(relation.getName());
- query_string.append(";");
+ std::string query_string = "SELECT COUNT(*) FROM \"";
+ query_string.append(rel_name);
+ query_string.append("\";");
TypedValue num_tuples =
- executeQueryForSingleResult(main_thread_client_id,
+ ExecuteQueryForSingleResult(main_thread_client_id,
foreman_client_id,
query_string,
bus,
@@ -362,8 +412,9 @@ void executeAnalyze(const PtrVector<ParseString> *arguments,
parser_wrapper.get());
DCHECK(num_tuples.getTypeID() == TypeID::kLong);
- mutable_relation->getStatisticsMutable()->setNumTuples(
- num_tuples.getLiteral<std::int64_t>());
+ mutable_stat->setNumTuples(num_tuples.getLiteral<std::int64_t>());
+
+ mutable_stat->setExactness(true);
fprintf(out, "done\n");
fflush(out);
@@ -386,15 +437,15 @@ void executeCommand(const ParseStatement &statement,
const PtrVector<ParseString> *arguments = command.arguments();
const std::string &command_str = command.command()->value();
if (command_str == C::kDescribeDatabaseCommand) {
- executeDescribeDatabase(arguments, catalog_database, storage_manager, out);
+ ExecuteDescribeDatabase(arguments, catalog_database, storage_manager, out);
} else if (command_str == C::kDescribeTableCommand) {
if (arguments->size() == 0) {
- executeDescribeDatabase(arguments, catalog_database, storage_manager, out);
+ ExecuteDescribeDatabase(arguments, catalog_database, storage_manager, out);
} else {
- executeDescribeTable(arguments, catalog_database, out);
+ ExecuteDescribeTable(arguments, catalog_database, out);
}
} else if (command_str == C::kAnalyzeCommand) {
- executeAnalyze(arguments,
+ ExecuteAnalyze(arguments,
main_thread_client_id,
foreman_client_id,
bus,
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/9fcb0aca/query_optimizer/ExecutionGenerator.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/ExecutionGenerator.cpp b/query_optimizer/ExecutionGenerator.cpp
index 5a2c450..d24f498 100644
--- a/query_optimizer/ExecutionGenerator.cpp
+++ b/query_optimizer/ExecutionGenerator.cpp
@@ -924,9 +924,11 @@ void ExecutionGenerator::convertCopyFrom(
insert_destination_index));
insert_destination_proto->set_relational_op_index(scan_operator_index);
+ CatalogRelation *mutable_output_relation =
+ catalog_database_->getRelationByIdMutable(output_relation->getID());
const QueryPlan::DAGNodeIndex save_blocks_operator_index =
execution_plan_->addRelationalOperator(
- new SaveBlocksOperator(query_handle_->query_id()));
+ new SaveBlocksOperator(query_handle_->query_id(), mutable_output_relation));
execution_plan_->addDirectDependency(save_blocks_operator_index,
scan_operator_index,
false /* is_pipeline_breaker */);
@@ -1038,13 +1040,16 @@ void ExecutionGenerator::convertDeleteTuples(
const CatalogRelationInfo *input_relation_info =
findRelationInfoOutputByPhysical(physical_plan->input());
DCHECK(input_relation_info != nullptr);
+
+ const CatalogRelation *input_relation = input_relation_info->relation;
+
if (execution_predicate == nullptr ||
(execution_predicate->hasStaticResult() &&
execution_predicate->getStaticResult())) {
const QueryPlan::DAGNodeIndex drop_table_index =
execution_plan_->addRelationalOperator(
new DropTableOperator(query_handle_->query_id(),
- *input_relation_info->relation,
+ *input_relation,
catalog_database_,
true /* only_drop_blocks */));
if (!input_relation_info->isStoredRelation()) {
@@ -1059,7 +1064,7 @@ void ExecutionGenerator::convertDeleteTuples(
const QueryPlan::DAGNodeIndex delete_tuples_index =
execution_plan_->addRelationalOperator(
new DeleteOperator(query_handle_->query_id(),
- *input_relation_info->relation,
+ *input_relation,
execution_predicate_index,
input_relation_info->isStoredRelation()));
@@ -1069,9 +1074,11 @@ void ExecutionGenerator::convertDeleteTuples(
false /* is_pipeline_breaker */);
}
+ CatalogRelation *mutable_relation =
+ catalog_database_->getRelationByIdMutable(input_relation->getID());
const QueryPlan::DAGNodeIndex save_blocks_index =
execution_plan_->addRelationalOperator(
- new SaveBlocksOperator(query_handle_->query_id()));
+ new SaveBlocksOperator(query_handle_->query_id(), mutable_relation));
execution_plan_->addDirectDependency(save_blocks_index,
delete_tuples_index,
false /* is_pipeline_breaker */);
@@ -1150,9 +1157,11 @@ void ExecutionGenerator::convertInsertTuple(
tuple_index));
insert_destination_proto->set_relational_op_index(insert_operator_index);
+ CatalogRelation *mutable_relation =
+ catalog_database_->getRelationByIdMutable(input_relation.getID());
const QueryPlan::DAGNodeIndex save_blocks_index =
execution_plan_->addRelationalOperator(
- new SaveBlocksOperator(query_handle_->query_id()));
+ new SaveBlocksOperator(query_handle_->query_id(), mutable_relation));
if (!input_relation_info->isStoredRelation()) {
execution_plan_->addDirectDependency(insert_operator_index,
input_relation_info->producer_operator_index,
@@ -1201,6 +1210,7 @@ void ExecutionGenerator::convertInsertSelection(
const CatalogRelationInfo *selection_relation_info =
findRelationInfoOutputByPhysical(physical_plan->selection());
+ const CatalogRelation *selection_relation = selection_relation_info->relation;
// Prepare the attributes, which are output columns of the selection relation.
std::vector<attribute_id> attributes;
@@ -1221,7 +1231,7 @@ void ExecutionGenerator::convertInsertSelection(
// physical plan by modifying class Physical.
SelectOperator *insert_selection_op =
new SelectOperator(query_handle_->query_id(),
- *selection_relation_info->relation,
+ *selection_relation,
destination_relation,
insert_destination_index,
QueryContext::kInvalidPredicateId,
@@ -1232,8 +1242,11 @@ void ExecutionGenerator::convertInsertSelection(
execution_plan_->addRelationalOperator(insert_selection_op);
insert_destination_proto->set_relational_op_index(insert_selection_index);
+ CatalogRelation *mutable_relation =
+ catalog_database_->getRelationByIdMutable(selection_relation->getID());
const QueryPlan::DAGNodeIndex save_blocks_index =
- execution_plan_->addRelationalOperator(new SaveBlocksOperator(query_handle_->query_id()));
+ execution_plan_->addRelationalOperator(
+ new SaveBlocksOperator(query_handle_->query_id(), mutable_relation));
if (!selection_relation_info->isStoredRelation()) {
execution_plan_->addDirectDependency(insert_selection_index,
@@ -1253,7 +1266,8 @@ void ExecutionGenerator::convertUpdateTable(
findRelationInfoOutputByPhysical(physical_plan->input());
DCHECK(input_relation_info != nullptr);
- const relation_id input_rel_id = input_relation_info->relation->getID();
+ const CatalogRelation *input_relation = input_relation_info->relation;
+ const relation_id input_rel_id = input_relation->getID();
// Create InsertDestination proto.
const QueryContext::insert_destination_id relocation_destination_index =
@@ -1303,8 +1317,7 @@ void ExecutionGenerator::convertUpdateTable(
const QueryPlan::DAGNodeIndex update_operator_index =
execution_plan_->addRelationalOperator(new UpdateOperator(
query_handle_->query_id(),
- *catalog_database_->getRelationById(
- input_rel_id),
+ *input_relation,
relocation_destination_index,
execution_predicate_index,
update_group_index));
@@ -1312,7 +1325,8 @@ void ExecutionGenerator::convertUpdateTable(
const QueryPlan::DAGNodeIndex save_blocks_index =
execution_plan_->addRelationalOperator(
- new SaveBlocksOperator(query_handle_->query_id()));
+ new SaveBlocksOperator(query_handle_->query_id(),
+ catalog_database_->getRelationByIdMutable(input_rel_id)));
if (!input_relation_info->isStoredRelation()) {
execution_plan_->addDirectDependency(update_operator_index,
input_relation_info->producer_operator_index,
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/9fcb0aca/query_optimizer/cost_model/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_optimizer/cost_model/CMakeLists.txt b/query_optimizer/cost_model/CMakeLists.txt
index 032e34c..90133e7 100644
--- a/query_optimizer/cost_model/CMakeLists.txt
+++ b/query_optimizer/cost_model/CMakeLists.txt
@@ -48,6 +48,7 @@ target_link_libraries(quickstep_queryoptimizer_costmodel_SimpleCostModel
target_link_libraries(quickstep_queryoptimizer_costmodel_StarSchemaSimpleCostModel
glog
quickstep_catalog_CatalogRelation
+ quickstep_catalog_CatalogRelationStatistics
quickstep_queryoptimizer_costmodel_CostModel
quickstep_queryoptimizer_expressions_AttributeReference
quickstep_queryoptimizer_expressions_ComparisonExpression
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/9fcb0aca/query_optimizer/cost_model/SimpleCostModel.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/cost_model/SimpleCostModel.cpp b/query_optimizer/cost_model/SimpleCostModel.cpp
index bf6da6a..7808898 100644
--- a/query_optimizer/cost_model/SimpleCostModel.cpp
+++ b/query_optimizer/cost_model/SimpleCostModel.cpp
@@ -94,13 +94,12 @@ std::size_t SimpleCostModel::estimateCardinalityForTopLevelPlan(
std::size_t SimpleCostModel::estimateCardinalityForTableReference(
const P::TableReferencePtr &physical_plan) {
- const std::size_t num_tuples_in_relation =
- physical_plan->relation()->getStatistics().getNumTuples();
- if (num_tuples_in_relation == 0) {
- return physical_plan->relation()->estimateTupleCardinality();
- } else {
- return num_tuples_in_relation;
- }
+ const CatalogRelation *relation = physical_plan->relation();
+ const CatalogRelationStatistics &stat = relation->getStatistics();
+ const std::size_t num_tuples =
+ stat.hasNumTuples() ? stat.getNumTuples()
+ : relation->estimateTupleCardinality();
+ return num_tuples;
}
std::size_t SimpleCostModel::estimateCardinalityForSelection(
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/9fcb0aca/query_optimizer/cost_model/StarSchemaSimpleCostModel.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/cost_model/StarSchemaSimpleCostModel.cpp b/query_optimizer/cost_model/StarSchemaSimpleCostModel.cpp
index dfafa7d..75b1b2b 100644
--- a/query_optimizer/cost_model/StarSchemaSimpleCostModel.cpp
+++ b/query_optimizer/cost_model/StarSchemaSimpleCostModel.cpp
@@ -25,6 +25,7 @@
#include <vector>
#include "catalog/CatalogRelation.hpp"
+#include "catalog/CatalogRelationStatistics.hpp"
#include "query_optimizer/cost_model/CostModel.hpp"
#include "query_optimizer/expressions/AttributeReference.hpp"
#include "query_optimizer/expressions/ComparisonExpression.hpp"
@@ -105,10 +106,11 @@ std::size_t StarSchemaSimpleCostModel::estimateCardinalityForTopLevelPlan(
std::size_t StarSchemaSimpleCostModel::estimateCardinalityForTableReference(
const P::TableReferencePtr &physical_plan) {
- std::size_t num_tuples = physical_plan->relation()->getStatistics().getNumTuples();
- if (num_tuples == 0) {
- num_tuples = physical_plan->relation()->estimateTupleCardinality();
- }
+ const CatalogRelation *relation = physical_plan->relation();
+ const CatalogRelationStatistics &stat = relation->getStatistics();
+ const std::size_t num_tuples =
+ stat.hasNumTuples() ? stat.getNumTuples()
+ : relation->estimateTupleCardinality();
return num_tuples;
}
@@ -385,9 +387,9 @@ std::size_t StarSchemaSimpleCostModel::getNumDistinctValues(
const std::vector<E::AttributeReferencePtr> &attributes = table_reference->attribute_list();
for (std::size_t i = 0; i < attributes.size(); ++i) {
if (attributes[i]->id() == attribute_id) {
- std::size_t num_distinct_values = relation.getStatistics().getNumDistinctValues(i);
- if (num_distinct_values > 0) {
- return num_distinct_values;
+ const CatalogRelationStatistics &stat = relation.getStatistics();
+ if (stat.hasNumDistinctValues(i)) {
+ return stat.getNumDistinctValues(i);
}
break;
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/9fcb0aca/relational_operators/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/relational_operators/CMakeLists.txt b/relational_operators/CMakeLists.txt
index 0735bce..9e4b1b6 100644
--- a/relational_operators/CMakeLists.txt
+++ b/relational_operators/CMakeLists.txt
@@ -303,6 +303,7 @@ target_link_libraries(quickstep_relationaloperators_SampleOperator
tmb)
target_link_libraries(quickstep_relationaloperators_SaveBlocksOperator
glog
+ quickstep_catalog_CatalogRelation
quickstep_catalog_CatalogTypedefs
quickstep_queryexecution_WorkOrderProtosContainer
quickstep_queryexecution_WorkOrdersContainer
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/9fcb0aca/relational_operators/SaveBlocksOperator.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/SaveBlocksOperator.cpp b/relational_operators/SaveBlocksOperator.cpp
index 04d4211..a5a96c4 100644
--- a/relational_operators/SaveBlocksOperator.cpp
+++ b/relational_operators/SaveBlocksOperator.cpp
@@ -21,6 +21,7 @@
#include <vector>
+#include "catalog/CatalogRelation.hpp"
#include "query_execution/WorkOrderProtosContainer.hpp"
#include "query_execution/WorkOrdersContainer.hpp"
#include "relational_operators/WorkOrder.pb.h"
@@ -70,6 +71,16 @@ void SaveBlocksOperator::feedInputBlock(const block_id input_block_id, const rel
destination_block_ids_.push_back(input_block_id);
}
+void SaveBlocksOperator::updateCatalogOnCompletion() {
+ // Note(jianqiao): We need to reset the exactness flag whenever a stored
+ // relation gets changed. Given the pre-condition that all the data manipulation
+ // operations (insert, delete, update, copy) use this SaveBlocksOperator as a
+ // common routine. It is valid to put the updating call here to minimize the
+ // changes to other relational operators (e.g. InsertOperator, DeleteOperator,
+ // TextScanOperator, SelectOperator, etc.).
+ relation_->getStatisticsMutable()->setExactness(false);
+}
+
void SaveBlocksWorkOrder::execute() {
// It may happen that the block gets saved to disk as a result of an eviction,
// before this invocation. In either case, we don't care about the return
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/9fcb0aca/relational_operators/SaveBlocksOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/SaveBlocksOperator.hpp b/relational_operators/SaveBlocksOperator.hpp
index e84b641..a8d5327 100644
--- a/relational_operators/SaveBlocksOperator.hpp
+++ b/relational_operators/SaveBlocksOperator.hpp
@@ -38,6 +38,7 @@ namespace tmb { class MessageBus; }
namespace quickstep {
+class CatalogRelation;
class QueryContext;
class StorageManager;
class WorkOrderProtosContainer;
@@ -56,13 +57,16 @@ class SaveBlocksOperator : public RelationalOperator {
* @brief Constructor for saving only modified blocks in a relation.
*
* @param query_id The ID of the query to which this operator belongs.
+ * @param relation The relation whose blocks will be saved.
* @param force If true, force writing of all blocks to disk, otherwise only
* write dirty blocks.
**/
explicit SaveBlocksOperator(const std::size_t query_id,
+ CatalogRelation *relation,
const bool force = false)
: RelationalOperator(query_id),
force_(force),
+ relation_(relation),
num_workorders_generated_(0) {}
~SaveBlocksOperator() override {}
@@ -89,9 +93,12 @@ class SaveBlocksOperator : public RelationalOperator {
}
}
+ void updateCatalogOnCompletion() override;
+
private:
const bool force_;
+ CatalogRelation *relation_;
std::vector<block_id> destination_block_ids_;
std::vector<block_id>::size_type num_workorders_generated_;
[11/50] incubator-quickstep git commit: Removed StorageManager from
QueryProcessor.
Posted by zu...@apache.org.
Removed StorageManager from QueryProcessor.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/9a005f36
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/9a005f36
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/9a005f36
Branch: refs/heads/quickstep_partition_parser_support
Commit: 9a005f36944de0eebd1bf764124fc7876e62d934
Parents: ea5f28b
Author: Zuyu Zhang <zu...@apache.org>
Authored: Mon Nov 14 21:09:28 2016 -0800
Committer: Harshad Deshmukh <hb...@apache.org>
Committed: Thu Nov 17 14:02:51 2016 -0600
----------------------------------------------------------------------
cli/CommandExecutor.cpp | 5 +++--
cli/QuickstepCli.cpp | 17 +++++++++--------
query_optimizer/CMakeLists.txt | 1 -
query_optimizer/QueryProcessor.hpp | 15 +--------------
4 files changed, 13 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/9a005f36/cli/CommandExecutor.cpp
----------------------------------------------------------------------
diff --git a/cli/CommandExecutor.cpp b/cli/CommandExecutor.cpp
index fea4b74..4ab32de 100644
--- a/cli/CommandExecutor.cpp
+++ b/cli/CommandExecutor.cpp
@@ -260,7 +260,7 @@ inline std::vector<TypedValue> executeQueryForSingleRow(
// Drop the result relation.
DropRelation::Drop(*query_result_relation,
query_processor->getDefaultDatabase(),
- query_processor->getStorageManager());
+ storage_manager);
return values;
}
@@ -292,10 +292,10 @@ void executeAnalyze(const PtrVector<ParseString> *arguments,
const tmb::client_id main_thread_client_id,
const tmb::client_id foreman_client_id,
MessageBus *bus,
+ StorageManager *storage_manager,
QueryProcessor *query_processor,
FILE *out) {
const CatalogDatabase &database = *query_processor->getDefaultDatabase();
- StorageManager *storage_manager = query_processor->getStorageManager();
std::unique_ptr<SqlParserWrapper> parser_wrapper(new SqlParserWrapper());
std::vector<std::reference_wrapper<const CatalogRelation>> relations;
@@ -396,6 +396,7 @@ void executeCommand(const ParseStatement &statement,
main_thread_client_id,
foreman_client_id,
bus,
+ storage_manager,
query_processor, out);
} else {
THROW_SQL_ERROR_AT(command.command()) << "Invalid Command";
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/9a005f36/cli/QuickstepCli.cpp
----------------------------------------------------------------------
diff --git a/cli/QuickstepCli.cpp b/cli/QuickstepCli.cpp
index 31d9834..31215f6 100644
--- a/cli/QuickstepCli.cpp
+++ b/cli/QuickstepCli.cpp
@@ -240,6 +240,7 @@ int main(int argc, char* argv[]) {
&& (fixed_storage_path.back() != quickstep::kPathSeparator)) {
fixed_storage_path.push_back(quickstep::kPathSeparator);
}
+ quickstep::StorageManager storage_manager(fixed_storage_path);
string catalog_path(fixed_storage_path);
catalog_path.append("catalog.pb.bin");
@@ -281,10 +282,10 @@ int main(int argc, char* argv[]) {
catalog_file.close();
}
- // Setup QueryProcessor, including CatalogDatabase and StorageManager.
+ // Setup QueryProcessor, including CatalogDatabase.
std::unique_ptr<QueryProcessor> query_processor;
try {
- query_processor.reset(new QueryProcessor(catalog_path, fixed_storage_path));
+ query_processor = std::make_unique<QueryProcessor>(catalog_path);
} catch (const std::exception &e) {
LOG(FATAL) << "FATAL ERROR DURING STARTUP: "
<< e.what()
@@ -308,7 +309,7 @@ int main(int argc, char* argv[]) {
printf("Preloading the buffer pool ... ");
fflush(stdout);
quickstep::PreloaderThread preloader(*query_processor->getDefaultDatabase(),
- query_processor->getStorageManager(),
+ &storage_manager,
worker_cpu_affinities.front());
preloader.start();
@@ -357,7 +358,7 @@ int main(int argc, char* argv[]) {
&worker_directory,
&bus,
query_processor->getDefaultDatabase(),
- query_processor->getStorageManager(),
+ &storage_manager,
-1, // Don't pin the Foreman thread.
num_numa_nodes_system,
quickstep::FLAGS_profile_and_report_workorder_perf || quickstep::FLAGS_visualize_execution_dag);
@@ -412,7 +413,7 @@ int main(int argc, char* argv[]) {
main_thread_client_id,
foreman.getBusClientID(),
&bus,
- query_processor->getStorageManager(),
+ &storage_manager,
query_processor.get(),
stdout);
} catch (const quickstep::SqlError &sql_error) {
@@ -458,16 +459,16 @@ int main(int argc, char* argv[]) {
const CatalogRelation *query_result_relation = query_handle->getQueryResultRelation();
if (query_result_relation) {
PrintToScreen::PrintRelation(*query_result_relation,
- query_processor->getStorageManager(),
+ &storage_manager,
stdout);
PrintToScreen::PrintOutputSize(
*query_result_relation,
- query_processor->getStorageManager(),
+ &storage_manager,
stdout);
DropRelation::Drop(*query_result_relation,
query_processor->getDefaultDatabase(),
- query_processor->getStorageManager());
+ &storage_manager);
}
query_processor->saveCatalog();
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/9a005f36/query_optimizer/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_optimizer/CMakeLists.txt b/query_optimizer/CMakeLists.txt
index 00d5163..10c52a1 100644
--- a/query_optimizer/CMakeLists.txt
+++ b/query_optimizer/CMakeLists.txt
@@ -234,7 +234,6 @@ target_link_libraries(quickstep_queryoptimizer_QueryProcessor
quickstep_catalog_Catalog_proto
quickstep_queryoptimizer_Optimizer
quickstep_queryoptimizer_OptimizerContext
- quickstep_storage_StorageManager
quickstep_utility_Macros)
target_link_libraries(quickstep_queryoptimizer_Validator
glog
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/9a005f36/query_optimizer/QueryProcessor.hpp
----------------------------------------------------------------------
diff --git a/query_optimizer/QueryProcessor.hpp b/query_optimizer/QueryProcessor.hpp
index 5d3818e..5e5c115 100644
--- a/query_optimizer/QueryProcessor.hpp
+++ b/query_optimizer/QueryProcessor.hpp
@@ -27,7 +27,6 @@
#include "catalog/Catalog.hpp"
#include "query_optimizer/Optimizer.hpp"
-#include "storage/StorageManager.hpp"
#include "utility/Macros.hpp"
namespace quickstep {
@@ -130,13 +129,9 @@ class QueryProcessor {
* @brief Constructor.
*
* @param catalog_filename The file to read the serialized catalog from.
- * @param storage_path The filesystem directory where blocks are stored on
- * disk.
**/
- QueryProcessor(const std::string &catalog_filename,
- const std::string &storage_path)
+ explicit QueryProcessor(const std::string &catalog_filename)
: catalog_filename_(catalog_filename),
- storage_manager_(std::make_unique<StorageManager>(storage_path)),
catalog_altered_(false),
query_id_(0) {
loadCatalog();
@@ -185,13 +180,6 @@ class QueryProcessor {
return catalog_->getDatabaseByNameMutable("default");
}
- /**
- * @brief Get the StorageManager held by this QueryProcessor.
- **/
- StorageManager* getStorageManager() const {
- return storage_manager_.get();
- }
-
private:
void loadCatalog(); // If it exists, free catalog_ before calling this
@@ -200,7 +188,6 @@ class QueryProcessor {
std::string catalog_filename_;
std::unique_ptr<Catalog> catalog_;
- std::unique_ptr<StorageManager> storage_manager_;
bool catalog_altered_;
[28/50] incubator-quickstep git commit: Refactored the database init
code.
Posted by zu...@apache.org.
Refactored the database init code.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/f673f945
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/f673f945
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/f673f945
Branch: refs/heads/quickstep_partition_parser_support
Commit: f673f94561e104c62d0441f1ded04737e23b0ba0
Parents: 54e0654
Author: Zuyu Zhang <zu...@apache.org>
Authored: Sun Nov 20 23:55:22 2016 -0800
Committer: Zuyu Zhang <zu...@apache.org>
Committed: Sun Nov 20 23:55:22 2016 -0800
----------------------------------------------------------------------
cli/CMakeLists.txt | 8 +++-
cli/DefaultsConfigurator.cpp | 77 +++++++++++++++++++++++++++++++++++++++
cli/DefaultsConfigurator.hpp | 10 +++++
cli/QuickstepCli.cpp | 36 +-----------------
4 files changed, 94 insertions(+), 37 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f673f945/cli/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/cli/CMakeLists.txt b/cli/CMakeLists.txt
index 9847787..9b62af9 100644
--- a/cli/CMakeLists.txt
+++ b/cli/CMakeLists.txt
@@ -75,7 +75,7 @@ else()
quickstep_utility_Macros)
endif()
-add_library(quickstep_cli_DefaultsConfigurator ../empty_src.cpp DefaultsConfigurator.hpp)
+add_library(quickstep_cli_DefaultsConfigurator DefaultsConfigurator.cpp DefaultsConfigurator.hpp)
add_library(quickstep_cli_InputParserUtil InputParserUtil.cpp InputParserUtil.hpp)
add_library(quickstep_cli_PrintToScreen PrintToScreen.cpp PrintToScreen.hpp)
@@ -106,10 +106,14 @@ target_link_libraries(quickstep_cli_CommandExecutor
quickstep_utility_SqlError)
target_link_libraries(quickstep_cli_DefaultsConfigurator
+ glog
+ quickstep_catalog_Catalog
+ quickstep_catalog_Catalog_proto
+ quickstep_catalog_CatalogDatabase
quickstep_utility_Macros)
if(QUICKSTEP_HAVE_LIBNUMA)
target_link_libraries(quickstep_cli_DefaultsConfigurator
- ${LIBNUMA_LIBRARY})
+ ${LIBNUMA_LIBRARY})
endif()
target_link_libraries(quickstep_cli_Flags
quickstep_cli_DefaultsConfigurator
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f673f945/cli/DefaultsConfigurator.cpp
----------------------------------------------------------------------
diff --git a/cli/DefaultsConfigurator.cpp b/cli/DefaultsConfigurator.cpp
new file mode 100644
index 0000000..355031c
--- /dev/null
+++ b/cli/DefaultsConfigurator.cpp
@@ -0,0 +1,77 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ **/
+
+#include "cli/DefaultsConfigurator.hpp"
+
+#include "cli/CliConfig.h" // For QUICKSTEP_OS_WINDOWS.
+
+#ifdef QUICKSTEP_OS_WINDOWS
+#include <experimental/filesystem>
+#else
+#include <cstdlib>
+#endif // QUICKSTEP_OS_WINDOWS
+
+#include <fstream>
+#include <string>
+
+#include "catalog/Catalog.hpp"
+#include "catalog/Catalog.pb.h"
+#include "catalog/CatalogDatabase.hpp"
+
+#include "glog/logging.h"
+
+using std::string;
+
+namespace quickstep {
+
+void DefaultsConfigurator::InitializeDefaultDatabase(const string &storage_path, const string &catalog_path) {
+ // TODO(jmp): Refactor the code in this file!
+ LOG(INFO) << "Initializing the database, creating a new catalog file and storage directory";
+
+ // Create the directory
+ // TODO(jmp): At some point, likely in C++-17, we will just have the
+ // filesystem path, and we can clean this up
+#ifdef QUICKSTEP_OS_WINDOWS
+ CHECK(std::experimental::filesystem::create_directories(storage_path))
+ << "Failed when attempting to create the directory: " << storage_path
+ << "\nCheck if the directory already exists. If so, delete it or move it before initializing";
+#else
+ {
+ const string path_name = "mkdir " + storage_path;
+ CHECK(std::system(path_name.c_str()))
+ << "Failed when attempting to create the directory: " << storage_path;
+ }
+#endif // QUICKSTEP_OS_WINDOWS
+
+ // Create the default catalog file.
+ std::ofstream catalog_file(catalog_path.c_str());
+ CHECK(catalog_file.good())
+ << "ERROR: Unable to open " << catalog_path << " for writing.";
+
+ Catalog catalog;
+ catalog.addDatabase(new CatalogDatabase(nullptr, "default"));
+
+ CHECK(catalog.getProto().SerializeToOstream(&catalog_file))
+ << "ERROR: Unable to serialize catalog proto to file " << catalog_path;
+
+ // Close the catalog file - it will be reopened below by the QueryProcessor.
+ catalog_file.close();
+}
+
+} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f673f945/cli/DefaultsConfigurator.hpp
----------------------------------------------------------------------
diff --git a/cli/DefaultsConfigurator.hpp b/cli/DefaultsConfigurator.hpp
index 4fd62ac..4b534d6 100644
--- a/cli/DefaultsConfigurator.hpp
+++ b/cli/DefaultsConfigurator.hpp
@@ -27,6 +27,7 @@
#endif // QUICKSTEP_HAVE_LIBNUMA
#include <cstddef>
+#include <string>
#include <thread> // NOLINT(build/c++11)
#ifdef QUICKSTEP_HAVE_LIBNUMA
@@ -106,6 +107,15 @@ class DefaultsConfigurator {
return 1;
}
+ /**
+ * @brief Initialize the default database with no relations.
+ *
+ * @param storage_path The filesystem directory to store catalog.
+ * @param catalog_path The full path of the catalog file.
+ **/
+ static void InitializeDefaultDatabase(const std::string &storage_path,
+ const std::string &catalog_path);
+
private:
/**
* @brief Private constructor to disable instantiation of the class.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f673f945/cli/QuickstepCli.cpp
----------------------------------------------------------------------
diff --git a/cli/QuickstepCli.cpp b/cli/QuickstepCli.cpp
index 6a7c500..465793c 100644
--- a/cli/QuickstepCli.cpp
+++ b/cli/QuickstepCli.cpp
@@ -216,41 +216,7 @@ int main(int argc, char* argv[]) {
string catalog_path(FLAGS_storage_path);
catalog_path.append(kCatalogFilename);
if (quickstep::FLAGS_initialize_db) { // Initialize the database
- // TODO(jmp): Refactor the code in this file!
- LOG(INFO) << "Initializing the database, creating a new catalog file and storage directory\n";
-
- // Create the directory
- // TODO(jmp): At some point, likely in C++-17, we will just have the
- // filesystem path, and we can clean this up
-#ifdef QUICKSTEP_OS_WINDOWS
- std::filesystem::create_directories(FLAGS_storage_path);
- LOG(FATAL) << "Failed when attempting to create the directory: " << FLAGS_storage_path << "\n";
- LOG(FATAL) << "Check if the directory already exists. If so, delete it or move it before initializing \n";
-#else
- {
- string path_name = "mkdir " + FLAGS_storage_path;
- if (std::system(path_name.c_str())) {
- LOG(FATAL) << "Failed when attempting to create the directory: " << FLAGS_storage_path << "\n";
- }
- }
-#endif
-
- // Create the default catalog file.
- std::ofstream catalog_file(catalog_path);
- if (!catalog_file.good()) {
- LOG(FATAL) << "ERROR: Unable to open " << kCatalogFilename << " for writing.\n";
- }
-
- quickstep::Catalog catalog;
- catalog.addDatabase(new quickstep::CatalogDatabase(nullptr, "default"));
-
- if (!catalog.getProto().SerializeToOstream(&catalog_file)) {
- LOG(FATAL) << "ERROR: Unable to serialize catalog proto to file " << kCatalogFilename;
- return 1;
- }
-
- // Close the catalog file - it will be reopened below by the QueryProcessor.
- catalog_file.close();
+ DefaultsConfigurator::InitializeDefaultDatabase(FLAGS_storage_path, catalog_path);
}
// Setup QueryProcessor, including CatalogDatabase.
[02/50] incubator-quickstep git commit: Fixed the LIP bug in the
distributed execution engine unit tests.
Posted by zu...@apache.org.
Fixed the LIP bug in the distributed execution engine unit tests.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/fc898b11
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/fc898b11
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/fc898b11
Branch: refs/heads/quickstep_partition_parser_support
Commit: fc898b113efccdb77385297f27298da7b7441877
Parents: 7f0067b
Author: Zuyu Zhang <zu...@apache.org>
Authored: Mon Oct 31 21:19:58 2016 -0700
Committer: Zuyu Zhang <zu...@apache.org>
Committed: Mon Oct 31 21:19:58 2016 -0700
----------------------------------------------------------------------
relational_operators/WorkOrderFactory.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/fc898b11/relational_operators/WorkOrderFactory.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/WorkOrderFactory.cpp b/relational_operators/WorkOrderFactory.cpp
index 91a717f..6943a52 100644
--- a/relational_operators/WorkOrderFactory.cpp
+++ b/relational_operators/WorkOrderFactory.cpp
@@ -361,7 +361,7 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder
proto.GetExtension(serialization::SelectWorkOrder::insert_destination_index)),
storage_manager,
CreateLIPFilterAdaptiveProberHelper(
- proto.GetExtension(serialization::HashJoinWorkOrder::lip_deployment_index), query_context));
+ proto.GetExtension(serialization::SelectWorkOrder::lip_deployment_index), query_context));
}
case serialization::SORT_MERGE_RUN: {
LOG(INFO) << "Creating SortMergeRunWorkOrder";
[29/50] incubator-quickstep git commit: IWYU for QuickstepCli.
Posted by zu...@apache.org.
IWYU for QuickstepCli.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/172b51b5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/172b51b5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/172b51b5
Branch: refs/heads/quickstep_partition_parser_support
Commit: 172b51b5eaa9a2354792dc0a8a51fbad13ba10f2
Parents: f673f94
Author: Zuyu Zhang <zu...@apache.org>
Authored: Mon Nov 21 00:06:05 2016 -0800
Committer: Zuyu Zhang <zu...@apache.org>
Committed: Mon Nov 21 00:06:05 2016 -0800
----------------------------------------------------------------------
CMakeLists.txt | 3 ---
cli/DefaultsConfigurator.cpp | 1 +
cli/QuickstepCli.cpp | 21 +--------------------
3 files changed, 2 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/172b51b5/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/CMakeLists.txt b/CMakeLists.txt
index cd53967..126b47b 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -758,16 +758,13 @@ target_link_libraries(quickstep_cli_shell
quickstep_cli_PrintToScreen
quickstep_parser_ParseStatement
quickstep_parser_SqlParserWrapper
- quickstep_queryexecution_AdmitRequestMessage
quickstep_queryexecution_ForemanSingleNode
quickstep_queryexecution_QueryContext
quickstep_queryexecution_QueryExecutionTypedefs
quickstep_queryexecution_QueryExecutionUtil
quickstep_queryexecution_Worker
quickstep_queryexecution_WorkerDirectory
- quickstep_queryexecution_WorkerMessage
quickstep_queryoptimizer_QueryHandle
- quickstep_queryoptimizer_QueryPlan
quickstep_queryoptimizer_QueryProcessor
quickstep_storage_PreloaderThread
quickstep_storage_StorageConstants
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/172b51b5/cli/DefaultsConfigurator.cpp
----------------------------------------------------------------------
diff --git a/cli/DefaultsConfigurator.cpp b/cli/DefaultsConfigurator.cpp
index 355031c..94280a7 100644
--- a/cli/DefaultsConfigurator.cpp
+++ b/cli/DefaultsConfigurator.cpp
@@ -22,6 +22,7 @@
#include "cli/CliConfig.h" // For QUICKSTEP_OS_WINDOWS.
#ifdef QUICKSTEP_OS_WINDOWS
+// TODO(jmp): If filesystem shows up in C++-17, we can switch to just using that.
#include <experimental/filesystem>
#else
#include <cstdlib>
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/172b51b5/cli/QuickstepCli.cpp
----------------------------------------------------------------------
diff --git a/cli/QuickstepCli.cpp b/cli/QuickstepCli.cpp
index 465793c..656786a 100644
--- a/cli/QuickstepCli.cpp
+++ b/cli/QuickstepCli.cpp
@@ -27,16 +27,8 @@
#include <string>
#include <utility>
#include <vector>
-#include <fstream>
-#include "cli/CliConfig.h" // For QUICKSTEP_USE_LINENOISE, QUICKSTEP_ENABLE_GOOGLE_PROFILER, and QUICKSTEP_OS_WINDOWS.
-
-// TODO(jmp): If filesystem shows up in C++-17, we can switch to just using that.
-#ifdef QUICKSTEP_OS_WINDOWS
-#include <filesystem>
-#else
-#include <stdlib.h>
-#endif
+#include "cli/CliConfig.h" // For QUICKSTEP_USE_LINENOISE, QUICKSTEP_ENABLE_GOOGLE_PROFILER.
#include "cli/CommandExecutor.hpp"
#include "cli/DropRelation.hpp"
@@ -59,15 +51,12 @@ typedef quickstep::LineReaderDumb LineReaderImpl;
#include "cli/PrintToScreen.hpp"
#include "parser/ParseStatement.hpp"
#include "parser/SqlParserWrapper.hpp"
-#include "query_execution/AdmitRequestMessage.hpp"
#include "query_execution/ForemanSingleNode.hpp"
#include "query_execution/QueryExecutionTypedefs.hpp"
#include "query_execution/QueryExecutionUtil.hpp"
#include "query_execution/Worker.hpp"
#include "query_execution/WorkerDirectory.hpp"
-#include "query_execution/WorkerMessage.hpp"
#include "query_optimizer/QueryHandle.hpp"
-#include "query_optimizer/QueryPlan.hpp"
#include "query_optimizer/QueryProcessor.hpp"
#include "storage/StorageConfig.h" // For QUICKSTEP_HAVE_FILE_MANAGER_HDFS.
@@ -89,10 +78,8 @@ typedef quickstep::LineReaderDumb LineReaderImpl;
#include "glog/logging.h"
-#include "tmb/address.h"
#include "tmb/id_typedefs.h"
#include "tmb/message_bus.h"
-#include "tmb/message_style.h"
namespace quickstep {
class CatalogRelation;
@@ -104,8 +91,6 @@ using std::printf;
using std::string;
using std::vector;
-using quickstep::Address;
-using quickstep::AdmitRequestMessage;
using quickstep::CatalogRelation;
using quickstep::DefaultsConfigurator;
using quickstep::DropRelation;
@@ -114,20 +99,16 @@ using quickstep::FLAGS_storage_path;
using quickstep::ForemanSingleNode;
using quickstep::InputParserUtil;
using quickstep::MessageBusImpl;
-using quickstep::MessageStyle;
-using quickstep::ParseCommand;
using quickstep::ParseResult;
using quickstep::ParseStatement;
using quickstep::PrintToScreen;
using quickstep::PtrVector;
using quickstep::QueryExecutionUtil;
using quickstep::QueryHandle;
-using quickstep::QueryPlan;
using quickstep::QueryProcessor;
using quickstep::SqlParserWrapper;
using quickstep::Worker;
using quickstep::WorkerDirectory;
-using quickstep::WorkerMessage;
using quickstep::kAdmitRequestMessage;
using quickstep::kCatalogFilename;
using quickstep::kPoisonMessage;
[40/50] incubator-quickstep git commit: LOG only in the debug mode
for BlockLocator.
Posted by zu...@apache.org.
LOG only in the debug mode for BlockLocator.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/e50a2b7a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/e50a2b7a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/e50a2b7a
Branch: refs/heads/quickstep_partition_parser_support
Commit: e50a2b7aaed97f3282c338be0b36071a5cffd523
Parents: 5ff89dd
Author: Zuyu Zhang <zu...@apache.org>
Authored: Sat Dec 3 21:24:25 2016 -0800
Committer: Zuyu Zhang <zu...@apache.org>
Committed: Sat Dec 3 21:24:25 2016 -0800
----------------------------------------------------------------------
query_execution/BlockLocator.cpp | 48 ++++++++++++------------
query_execution/QueryManagerDistributed.cpp | 4 +-
2 files changed, 26 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/e50a2b7a/query_execution/BlockLocator.cpp
----------------------------------------------------------------------
diff --git a/query_execution/BlockLocator.cpp b/query_execution/BlockLocator.cpp
index 81684ba..5de6a54 100644
--- a/query_execution/BlockLocator.cpp
+++ b/query_execution/BlockLocator.cpp
@@ -55,8 +55,8 @@ void BlockLocator::run() {
const tmb::AnnotatedMessage annotated_message = bus_->Receive(locator_client_id_, 0, true);
const TaggedMessage &tagged_message = annotated_message.tagged_message;
const client_id sender = annotated_message.sender;
- LOG(INFO) << "BlockLocator received the typed '" << tagged_message.message_type()
- << "' message from TMB Client " << sender;
+ DLOG(INFO) << "BlockLocator received the typed '" << tagged_message.message_type()
+ << "' message from TMB Client " << sender;
switch (tagged_message.message_type()) {
case kBlockDomainRegistrationMessage: {
serialization::BlockDomainRegistrationMessage proto;
@@ -77,9 +77,9 @@ void BlockLocator::run() {
DCHECK_EQ(result_block_locations.second, result_domain_blocks.second);
if (result_domain_blocks.second) {
- LOG(INFO) << "Block " << BlockIdUtil::ToString(block) << " loaded in Domain " << domain;
+ DLOG(INFO) << "Block " << BlockIdUtil::ToString(block) << " loaded in Domain " << domain;
} else {
- LOG(INFO) << "Block " << BlockIdUtil::ToString(block) << " existed in Domain " << domain;
+ DLOG(INFO) << "Block " << BlockIdUtil::ToString(block) << " existed in Domain " << domain;
}
break;
}
@@ -95,9 +95,9 @@ void BlockLocator::run() {
block_locations_[block].erase(domain);
domain_blocks_[domain].erase(block);
- LOG(INFO) << "Block " << BlockIdUtil::ToString(block) << " evicted in Domain " << domain;
+ DLOG(INFO) << "Block " << BlockIdUtil::ToString(block) << " evicted in Domain " << domain;
} else {
- LOG(INFO) << "Block " << BlockIdUtil::ToString(block) << " not found in Domain " << domain;
+ DLOG(INFO) << "Block " << BlockIdUtil::ToString(block) << " not found in Domain " << domain;
}
break;
}
@@ -128,7 +128,7 @@ void BlockLocator::run() {
}
domain_blocks_.erase(domain);
- LOG(INFO) << "Unregistered Domain " << domain;
+ DLOG(INFO) << "Unregistered Domain " << domain;
break;
}
case kPoisonMessage: {
@@ -153,14 +153,14 @@ void BlockLocator::processBlockDomainRegistrationMessage(const client_id receive
CHECK(proto.SerializeToArray(proto_bytes, proto_length));
TaggedMessage message(static_cast<const void*>(proto_bytes),
- proto_length,
- kBlockDomainRegistrationResponseMessage);
+ proto_length,
+ kBlockDomainRegistrationResponseMessage);
free(proto_bytes);
- LOG(INFO) << "BlockLocator (id '" << locator_client_id_
- << "') sent BlockDomainRegistrationResponseMessage (typed '"
- << kBlockDomainRegistrationResponseMessage
- << "') to Worker (id '" << receiver << "')";
+ DLOG(INFO) << "BlockLocator (id '" << locator_client_id_
+ << "') sent BlockDomainRegistrationResponseMessage (typed '"
+ << kBlockDomainRegistrationResponseMessage
+ << "') to TMB Client (id '" << receiver << "')";
CHECK(tmb::MessageBus::SendStatus::kOK ==
QueryExecutionUtil::SendTMBMessage(bus_,
locator_client_id_,
@@ -181,13 +181,13 @@ void BlockLocator::processLocateBlockMessage(const client_id receiver,
CHECK(proto.SerializeToArray(proto_bytes, proto_length));
TaggedMessage message(static_cast<const void*>(proto_bytes),
- proto_length,
- kLocateBlockResponseMessage);
+ proto_length,
+ kLocateBlockResponseMessage);
free(proto_bytes);
- LOG(INFO) << "BlockLocator (id '" << locator_client_id_
- << "') sent LocateBlockResponseMessage (typed '" << kLocateBlockResponseMessage
- << "') to StorageManager (id '" << receiver << "')";
+ DLOG(INFO) << "BlockLocator (id '" << locator_client_id_
+ << "') sent LocateBlockResponseMessage (typed '" << kLocateBlockResponseMessage
+ << "') to StorageManager (id '" << receiver << "')";
CHECK(tmb::MessageBus::SendStatus::kOK ==
QueryExecutionUtil::SendTMBMessage(bus_,
locator_client_id_,
@@ -208,14 +208,14 @@ void BlockLocator::processGetPeerDomainNetworkAddressesMessage(const client_id r
CHECK(proto.SerializeToArray(proto_bytes, proto_length));
TaggedMessage message(static_cast<const void*>(proto_bytes),
- proto_length,
- kGetPeerDomainNetworkAddressesResponseMessage);
+ proto_length,
+ kGetPeerDomainNetworkAddressesResponseMessage);
free(proto_bytes);
- LOG(INFO) << "BlockLocator (id '" << locator_client_id_
- << "') sent GetPeerDomainNetworkAddressesResponseMessage (typed '"
- << kGetPeerDomainNetworkAddressesResponseMessage
- << "') to StorageManager (id '" << receiver << "')";
+ DLOG(INFO) << "BlockLocator (id '" << locator_client_id_
+ << "') sent GetPeerDomainNetworkAddressesResponseMessage (typed '"
+ << kGetPeerDomainNetworkAddressesResponseMessage
+ << "') to StorageManager (id '" << receiver << "')";
CHECK(tmb::MessageBus::SendStatus::kOK ==
QueryExecutionUtil::SendTMBMessage(bus_,
locator_client_id_,
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/e50a2b7a/query_execution/QueryManagerDistributed.cpp
----------------------------------------------------------------------
diff --git a/query_execution/QueryManagerDistributed.cpp b/query_execution/QueryManagerDistributed.cpp
index 20650d0..5c7e0d8 100644
--- a/query_execution/QueryManagerDistributed.cpp
+++ b/query_execution/QueryManagerDistributed.cpp
@@ -176,8 +176,8 @@ bool QueryManagerDistributed::initiateRebuild(const dag_node_index index) {
shiftboss_addresses.AddRecipient(shiftboss_directory_->getClientId(i));
}
- LOG(INFO) << "ForemanDistributed sent InitiateRebuildMessage (typed '" << kInitiateRebuildMessage
- << "') to all Shiftbosses";
+ DLOG(INFO) << "ForemanDistributed sent InitiateRebuildMessage (typed '" << kInitiateRebuildMessage
+ << "') to all Shiftbosses";
QueryExecutionUtil::BroadcastMessage(foreman_client_id_,
shiftboss_addresses,
move(tagged_msg),
[20/50] incubator-quickstep git commit: Refactor regarding the
profile flag.
Posted by zu...@apache.org.
Refactor regarding the profile flag.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/e7c664ba
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/e7c664ba
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/e7c664ba
Branch: refs/heads/quickstep_partition_parser_support
Commit: e7c664baa216fad0c7d04f08cad8398d6d9db2bd
Parents: 7fb96f3
Author: Zuyu Zhang <zu...@apache.org>
Authored: Fri Nov 18 17:38:17 2016 -0800
Committer: Zuyu Zhang <zu...@apache.org>
Committed: Sun Nov 20 19:51:25 2016 -0800
----------------------------------------------------------------------
cli/QuickstepCli.cpp | 13 ++++---------
query_execution/CMakeLists.txt | 3 ++-
query_execution/ForemanDistributed.cpp | 6 ++----
query_execution/ForemanDistributed.hpp | 5 +----
query_execution/ForemanSingleNode.cpp | 6 ++----
query_execution/ForemanSingleNode.hpp | 5 +----
query_execution/PolicyEnforcerBase.cpp | 15 +++++++++++++++
query_execution/PolicyEnforcerBase.hpp | 6 +-----
query_execution/PolicyEnforcerDistributed.hpp | 5 ++---
query_execution/PolicyEnforcerSingleNode.hpp | 5 ++---
10 files changed, 32 insertions(+), 37 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/e7c664ba/cli/QuickstepCli.cpp
----------------------------------------------------------------------
diff --git a/cli/QuickstepCli.cpp b/cli/QuickstepCli.cpp
index 0e8468e..a4e55d7 100644
--- a/cli/QuickstepCli.cpp
+++ b/cli/QuickstepCli.cpp
@@ -133,9 +133,6 @@ using tmb::client_id;
namespace quickstep {
-DEFINE_bool(profile_and_report_workorder_perf, false,
- "If true, Quickstep will record the exceution time of all the individual "
- "normal work orders and report it at the end of query execution.");
DEFINE_int32(num_workers, 0, "Number of worker threads. If this value is "
"specified and is greater than 0, then this "
"user-supplied value is used. Else (i.e. the"
@@ -181,10 +178,9 @@ DEFINE_string(profile_file_name, "",
// To put things in perspective, the first run is, in my experiments, about 5-10
// times more expensive than the average run. That means the query needs to be
// run at least a hundred times to make the impact of the first run small (< 5 %).
-DEFINE_bool(visualize_execution_dag, false,
- "If true, visualize the execution plan DAG into a graph in DOT "
- "format (DOT is a plain text graph description language) which is "
- "then printed via stderr.");
+
+DECLARE_bool(profile_and_report_workorder_perf);
+DECLARE_bool(visualize_execution_dag);
} // namespace quickstep
@@ -361,8 +357,7 @@ int main(int argc, char* argv[]) {
query_processor->getDefaultDatabase(),
&storage_manager,
-1, // Don't pin the Foreman thread.
- num_numa_nodes_system,
- quickstep::FLAGS_profile_and_report_workorder_perf || quickstep::FLAGS_visualize_execution_dag);
+ num_numa_nodes_system);
// Start the worker threads.
for (Worker &worker : workers) {
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/e7c664ba/query_execution/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_execution/CMakeLists.txt b/query_execution/CMakeLists.txt
index cf9d5b0..eec0029 100644
--- a/query_execution/CMakeLists.txt
+++ b/query_execution/CMakeLists.txt
@@ -138,7 +138,8 @@ target_link_libraries(quickstep_queryexecution_PolicyEnforcerBase
quickstep_relationaloperators_WorkOrder
quickstep_storage_StorageBlockInfo
quickstep_utility_Macros
- tmb)
+ tmb
+ ${GFLAGS_LIB_NAME})
if (ENABLE_DISTRIBUTED)
target_link_libraries(quickstep_queryexecution_PolicyEnforcerDistributed
glog
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/e7c664ba/query_execution/ForemanDistributed.cpp
----------------------------------------------------------------------
diff --git a/query_execution/ForemanDistributed.cpp b/query_execution/ForemanDistributed.cpp
index aa4db17..d619657 100644
--- a/query_execution/ForemanDistributed.cpp
+++ b/query_execution/ForemanDistributed.cpp
@@ -64,8 +64,7 @@ class QueryHandle;
ForemanDistributed::ForemanDistributed(
MessageBus *bus,
CatalogDatabaseLite *catalog_database,
- const int cpu_id,
- const bool profile_individual_workorders)
+ const int cpu_id)
: ForemanBase(bus, cpu_id),
catalog_database_(DCHECK_NOTNULL(catalog_database)) {
const std::vector<QueryExecutionMessageType> sender_message_types{
@@ -103,8 +102,7 @@ ForemanDistributed::ForemanDistributed(
foreman_client_id_,
catalog_database_,
&shiftboss_directory_,
- bus_,
- profile_individual_workorders);
+ bus_);
}
void ForemanDistributed::run() {
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/e7c664ba/query_execution/ForemanDistributed.hpp
----------------------------------------------------------------------
diff --git a/query_execution/ForemanDistributed.hpp b/query_execution/ForemanDistributed.hpp
index 7f2d2f9..ccdd0ae 100644
--- a/query_execution/ForemanDistributed.hpp
+++ b/query_execution/ForemanDistributed.hpp
@@ -54,16 +54,13 @@ class ForemanDistributed final : public ForemanBase {
* @param bus A pointer to the TMB.
* @param catalog_database The catalog database where this query is executed.
* @param cpu_id The ID of the CPU to which the Foreman thread can be pinned.
- * @param profile_individual_workorders Whether every workorder's execution
- * be profiled or not.
*
* @note If cpu_id is not specified, Foreman thread can be possibly moved
* around on different CPUs by the OS.
**/
ForemanDistributed(tmb::MessageBus *bus,
CatalogDatabaseLite *catalog_database,
- const int cpu_id = -1,
- const bool profile_individual_workorders = false);
+ const int cpu_id = -1);
~ForemanDistributed() override {}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/e7c664ba/query_execution/ForemanSingleNode.cpp
----------------------------------------------------------------------
diff --git a/query_execution/ForemanSingleNode.cpp b/query_execution/ForemanSingleNode.cpp
index dfdfb71..02799c7 100644
--- a/query_execution/ForemanSingleNode.cpp
+++ b/query_execution/ForemanSingleNode.cpp
@@ -63,8 +63,7 @@ ForemanSingleNode::ForemanSingleNode(
CatalogDatabaseLite *catalog_database,
StorageManager *storage_manager,
const int cpu_id,
- const size_t num_numa_nodes,
- const bool profile_individual_workorders)
+ const size_t num_numa_nodes)
: ForemanBase(bus, cpu_id),
main_thread_client_id_(main_thread_client_id),
worker_directory_(DCHECK_NOTNULL(worker_directory)),
@@ -99,8 +98,7 @@ ForemanSingleNode::ForemanSingleNode(
catalog_database_,
storage_manager_,
worker_directory_,
- bus_,
- profile_individual_workorders);
+ bus_);
}
void ForemanSingleNode::run() {
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/e7c664ba/query_execution/ForemanSingleNode.hpp
----------------------------------------------------------------------
diff --git a/query_execution/ForemanSingleNode.hpp b/query_execution/ForemanSingleNode.hpp
index 4cc7a63..d2db51b 100644
--- a/query_execution/ForemanSingleNode.hpp
+++ b/query_execution/ForemanSingleNode.hpp
@@ -60,8 +60,6 @@ class ForemanSingleNode final : public ForemanBase {
* @param storage_manager The StorageManager to use.
* @param cpu_id The ID of the CPU to which the Foreman thread can be pinned.
* @param num_numa_nodes The number of NUMA nodes in the system.
- * @param profile_individual_workorders Whether every workorder's execution
- * be profiled or not.
*
* @note If cpu_id is not specified, Foreman thread can be possibly moved
* around on different CPUs by the OS.
@@ -72,8 +70,7 @@ class ForemanSingleNode final : public ForemanBase {
CatalogDatabaseLite *catalog_database,
StorageManager *storage_manager,
const int cpu_id = -1,
- const std::size_t num_numa_nodes = 1,
- const bool profile_individual_workorders = false);
+ const std::size_t num_numa_nodes = 1);
~ForemanSingleNode() override {}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/e7c664ba/query_execution/PolicyEnforcerBase.cpp
----------------------------------------------------------------------
diff --git a/query_execution/PolicyEnforcerBase.cpp b/query_execution/PolicyEnforcerBase.cpp
index 6e922a8..b799d5f 100644
--- a/query_execution/PolicyEnforcerBase.cpp
+++ b/query_execution/PolicyEnforcerBase.cpp
@@ -35,10 +35,25 @@
#include "relational_operators/WorkOrder.hpp"
#include "storage/StorageBlockInfo.hpp"
+#include "gflags/gflags.h"
#include "glog/logging.h"
namespace quickstep {
+DEFINE_bool(profile_and_report_workorder_perf, false,
+ "If true, Quickstep will record the exceution time of all the individual "
+ "normal work orders and report it at the end of query execution.");
+
+DEFINE_bool(visualize_execution_dag, false,
+ "If true, visualize the execution plan DAG into a graph in DOT "
+ "format (DOT is a plain text graph description language) which is "
+ "then printed via stderr.");
+
+PolicyEnforcerBase::PolicyEnforcerBase(CatalogDatabaseLite *catalog_database)
+ : catalog_database_(catalog_database),
+ profile_individual_workorders_(FLAGS_profile_and_report_workorder_perf || FLAGS_visualize_execution_dag) {
+}
+
void PolicyEnforcerBase::processMessage(const TaggedMessage &tagged_message) {
std::size_t query_id;
QueryManagerBase::dag_node_index op_index;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/e7c664ba/query_execution/PolicyEnforcerBase.hpp
----------------------------------------------------------------------
diff --git a/query_execution/PolicyEnforcerBase.hpp b/query_execution/PolicyEnforcerBase.hpp
index aa070b8..baf9c68 100644
--- a/query_execution/PolicyEnforcerBase.hpp
+++ b/query_execution/PolicyEnforcerBase.hpp
@@ -54,12 +54,8 @@ class PolicyEnforcerBase {
* @brief Constructor.
*
* @param catalog_database The CatalogDatabase used.
- * @param profile_individual_workorders If true, profile each normal work order.
**/
- PolicyEnforcerBase(CatalogDatabaseLite *catalog_database,
- const bool profile_individual_workorders)
- : catalog_database_(catalog_database),
- profile_individual_workorders_(profile_individual_workorders) {}
+ explicit PolicyEnforcerBase(CatalogDatabaseLite *catalog_database);
/**
* @brief Virtual Destructor.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/e7c664ba/query_execution/PolicyEnforcerDistributed.hpp
----------------------------------------------------------------------
diff --git a/query_execution/PolicyEnforcerDistributed.hpp b/query_execution/PolicyEnforcerDistributed.hpp
index 0bf249c..146e9af 100644
--- a/query_execution/PolicyEnforcerDistributed.hpp
+++ b/query_execution/PolicyEnforcerDistributed.hpp
@@ -59,9 +59,8 @@ class PolicyEnforcerDistributed final : public PolicyEnforcerBase {
PolicyEnforcerDistributed(const tmb::client_id foreman_client_id,
CatalogDatabaseLite *catalog_database,
ShiftbossDirectory *shiftboss_directory,
- tmb::MessageBus *bus,
- const bool profile_individual_workorders = false)
- : PolicyEnforcerBase(catalog_database, profile_individual_workorders),
+ tmb::MessageBus *bus)
+ : PolicyEnforcerBase(catalog_database),
foreman_client_id_(foreman_client_id),
shiftboss_directory_(shiftboss_directory),
bus_(bus) {}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/e7c664ba/query_execution/PolicyEnforcerSingleNode.hpp
----------------------------------------------------------------------
diff --git a/query_execution/PolicyEnforcerSingleNode.hpp b/query_execution/PolicyEnforcerSingleNode.hpp
index 16c7a0d..f87d670 100644
--- a/query_execution/PolicyEnforcerSingleNode.hpp
+++ b/query_execution/PolicyEnforcerSingleNode.hpp
@@ -64,9 +64,8 @@ class PolicyEnforcerSingleNode final : public PolicyEnforcerBase {
CatalogDatabaseLite *catalog_database,
StorageManager *storage_manager,
WorkerDirectory *worker_directory,
- tmb::MessageBus *bus,
- const bool profile_individual_workorders = false)
- : PolicyEnforcerBase(catalog_database, profile_individual_workorders),
+ tmb::MessageBus *bus)
+ : PolicyEnforcerBase(catalog_database),
foreman_client_id_(foreman_client_id),
num_numa_nodes_(num_numa_nodes),
storage_manager_(storage_manager),
[49/50] incubator-quickstep git commit: Removes packed row store.
Posted by zu...@apache.org.
Removes packed row store.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/5fee8218
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/5fee8218
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/5fee8218
Branch: refs/heads/quickstep_partition_parser_support
Commit: 5fee82186da965eb204ec2435dab5b0f7690920e
Parents: c394405
Author: cramja <ma...@gmail.com>
Authored: Thu Nov 17 14:16:50 2016 -0600
Committer: cramja <ma...@gmail.com>
Committed: Sun Jan 8 14:55:21 2017 -0600
----------------------------------------------------------------------
catalog/tests/Catalog_unittest.cpp | 26 +-
query_optimizer/OptimizerTree.hpp | 4 -
query_optimizer/resolver/Resolver.cpp | 5 +-
.../tests/logical_generator/Create.test | 4 +-
query_optimizer/tests/resolver/Create.test | 56 +-
relational_operators/CMakeLists.txt | 2 -
.../tests/SortMergeRunOperator_unittest.cpp | 1 -
.../SortRunGenerationOperator_unittest.cpp | 1 -
storage/PackedRowStoreTupleStorageSubBlock.cpp | 491 ----------------
storage/PackedRowStoreTupleStorageSubBlock.hpp | 216 -------
storage/PackedRowStoreValueAccessor.hpp | 150 -----
storage/SplitRowStoreTupleStorageSubBlock.cpp | 4 +-
storage/SplitRowStoreTupleStorageSubBlock.hpp | 2 +-
storage/StorageBlock.cpp | 7 -
storage/StorageBlockInfo.cpp | 3 +-
storage/StorageBlockInfo.hpp | 1 -
storage/StorageBlockLayout.hpp | 4 +-
storage/StorageBlockLayout.proto | 9 +-
storage/SubBlockTypeRegistry.hpp | 4 +-
storage/ValueAccessor.hpp | 1 -
storage/ValueAccessorUtil.hpp | 11 -
...ColumnStoreTupleStorageSubBlock_unittest.cpp | 2 +-
...kedRowStoreTupleStorageSubBlock_unittest.cpp | 2 +-
...kedRowStoreTupleStorageSubBlock_unittest.cpp | 584 -------------------
24 files changed, 44 insertions(+), 1546 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5fee8218/catalog/tests/Catalog_unittest.cpp
----------------------------------------------------------------------
diff --git a/catalog/tests/Catalog_unittest.cpp b/catalog/tests/Catalog_unittest.cpp
index e430b89..276bfa9 100644
--- a/catalog/tests/Catalog_unittest.cpp
+++ b/catalog/tests/Catalog_unittest.cpp
@@ -52,7 +52,6 @@ QUICKSTEP_FORCE_SUB_BLOCK_REGISTRATION(BasicColumnStoreTupleStorageSubBlock);
QUICKSTEP_FORCE_SUB_BLOCK_REGISTRATION(CSBTreeIndexSubBlock);
QUICKSTEP_FORCE_SUB_BLOCK_REGISTRATION(CompressedColumnStoreTupleStorageSubBlock);
QUICKSTEP_FORCE_SUB_BLOCK_REGISTRATION(CompressedPackedRowStoreTupleStorageSubBlock);
-QUICKSTEP_FORCE_SUB_BLOCK_REGISTRATION(PackedRowStoreTupleStorageSubBlock);
QUICKSTEP_FORCE_SUB_BLOCK_REGISTRATION(SMAIndexSubBlock);
QUICKSTEP_FORCE_SUB_BLOCK_REGISTRATION(SplitRowStoreTupleStorageSubBlock);
@@ -79,8 +78,6 @@ class CatalogTest : public ::testing::Test {
EXPECT_EQ(expected.sub_block_type(), checked.sub_block_type());
switch (expected.sub_block_type()) {
- case TupleStorageSubBlockDescription::PACKED_ROW_STORE:
- break;
case TupleStorageSubBlockDescription::BASIC_COLUMN_STORE:
EXPECT_TRUE(expected.HasExtension(BasicColumnStoreTupleStorageSubBlockDescription::sort_attribute_id));
EXPECT_TRUE(checked.HasExtension(BasicColumnStoreTupleStorageSubBlockDescription::sort_attribute_id));
@@ -436,27 +433,6 @@ TEST_F(CatalogTest, DroppedCatalogRelationSerializationTest) {
checkCatalogSerialization();
}
-TEST_F(CatalogTest, CatalogPackedRowStoreSerializationTest) {
- CatalogRelation* const rel = createCatalogRelation("rel");
- StorageBlockLayoutDescription layout_description(rel->getDefaultStorageBlockLayout().getDescription());
-
- rel->addAttribute(new CatalogAttribute(nullptr, "attr_int", TypeFactory::GetType(kInt)));
- rel->addAttribute(new CatalogAttribute(nullptr, "attr_long", TypeFactory::GetType(kLong)));
- rel->addAttribute(new CatalogAttribute(nullptr, "attr_float", TypeFactory::GetType(kFloat)));
- rel->addAttribute(new CatalogAttribute(nullptr, "attr_double", TypeFactory::GetType(kDouble)));
-
- const std::size_t str_type_length = 20;
- rel->addAttribute(new CatalogAttribute(nullptr, "attr_char", TypeFactory::GetType(kChar, str_type_length)));
- // NOTE(zuyu): PackedRowStoreTupleStorageSubBlock does NOT support variable-length attributes.
-
- layout_description.mutable_tuple_store_description()->set_sub_block_type(
- TupleStorageSubBlockDescription::PACKED_ROW_STORE);
-
- rel->setDefaultStorageBlockLayout(new StorageBlockLayout(*rel, layout_description));
-
- checkCatalogSerialization();
-}
-
TEST_F(CatalogTest, CatalogBasicColumnStoreSerializationTest) {
CatalogRelation* const rel = createCatalogRelation("rel");
StorageBlockLayoutDescription layout_description(rel->getDefaultStorageBlockLayout().getDescription());
@@ -569,7 +545,7 @@ TEST_F(CatalogTest, CatalogIndexTest) {
rel->addAttribute(new CatalogAttribute(nullptr, "attr_idx2", TypeFactory::GetType(kInt)));
layout_description.mutable_tuple_store_description()->set_sub_block_type(
- TupleStorageSubBlockDescription::PACKED_ROW_STORE);
+ TupleStorageSubBlockDescription::SPLIT_ROW_STORE);
rel->setDefaultStorageBlockLayout(new StorageBlockLayout(*rel, layout_description));
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5fee8218/query_optimizer/OptimizerTree.hpp
----------------------------------------------------------------------
diff --git a/query_optimizer/OptimizerTree.hpp b/query_optimizer/OptimizerTree.hpp
index 8ea2c6f..62df66d 100644
--- a/query_optimizer/OptimizerTree.hpp
+++ b/query_optimizer/OptimizerTree.hpp
@@ -233,10 +233,6 @@ OptimizerProtoRepresentation<TreeNodeType>* getOptimizerRepresentationForProto(
const ::quickstep::TupleStorageSubBlockDescription &storage_block_description
= description->tuple_store_description();
switch (storage_block_description.sub_block_type()) {
- case TupleStorageSubBlockDescription::PACKED_ROW_STORE: {
- node->addProperty("blocktype", "rowstore");
- break;
- }
case TupleStorageSubBlockDescription::SPLIT_ROW_STORE: {
node->addProperty("blocktype", "split_rowstore");
break;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5fee8218/query_optimizer/resolver/Resolver.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/resolver/Resolver.cpp b/query_optimizer/resolver/Resolver.cpp
index 38ec24a..2580342 100644
--- a/query_optimizer/resolver/Resolver.cpp
+++ b/query_optimizer/resolver/Resolver.cpp
@@ -520,10 +520,7 @@ StorageBlockLayoutDescription* Resolver::resolveBlockProperties(
<< "TYPE property must be specified and be a string.";
}
const std::string type_string = ToLower(type_parse_string->value());
- if (type_string.compare("rowstore") == 0) {
- description->set_sub_block_type(
- quickstep::TupleStorageSubBlockDescription::PACKED_ROW_STORE);
- } else if (type_string.compare("split_rowstore") == 0) {
+ if (type_string.compare("split_rowstore") == 0) {
description->set_sub_block_type(
quickstep::TupleStorageSubBlockDescription::SPLIT_ROW_STORE);
} else if (type_string.compare("columnstore") == 0) {
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5fee8218/query_optimizer/tests/logical_generator/Create.test
----------------------------------------------------------------------
diff --git a/query_optimizer/tests/logical_generator/Create.test b/query_optimizer/tests/logical_generator/Create.test
index a35a92a..aeff9ec 100644
--- a/query_optimizer/tests/logical_generator/Create.test
+++ b/query_optimizer/tests/logical_generator/Create.test
@@ -16,12 +16,12 @@
# under the License.
[default optimized_logical_plan]
-CREATE TABLE foo (attr int) WITH BLOCKPROPERTIES (TYPE rowstore, BLOCKSIZEMB 10)
+CREATE TABLE foo (attr int) WITH BLOCKPROPERTIES (TYPE split_rowstore, BLOCKSIZEMB 10)
--
TopLevelPlan
+-plan=CreateTable[relation=foo]
| +-block_properties=ProtoDescription
-| | +-Property=ProtoProperty[Property=blocktype,Value=rowstore]
+| | +-Property=ProtoProperty[Property=blocktype,Value=split_rowstore]
| | +-Property=ProtoProperty[Property=slots,Value=5]
| +-attributes=
| +-AttributeReference[id=0,name=attr,relation=foo,type=Int]
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5fee8218/query_optimizer/tests/resolver/Create.test
----------------------------------------------------------------------
diff --git a/query_optimizer/tests/resolver/Create.test b/query_optimizer/tests/resolver/Create.test
index 7cd980f..b04d785 100644
--- a/query_optimizer/tests/resolver/Create.test
+++ b/query_optimizer/tests/resolver/Create.test
@@ -90,20 +90,20 @@ create table "_qstemp_result_s" (col int)
# Cannot repeat Block properties.
CREATE TABLE foo (attr INT) WITH BLOCKPROPERTIES
-(TYPE rowstore, TYPE columnstore);
+(TYPE columnstore, TYPE columnstore);
--
-ERROR: Properties must be specified at most once. (2 : 17)
-(TYPE rowstore, TYPE columnstore);
- ^
+ERROR: Properties must be specified at most once. (2 : 20)
+(TYPE columnstore, TYPE columnstore);
+ ^
==
# Unrecognized BLOCKPROPERTIES should throw.
CREATE TABLE foo (attr INT) WITH BLOCKPROPERTIES
-(TYPE rowstore, INVALID_PROP val);
+(TYPE split_rowstore, INVALID_PROP val);
--
-ERROR: Unrecognized property name. (2 : 17)
-(TYPE rowstore, INVALID_PROP val);
- ^
+ERROR: Unrecognized property name. (2 : 23)
+(TYPE split_rowstore, INVALID_PROP val);
+ ^
==
# TYPE property only accepts names of storage blocks.
@@ -126,11 +126,11 @@ BLOCKPROPERTIES (BLOCKSIZEMB 1...
# Rowstores cannot have a sorted attribute.
CREATE TABLE foo (attr INT) WITH BLOCKPROPERTIES
-(TYPE rowstore, SORT attr);
+(TYPE split_rowstore, SORT attr);
--
-ERROR: The SORT property does not apply to this block type. (2 : 22)
-(TYPE rowstore, SORT attr);
- ^
+ERROR: The SORT property does not apply to this block type. (2 : 28)
+(TYPE split_rowstore, SORT attr);
+ ^
==
# Columnstores require a sort attribute.
@@ -197,35 +197,35 @@ ERROR: The given attribute was not found. (2 : 44)
# BLOCKSIZEMB property must be an integer.
CREATE TABLE foo (attr INT) WITH BLOCKPROPERTIES
-(TYPE rowstore, BLOCKSIZEMB ten);
+(TYPE split_rowstore, BLOCKSIZEMB ten);
--
-ERROR: The BLOCKSIZEMB property must be an integer. (2 : 17)
-(TYPE rowstore, BLOCKSIZEMB ten);
- ^
+ERROR: The BLOCKSIZEMB property must be an integer. (2 : 23)
+(TYPE split_rowstore, BLOCKSIZEMB ten);
+ ^
==
# BLOCKSIZEMB property must be multiple times of the slot size.
CREATE TABLE foo (attr INT) WITH BLOCKPROPERTIES
-(TYPE rowstore, BLOCKSIZEMB 25);
+(TYPE split_rowstore, BLOCKSIZEMB 25);
--
-ERROR: The BLOCKSIZEMB property must be multiple times of 2MB. (2 : 17)
-(TYPE rowstore, BLOCKSIZEMB 25);
- ^
+ERROR: The BLOCKSIZEMB property must be multiple times of 2MB. (2 : 23)
+(TYPE split_rowstore, BLOCKSIZEMB 25);
+ ^
==
# BLOCKSIZEMB must be greater than the minimum (defined in StorageConstants.hpp).
CREATE TABLE foo (attr INT) WITH BLOCKPROPERTIES
-(TYPE rowstore, BLOCKSIZEMB 0);
+(TYPE split_rowstore, BLOCKSIZEMB 0);
--
-ERROR: The BLOCKSIZEMB property must be between 2MB and 1024MB. (2 : 17)
-(TYPE rowstore, BLOCKSIZEMB 0);
- ^
+ERROR: The BLOCKSIZEMB property must be between 2MB and 1024MB. (2 : 23)
+(TYPE split_rowstore, BLOCKSIZEMB 0);
+ ^
==
# BLOCKSIZEMB must be less than the maximum (defined in StorageConstants.hpp).
CREATE TABLE foo (attr INT) WITH BLOCKPROPERTIES
-(TYPE rowstore, BLOCKSIZEMB 2000);
+(TYPE split_rowstore, BLOCKSIZEMB 2000);
--
-ERROR: The BLOCKSIZEMB property must be between 2MB and 1024MB. (2 : 17)
-(TYPE rowstore, BLOCKSIZEMB 2000);
- ^
+ERROR: The BLOCKSIZEMB property must be between 2MB and 1024MB. (2 : 23)
+(TYPE split_rowstore, BLOCKSIZEMB 2000);
+ ^
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5fee8218/relational_operators/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/relational_operators/CMakeLists.txt b/relational_operators/CMakeLists.txt
index 9e4b1b6..526820e 100644
--- a/relational_operators/CMakeLists.txt
+++ b/relational_operators/CMakeLists.txt
@@ -674,7 +674,6 @@ target_link_libraries(SortMergeRunOperator_unittest
quickstep_storage_CountedReference
quickstep_storage_InsertDestination
quickstep_storage_InsertDestination_proto
- quickstep_storage_PackedRowStoreValueAccessor
quickstep_storage_SplitRowStoreValueAccessor
quickstep_storage_StorageBlock
quickstep_storage_StorageBlockInfo
@@ -723,7 +722,6 @@ target_link_libraries(SortRunGenerationOperator_unittest
quickstep_storage_CompressedPackedRowStoreValueAccessor
quickstep_storage_CountedReference
quickstep_storage_InsertDestination
- quickstep_storage_PackedRowStoreValueAccessor
quickstep_storage_SplitRowStoreValueAccessor
quickstep_storage_StorageBlock
quickstep_storage_StorageBlockInfo
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5fee8218/relational_operators/tests/SortMergeRunOperator_unittest.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/tests/SortMergeRunOperator_unittest.cpp b/relational_operators/tests/SortMergeRunOperator_unittest.cpp
index b37d81f..a9128b1 100644
--- a/relational_operators/tests/SortMergeRunOperator_unittest.cpp
+++ b/relational_operators/tests/SortMergeRunOperator_unittest.cpp
@@ -51,7 +51,6 @@
#include "storage/CountedReference.hpp"
#include "storage/InsertDestination.hpp"
#include "storage/InsertDestination.pb.h"
-#include "storage/PackedRowStoreValueAccessor.hpp"
#include "storage/SplitRowStoreValueAccessor.hpp"
#include "storage/StorageBlock.hpp"
#include "storage/StorageBlockInfo.hpp"
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5fee8218/relational_operators/tests/SortRunGenerationOperator_unittest.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/tests/SortRunGenerationOperator_unittest.cpp b/relational_operators/tests/SortRunGenerationOperator_unittest.cpp
index d09ff07..99fafa8 100644
--- a/relational_operators/tests/SortRunGenerationOperator_unittest.cpp
+++ b/relational_operators/tests/SortRunGenerationOperator_unittest.cpp
@@ -47,7 +47,6 @@
#include "storage/CountedReference.hpp"
#include "storage/InsertDestination.hpp"
#include "storage/InsertDestination.pb.h"
-#include "storage/PackedRowStoreValueAccessor.hpp"
#include "storage/SplitRowStoreValueAccessor.hpp"
#include "storage/StorageBlock.hpp"
#include "storage/StorageBlockInfo.hpp"
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5fee8218/storage/PackedRowStoreTupleStorageSubBlock.cpp
----------------------------------------------------------------------
diff --git a/storage/PackedRowStoreTupleStorageSubBlock.cpp b/storage/PackedRowStoreTupleStorageSubBlock.cpp
deleted file mode 100644
index 0ad4a4c..0000000
--- a/storage/PackedRowStoreTupleStorageSubBlock.cpp
+++ /dev/null
@@ -1,491 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- **/
-
-#include "storage/PackedRowStoreTupleStorageSubBlock.hpp"
-
-#include <cstddef>
-#include <cstring>
-#include <vector>
-
-#include "catalog/CatalogAttribute.hpp"
-#include "catalog/CatalogRelationSchema.hpp"
-#include "catalog/CatalogTypedefs.hpp"
-#include "storage/PackedRowStoreValueAccessor.hpp"
-#include "storage/StorageBlockInfo.hpp"
-#include "storage/StorageBlockLayout.pb.h"
-#include "storage/StorageErrors.hpp"
-#include "storage/SubBlockTypeRegistry.hpp"
-#include "storage/TupleIdSequence.hpp"
-#include "storage/ValueAccessor.hpp"
-#include "storage/ValueAccessorUtil.hpp"
-#include "types/Type.hpp"
-#include "types/TypedValue.hpp"
-#include "types/containers/Tuple.hpp"
-#include "utility/BitVector.hpp"
-#include "utility/Macros.hpp"
-
-using std::vector;
-using std::memcpy;
-using std::size_t;
-
-namespace quickstep {
-
-QUICKSTEP_REGISTER_TUPLE_STORE(PackedRowStoreTupleStorageSubBlock, PACKED_ROW_STORE);
-
-PackedRowStoreTupleStorageSubBlock::PackedRowStoreTupleStorageSubBlock(
- const CatalogRelationSchema &relation,
- const TupleStorageSubBlockDescription &description,
- const bool new_block,
- void *sub_block_memory,
- const std::size_t sub_block_memory_size)
- : TupleStorageSubBlock(relation,
- description,
- new_block,
- sub_block_memory,
- sub_block_memory_size),
- header_(static_cast<PackedRowStoreHeader*>(sub_block_memory)),
- null_bitmap_bytes_(0) {
- if (!DescriptionIsValid(relation_, description_)) {
- FATAL_ERROR("Attempted to construct a PackedRowStoreTupleStorageSubBlock from an invalid description.");
- }
-
- if (sub_block_memory_size < sizeof(PackedRowStoreHeader)) {
- throw BlockMemoryTooSmall("PackedRowStoreTupleStorageSubBlock", sub_block_memory_size);
- }
-
- if (relation_.hasNullableAttributes()) {
- // Compute on the order of bits to account for bits in null_bitmap_.
- tuple_id row_capacity = ((sub_block_memory_size_ - sizeof(PackedRowStoreHeader)) << 3)
- / ((relation.getFixedByteLength() << 3) + relation.numNullableAttributes());
- null_bitmap_bytes_ = BitVector<false>::BytesNeeded(row_capacity * relation.numNullableAttributes());
-
- if (sub_block_memory_size < sizeof(PackedRowStoreHeader) + null_bitmap_bytes_) {
- if (relation_.getFixedByteLength() == 0) {
- // Special case: relation consists entirely of NullType attributes.
- row_capacity = BitVector<false>::MaxCapacityForBytes(
- sub_block_memory_size - sizeof(PackedRowStoreHeader))
- / relation.numNullableAttributes();
- null_bitmap_bytes_ = sub_block_memory_size - sizeof(PackedRowStoreHeader);
- } else {
- throw BlockMemoryTooSmall("PackedRowStoreTupleStorageSubBlock", sub_block_memory_size);
- }
- }
-
- null_bitmap_.reset(new BitVector<false>(static_cast<char*>(sub_block_memory_)
- + sizeof(PackedRowStoreHeader),
- row_capacity * relation.numNullableAttributes()));
- tuple_storage_ = static_cast<char*>(sub_block_memory_)
- + sizeof(PackedRowStoreHeader)
- + null_bitmap_bytes_;
- } else {
- tuple_storage_ = static_cast<char*>(sub_block_memory_)
- + sizeof(PackedRowStoreHeader);
- }
-
- if (new_block) {
- header_->num_tuples = 0;
- if (relation_.hasNullableAttributes()) {
- null_bitmap_->clear();
- }
- }
-}
-
-bool PackedRowStoreTupleStorageSubBlock::DescriptionIsValid(
- const CatalogRelationSchema &relation,
- const TupleStorageSubBlockDescription &description) {
- // Make sure description is initialized and specifies PackedRowStore.
- if (!description.IsInitialized()) {
- return false;
- }
- if (description.sub_block_type() != TupleStorageSubBlockDescription::PACKED_ROW_STORE) {
- return false;
- }
-
- // Make sure relation is not variable-length.
- if (relation.isVariableLength()) {
- return false;
- }
-
- return true;
-}
-
-std::size_t PackedRowStoreTupleStorageSubBlock::EstimateBytesPerTuple(
- const CatalogRelationSchema &relation,
- const TupleStorageSubBlockDescription &description) {
- DEBUG_ASSERT(DescriptionIsValid(relation, description));
-
- // NOTE(chasseur): We round-up the number of bytes needed in the NULL bitmap
- // to avoid estimating 0 bytes needed for a relation with less than 8
- // attributes which are all NullType.
- return relation.getFixedByteLength()
- + ((relation.numNullableAttributes() + 7) >> 3);
-}
-
-tuple_id PackedRowStoreTupleStorageSubBlock::bulkInsertTuples(ValueAccessor *accessor) {
- const tuple_id original_num_tuples = header_->num_tuples;
- char *dest_addr = static_cast<char*>(tuple_storage_)
- + header_->num_tuples * relation_.getFixedByteLength();
- const unsigned num_nullable_attrs = relation_.numNullableAttributes();
-
- InvokeOnAnyValueAccessor(
- accessor,
- [this, &dest_addr, &num_nullable_attrs](auto *accessor) -> void { // NOLINT(build/c++11)
- const std::size_t num_attrs = relation_.size();
- const std::vector<std::size_t> &attrs_max_size =
- relation_.getMaximumAttributeByteLengths();
-
- if (num_nullable_attrs != 0) {
- while (this->hasSpaceToInsert<true>(1) && accessor->next()) {
- for (std::size_t curr_attr = 0; curr_attr < num_attrs; ++curr_attr) {
- const std::size_t attr_size = attrs_max_size[curr_attr];
- const attribute_id nullable_idx = relation_.getNullableAttributeIndex(curr_attr);
- // If this attribute is nullable, check for a returned null value.
- if (nullable_idx != kInvalidCatalogId) {
- const void *attr_value
- = accessor->template getUntypedValue<true>(curr_attr);
- if (attr_value == nullptr) {
- null_bitmap_->setBit(
- header_->num_tuples * num_nullable_attrs + nullable_idx,
- true);
- } else {
- memcpy(dest_addr, attr_value, attr_size);
- }
- } else {
- memcpy(dest_addr,
- accessor->template getUntypedValue<false>(curr_attr),
- attr_size);
- }
- dest_addr += attr_size;
- }
- ++(header_->num_tuples);
- }
- } else {
- // If the accessor is from a packed row store, we can optimize the
- // memcpy by avoiding iterating over each attribute.
- const bool fast_copy =
- (accessor->getImplementationType() ==
- ValueAccessor::Implementation::kCompressedPackedRowStore);
- const std::size_t attrs_total_size = relation_.getMaximumByteLength();
- while (this->hasSpaceToInsert<false>(1) && accessor->next()) {
- if (fast_copy) {
- memcpy(dest_addr,
- accessor->template getUntypedValue<false>(0),
- attrs_total_size);
- } else {
- for (std::size_t curr_attr = 0; curr_attr < num_attrs; ++curr_attr) {
- const std::size_t attr_size = attrs_max_size[curr_attr];
- memcpy(dest_addr,
- accessor->template getUntypedValue<false>(curr_attr),
- attr_size);
- dest_addr += attr_size;
- }
- }
- ++(header_->num_tuples);
- }
- }
- });
-
- return header_->num_tuples - original_num_tuples;
-}
-
-tuple_id PackedRowStoreTupleStorageSubBlock::bulkInsertTuplesWithRemappedAttributes(
- const std::vector<attribute_id> &attribute_map,
- ValueAccessor *accessor) {
- DEBUG_ASSERT(attribute_map.size() == relation_.size());
-
- const tuple_id original_num_tuples = header_->num_tuples;
- char *dest_addr = static_cast<char*>(tuple_storage_)
- + header_->num_tuples * relation_.getFixedByteLength();
- const unsigned num_nullable_attrs = relation_.numNullableAttributes();
-
- InvokeOnAnyValueAccessor(
- accessor,
- [this, &num_nullable_attrs, &attribute_map, &dest_addr](auto *accessor) -> void { // NOLINT(build/c++11)
- const std::size_t num_attrs = relation_.size();
- const std::vector<std::size_t> &attrs_max_size =
- relation_.getMaximumAttributeByteLengths();
-
- if (num_nullable_attrs != 0) {
- while (this->hasSpaceToInsert<true>(1) && accessor->next()) {
- for (std::size_t curr_attr = 0; curr_attr < num_attrs; ++curr_attr) {
- const std::size_t attr_size = attrs_max_size[curr_attr];
- const attribute_id nullable_idx = relation_.getNullableAttributeIndex(curr_attr);
- // If this attribute is nullable, check for a returned null value.
- if (nullable_idx != kInvalidCatalogId) {
- const void *attr_value
- = accessor->template getUntypedValue<true>(attribute_map[curr_attr]);
- if (attr_value == nullptr) {
- null_bitmap_->setBit(
- header_->num_tuples * num_nullable_attrs + nullable_idx,
- true);
- } else {
- memcpy(dest_addr, attr_value, attr_size);
- }
- } else {
- memcpy(dest_addr,
- accessor->template getUntypedValue<false>(attribute_map[curr_attr]),
- attr_size);
- }
- dest_addr += attr_size;
- }
- ++(header_->num_tuples);
- }
- } else {
- while (this->hasSpaceToInsert<false>(1) && accessor->next()) {
- for (std::size_t curr_attr = 0; curr_attr < num_attrs; ++curr_attr) {
- const std::size_t attr_size = attrs_max_size[curr_attr];
- memcpy(dest_addr,
- accessor->template getUntypedValue<false>(attribute_map[curr_attr]),
- attr_size);
- dest_addr += attr_size;
- }
- ++(header_->num_tuples);
- }
- }
- });
-
- return header_->num_tuples - original_num_tuples;
-}
-
-const void* PackedRowStoreTupleStorageSubBlock::getAttributeValue(
- const tuple_id tuple,
- const attribute_id attr) const {
- DEBUG_ASSERT(hasTupleWithID(tuple));
- DEBUG_ASSERT(relation_.hasAttributeWithId(attr));
-
- const int nullable_idx = relation_.getNullableAttributeIndex(attr);
- if ((nullable_idx != -1)
- && null_bitmap_->getBit(tuple * relation_.numNullableAttributes() + nullable_idx)) {
- return nullptr;
- }
-
- return static_cast<char*>(tuple_storage_) // Start of actual tuple storage.
- + (tuple * relation_.getFixedByteLength()) // Tuples prior to 'tuple'.
- + relation_.getFixedLengthAttributeOffset(attr); // Attribute offset within tuple.
-}
-
-TypedValue PackedRowStoreTupleStorageSubBlock::getAttributeValueTyped(
- const tuple_id tuple,
- const attribute_id attr) const {
- const Type &attr_type = relation_.getAttributeById(attr)->getType();
- const void *untyped_value = getAttributeValue(tuple, attr);
- return (untyped_value == nullptr)
- ? attr_type.makeNullValue()
- : attr_type.makeValue(untyped_value, attr_type.maximumByteLength());
-}
-
-ValueAccessor* PackedRowStoreTupleStorageSubBlock::createValueAccessor(
- const TupleIdSequence *sequence) const {
- PackedRowStoreValueAccessor *base_accessor
- = new PackedRowStoreValueAccessor(relation_,
- relation_,
- header_->num_tuples,
- tuple_storage_,
- null_bitmap_.get());
- if (sequence == nullptr) {
- return base_accessor;
- } else {
- return new TupleIdSequenceAdapterValueAccessor<PackedRowStoreValueAccessor>(
- base_accessor,
- *sequence);
- }
-}
-
-void PackedRowStoreTupleStorageSubBlock::setAttributeValueInPlaceTyped(const tuple_id tuple,
- const attribute_id attr,
- const TypedValue &value) {
- DEBUG_ASSERT(hasTupleWithID(tuple));
- DEBUG_ASSERT(relation_.hasAttributeWithId(attr));
- DEBUG_ASSERT(value.isPlausibleInstanceOf(relation_.getAttributeById(attr)->getType().getSignature()));
-
- const int nullable_idx = relation_.getNullableAttributeIndex(attr);
- if (nullable_idx != -1) {
- if (value.isNull()) {
- null_bitmap_->setBit(tuple * relation_.numNullableAttributes() + nullable_idx, true);
- return;
- } else {
- null_bitmap_->setBit(tuple * relation_.numNullableAttributes() + nullable_idx, false);
- }
- }
-
- char *base_addr = static_cast<char*>(tuple_storage_) // Start of actual tuple storage.
- + (tuple * relation_.getFixedByteLength()) // Tuples prior to 'tuple'.
- + relation_.getFixedLengthAttributeOffset(attr); // Attribute offset within tuple.
-
- value.copyInto(base_addr);
-}
-
-bool PackedRowStoreTupleStorageSubBlock::deleteTuple(const tuple_id tuple) {
- DEBUG_ASSERT(hasTupleWithID(tuple));
-
- if (tuple == header_->num_tuples - 1) {
- // If deleting the last tuple, simply truncate.
- --(header_->num_tuples);
- if (null_bitmap_.get() != nullptr) {
- null_bitmap_->setBitRange(tuple * relation_.numNullableAttributes(),
- relation_.numNullableAttributes(),
- false);
- }
- return false;
- } else {
- const size_t tuple_length = relation_.getFixedByteLength();
-
- char *dest_addr = static_cast<char*>(tuple_storage_) // Start of actual tuple storage.
- + (tuple * tuple_length); // Prior tuples.
- char *src_addr = dest_addr + tuple_length; // Start of subsequent tuples.
- const size_t copy_bytes = (header_->num_tuples - tuple - 1) * tuple_length; // Bytes in subsequent tuples.
- memmove(dest_addr, src_addr, copy_bytes);
-
- if (null_bitmap_.get() != nullptr) {
- null_bitmap_->shiftTailForward(tuple * relation_.numNullableAttributes(),
- relation_.numNullableAttributes());
- }
-
- --(header_->num_tuples);
-
- return true;
- }
-}
-
-bool PackedRowStoreTupleStorageSubBlock::bulkDeleteTuples(TupleIdSequence *tuples) {
- if (tuples->empty()) {
- // Nothing to do.
- return false;
- }
-
- const tuple_id front = tuples->front();
- const tuple_id back = tuples->back();
- const tuple_id num_tuples = tuples->numTuples();
-
- if ((back == header_->num_tuples - 1)
- && (back - front == num_tuples - 1)) {
- // Just truncate the back.
- header_->num_tuples = front;
- if (null_bitmap_.get() != nullptr) {
- null_bitmap_->setBitRange(header_->num_tuples * relation_.numNullableAttributes(),
- num_tuples * relation_.numNullableAttributes(),
- false);
- }
- return false;
- }
-
- // Pack the non-deleted tuples.
- const size_t tuple_length = relation_.getFixedByteLength();
- tuple_id dest_tid = front;
- tuple_id src_tid = dest_tid;
-
- TupleIdSequence::const_iterator it = tuples->begin();
- for (tuple_id current_id = front;
- current_id < header_->num_tuples;
- ++current_id, ++src_tid) {
- if (current_id == *it) {
- // Don't copy a deleted tuple.
-
- if (null_bitmap_.get() != nullptr) {
- // Erase the deleted tuple's entries in the null bitmap.
- null_bitmap_->shiftTailForward(dest_tid * relation_.numNullableAttributes(),
- relation_.numNullableAttributes());
- }
-
- ++it;
- if (it == tuples->end()) {
- // No more to delete, so copy all the remaining tuples in one go.
- memmove(static_cast<char*>(tuple_storage_) + dest_tid * tuple_length,
- static_cast<char*>(tuple_storage_) + (src_tid + 1) * tuple_length,
- (header_->num_tuples - back - 1) * tuple_length);
- break;
- }
- } else {
- // Copy the next tuple to the packed region.
- memmove(static_cast<char*>(tuple_storage_) + dest_tid * tuple_length,
- static_cast<char*>(tuple_storage_) + src_tid * tuple_length,
- tuple_length);
- ++dest_tid;
- }
- }
-
- header_->num_tuples -= static_cast<tuple_id>(num_tuples);
-
- return true;
-}
-
-template <bool nullable_attrs>
-bool PackedRowStoreTupleStorageSubBlock::hasSpaceToInsert(const tuple_id num_tuples) const {
- if (sizeof(PackedRowStoreHeader)
- + null_bitmap_bytes_
- + (header_->num_tuples + num_tuples) * relation_.getFixedByteLength()
- <= sub_block_memory_size_) {
- if (nullable_attrs) {
- return static_cast<std::size_t>(header_->num_tuples + num_tuples) < null_bitmap_->size();
- } else {
- return true;
- }
- } else {
- return false;
- }
-}
-
-// Make sure both versions get compiled in.
-template bool PackedRowStoreTupleStorageSubBlock::hasSpaceToInsert<false>(
- const tuple_id num_tuples) const;
-template bool PackedRowStoreTupleStorageSubBlock::hasSpaceToInsert<true>(
- const tuple_id num_tuples) const;
-
-template <bool nullable_attrs>
-TupleStorageSubBlock::InsertResult PackedRowStoreTupleStorageSubBlock::insertTupleImpl(
- const Tuple &tuple) {
-#ifdef QUICKSTEP_DEBUG
- paranoidInsertTypeCheck(tuple);
-#endif
- if (!hasSpaceToInsert<nullable_attrs>(1)) {
- return InsertResult(-1, false);
- }
-
- char *base_addr = static_cast<char*>(tuple_storage_) // Start of actual tuple-storage region.
- + header_->num_tuples * relation_.getFixedByteLength(); // Existing tuples.
-
- Tuple::const_iterator value_it = tuple.begin();
- CatalogRelationSchema::const_iterator attr_it = relation_.begin();
-
- while (value_it != tuple.end()) {
- if (nullable_attrs) {
- const int nullable_idx = relation_.getNullableAttributeIndex(attr_it->getID());
- if ((nullable_idx != -1) && value_it->isNull()) {
- null_bitmap_->setBit(header_->num_tuples * relation_.numNullableAttributes()
- + nullable_idx,
- true);
- } else {
- value_it->copyInto(base_addr);
- }
- } else {
- value_it->copyInto(base_addr);
- }
-
- base_addr += attr_it->getType().maximumByteLength();
-
- ++value_it;
- ++attr_it;
- }
-
- ++(header_->num_tuples);
-
- return InsertResult(header_->num_tuples - 1, false);
-}
-
-} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5fee8218/storage/PackedRowStoreTupleStorageSubBlock.hpp
----------------------------------------------------------------------
diff --git a/storage/PackedRowStoreTupleStorageSubBlock.hpp b/storage/PackedRowStoreTupleStorageSubBlock.hpp
deleted file mode 100644
index 0cd41f4..0000000
--- a/storage/PackedRowStoreTupleStorageSubBlock.hpp
+++ /dev/null
@@ -1,216 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- **/
-
-#ifndef QUICKSTEP_STORAGE_PACKED_ROW_STORE_TUPLE_STORAGE_SUB_BLOCK_HPP_
-#define QUICKSTEP_STORAGE_PACKED_ROW_STORE_TUPLE_STORAGE_SUB_BLOCK_HPP_
-
-#include <memory>
-#include <unordered_map>
-#include <vector>
-
-#include "expressions/predicate/PredicateCost.hpp"
-#include "storage/SubBlockTypeRegistryMacros.hpp"
-#include "storage/TupleStorageSubBlock.hpp"
-#include "types/TypedValue.hpp"
-#include "utility/BitVector.hpp"
-#include "utility/Macros.hpp"
-
-namespace quickstep {
-
-class CatalogRelationSchema;
-class ComparisonPredicate;
-class TupleStorageSubBlockDescription;
-class ValueAccessor;
-
-QUICKSTEP_DECLARE_SUB_BLOCK_TYPE_REGISTERED(PackedRowStoreTupleStorageSubBlock);
-
-/** \addtogroup Storage
- * @{
- */
-
-/**
- * @brief An implementation of TupleStorageSubBlock as a packed row-store (i.e.
- * an array of fixed-length values with no holes).
- * @warning This implementation does NOT support variable-length attributes. It
- * is an error to attempt to construct a
- * PackedRowStoreTupleStorageSubBlock for a relation with any
- * variable-length attributes.
- **/
-class PackedRowStoreTupleStorageSubBlock: public TupleStorageSubBlock {
- public:
- PackedRowStoreTupleStorageSubBlock(const CatalogRelationSchema &relation,
- const TupleStorageSubBlockDescription &description,
- const bool new_block,
- void *sub_block_memory,
- const std::size_t sub_block_memory_size);
-
- ~PackedRowStoreTupleStorageSubBlock() override {
- }
-
- /**
- * @brief Determine whether a TupleStorageSubBlockDescription is valid for
- * this type of TupleStorageSubBlock.
- *
- * @param relation The relation a tuple store described by description would
- * belong to.
- * @param description A description of the parameters for this type of
- * TupleStorageSubBlock, which will be checked for validity.
- * @return Whether description is well-formed and valid for this type of
- * TupleStorageSubBlock belonging to relation (i.e. whether a
- * TupleStorageSubBlock of this type, belonging to relation, can be
- * constructed according to description).
- **/
- static bool DescriptionIsValid(const CatalogRelationSchema &relation,
- const TupleStorageSubBlockDescription &description);
-
- /**
- * @brief Estimate the average number of bytes (including any applicable
- * overhead) used to store a single tuple in this type of
- * TupleStorageSubBlock. Used by StorageBlockLayout::finalize() to
- * divide block memory amongst sub-blocks.
- * @warning description must be valid. DescriptionIsValid() should be called
- * first if necessary.
- *
- * @param relation The relation tuples belong to.
- * @param description A description of the parameters for this type of
- * TupleStorageSubBlock.
- * @return The average/ammortized number of bytes used to store a single
- * tuple of relation in a TupleStorageSubBlock of this type described
- * by description.
- **/
- static std::size_t EstimateBytesPerTuple(const CatalogRelationSchema &relation,
- const TupleStorageSubBlockDescription &description);
-
- bool supportsUntypedGetAttributeValue(const attribute_id attr) const override {
- return true;
- }
-
- bool supportsAdHocInsert() const override {
- return true;
- }
-
- bool adHocInsertIsEfficient() const override {
- return true;
- }
-
- TupleStorageSubBlockType getTupleStorageSubBlockType() const override {
- return kPackedRowStore;
- }
-
- bool isEmpty() const override {
- return (header_->num_tuples == 0);
- }
-
- bool isPacked() const override {
- return true;
- }
-
- tuple_id getMaxTupleID() const override {
- return header_->num_tuples - 1;
- }
-
- bool hasTupleWithID(const tuple_id tuple) const override {
- return ((tuple >=0) && (tuple < header_->num_tuples));
- }
-
- InsertResult insertTuple(const Tuple &tuple) override {
- if (null_bitmap_.get() == nullptr) {
- return insertTupleImpl<false>(tuple);
- } else {
- return insertTupleImpl<true>(tuple);
- }
- }
-
- inline bool insertTupleInBatch(const Tuple &tuple) override {
- const InsertResult result = insertTuple(tuple);
- return (result.inserted_id >= 0);
- }
-
- tuple_id bulkInsertTuples(ValueAccessor *accessor) override;
-
- tuple_id bulkInsertTuplesWithRemappedAttributes(
- const std::vector<attribute_id> &attribute_map,
- ValueAccessor *accessor) override;
-
- const void* getAttributeValue(const tuple_id tuple,
- const attribute_id attr) const override;
-
- TypedValue getAttributeValueTyped(const tuple_id tuple,
- const attribute_id attr) const override;
-
- ValueAccessor* createValueAccessor(
- const TupleIdSequence *sequence = nullptr) const override;
-
- bool canSetAttributeValuesInPlaceTyped(
- const tuple_id tuple,
- const std::unordered_map<attribute_id, TypedValue> &new_values) const override {
- return true;
- }
-
- void setAttributeValueInPlaceTyped(const tuple_id tuple,
- const attribute_id attr,
- const TypedValue &value) override;
-
- bool deleteTuple(const tuple_id tuple) override;
- bool bulkDeleteTuples(TupleIdSequence *tuples) override;
-
- predicate_cost_t estimatePredicateEvaluationCost(
- const ComparisonPredicate &predicate) const override {
- return predicate_cost::kRowScan;
- }
-
- void rebuild() override {
- }
-
- bool isInsertOrderPreserving() const override {
- return true;
- }
-
- private:
- struct PackedRowStoreHeader {
- tuple_id num_tuples;
- };
-
- // If 'nullable_attrs' is true, extra branches are enabled to deal with NULLs
- // and set bits in '*null_bitmap_' appropriately. If no attributes of the
- // relation are nullable, then 'nullable_attrs' is false and this function
- // has a simpler and faster implementation with no NULL-checking.
- template <bool nullable_attrs>
- InsertResult insertTupleImpl(const Tuple &tuple);
-
- // Similar to insertTupleImpl(), the 'nullable_attrs' template parameter
- // distinguishes between a version of this function that includes a check for
- // space in '*null_bitmap_' and a simpler version that skips the check for
- // relations that have no nullable attributes (and therefore no NULL-bitmap).
- template <bool nullable_attrs>
- bool hasSpaceToInsert(const tuple_id num_tuples) const;
-
- PackedRowStoreHeader *header_;
- std::unique_ptr<BitVector<false>> null_bitmap_;
- std::size_t null_bitmap_bytes_;
- void *tuple_storage_;
-
- DISALLOW_COPY_AND_ASSIGN(PackedRowStoreTupleStorageSubBlock);
-};
-
-/** @} */
-
-} // namespace quickstep
-
-#endif // QUICKSTEP_STORAGE_PACKED_ROW_STORE_TUPLE_STORAGE_SUB_BLOCK_HPP_
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5fee8218/storage/PackedRowStoreValueAccessor.hpp
----------------------------------------------------------------------
diff --git a/storage/PackedRowStoreValueAccessor.hpp b/storage/PackedRowStoreValueAccessor.hpp
deleted file mode 100644
index 9d43955..0000000
--- a/storage/PackedRowStoreValueAccessor.hpp
+++ /dev/null
@@ -1,150 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- **/
-
-#ifndef QUICKSTEP_STORAGE_PACKED_ROW_STORE_VALUE_ACCESSOR_HPP_
-#define QUICKSTEP_STORAGE_PACKED_ROW_STORE_VALUE_ACCESSOR_HPP_
-
-#include "catalog/CatalogRelationSchema.hpp"
-#include "catalog/CatalogTypedefs.hpp"
-#include "storage/StorageBlockInfo.hpp"
-#include "storage/ValueAccessor.hpp"
-#include "types/Type.hpp"
-#include "types/TypedValue.hpp"
-#include "utility/BitVector.hpp"
-#include "utility/Macros.hpp"
-
-namespace quickstep {
-
-class PackedRowStoreTupleStorageSubBlock;
-
-class PackedRowStoreValueAccessorHelper {
- public:
- PackedRowStoreValueAccessorHelper(const CatalogRelationSchema &relation,
- const tuple_id num_tuples,
- const void *tuple_storage,
- const BitVector<false> *null_bitmap)
- : relation_(relation),
- num_tuples_(num_tuples),
- tuple_storage_(tuple_storage),
- null_bitmap_(null_bitmap) {
- }
-
- inline tuple_id numPackedTuples() const {
- return num_tuples_;
- }
-
- /**
- * @brief Returns whether this accessor has a fast strided ColumnAccessor available
- * that can be used to optimize memory access in a tight loop iteration
- * over the underlying storage block.
- *
- * @return true if fast ColumnAccessor is supported, otherwise false.
- */
- inline bool isColumnAccessorSupported() const {
- return true;
- }
-
- /**
- * @brief Get a pointer to a ColumnAccessor object that provides a fast strided memory
- * access on the underlying storage block.
- * @note The ownership of the returned object lies with the caller.
- * @warning This method should only be called if isColumnAccessorSupported() method
- * returned true. If ColumnAccessor is not supported this method will return a nullptr.
- *
- * @param current_tuple_position A constant reference to the tuple position in the containing
- * ValueAccessor. This reference value is shared between the containing ValueAccessor &
- * a ColumnAccessor. However, a ColumnAccessor *CANNOT* modify this tuple position.
- * @param attr_id The attribute id on which this ColumnAccessor will be created.
- *
- * @return A pointer to a ColumnAccessor object with specific properties set that can be used
- * in a tight loop iterations over the underlying storage block.
- **/
- template <bool check_null = true>
- inline const ColumnAccessor<check_null>* getColumnAccessor(const tuple_id ¤t_tuple_position,
- const attribute_id attr_id) const {
- DCHECK(relation_.hasAttributeWithId(attr_id));
- const void* base_location = static_cast<const char*>(tuple_storage_)
- + relation_.getFixedLengthAttributeOffset(attr_id);
- const std::size_t stride = relation_.getFixedByteLength();
-
- std::unique_ptr<ColumnAccessor<check_null>> column_accessor;
- if (check_null) {
- const int nullable_base = relation_.getNullableAttributeIndex(attr_id);
- const unsigned nullable_stride = relation_.numNullableAttributes();
- column_accessor.reset(new ColumnAccessor<check_null>(current_tuple_position,
- num_tuples_,
- base_location,
- stride,
- null_bitmap_,
- nullable_base,
- nullable_stride));
- } else {
- column_accessor.reset(new ColumnAccessor<check_null>(current_tuple_position,
- num_tuples_,
- base_location,
- stride));
- }
- return column_accessor.release();
- }
-
- template <bool check_null>
- inline const void* getAttributeValue(const tuple_id tuple,
- const attribute_id attr) const {
- DEBUG_ASSERT(tuple < num_tuples_);
- DEBUG_ASSERT(relation_.hasAttributeWithId(attr));
- if (check_null) {
- const int nullable_idx = relation_.getNullableAttributeIndex(attr);
- if ((nullable_idx != -1)
- && null_bitmap_->getBit(tuple * relation_.numNullableAttributes() + nullable_idx)) {
- return nullptr;
- }
- }
-
- return static_cast<const char*>(tuple_storage_) // Start of actual tuple storage.
- + (tuple * relation_.getFixedByteLength()) // Tuples prior to 'tuple'.
- + relation_.getFixedLengthAttributeOffset(attr); // Attribute offset within tuple.
- }
-
- inline TypedValue getAttributeValueTyped(const tuple_id tuple,
- const attribute_id attr) const {
- const Type &attr_type = relation_.getAttributeById(attr)->getType();
- const void *untyped_value = getAttributeValue<true>(tuple, attr);
- return (untyped_value == nullptr)
- ? attr_type.makeNullValue()
- : attr_type.makeValue(untyped_value, attr_type.maximumByteLength());
- }
-
- private:
- const CatalogRelationSchema &relation_;
- const tuple_id num_tuples_;
- const void *tuple_storage_;
- const BitVector<false> *null_bitmap_;
-
- DISALLOW_COPY_AND_ASSIGN(PackedRowStoreValueAccessorHelper);
-};
-
-typedef PackedTupleStorageSubBlockValueAccessor<
- PackedRowStoreTupleStorageSubBlock,
- PackedRowStoreValueAccessorHelper,
- ValueAccessor::Implementation::kPackedRowStore>
- PackedRowStoreValueAccessor;
-
-} // namespace quickstep
-
-#endif // QUICKSTEP_STORAGE_PACKED_ROW_STORE_VALUE_ACCESSOR_HPP_
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5fee8218/storage/SplitRowStoreTupleStorageSubBlock.cpp
----------------------------------------------------------------------
diff --git a/storage/SplitRowStoreTupleStorageSubBlock.cpp b/storage/SplitRowStoreTupleStorageSubBlock.cpp
index 1e6f7ff..ad583eb 100644
--- a/storage/SplitRowStoreTupleStorageSubBlock.cpp
+++ b/storage/SplitRowStoreTupleStorageSubBlock.cpp
@@ -245,9 +245,7 @@ tuple_id SplitRowStoreTupleStorageSubBlock::bulkInsertDispatcher(
CopyGroupList copy_groups;
getCopyGroupsForAttributeMap(attribute_map, ©_groups);
auto impl = accessor->getImplementationType();
- const bool is_rowstore_source =
- (impl == ValueAccessor::Implementation::kPackedRowStore ||
- impl == ValueAccessor::Implementation::kSplitRowStore);
+ const bool is_rowstore_source = impl == ValueAccessor::Implementation::kSplitRowStore;
if (is_rowstore_source) {
copy_groups.merge_contiguous();
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5fee8218/storage/SplitRowStoreTupleStorageSubBlock.hpp
----------------------------------------------------------------------
diff --git a/storage/SplitRowStoreTupleStorageSubBlock.hpp b/storage/SplitRowStoreTupleStorageSubBlock.hpp
index 89c756d..67d36fe 100644
--- a/storage/SplitRowStoreTupleStorageSubBlock.hpp
+++ b/storage/SplitRowStoreTupleStorageSubBlock.hpp
@@ -421,4 +421,4 @@ class SplitRowStoreTupleStorageSubBlock: public TupleStorageSubBlock {
} // namespace quickstep
-#endif // QUICKSTEP_STORAGE_PACKED_ROW_STORE_TUPLE_STORAGE_SUB_BLOCK_HPP_
+#endif // QUICKSTEP_STORAGE_SPLIT_ROW_STORE_TUPLE_STORAGE_SUB_BLOCK_HPP_
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5fee8218/storage/StorageBlock.cpp
----------------------------------------------------------------------
diff --git a/storage/StorageBlock.cpp b/storage/StorageBlock.cpp
index 6267d6b..de2d25b 100644
--- a/storage/StorageBlock.cpp
+++ b/storage/StorageBlock.cpp
@@ -40,7 +40,6 @@
#include "storage/HashTableBase.hpp"
#include "storage/IndexSubBlock.hpp"
#include "storage/InsertDestinationInterface.hpp"
-#include "storage/PackedRowStoreTupleStorageSubBlock.hpp"
#include "storage/SMAIndexSubBlock.hpp"
#include "storage/SplitRowStoreTupleStorageSubBlock.hpp"
#include "storage/StorageBlockBase.hpp"
@@ -958,12 +957,6 @@ TupleStorageSubBlock* StorageBlock::CreateTupleStorageSubBlock(
const std::size_t sub_block_memory_size) {
DEBUG_ASSERT(description.IsInitialized());
switch (description.sub_block_type()) {
- case TupleStorageSubBlockDescription::PACKED_ROW_STORE:
- return new PackedRowStoreTupleStorageSubBlock(relation,
- description,
- new_block,
- sub_block_memory,
- sub_block_memory_size);
case TupleStorageSubBlockDescription::BASIC_COLUMN_STORE:
return new BasicColumnStoreTupleStorageSubBlock(relation,
description,
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5fee8218/storage/StorageBlockInfo.cpp
----------------------------------------------------------------------
diff --git a/storage/StorageBlockInfo.cpp b/storage/StorageBlockInfo.cpp
index 8c40be6..2646e5f 100644
--- a/storage/StorageBlockInfo.cpp
+++ b/storage/StorageBlockInfo.cpp
@@ -37,7 +37,6 @@ string BlockIdUtil::ToString(const block_id block) {
}
const char *kTupleStorageSubBlockTypeNames[] = {
- "PackedRowStore",
"BasicColumnStore",
"CompressedPackedRowStore",
"CompressedColumnStore",
@@ -46,7 +45,7 @@ const char *kTupleStorageSubBlockTypeNames[] = {
const char *kIndexSubBlockTypeNames[] = {
"CSBTree",
+ "SMA",
};
} // namespace quickstep
-
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5fee8218/storage/StorageBlockInfo.hpp
----------------------------------------------------------------------
diff --git a/storage/StorageBlockInfo.hpp b/storage/StorageBlockInfo.hpp
index 8d443d5..361648f 100644
--- a/storage/StorageBlockInfo.hpp
+++ b/storage/StorageBlockInfo.hpp
@@ -118,7 +118,6 @@ const tuple_id kMaxTupleID = INT_MAX;
* @brief Codes for the different implementations of TupleStorageSubBlock.
**/
enum TupleStorageSubBlockType {
- kPackedRowStore = 0,
kBasicColumnStore,
kCompressedPackedRowStore,
kCompressedColumnStore,
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5fee8218/storage/StorageBlockLayout.hpp
----------------------------------------------------------------------
diff --git a/storage/StorageBlockLayout.hpp b/storage/StorageBlockLayout.hpp
index 9827a4c..30a00f5 100644
--- a/storage/StorageBlockLayout.hpp
+++ b/storage/StorageBlockLayout.hpp
@@ -73,9 +73,7 @@ class StorageBlockLayout {
* @brief Static method to generate a default layout for a particular
* relation.
* @note The current policy is that a default layout takes up one slot, uses
- * PackedRowStoreTupleStorageSubBlock for fixed-length relations or
- * SplitRowStoreTupleStorageSubBlock for variable-length relations, and
- * has no indices.
+ * SplitRowStoreTupleStorageSubBlock and uses no indices.
*
* @param relation The relation to generate a layout for.
* @param relation_variable_length Whether relation is variable-length.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5fee8218/storage/StorageBlockLayout.proto
----------------------------------------------------------------------
diff --git a/storage/StorageBlockLayout.proto b/storage/StorageBlockLayout.proto
index 6573470..8919505 100644
--- a/storage/StorageBlockLayout.proto
+++ b/storage/StorageBlockLayout.proto
@@ -22,11 +22,10 @@ package quickstep;
// Options for TupleStorageSubBlocks.
message TupleStorageSubBlockDescription {
enum TupleStorageSubBlockType {
- PACKED_ROW_STORE = 0;
- BASIC_COLUMN_STORE = 1;
- COMPRESSED_PACKED_ROW_STORE = 2;
- COMPRESSED_COLUMN_STORE = 3;
- SPLIT_ROW_STORE = 4;
+ BASIC_COLUMN_STORE = 0;
+ COMPRESSED_PACKED_ROW_STORE = 1;
+ COMPRESSED_COLUMN_STORE = 2;
+ SPLIT_ROW_STORE = 3;
}
required TupleStorageSubBlockType sub_block_type = 1;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5fee8218/storage/SubBlockTypeRegistry.hpp
----------------------------------------------------------------------
diff --git a/storage/SubBlockTypeRegistry.hpp b/storage/SubBlockTypeRegistry.hpp
index c362d9f..71e0629 100644
--- a/storage/SubBlockTypeRegistry.hpp
+++ b/storage/SubBlockTypeRegistry.hpp
@@ -47,7 +47,7 @@ class CatalogRelationSchema;
* (.cpp) file, and put an invocation of the
* QUICKSTEP_REGISTER_TUPLE_STORE() macro in the quickstep namespace.
* For example:
- * QUICKSTEP_REGISTER_TUPLE_STORE(PackedRowStoreTupleStorageSubBlock, PACKED_ROW_STORE);
+ * QUICKSTEP_REGISTER_TUPLE_STORE(SplitRowStoreTupleStorageSubBlock, SPLIT_ROW_STORE);
* The first argument to the macro is the name of the class, the second
* is the name of the class' corresponding case in the
* TupleStorageSubBlockDescription::TupleStorageSubBlockType enum.
@@ -55,7 +55,7 @@ class CatalogRelationSchema;
* file, and put an invocation of the
* QUICKSTEP_DECLARE_SUB_BLOCK_TYPE_REGISTERED() macro in the quickstep
* namespace like so:
- * QUICKSTEP_DECLARE_SUB_BLOCK_TYPE_REGISTERED(PackedRowStoreTupleStorageSubBlock);
+ * QUICKSTEP_DECLARE_SUB_BLOCK_TYPE_REGISTERED(SplitRowStoreTupleStorageSubBlock);
*
* Registration of IndexSubBlock implementations works the same way, except the
* first macro used should be QUICKSTEP_REGISTER_INDEX() instead of
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5fee8218/storage/ValueAccessor.hpp
----------------------------------------------------------------------
diff --git a/storage/ValueAccessor.hpp b/storage/ValueAccessor.hpp
index e4a2906..654bbf9 100644
--- a/storage/ValueAccessor.hpp
+++ b/storage/ValueAccessor.hpp
@@ -78,7 +78,6 @@ class ValueAccessor {
kBasicColumnStore,
kCompressedColumnStore,
kCompressedPackedRowStore,
- kPackedRowStore,
kSplitRowStore,
// Accessor for a group of ColumnVectors:
kColumnVectors
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5fee8218/storage/ValueAccessorUtil.hpp
----------------------------------------------------------------------
diff --git a/storage/ValueAccessorUtil.hpp b/storage/ValueAccessorUtil.hpp
index 8be3785..4969156 100644
--- a/storage/ValueAccessorUtil.hpp
+++ b/storage/ValueAccessorUtil.hpp
@@ -25,7 +25,6 @@
#include "storage/BasicColumnStoreValueAccessor.hpp"
#include "storage/CompressedColumnStoreValueAccessor.hpp"
#include "storage/CompressedPackedRowStoreValueAccessor.hpp"
-#include "storage/PackedRowStoreValueAccessor.hpp"
#include "storage/SplitRowStoreValueAccessor.hpp"
#include "storage/ValueAccessor.hpp"
#include "types/containers/ColumnVectorsValueAccessor.hpp"
@@ -70,8 +69,6 @@ auto InvokeOnValueAccessorNotAdapter(
return functor(static_cast<CompressedColumnStoreValueAccessor*>(accessor));
case ValueAccessor::Implementation::kCompressedPackedRowStore:
return functor(static_cast<CompressedPackedRowStoreValueAccessor*>(accessor));
- case ValueAccessor::Implementation::kPackedRowStore:
- return functor(static_cast<PackedRowStoreValueAccessor*>(accessor));
case ValueAccessor::Implementation::kSplitRowStore:
return functor(static_cast<SplitRowStoreValueAccessor*>(accessor));
case ValueAccessor::Implementation::kColumnVectors:
@@ -121,10 +118,6 @@ auto InvokeOnTupleIdSequenceAdapterValueAccessor(
return functor(
static_cast<TupleIdSequenceAdapterValueAccessor<CompressedPackedRowStoreValueAccessor>*>(
accessor));
- case ValueAccessor::Implementation::kPackedRowStore:
- return functor(
- static_cast<TupleIdSequenceAdapterValueAccessor<PackedRowStoreValueAccessor>*>(
- accessor));
case ValueAccessor::Implementation::kSplitRowStore:
return functor(
static_cast<TupleIdSequenceAdapterValueAccessor<SplitRowStoreValueAccessor>*>(
@@ -177,10 +170,6 @@ auto InvokeOnOrderedTupleIdSequenceAdapterValueAccessor(
return functor(
static_cast<OrderedTupleIdSequenceAdapterValueAccessor<CompressedPackedRowStoreValueAccessor>*>(
accessor));
- case ValueAccessor::Implementation::kPackedRowStore:
- return functor(
- static_cast<OrderedTupleIdSequenceAdapterValueAccessor<PackedRowStoreValueAccessor>*>(
- accessor));
case ValueAccessor::Implementation::kSplitRowStore:
return functor(
static_cast<OrderedTupleIdSequenceAdapterValueAccessor<SplitRowStoreValueAccessor>*>(
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5fee8218/storage/tests/BasicColumnStoreTupleStorageSubBlock_unittest.cpp
----------------------------------------------------------------------
diff --git a/storage/tests/BasicColumnStoreTupleStorageSubBlock_unittest.cpp b/storage/tests/BasicColumnStoreTupleStorageSubBlock_unittest.cpp
index a511fcb..d41a457 100644
--- a/storage/tests/BasicColumnStoreTupleStorageSubBlock_unittest.cpp
+++ b/storage/tests/BasicColumnStoreTupleStorageSubBlock_unittest.cpp
@@ -859,7 +859,7 @@ TEST_P(BasicColumnStoreTupleStorageSubBlockTest, DescriptionIsValidTest) {
// A description that specifies the wrong sub_block_type is not valid.
tuple_store_description_->set_sub_block_type(
- TupleStorageSubBlockDescription::PACKED_ROW_STORE);
+ TupleStorageSubBlockDescription::SPLIT_ROW_STORE);
EXPECT_FALSE(BasicColumnStoreTupleStorageSubBlock::DescriptionIsValid(
*relation_,
*tuple_store_description_));
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5fee8218/storage/tests/CompressedPackedRowStoreTupleStorageSubBlock_unittest.cpp
----------------------------------------------------------------------
diff --git a/storage/tests/CompressedPackedRowStoreTupleStorageSubBlock_unittest.cpp b/storage/tests/CompressedPackedRowStoreTupleStorageSubBlock_unittest.cpp
index 530507b..3feee5e 100644
--- a/storage/tests/CompressedPackedRowStoreTupleStorageSubBlock_unittest.cpp
+++ b/storage/tests/CompressedPackedRowStoreTupleStorageSubBlock_unittest.cpp
@@ -679,7 +679,7 @@ TEST_P(CompressedPackedRowStoreTupleStorageSubBlockTest, DescriptionIsValidTest)
*tuple_store_description_));
// A description that specifies the wrong sub_block_type is not valid.
- tuple_store_description_->set_sub_block_type(TupleStorageSubBlockDescription::PACKED_ROW_STORE);
+ tuple_store_description_->set_sub_block_type(TupleStorageSubBlockDescription::SPLIT_ROW_STORE);
EXPECT_FALSE(CompressedPackedRowStoreTupleStorageSubBlock::DescriptionIsValid(*relation_,
*tuple_store_description_));
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5fee8218/storage/tests/PackedRowStoreTupleStorageSubBlock_unittest.cpp
----------------------------------------------------------------------
diff --git a/storage/tests/PackedRowStoreTupleStorageSubBlock_unittest.cpp b/storage/tests/PackedRowStoreTupleStorageSubBlock_unittest.cpp
deleted file mode 100644
index bf3c605..0000000
--- a/storage/tests/PackedRowStoreTupleStorageSubBlock_unittest.cpp
+++ /dev/null
@@ -1,584 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- **/
-
-#include <cstring>
-#include <memory>
-#include <sstream>
-#include <string>
-#include <unordered_map>
-#include <utility>
-#include <vector>
-
-#include "gtest/gtest.h"
-
-#include "catalog/CatalogAttribute.hpp"
-#include "catalog/CatalogRelation.hpp"
-#include "storage/PackedRowStoreTupleStorageSubBlock.hpp"
-#include "storage/StorageBlockInfo.hpp"
-#include "storage/StorageBlockLayout.hpp"
-#include "storage/StorageBlockLayout.pb.h"
-#include "storage/StorageConstants.hpp"
-#include "storage/StorageErrors.hpp"
-#include "storage/TupleIdSequence.hpp"
-#include "storage/TupleStorageSubBlock.hpp"
-#include "storage/ValueAccessor.hpp"
-#include "storage/ValueAccessorUtil.hpp"
-#include "types/CharType.hpp"
-#include "types/DoubleType.hpp"
-#include "types/IntType.hpp"
-#include "types/Type.hpp"
-#include "types/TypeFactory.hpp"
-#include "types/TypedValue.hpp"
-#include "types/TypeID.hpp"
-#include "types/containers/Tuple.hpp"
-#include "types/operations/comparisons/Comparison.hpp"
-#include "types/operations/comparisons/ComparisonFactory.hpp"
-#include "types/operations/comparisons/ComparisonID.hpp"
-#include "utility/BitVector.hpp"
-#include "utility/ScopedBuffer.hpp"
-
-using std::make_pair;
-using std::ostringstream;
-using std::pair;
-
-namespace quickstep {
-
-class PackedRowStoreTupleStorageSubBlockTest : public ::testing::TestWithParam<bool> {
- protected:
- static const size_t kSubBlockSize = 0x100000; // 1 MB
- static const size_t kTupleLength = 24;
-
- virtual void SetUp() {
- // Create a sample relation with a variety of attribute types.
- relation_.reset(new CatalogRelation(nullptr, "TestRelation"));
-
- // An integer.
- CatalogAttribute *current_attr = new CatalogAttribute(relation_.get(),
- "int_attr",
- TypeFactory::GetType(kInt, GetParam()));
- ASSERT_EQ(0, relation_->addAttribute(current_attr));
- eq_comp_int_.reset(ComparisonFactory::GetComparison(ComparisonID::kEqual).makeUncheckedComparatorForTypes(
- current_attr->getType(),
- current_attr->getType()));
-
- // A double.
- current_attr = new CatalogAttribute(relation_.get(),
- "double_attr",
- TypeFactory::GetType(kDouble, GetParam()));
- ASSERT_EQ(1, relation_->addAttribute(current_attr));
- eq_comp_double_.reset(ComparisonFactory::GetComparison(ComparisonID::kEqual).makeUncheckedComparatorForTypes(
- current_attr->getType(),
- current_attr->getType()));
-
- // A string.
- current_attr = new CatalogAttribute(relation_.get(),
- "char_attr",
- TypeFactory::GetType(kChar, 12, GetParam()));
- ASSERT_EQ(2, relation_->addAttribute(current_attr));
- eq_comp_char_.reset(ComparisonFactory::GetComparison(ComparisonID::kEqual).makeUncheckedComparatorForTypes(
- current_attr->getType(),
- current_attr->getType()));
-
- tuple_store_description_.reset(new TupleStorageSubBlockDescription());
- tuple_store_description_->set_sub_block_type(TupleStorageSubBlockDescription::PACKED_ROW_STORE);
-
- // Don't initialize the block yet. Different tests will use different
- // params.
- tuple_store_memory_.reset();
- tuple_store_.reset();
- }
-
- void initializeNewBlock(const size_t block_size) {
- tuple_store_memory_.reset(block_size);
- tuple_store_.reset(new PackedRowStoreTupleStorageSubBlock(*relation_,
- *tuple_store_description_,
- true,
- tuple_store_memory_.get(),
- block_size));
- }
-
- int computeRowCapacity() {
- if (GetParam()) {
- // Estimate using the same heuristic as
- // PackedRowStoreTupleStorageSubBlock's constructor.
- int row_capacity = ((kSubBlockSize - sizeof(tuple_id)) << 3)
- / ((kTupleLength << 3) + relation_->numNullableAttributes());
- const size_t null_bitmap_bytes_ = BitVector<false>::BytesNeeded(
- row_capacity * relation_->numNullableAttributes());
- return static_cast<int>(kSubBlockSize - sizeof(tuple_id) - null_bitmap_bytes_) / kTupleLength;
- } else {
- return static_cast<int>(kSubBlockSize - sizeof(tuple_id)) / kTupleLength;
- }
- }
-
- // Caller takes ownership of new heap-created Tuple.
- Tuple* createSampleTuple(const int base_value) const {
- std::vector<TypedValue> attrs;
-
- // int_attr
- if (GetParam() && (base_value % 6 == 0)) {
- // Throw in a NULL integer for every sixth value.
- attrs.emplace_back(kInt);
- } else {
- attrs.emplace_back(base_value);
- }
-
- // double_attr
- if (GetParam() && (base_value % 6 == 2)) {
- // NULL very sixth value.
- attrs.emplace_back(kDouble);
- } else {
- attrs.emplace_back(static_cast<double>(0.25 * base_value));
- }
-
- // char_attr
- if (GetParam() && (base_value % 6 == 4)) {
- // NULL very sixth value.
- attrs.emplace_back(CharType::InstanceNullable(12).makeNullValue());
- } else {
- ostringstream char_buffer;
- char_buffer << base_value;
- std::string string_literal(char_buffer.str());
- attrs.emplace_back(CharType::InstanceNonNullable(12).makeValue(string_literal.c_str(),
- string_literal.size() + 1));
- attrs.back().ensureNotReference();
- }
-
- return new Tuple(std::move(attrs));
- }
-
- void fillBlockWithSampleData() {
- tuple_id current_tid = 0;
- std::unique_ptr<Tuple> current_tuple(createSampleTuple(current_tid));
- while (tuple_store_->insertTupleInBatch(*current_tuple)) {
- ++current_tid;
- current_tuple.reset(createSampleTuple(current_tid));
- }
-
- tuple_store_->rebuild();
- }
-
- void checkTupleValuesUntyped(const tuple_id tid,
- const int base_value) {
- ASSERT_TRUE(tuple_store_->hasTupleWithID(tid));
- ASSERT_TRUE(tuple_store_->supportsUntypedGetAttributeValue(0));
- ASSERT_TRUE(tuple_store_->supportsUntypedGetAttributeValue(1));
- ASSERT_TRUE(tuple_store_->supportsUntypedGetAttributeValue(2));
-
- std::unique_ptr<Tuple> comp_tuple(createSampleTuple(base_value));
-
- if (comp_tuple->getAttributeValue(0).isNull()) {
- assert(nullptr == tuple_store_->getAttributeValue(tid, 0));
- // NULL comparisons are always false (for now).
- EXPECT_FALSE(eq_comp_int_->compareDataPtrs(nullptr,
- tuple_store_->getAttributeValue(tid, 0)));
- } else {
- assert(eq_comp_int_->compareDataPtrs(comp_tuple->getAttributeValue(0).getDataPtr(),
- tuple_store_->getAttributeValue(tid, 0)));
- }
-
- if (comp_tuple->getAttributeValue(1).isNull()) {
- EXPECT_EQ(nullptr, tuple_store_->getAttributeValue(tid, 1));
- // NULL comparisons are always false (for now).
- EXPECT_FALSE(eq_comp_double_->compareDataPtrs(nullptr,
- tuple_store_->getAttributeValue(tid, 1)));
- } else {
- EXPECT_TRUE(eq_comp_double_->compareDataPtrs(comp_tuple->getAttributeValue(1).getDataPtr(),
- tuple_store_->getAttributeValue(tid, 1)));
- }
-
- if (comp_tuple->getAttributeValue(2).isNull()) {
- EXPECT_EQ(nullptr, tuple_store_->getAttributeValue(tid, 2));
- // NULL comparisons are always false (for now).
- EXPECT_FALSE(eq_comp_char_->compareDataPtrs(nullptr,
- tuple_store_->getAttributeValue(tid, 2)));
- } else {
- EXPECT_TRUE(eq_comp_char_->compareDataPtrs(comp_tuple->getAttributeValue(2).getDataPtr(),
- tuple_store_->getAttributeValue(tid, 2)));
- }
- }
-
- void checkTupleValuesTyped(const tuple_id tid,
- const int base_value) {
- ASSERT_TRUE(tuple_store_->hasTupleWithID(tid));
-
- std::unique_ptr<Tuple> comp_tuple(createSampleTuple(base_value));
-
- if (comp_tuple->getAttributeValue(0).isNull()) {
- EXPECT_TRUE(tuple_store_->getAttributeValueTyped(tid, 0).isNull());
- // NULL comparisons are always false (for now).
- EXPECT_FALSE(eq_comp_int_->compareTypedValues(comp_tuple->getAttributeValue(0),
- tuple_store_->getAttributeValueTyped(tid, 0)));
- } else {
- EXPECT_TRUE(eq_comp_int_->compareTypedValues(comp_tuple->getAttributeValue(0),
- tuple_store_->getAttributeValueTyped(tid, 0)));
- }
-
- if (comp_tuple->getAttributeValue(1).isNull()) {
- EXPECT_TRUE(tuple_store_->getAttributeValueTyped(tid, 1).isNull());
- // NULL comparisons are always false (for now).
- EXPECT_FALSE(eq_comp_double_->compareTypedValues(comp_tuple->getAttributeValue(1),
- tuple_store_->getAttributeValueTyped(tid, 1)));
- } else {
- EXPECT_TRUE(eq_comp_double_->compareTypedValues(comp_tuple->getAttributeValue(1),
- tuple_store_->getAttributeValueTyped(tid, 1)));
- }
-
- if (comp_tuple->getAttributeValue(2).isNull()) {
- EXPECT_TRUE(tuple_store_->getAttributeValueTyped(tid, 2).isNull());
- // NULL comparisons are always false (for now).
- EXPECT_FALSE(eq_comp_char_->compareTypedValues(comp_tuple->getAttributeValue(2),
- tuple_store_->getAttributeValueTyped(tid, 2)));
- } else {
- EXPECT_TRUE(eq_comp_char_->compareTypedValues(comp_tuple->getAttributeValue(2),
- tuple_store_->getAttributeValueTyped(tid, 2)));
- }
- }
-
- template<bool check_null>
- void checkColumnAccessor() {
- initializeNewBlock(kSubBlockSize);
- fillBlockWithSampleData();
- ASSERT_TRUE(tuple_store_->isPacked());
- std::unique_ptr<PackedRowStoreValueAccessor> accessor(
- static_cast<PackedRowStoreValueAccessor*>(tuple_store_->createValueAccessor()));
- attribute_id value_accessor_id = 0;
- tuple_id tid = 0;
- accessor->beginIteration();
- ASSERT_TRUE(accessor->isColumnAccessorSupported());
- std::unique_ptr<const ColumnAccessor<check_null>>
- column_accessor(accessor->template getColumnAccessor<check_null>(value_accessor_id));
- ASSERT_TRUE(column_accessor != nullptr);
- while (accessor->next()) {
- const void *va_value = column_accessor->getUntypedValue();
- std::unique_ptr<Tuple> expected_tuple(createSampleTuple(tid));
-
- if (expected_tuple->getAttributeValue(value_accessor_id).isNull()) {
- ASSERT_TRUE(va_value == nullptr);
- } else {
- ASSERT_TRUE(eq_comp_int_->compareDataPtrs(expected_tuple->getAttributeValue(value_accessor_id).getDataPtr(),
- va_value));
- }
- ++tid;
- }
- }
-
- std::unique_ptr<CatalogRelation> relation_;
- ScopedBuffer tuple_store_memory_;
- std::unique_ptr<TupleStorageSubBlockDescription> tuple_store_description_;
- std::unique_ptr<PackedRowStoreTupleStorageSubBlock> tuple_store_;
-
- std::unique_ptr<UncheckedComparator> eq_comp_int_;
- std::unique_ptr<UncheckedComparator> eq_comp_double_;
- std::unique_ptr<UncheckedComparator> eq_comp_char_;
-};
-
-typedef PackedRowStoreTupleStorageSubBlockTest PackedRowStoreTupleStorageSubBlockDeathTest;
-
-TEST_P(PackedRowStoreTupleStorageSubBlockTest, DescriptionIsValidTest) {
- // The descriptions we use for the other tests should be valid.
- EXPECT_TRUE(PackedRowStoreTupleStorageSubBlock::DescriptionIsValid(*relation_,
- *tuple_store_description_));
-
- // An uninitialized description is not valid.
- tuple_store_description_.reset(new TupleStorageSubBlockDescription());
- EXPECT_FALSE(PackedRowStoreTupleStorageSubBlock::DescriptionIsValid(*relation_,
- *tuple_store_description_));
-
- // A description that specifies the wrong sub_block_type is not valid.
- tuple_store_description_->set_sub_block_type(TupleStorageSubBlockDescription::BASIC_COLUMN_STORE);
- EXPECT_FALSE(PackedRowStoreTupleStorageSubBlock::DescriptionIsValid(*relation_,
- *tuple_store_description_));
-
- // A relation with a nullable attribute is OK.
- std::unique_ptr<CatalogRelation> nullable_relation(new CatalogRelation(NULL, "nullable_relation"));
- CatalogAttribute *nullable_attribute = new CatalogAttribute(nullable_relation.get(),
- "nullable_attr",
- TypeFactory::GetType(kInt, true));
- ASSERT_EQ(0, nullable_relation->addAttribute(nullable_attribute));
- tuple_store_description_.reset(new TupleStorageSubBlockDescription());
- tuple_store_description_->set_sub_block_type(TupleStorageSubBlockDescription::PACKED_ROW_STORE);
- EXPECT_TRUE(PackedRowStoreTupleStorageSubBlock::DescriptionIsValid(*nullable_relation,
- *tuple_store_description_));
-
- // A relation with a variable-length attribute can't be used with this block type.
- std::unique_ptr<CatalogRelation> variable_length_relation(new CatalogRelation(NULL, "variable_length_relation"));
- CatalogAttribute *variable_length_attribute = new CatalogAttribute(variable_length_relation.get(),
- "variable_length_attr",
- TypeFactory::GetType(kVarChar, 20, false));
- ASSERT_EQ(0, variable_length_relation->addAttribute(variable_length_attribute));
- EXPECT_FALSE(PackedRowStoreTupleStorageSubBlock::DescriptionIsValid(*variable_length_relation,
- *tuple_store_description_));
-}
-
-TEST_P(PackedRowStoreTupleStorageSubBlockDeathTest, ConstructWithInvalidDescriptionTest) {
- tuple_store_description_.reset(new TupleStorageSubBlockDescription());
- tuple_store_description_->set_sub_block_type(TupleStorageSubBlockDescription::BASIC_COLUMN_STORE);
- EXPECT_DEATH(initializeNewBlock(kSubBlockSize), "");
-}
-
-TEST_P(PackedRowStoreTupleStorageSubBlockTest, MemoryTooSmallTest) {
- // 1 byte short.
- EXPECT_THROW(initializeNewBlock(sizeof(tuple_id) - 1),
- BlockMemoryTooSmall);
-}
-
-TEST_P(PackedRowStoreTupleStorageSubBlockTest, InsertTest) {
- initializeNewBlock(kSubBlockSize);
-
- int row_capacity = computeRowCapacity();
-
- EXPECT_TRUE(tuple_store_->supportsAdHocInsert());
- EXPECT_TRUE(tuple_store_->adHocInsertIsEfficient());
- EXPECT_FALSE(tuple_store_->isCompressed());
- EXPECT_TRUE(tuple_store_->isEmpty());
-
- std::unique_ptr<Tuple> current_tuple;
- for (int tuple_num = 0;
- tuple_num < row_capacity;
- ++tuple_num) {
- current_tuple.reset(createSampleTuple(tuple_num));
- TupleStorageSubBlock::InsertResult result = tuple_store_->insertTuple(*current_tuple);
- ASSERT_EQ(tuple_num, result.inserted_id);
- ASSERT_FALSE(result.ids_mutated);
-
- EXPECT_FALSE(tuple_store_->isEmpty());
- EXPECT_TRUE(tuple_store_->isPacked());
- EXPECT_EQ(tuple_num, tuple_store_->getMaxTupleID());
- EXPECT_EQ(tuple_num + 1, tuple_store_->numTuples());
- }
-
- current_tuple.reset(createSampleTuple(0));
- TupleStorageSubBlock::InsertResult result = tuple_store_->insertTuple(*current_tuple);
- EXPECT_EQ(-1, result.inserted_id);
- EXPECT_FALSE(result.ids_mutated);
-
- EXPECT_TRUE(tuple_store_->isPacked());
- EXPECT_EQ(row_capacity - 1, tuple_store_->getMaxTupleID());
- EXPECT_EQ(row_capacity, tuple_store_->numTuples());
-}
-
-TEST_P(PackedRowStoreTupleStorageSubBlockTest, InsertInBatchTest) {
- initializeNewBlock(kSubBlockSize);
-
- int row_capacity = computeRowCapacity();
-
- EXPECT_TRUE(tuple_store_->supportsAdHocInsert());
- EXPECT_TRUE(tuple_store_->adHocInsertIsEfficient());
- EXPECT_FALSE(tuple_store_->isCompressed());
- EXPECT_TRUE(tuple_store_->isEmpty());
-
- std::unique_ptr<Tuple> current_tuple;
- for (int tuple_num = 0;
- tuple_num < row_capacity;
- ++tuple_num) {
- current_tuple.reset(createSampleTuple(tuple_num));
- EXPECT_TRUE(tuple_store_->insertTupleInBatch(*current_tuple));
-
- EXPECT_FALSE(tuple_store_->isEmpty());
- EXPECT_TRUE(tuple_store_->isPacked());
- EXPECT_EQ(tuple_num, tuple_store_->getMaxTupleID());
- EXPECT_EQ(tuple_num + 1, tuple_store_->numTuples());
- }
-
- current_tuple.reset(createSampleTuple(0));
- EXPECT_FALSE(tuple_store_->insertTupleInBatch(*current_tuple));
-
- EXPECT_TRUE(tuple_store_->isPacked());
- EXPECT_EQ(row_capacity - 1, tuple_store_->getMaxTupleID());
- EXPECT_EQ(row_capacity, tuple_store_->numTuples());
-}
-
-TEST_P(PackedRowStoreTupleStorageSubBlockTest, ColumnAccessorTest) {
- if (GetParam()) { // when true, the attributes can be nullable.
- checkColumnAccessor<true>();
- } else { // when false, the attributes are non-null.
- checkColumnAccessor<false>();
- }
-}
-
-TEST_P(PackedRowStoreTupleStorageSubBlockTest, GetAttributeValueTest) {
- initializeNewBlock(kSubBlockSize);
- fillBlockWithSampleData();
- ASSERT_TRUE(tuple_store_->isPacked());
-
- for (tuple_id tid = 0;
- tid <= tuple_store_->getMaxTupleID();
- ++tid) {
- checkTupleValuesUntyped(tid, tid);
- }
-}
-
-TEST_P(PackedRowStoreTupleStorageSubBlockTest, GetAttributeValueTypedTest) {
- initializeNewBlock(kSubBlockSize);
- fillBlockWithSampleData();
- ASSERT_TRUE(tuple_store_->isPacked());
-
- for (tuple_id tid = 0;
- tid <= tuple_store_->getMaxTupleID();
- ++tid) {
- checkTupleValuesTyped(tid, tid);
- }
-}
-
-TEST_P(PackedRowStoreTupleStorageSubBlockTest, SetAttributeValueTypedTest) {
- initializeNewBlock(kSubBlockSize);
- fillBlockWithSampleData();
- ASSERT_TRUE(tuple_store_->isPacked());
-
- // Alter every 16th tuple.
- for (tuple_id tid = 0;
- tid <= tuple_store_->getMaxTupleID();
- tid += 16) {
- std::unique_ptr<Tuple> mod_tuple(createSampleTuple(-tid));
-
- std::unordered_map<attribute_id, TypedValue> new_values;
- new_values.insert(make_pair(0, mod_tuple->getAttributeValue(0)));
- new_values.insert(make_pair(1, mod_tuple->getAttributeValue(1)));
- new_values.insert(make_pair(2, mod_tuple->getAttributeValue(2)));
- ASSERT_TRUE(tuple_store_->canSetAttributeValuesInPlaceTyped(tid, new_values));
-
- tuple_store_->setAttributeValueInPlaceTyped(tid, 0, mod_tuple->getAttributeValue(0));
- tuple_store_->setAttributeValueInPlaceTyped(tid, 1, mod_tuple->getAttributeValue(1));
- tuple_store_->setAttributeValueInPlaceTyped(tid, 2, mod_tuple->getAttributeValue(2));
- }
-
- // Check all values.
- for (tuple_id tid = 0;
- tid <= tuple_store_->getMaxTupleID();
- ++tid) {
- if (tid & 0xF) {
- checkTupleValuesTyped(tid, tid);
- } else {
- checkTupleValuesTyped(tid, -tid);
- }
- }
-}
-
-TEST_P(PackedRowStoreTupleStorageSubBlockTest, DeleteTest) {
- initializeNewBlock(kSubBlockSize);
- fillBlockWithSampleData();
- ASSERT_TRUE(tuple_store_->isPacked());
-
- tuple_id original_num_tuples = tuple_store_->numTuples();
-
- // Delete the last tuple.
- EXPECT_FALSE(tuple_store_->deleteTuple(original_num_tuples - 1));
-
- // Delete the first tuple.
- EXPECT_TRUE(tuple_store_->deleteTuple(0));
-
- // Delete a sequence of tuples.
- TupleIdSequence delete_sequence(tuple_store_->getMaxTupleID() + 1);
- for (tuple_id tid = 63;
- tid <= tuple_store_->getMaxTupleID();
- tid += 64) {
- delete_sequence.set(tid, true);
- }
- EXPECT_TRUE(tuple_store_->bulkDeleteTuples(&delete_sequence));
-
- EXPECT_EQ(static_cast<tuple_id>(original_num_tuples - 2 - delete_sequence.numTuples()),
- tuple_store_->numTuples());
-
- tuple_id remaining_tid = 0;
- for (tuple_id original_tid = 0;
- original_tid < (original_num_tuples - 1);
- ++original_tid) {
- if (original_tid & 0x3F) {
- checkTupleValuesUntyped(remaining_tid, original_tid);
- ++remaining_tid;
- }
- }
-}
-
-TEST(PackedRowStoreTupleStorageSubBlockNullTypeTest, NullTypeTest) {
- // Set up a relation with a single NullType attribute.
- CatalogRelation test_relation(nullptr, "TestRelation");
- CatalogAttribute *nulltype_attr = new CatalogAttribute(&test_relation,
- "nulltype_attr",
- TypeFactory::GetType(kNullType, true));
- ASSERT_EQ(0, test_relation.addAttribute(nulltype_attr));
-
- // Set up a minimal StorageBlockLayoutDescription.
- StorageBlockLayoutDescription layout_desc;
- layout_desc.set_num_slots(1);
- layout_desc.mutable_tuple_store_description()->set_sub_block_type(
- TupleStorageSubBlockDescription::PACKED_ROW_STORE);
-
- // Check that the description is considered valid.
- EXPECT_TRUE(StorageBlockLayout::DescriptionIsValid(test_relation, layout_desc));
-
- StorageBlockLayout layout(test_relation, layout_desc);
-
- // Construct an actual PackedRowStoreTupleStorageSubBlock.
- ScopedBuffer tuple_store_memory(kSlotSizeBytes);
- PackedRowStoreTupleStorageSubBlock tuple_store(test_relation,
- layout_desc.tuple_store_description(),
- true,
- tuple_store_memory.get(),
- kSlotSizeBytes);
-
- // Insert some NullType values.
- std::vector<TypedValue> attr_values;
- attr_values.emplace_back(kNullType);
- Tuple tuple(std::move(attr_values));
-
- for (tuple_id tid = 0; tid < 100; ++tid) {
- tuple_store.insertTuple(tuple);
- }
-
- EXPECT_EQ(100, tuple_store.numTuples());
-
- // Delete some values.
- TupleIdSequence delete_sequence(100);
- delete_sequence.set(5, true);
- delete_sequence.set(25, true);
- delete_sequence.set(45, true);
- delete_sequence.set(65, true);
- delete_sequence.set(85, true);
-
- EXPECT_TRUE(tuple_store.bulkDeleteTuples(&delete_sequence));
- EXPECT_EQ(95, tuple_store.numTuples());
- ASSERT_EQ(94, tuple_store.getMaxTupleID());
-
- // Read out values.
- for (tuple_id tid = 0; tid < 95; ++tid) {
- ASSERT_TRUE(tuple_store.hasTupleWithID(tid));
- EXPECT_EQ(nullptr, tuple_store.getAttributeValue(tid, 0));
-
- TypedValue value = tuple_store.getAttributeValueTyped(tid, 0);
- EXPECT_TRUE(value.isNull());
- EXPECT_EQ(kNullType, value.getTypeID());
- }
-}
-
-// Note: INSTANTIATE_TEST_CASE_P has variadic arguments part. If the variable argument part
-// is empty, C++11 standard says it should produce a warning. A warning is converted
-// to an error since we use -Werror as a compiler parameter. It causes Travis to build.
-// This is the reason that we must give an empty string argument as a last parameter
-// to supress warning that clang gives.
-INSTANTIATE_TEST_CASE_P(WithAndWithoutNullableAttributes,
- PackedRowStoreTupleStorageSubBlockTest,
- ::testing::Bool(),); // NOLINT(whitespace/comma)
-
-INSTANTIATE_TEST_CASE_P(WithAndWithoutNullableAttributes,
- PackedRowStoreTupleStorageSubBlockDeathTest,
- ::testing::Bool(),); // NOLINT(whitespace/comma)
-
-} // namespace quickstep
[23/50] incubator-quickstep git commit: Added an util method to get
the ip address.
Posted by zu...@apache.org.
Added an util method to get the ip address.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/8f1b0e56
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/8f1b0e56
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/8f1b0e56
Branch: refs/heads/quickstep_partition_parser_support
Commit: 8f1b0e56947b49f544bb6e0f5a4b425e8735ccf9
Parents: 5655511
Author: Zuyu Zhang <zu...@apache.org>
Authored: Sun Nov 13 15:13:09 2016 -0800
Committer: Zuyu Zhang <zu...@apache.org>
Committed: Sun Nov 20 20:04:21 2016 -0800
----------------------------------------------------------------------
utility/CMakeLists.txt | 20 ++++++++
utility/NetworkUtil.cpp | 78 +++++++++++++++++++++++++++++
utility/NetworkUtil.hpp | 42 ++++++++++++++++
utility/tests/NetworkUtil_unittest.cpp | 44 ++++++++++++++++
4 files changed, 184 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/8f1b0e56/utility/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/utility/CMakeLists.txt b/utility/CMakeLists.txt
index e9be2ec..6a656fb 100644
--- a/utility/CMakeLists.txt
+++ b/utility/CMakeLists.txt
@@ -15,6 +15,12 @@
# specific language governing permissions and limitations
# under the License.
+if (BUILD_SHARED_LIBS)
+ set(GFLAGS_LIB_NAME gflags_nothreads-shared)
+else()
+ set(GFLAGS_LIB_NAME gflags_nothreads-static)
+endif()
+
include(CheckCXXSourceCompiles)
# Look for GCC-style builtins (also in clang, ICC, and possibly others) that
@@ -180,6 +186,7 @@ add_library(quickstep_utility_Glob Glob.cpp Glob.hpp)
add_library(quickstep_utility_HashPair ../empty_src.cpp HashPair.hpp)
add_library(quickstep_utility_Macros ../empty_src.cpp Macros.hpp)
add_library(quickstep_utility_MemStream ../empty_src.cpp MemStream.hpp)
+add_library(quickstep_utility_NetworkUtil NetworkUtil.cpp NetworkUtil.hpp)
add_library(quickstep_utility_PlanVisualizer PlanVisualizer.cpp PlanVisualizer.hpp)
add_library(quickstep_utility_PrimeNumber PrimeNumber.cpp PrimeNumber.hpp)
add_library(quickstep_utility_PtrList ../empty_src.cpp PtrList.hpp)
@@ -257,6 +264,9 @@ target_link_libraries(quickstep_utility_Glob
target_link_libraries(quickstep_utility_MemStream
glog
quickstep_utility_Macros)
+target_link_libraries(quickstep_utility_NetworkUtil
+ glog
+ ${GFLAGS_LIB_NAME})
target_link_libraries(quickstep_utility_PrimeNumber
glog)
target_link_libraries(quickstep_utility_PlanVisualizer
@@ -339,6 +349,7 @@ target_link_libraries(quickstep_utility
quickstep_utility_HashPair
quickstep_utility_Macros
quickstep_utility_MemStream
+ quickstep_utility_NetworkUtil
quickstep_utility_PlanVisualizer
quickstep_utility_PrimeNumber
quickstep_utility_PtrList
@@ -404,6 +415,15 @@ target_link_libraries(EqualsAnyConstant_unittest
${LIBS})
add_test(EqualsAnyConstant_unittest EqualsAnyConstant_unittest)
+add_executable(NetworkUtil_unittest "${CMAKE_CURRENT_SOURCE_DIR}/tests/NetworkUtil_unittest.cpp")
+target_link_libraries(NetworkUtil_unittest
+ gtest
+ gtest_main
+ quickstep_utility_NetworkUtil
+ ${GFLAGS_LIB_NAME}
+ ${LIBS})
+add_test(NetworkUtil_unittest NetworkUtil_unittest)
+
add_executable(PrimeNumber_unittest "${CMAKE_CURRENT_SOURCE_DIR}/tests/PrimeNumber_unittest.cpp")
target_link_libraries(PrimeNumber_unittest
gtest
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/8f1b0e56/utility/NetworkUtil.cpp
----------------------------------------------------------------------
diff --git a/utility/NetworkUtil.cpp b/utility/NetworkUtil.cpp
new file mode 100644
index 0000000..f665b62
--- /dev/null
+++ b/utility/NetworkUtil.cpp
@@ -0,0 +1,78 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ **/
+
+#include "utility/NetworkUtil.hpp"
+
+#include "utility/UtilityConfig.h" // For QUICKSTEP_HAVE_WINDOWS.
+
+#ifndef QUICKSTEP_HAVE_WINDOWS
+#include <arpa/inet.h>
+#include <net/if.h> // Need to include before <ifaddrs.h> for NetBSD.
+#include <netinet/in.h>
+#include <sys/socket.h>
+#include <ifaddrs.h>
+#include <netdb.h>
+
+#include <cstring>
+#endif // QUICKSTEP_HAVE_WINDOWS
+
+#include <string>
+
+#include "gflags/gflags.h"
+#include "glog/logging.h"
+
+namespace quickstep {
+
+DEFINE_bool(use_ethernet_ip, false,
+ "Use the Ethernet ip address, instead of the loopback.");
+
+std::string GetIpv4Address() {
+#ifndef QUICKSTEP_HAVE_WINDOWS
+ struct ifaddrs *ifaddrs;
+ CHECK_EQ(0, getifaddrs(&ifaddrs));
+
+ char ip_address[NI_MAXHOST] = { '\0' };
+
+ for (struct ifaddrs *ifa = ifaddrs; ifa; ifa = ifa->ifa_next) {
+ if (ifa->ifa_addr == nullptr ||
+ ifa->ifa_addr->sa_family != AF_INET ||
+ (ifa->ifa_flags & IFF_UP) == 0) {
+ continue;
+ }
+
+ if (FLAGS_use_ethernet_ip && !std::strncmp(ifa->ifa_name, "lo", 2)) {
+ // NOTE: On Linux it starts with 'eth', while on Mac OS X, 'en'.
+ continue;
+ }
+
+ struct sockaddr_in *s4 = (struct sockaddr_in *)(ifa->ifa_addr);
+ CHECK(inet_ntop(AF_INET, reinterpret_cast<void *>(&(s4->sin_addr)), ip_address, sizeof(ip_address)) != nullptr);
+ break;
+ }
+
+ freeifaddrs(ifaddrs);
+
+ return ip_address;
+#else
+ CHECK(!FLAGS_use_ethernet_ip);
+ return kLocalIpv4Address;
+#endif // QUICKSTEP_HAVE_WINDOWS
+}
+
+} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/8f1b0e56/utility/NetworkUtil.hpp
----------------------------------------------------------------------
diff --git a/utility/NetworkUtil.hpp b/utility/NetworkUtil.hpp
new file mode 100644
index 0000000..b825fd8
--- /dev/null
+++ b/utility/NetworkUtil.hpp
@@ -0,0 +1,42 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ **/
+
+#ifndef QUICKSTEP_UTILITY_NETWORK_UTIL_HPP_
+#define QUICKSTEP_UTILITY_NETWORK_UTIL_HPP_
+
+#include <string>
+
+namespace quickstep {
+
+/** \addtogroup Utility
+ * @{
+ */
+
+constexpr char kLocalIpv4Address[] = "127.0.0.1";
+
+/**
+ * @brief Get the IPv4 network address in the text format, i.e., "127.0.0.1".
+ */
+extern std::string GetIpv4Address();
+
+/** @} */
+
+} // namespace quickstep
+
+#endif // QUICKSTEP_UTILITY_NETWORK_UTIL_HPP_
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/8f1b0e56/utility/tests/NetworkUtil_unittest.cpp
----------------------------------------------------------------------
diff --git a/utility/tests/NetworkUtil_unittest.cpp b/utility/tests/NetworkUtil_unittest.cpp
new file mode 100644
index 0000000..0013d7c
--- /dev/null
+++ b/utility/tests/NetworkUtil_unittest.cpp
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ **/
+
+#include <string>
+
+#include "utility/NetworkUtil.hpp"
+
+#include "gflags/gflags_declare.h"
+#include "gtest/gtest.h"
+
+using std::string;
+
+namespace quickstep {
+
+DECLARE_bool(use_ethernet_ip);
+
+TEST(NetworkUtilTest, LoopbackTest) {
+ string lo_ip_address = GetIpv4Address();
+ EXPECT_STREQ(kLocalIpv4Address, lo_ip_address.c_str());
+}
+
+TEST(NetworkUtilTest, EthernetTest) {
+ FLAGS_use_ethernet_ip = true;
+ string ethernet_ip_address = GetIpv4Address();
+ EXPECT_STRNE(kLocalIpv4Address, ethernet_ip_address.c_str());
+}
+
+} // namespace quickstep