You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@quickstep.apache.org by zu...@apache.org on 2016/07/29 23:44:04 UTC
[01/25] incubator-quickstep git commit: QUICKSTEP-33: Fixed the bug
in NumericCast.
Repository: incubator-quickstep
Updated Branches:
refs/heads/dist-exe-test-new [created] 603b61498
QUICKSTEP-33: Fixed the bug in NumericCast.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/8af55df5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/8af55df5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/8af55df5
Branch: refs/heads/dist-exe-test-new
Commit: 8af55df5790f7313896c9ee94a4e8e0278c5f9fc
Parents: ba2e5bf
Author: Hakan Memisoglu <ha...@gmail.com>
Authored: Wed Jun 29 14:01:13 2016 -0500
Committer: Zuyu Zhang <zu...@twitter.com>
Committed: Fri Jul 29 16:42:09 2016 -0700
----------------------------------------------------------------------
types/operations/unary_operations/NumericCastOperation.hpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/8af55df5/types/operations/unary_operations/NumericCastOperation.hpp
----------------------------------------------------------------------
diff --git a/types/operations/unary_operations/NumericCastOperation.hpp b/types/operations/unary_operations/NumericCastOperation.hpp
index 250df6d..6662796 100644
--- a/types/operations/unary_operations/NumericCastOperation.hpp
+++ b/types/operations/unary_operations/NumericCastOperation.hpp
@@ -126,7 +126,7 @@ class UncheckedNumericCastOperator : public UncheckedUnaryOperator {
result->appendNullValue();
} else {
*static_cast<typename TargetType::cpptype*>(result->getPtrForDirectWrite())
- = static_cast<typename SourceType::cpptype>(*scalar_arg);
+ = static_cast<typename TargetType::cpptype>(*scalar_arg);
}
}
return result;
[03/25] incubator-quickstep git commit: Added PhysicalGenerator
support for Window Aggregation.
Posted by zu...@apache.org.
Added PhysicalGenerator support for Window Aggregation.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/50edb49a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/50edb49a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/50edb49a
Branch: refs/heads/dist-exe-test-new
Commit: 50edb49a72e595fe8687c665a99ab310530dc1ae
Parents: 06177e8
Author: shixuan-fan <sh...@apache.org>
Authored: Tue Jun 28 16:25:52 2016 +0000
Committer: Zuyu Zhang <zu...@twitter.com>
Committed: Fri Jul 29 16:42:09 2016 -0700
----------------------------------------------------------------------
query_optimizer/CMakeLists.txt | 1 +
query_optimizer/ExecutionGenerator.cpp | 4 +
query_optimizer/physical/CMakeLists.txt | 15 +-
query_optimizer/physical/PhysicalType.hpp | 3 +-
query_optimizer/physical/WindowAggregate.cpp | 66 +++++
query_optimizer/physical/WindowAggregate.hpp | 133 ++++++++++
query_optimizer/strategy/CMakeLists.txt | 5 +-
query_optimizer/strategy/OneToOne.cpp | 11 +-
.../tests/physical_generator/Select.test | 264 +++++++++++++++++++
9 files changed, 495 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/50edb49a/query_optimizer/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_optimizer/CMakeLists.txt b/query_optimizer/CMakeLists.txt
index 8f08130..8912414 100644
--- a/query_optimizer/CMakeLists.txt
+++ b/query_optimizer/CMakeLists.txt
@@ -109,6 +109,7 @@ target_link_libraries(quickstep_queryoptimizer_ExecutionGenerator
quickstep_queryoptimizer_physical_TableReference
quickstep_queryoptimizer_physical_TopLevelPlan
quickstep_queryoptimizer_physical_UpdateTable
+ quickstep_queryoptimizer_physical_WindowAggregate
quickstep_relationaloperators_AggregationOperator
quickstep_relationaloperators_BuildHashOperator
quickstep_relationaloperators_CreateIndexOperator
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/50edb49a/query_optimizer/ExecutionGenerator.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/ExecutionGenerator.cpp b/query_optimizer/ExecutionGenerator.cpp
index f9fd742..45f5f78 100644
--- a/query_optimizer/ExecutionGenerator.cpp
+++ b/query_optimizer/ExecutionGenerator.cpp
@@ -83,6 +83,7 @@
#include "query_optimizer/physical/TableReference.hpp"
#include "query_optimizer/physical/TopLevelPlan.hpp"
#include "query_optimizer/physical/UpdateTable.hpp"
+#include "query_optimizer/physical/WindowAggregate.hpp"
#include "relational_operators/AggregationOperator.hpp"
#include "relational_operators/BuildHashOperator.hpp"
#include "relational_operators/CreateIndexOperator.hpp"
@@ -282,6 +283,9 @@ void ExecutionGenerator::generatePlanInternal(
case P::PhysicalType::kUpdateTable:
return convertUpdateTable(
std::static_pointer_cast<const P::UpdateTable>(physical_plan));
+ case P::PhysicalType::kWindowAggregate:
+ THROW_SQL_ERROR()
+ << "Window aggregate function is not supported yet :(";
default:
LOG(FATAL) << "Unknown physical plan node "
<< physical_plan->getShortString();
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/50edb49a/query_optimizer/physical/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_optimizer/physical/CMakeLists.txt b/query_optimizer/physical/CMakeLists.txt
index ea3752d..1ad30e4 100644
--- a/query_optimizer/physical/CMakeLists.txt
+++ b/query_optimizer/physical/CMakeLists.txt
@@ -39,6 +39,7 @@ add_library(quickstep_queryoptimizer_physical_TableGenerator ../../empty_src.cpp
add_library(quickstep_queryoptimizer_physical_TableReference TableReference.cpp TableReference.hpp)
add_library(quickstep_queryoptimizer_physical_TopLevelPlan TopLevelPlan.cpp TopLevelPlan.hpp)
add_library(quickstep_queryoptimizer_physical_UpdateTable UpdateTable.cpp UpdateTable.hpp)
+add_library(quickstep_queryoptimizer_physical_WindowAggregate WindowAggregate.cpp WindowAggregate.hpp)
# Link dependencies:
target_link_libraries(quickstep_queryoptimizer_physical_Aggregate
@@ -250,6 +251,17 @@ target_link_libraries(quickstep_queryoptimizer_physical_UpdateTable
quickstep_queryoptimizer_physical_PhysicalType
quickstep_utility_Cast
quickstep_utility_Macros)
+target_link_libraries(quickstep_queryoptimizer_physical_WindowAggregate
+ quickstep_queryoptimizer_OptimizerTree
+ quickstep_queryoptimizer_expressions_Alias
+ quickstep_queryoptimizer_expressions_AttributeReference
+ quickstep_queryoptimizer_expressions_ExpressionUtil
+ quickstep_queryoptimizer_expressions_NamedExpression
+ quickstep_queryoptimizer_expressions_Predicate
+ quickstep_queryoptimizer_physical_Physical
+ quickstep_queryoptimizer_physical_PhysicalType
+ quickstep_utility_Cast
+ quickstep_utility_Macros)
# Module all-in-one library:
add_library(quickstep_queryoptimizer_physical ../../empty_src.cpp OptimizerPhysicalModule.hpp)
@@ -276,4 +288,5 @@ target_link_libraries(quickstep_queryoptimizer_physical
quickstep_queryoptimizer_physical_TableGenerator
quickstep_queryoptimizer_physical_TableReference
quickstep_queryoptimizer_physical_TopLevelPlan
- quickstep_queryoptimizer_physical_UpdateTable)
+ quickstep_queryoptimizer_physical_UpdateTable
+ quickstep_queryoptimizer_physical_WindowAggregate)
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/50edb49a/query_optimizer/physical/PhysicalType.hpp
----------------------------------------------------------------------
diff --git a/query_optimizer/physical/PhysicalType.hpp b/query_optimizer/physical/PhysicalType.hpp
index b036557..14f8e1a 100644
--- a/query_optimizer/physical/PhysicalType.hpp
+++ b/query_optimizer/physical/PhysicalType.hpp
@@ -47,7 +47,8 @@ enum class PhysicalType {
kTableGenerator,
kTableReference,
kTopLevelPlan,
- kUpdateTable
+ kUpdateTable,
+ kWindowAggregate
};
/** @} */
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/50edb49a/query_optimizer/physical/WindowAggregate.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/physical/WindowAggregate.cpp b/query_optimizer/physical/WindowAggregate.cpp
new file mode 100644
index 0000000..2927107
--- /dev/null
+++ b/query_optimizer/physical/WindowAggregate.cpp
@@ -0,0 +1,66 @@
+/**
+ * Copyright 2011-2015 Quickstep Technologies LLC.
+ * Copyright 2015 Pivotal Software, Inc.
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#include "query_optimizer/physical/WindowAggregate.hpp"
+
+#include <string>
+#include <vector>
+
+#include "query_optimizer/OptimizerTree.hpp"
+#include "query_optimizer/expressions/AttributeReference.hpp"
+#include "query_optimizer/expressions/ExpressionUtil.hpp"
+#include "query_optimizer/expressions/NamedExpression.hpp"
+#include "query_optimizer/expressions/Predicate.hpp"
+#include "utility/Cast.hpp"
+
+namespace quickstep {
+namespace optimizer {
+namespace physical {
+
+namespace E = ::quickstep::optimizer::expressions;
+
+std::vector<E::AttributeReferencePtr> WindowAggregate::getOutputAttributes() const {
+ std::vector<E::AttributeReferencePtr> output_attributes(
+ input_->getOutputAttributes());
+ output_attributes.push_back(E::ToRef(window_aggregate_expression_));
+ return output_attributes;
+}
+
+std::vector<E::AttributeReferencePtr> WindowAggregate::getReferencedAttributes()
+ const {
+ return window_aggregate_expression_->getReferencedAttributes();
+}
+
+void WindowAggregate::getFieldStringItems(
+ std::vector<std::string> *inline_field_names,
+ std::vector<std::string> *inline_field_values,
+ std::vector<std::string> *non_container_child_field_names,
+ std::vector<OptimizerTreeBaseNodePtr> *non_container_child_fields,
+ std::vector<std::string> *container_child_field_names,
+ std::vector<std::vector<OptimizerTreeBaseNodePtr>> *container_child_fields) const {
+ non_container_child_field_names->push_back("input");
+ non_container_child_fields->push_back(input_);
+
+ non_container_child_field_names->push_back("window_aggregate_expression");
+ non_container_child_fields->push_back(window_aggregate_expression_);
+}
+
+} // namespace physical
+} // namespace optimizer
+} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/50edb49a/query_optimizer/physical/WindowAggregate.hpp
----------------------------------------------------------------------
diff --git a/query_optimizer/physical/WindowAggregate.hpp b/query_optimizer/physical/WindowAggregate.hpp
new file mode 100644
index 0000000..4b17d07
--- /dev/null
+++ b/query_optimizer/physical/WindowAggregate.hpp
@@ -0,0 +1,133 @@
+/**
+ * Copyright 2011-2015 Quickstep Technologies LLC.
+ * Copyright 2015 Pivotal Software, Inc.
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#ifndef QUICKSTEP_QUERY_OPTIMIZER_PHYSICAL_WINDOW_AGGREGATE_HPP_
+#define QUICKSTEP_QUERY_OPTIMIZER_PHYSICAL_WINDOW_AGGREGATE_HPP_
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "query_optimizer/OptimizerTree.hpp"
+#include "query_optimizer/expressions/Alias.hpp"
+#include "query_optimizer/expressions/AttributeReference.hpp"
+#include "query_optimizer/expressions/ExpressionUtil.hpp"
+#include "query_optimizer/expressions/NamedExpression.hpp"
+#include "query_optimizer/physical/Physical.hpp"
+#include "query_optimizer/physical/PhysicalType.hpp"
+#include "utility/Macros.hpp"
+
+#include "glog/logging.h"
+
+namespace quickstep {
+namespace optimizer {
+namespace physical {
+
+/** \addtogroup OptimizerLogical
+ * @{
+ */
+
+class WindowAggregate;
+typedef std::shared_ptr<const WindowAggregate> WindowAggregatePtr;
+
+/**
+ * @brief Window Aggregate operator that computes window aggregate expressions.
+ */
+class WindowAggregate : public Physical {
+ public:
+ PhysicalType getPhysicalType() const override {
+ return PhysicalType::kWindowAggregate;
+ }
+
+ std::string getName() const override { return "WindowAggregate"; }
+
+ /**
+ * @return The input physical node.
+ */
+ const PhysicalPtr input() const { return input_; }
+
+ /**
+ * @return Window aggregate expression.
+ */
+ inline const expressions::AliasPtr window_aggregate_expression() const {
+ return window_aggregate_expression_;
+ }
+
+ PhysicalPtr copyWithNewChildren(
+ const std::vector<PhysicalPtr> &new_children) const override {
+ DCHECK_EQ(getNumChildren(), new_children.size());
+ return Create(new_children[0], window_aggregate_expression_);
+ }
+
+ std::vector<expressions::AttributeReferencePtr> getOutputAttributes() const override;
+
+ std::vector<expressions::AttributeReferencePtr> getReferencedAttributes() const override;
+
+ bool maybeCopyWithPrunedExpressions(
+ const expressions::UnorderedNamedExpressionSet &referenced_expressions,
+ PhysicalPtr *output) const override {
+ // The project expressions for an WindowAggregate cannot be changed.
+ return false;
+ }
+
+ /**
+ * @brief Creates an WindowAggregate physical node.
+ *
+ * @param input The input node.
+ * @param window_aggregate_expression The window aggregate expression.
+ * @return An immutable WindowAggregate node.
+ */
+ static WindowAggregatePtr Create(
+ const PhysicalPtr &input,
+ const expressions::AliasPtr &window_aggregate_expression) {
+ return WindowAggregatePtr(new WindowAggregate(input,
+ window_aggregate_expression));
+ }
+
+ protected:
+ void getFieldStringItems(
+ std::vector<std::string> *inline_field_names,
+ std::vector<std::string> *inline_field_values,
+ std::vector<std::string> *non_container_child_field_names,
+ std::vector<OptimizerTreeBaseNodePtr> *non_container_child_fields,
+ std::vector<std::string> *container_child_field_names,
+ std::vector<std::vector<OptimizerTreeBaseNodePtr>> *container_child_fields) const override;
+
+ private:
+ WindowAggregate(
+ const PhysicalPtr &input,
+ const expressions::AliasPtr &window_aggregate_expression)
+ : input_(input),
+ window_aggregate_expression_(window_aggregate_expression) {
+ addChild(input_);
+ }
+
+ const PhysicalPtr input_;
+ const expressions::AliasPtr window_aggregate_expression_;
+
+ DISALLOW_COPY_AND_ASSIGN(WindowAggregate);
+};
+
+/** @} */
+
+} // namespace physical
+} // namespace optimizer
+} // namespace quickstep
+
+#endif // QUICKSTEP_QUERY_OPTIMIZER_PHYSICAL_WINDOW_AGGREGATE_HPP_
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/50edb49a/query_optimizer/strategy/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_optimizer/strategy/CMakeLists.txt b/query_optimizer/strategy/CMakeLists.txt
index 84e151e..517bea3 100644
--- a/query_optimizer/strategy/CMakeLists.txt
+++ b/query_optimizer/strategy/CMakeLists.txt
@@ -89,6 +89,7 @@ target_link_libraries(quickstep_queryoptimizer_strategy_OneToOne
quickstep_queryoptimizer_logical_TableReference
quickstep_queryoptimizer_logical_TopLevelPlan
quickstep_queryoptimizer_logical_UpdateTable
+ quickstep_queryoptimizer_logical_WindowAggregate
quickstep_queryoptimizer_physical_CopyFrom
quickstep_queryoptimizer_physical_CreateIndex
quickstep_queryoptimizer_physical_CreateTable
@@ -104,9 +105,9 @@ target_link_libraries(quickstep_queryoptimizer_strategy_OneToOne
quickstep_queryoptimizer_physical_TableReference
quickstep_queryoptimizer_physical_TopLevelPlan
quickstep_queryoptimizer_physical_UpdateTable
+ quickstep_queryoptimizer_physical_WindowAggregate
quickstep_queryoptimizer_strategy_Strategy
- quickstep_utility_Macros
- quickstep_utility_SqlError)
+ quickstep_utility_Macros)
target_link_libraries(quickstep_queryoptimizer_strategy_Selection
glog
quickstep_queryoptimizer_LogicalToPhysicalMapper
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/50edb49a/query_optimizer/strategy/OneToOne.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/strategy/OneToOne.cpp b/query_optimizer/strategy/OneToOne.cpp
index f49a25c..e9a8897 100644
--- a/query_optimizer/strategy/OneToOne.cpp
+++ b/query_optimizer/strategy/OneToOne.cpp
@@ -41,6 +41,7 @@
#include "query_optimizer/logical/TableReference.hpp"
#include "query_optimizer/logical/TopLevelPlan.hpp"
#include "query_optimizer/logical/UpdateTable.hpp"
+#include "query_optimizer/logical/WindowAggregate.hpp"
#include "query_optimizer/physical/CopyFrom.hpp"
#include "query_optimizer/physical/CreateIndex.hpp"
#include "query_optimizer/physical/CreateTable.hpp"
@@ -55,7 +56,7 @@
#include "query_optimizer/physical/TableReference.hpp"
#include "query_optimizer/physical/TopLevelPlan.hpp"
#include "query_optimizer/physical/UpdateTable.hpp"
-#include "utility/SqlError.hpp"
+#include "query_optimizer/physical/WindowAggregate.hpp"
namespace quickstep {
namespace optimizer {
@@ -210,8 +211,12 @@ bool OneToOne::generatePlan(const L::LogicalPtr &logical_input,
return true;
}
case L::LogicalType::kWindowAggregate: {
- THROW_SQL_ERROR()
- << "Window aggregate function is not supported currently :(";
+ const L::WindowAggregatePtr window_aggregate =
+ std::static_pointer_cast<const L::WindowAggregate>(logical_input);
+ *physical_output = P::WindowAggregate::Create(
+ physical_mapper_->createOrGetPhysicalFromLogical(window_aggregate->input()),
+ window_aggregate->window_aggregate_expression());
+ return true;
}
default:
return false;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/50edb49a/query_optimizer/tests/physical_generator/Select.test
----------------------------------------------------------------------
diff --git a/query_optimizer/tests/physical_generator/Select.test b/query_optimizer/tests/physical_generator/Select.test
index 3365206..d99916c 100644
--- a/query_optimizer/tests/physical_generator/Select.test
+++ b/query_optimizer/tests/physical_generator/Select.test
@@ -2876,3 +2876,267 @@ TopLevelPlan
+-output_attributes=
+-AttributeReference[id=5,name=x,relation=,type=Int]
+-AttributeReference[id=6,name=y,relation=,type=Int]
+==
+
+# Window Aggregate Function Test.
+SELECT avg(int_col) OVER w FROM test
+WINDOW w AS
+(PARTITION BY char_col
+ ORDER BY long_col DESC NULLS LAST
+ ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW);
+--
+[Optimized Logical Plan]
+TopLevelPlan
++-plan=Project
+| +-input=WindowAggregate
+| | +-input=Sort[is_ascending=[true,false],nulls_first=[false,false]]
+| | | +-input=TableReference[relation_name=Test,relation_alias=test]
+| | | | +-AttributeReference[id=0,name=int_col,relation=test,type=Int NULL]
+| | | | +-AttributeReference[id=1,name=long_col,relation=test,type=Long]
+| | | | +-AttributeReference[id=2,name=float_col,relation=test,type=Float]
+| | | | +-AttributeReference[id=3,name=double_col,relation=test,type=Double NULL]
+| | | | +-AttributeReference[id=4,name=char_col,relation=test,type=Char(20)]
+| | | | +-AttributeReference[id=5,name=vchar_col,relation=test,
+| | | | type=VarChar(20) NULL]
+| | | +-sort_expressions=
+| | | +-AttributeReference[id=4,name=char_col,relation=test,type=Char(20)]
+| | | +-AttributeReference[id=1,name=long_col,relation=test,type=Long]
+| | +-window_aggregate_expression=Alias[id=6,name=,alias=$window_aggregate0,
+| | relation=$window_aggregate,type=Double NULL]
+| | +-WindowAggregateFunction[function=AVG,window_name=w,is_ascending=[false],
+| | nulls_first=[false],frame_mode=row,num_preceding=-1,num_following=0]
+| | +-arguments=
+| | | +-AttributeReference[id=0,name=int_col,relation=test,type=Int NULL]
+| | +-partition_by=
+| | | +-AttributeReference[id=4,name=char_col,relation=test,type=Char(20)]
+| | +-order_by=
+| | +-AttributeReference[id=1,name=long_col,relation=test,type=Long]
+| +-project_list=
+| +-Alias[id=6,name=,alias=avg(int_col),relation=,type=Double NULL]
+| +-AttributeReference[id=6,name=,alias=$window_aggregate0,
+| relation=$window_aggregate,type=Double NULL]
++-output_attributes=
+ +-AttributeReference[id=6,name=,alias=avg(int_col),relation=,type=Double NULL]
+[Physical Plan]
+TopLevelPlan
++-plan=Selection
+| +-input=WindowAggregate
+| | +-input=Sort[is_ascending=[true,false],nulls_first=[false,false]]
+| | | +-input=TableReference[relation=Test,alias=test]
+| | | | +-AttributeReference[id=0,name=int_col,relation=test,type=Int NULL]
+| | | | +-AttributeReference[id=1,name=long_col,relation=test,type=Long]
+| | | | +-AttributeReference[id=2,name=float_col,relation=test,type=Float]
+| | | | +-AttributeReference[id=3,name=double_col,relation=test,type=Double NULL]
+| | | | +-AttributeReference[id=4,name=char_col,relation=test,type=Char(20)]
+| | | | +-AttributeReference[id=5,name=vchar_col,relation=test,
+| | | | type=VarChar(20) NULL]
+| | | +-sort_attributes=
+| | | +-AttributeReference[id=4,name=char_col,relation=test,type=Char(20)]
+| | | +-AttributeReference[id=1,name=long_col,relation=test,type=Long]
+| | +-window_aggregate_expression=Alias[id=6,name=,alias=$window_aggregate0,
+| | relation=$window_aggregate,type=Double NULL]
+| | +-WindowAggregateFunction[function=AVG,window_name=w,is_ascending=[false],
+| | nulls_first=[false],frame_mode=row,num_preceding=-1,num_following=0]
+| | +-arguments=
+| | | +-AttributeReference[id=0,name=int_col,relation=test,type=Int NULL]
+| | +-partition_by=
+| | | +-AttributeReference[id=4,name=char_col,relation=test,type=Char(20)]
+| | +-order_by=
+| | +-AttributeReference[id=1,name=long_col,relation=test,type=Long]
+| +-project_expressions=
+| +-Alias[id=6,name=,alias=avg(int_col),relation=,type=Double NULL]
+| +-AttributeReference[id=6,name=,alias=$window_aggregate0,
+| relation=$window_aggregate,type=Double NULL]
++-output_attributes=
+ +-AttributeReference[id=6,name=,alias=avg(int_col),relation=,type=Double NULL]
+==
+
+SELECT int_col, sum(float_col) OVER
+(PARTITION BY vchar_col, long_col
+ ORDER BY double_col DESC NULLS LAST, int_col ASC NULLS FIRST
+ RANGE BETWEEN 3 PRECEDING AND 3 FOLLOWING)
+FROM test;
+--
+[Optimized Logical Plan]
+TopLevelPlan
++-plan=Project
+| +-input=WindowAggregate
+| | +-input=Sort[is_ascending=[true,true,false,true],
+| | | nulls_first=[false,false,false,true]]
+| | | +-input=TableReference[relation_name=Test,relation_alias=test]
+| | | | +-AttributeReference[id=0,name=int_col,relation=test,type=Int NULL]
+| | | | +-AttributeReference[id=1,name=long_col,relation=test,type=Long]
+| | | | +-AttributeReference[id=2,name=float_col,relation=test,type=Float]
+| | | | +-AttributeReference[id=3,name=double_col,relation=test,type=Double NULL]
+| | | | +-AttributeReference[id=4,name=char_col,relation=test,type=Char(20)]
+| | | | +-AttributeReference[id=5,name=vchar_col,relation=test,
+| | | | type=VarChar(20) NULL]
+| | | +-sort_expressions=
+| | | +-AttributeReference[id=5,name=vchar_col,relation=test,
+| | | | type=VarChar(20) NULL]
+| | | +-AttributeReference[id=1,name=long_col,relation=test,type=Long]
+| | | +-AttributeReference[id=3,name=double_col,relation=test,type=Double NULL]
+| | | +-AttributeReference[id=0,name=int_col,relation=test,type=Int NULL]
+| | +-window_aggregate_expression=Alias[id=6,name=,alias=$window_aggregate0,
+| | relation=$window_aggregate,type=Double NULL]
+| | +-WindowAggregateFunction[function=SUM,window_name=,
+| | is_ascending=[false,true],nulls_first=[false,true],frame_mode=range,
+| | num_preceding=3,num_following=3]
+| | +-arguments=
+| | | +-AttributeReference[id=2,name=float_col,relation=test,type=Float]
+| | +-partition_by=
+| | | +-AttributeReference[id=5,name=vchar_col,relation=test,
+| | | | type=VarChar(20) NULL]
+| | | +-AttributeReference[id=1,name=long_col,relation=test,type=Long]
+| | +-order_by=
+| | +-AttributeReference[id=3,name=double_col,relation=test,
+| | | type=Double NULL]
+| | +-AttributeReference[id=0,name=int_col,relation=test,type=Int NULL]
+| +-project_list=
+| +-AttributeReference[id=0,name=int_col,relation=test,type=Int NULL]
+| +-Alias[id=6,name=,alias=sum(float_col),relation=,type=Double NULL]
+| +-AttributeReference[id=6,name=,alias=$window_aggregate0,
+| relation=$window_aggregate,type=Double NULL]
++-output_attributes=
+ +-AttributeReference[id=0,name=int_col,relation=test,type=Int NULL]
+ +-AttributeReference[id=6,name=,alias=sum(float_col),relation=,
+ type=Double NULL]
+[Physical Plan]
+TopLevelPlan
++-plan=Selection
+| +-input=WindowAggregate
+| | +-input=Sort[is_ascending=[true,true,false,true],
+| | | nulls_first=[false,false,false,true]]
+| | | +-input=TableReference[relation=Test,alias=test]
+| | | | +-AttributeReference[id=0,name=int_col,relation=test,type=Int NULL]
+| | | | +-AttributeReference[id=1,name=long_col,relation=test,type=Long]
+| | | | +-AttributeReference[id=2,name=float_col,relation=test,type=Float]
+| | | | +-AttributeReference[id=3,name=double_col,relation=test,type=Double NULL]
+| | | | +-AttributeReference[id=4,name=char_col,relation=test,type=Char(20)]
+| | | | +-AttributeReference[id=5,name=vchar_col,relation=test,
+| | | | type=VarChar(20) NULL]
+| | | +-sort_attributes=
+| | | +-AttributeReference[id=5,name=vchar_col,relation=test,
+| | | | type=VarChar(20) NULL]
+| | | +-AttributeReference[id=1,name=long_col,relation=test,type=Long]
+| | | +-AttributeReference[id=3,name=double_col,relation=test,type=Double NULL]
+| | | +-AttributeReference[id=0,name=int_col,relation=test,type=Int NULL]
+| | +-window_aggregate_expression=Alias[id=6,name=,alias=$window_aggregate0,
+| | relation=$window_aggregate,type=Double NULL]
+| | +-WindowAggregateFunction[function=SUM,window_name=,
+| | is_ascending=[false,true],nulls_first=[false,true],frame_mode=range,
+| | num_preceding=3,num_following=3]
+| | +-arguments=
+| | | +-AttributeReference[id=2,name=float_col,relation=test,type=Float]
+| | +-partition_by=
+| | | +-AttributeReference[id=5,name=vchar_col,relation=test,
+| | | | type=VarChar(20) NULL]
+| | | +-AttributeReference[id=1,name=long_col,relation=test,type=Long]
+| | +-order_by=
+| | +-AttributeReference[id=3,name=double_col,relation=test,
+| | | type=Double NULL]
+| | +-AttributeReference[id=0,name=int_col,relation=test,type=Int NULL]
+| +-project_expressions=
+| +-AttributeReference[id=0,name=int_col,relation=test,type=Int NULL]
+| +-Alias[id=6,name=,alias=sum(float_col),relation=,type=Double NULL]
+| +-AttributeReference[id=6,name=,alias=$window_aggregate0,
+| relation=$window_aggregate,type=Double NULL]
++-output_attributes=
+ +-AttributeReference[id=0,name=int_col,relation=test,type=Int NULL]
+ +-AttributeReference[id=6,name=,alias=sum(float_col),relation=,
+ type=Double NULL]
+==
+
+SELECT sum(avg(int_col) OVER w) FROM test
+WINDOW w AS
+(PARTITION BY char_col
+ ORDER BY long_col
+ ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW);
+--
+[Optimized Logical Plan]
+TopLevelPlan
++-plan=Project
+| +-input=Aggregate
+| | +-input=WindowAggregate
+| | | +-input=Sort[is_ascending=[true,true],nulls_first=[false,false]]
+| | | | +-input=TableReference[relation_name=Test,relation_alias=test]
+| | | | | +-AttributeReference[id=0,name=int_col,relation=test,type=Int NULL]
+| | | | | +-AttributeReference[id=1,name=long_col,relation=test,type=Long]
+| | | | | +-AttributeReference[id=2,name=float_col,relation=test,type=Float]
+| | | | | +-AttributeReference[id=3,name=double_col,relation=test,
+| | | | | | type=Double NULL]
+| | | | | +-AttributeReference[id=4,name=char_col,relation=test,type=Char(20)]
+| | | | | +-AttributeReference[id=5,name=vchar_col,relation=test,
+| | | | | type=VarChar(20) NULL]
+| | | | +-sort_expressions=
+| | | | +-AttributeReference[id=4,name=char_col,relation=test,type=Char(20)]
+| | | | +-AttributeReference[id=1,name=long_col,relation=test,type=Long]
+| | | +-window_aggregate_expression=Alias[id=6,name=,alias=$window_aggregate0,
+| | | relation=$window_aggregate,type=Double NULL]
+| | | +-WindowAggregateFunction[function=AVG,window_name=w,
+| | | is_ascending=[true],nulls_first=[false],frame_mode=row,
+| | | num_preceding=-1,num_following=0]
+| | | +-arguments=
+| | | | +-AttributeReference[id=0,name=int_col,relation=test,type=Int NULL]
+| | | +-partition_by=
+| | | | +-AttributeReference[id=4,name=char_col,relation=test,type=Char(20)]
+| | | +-order_by=
+| | | +-AttributeReference[id=1,name=long_col,relation=test,type=Long]
+| | +-grouping_expressions=
+| | | +-[]
+| | +-aggregate_expressions=
+| | +-Alias[id=7,name=,alias=$aggregate0,relation=$aggregate,type=Double NULL]
+| | +-AggregateFunction[function=SUM]
+| | +-AttributeReference[id=6,name=,alias=$window_aggregate0,
+| | relation=$window_aggregate,type=Double NULL]
+| +-project_list=
+| +-Alias[id=7,name=,alias=sum(avg(int_col)),relation=,type=Double NULL]
+| +-AttributeReference[id=7,name=,alias=$aggregate0,relation=$aggregate,
+| type=Double NULL]
++-output_attributes=
+ +-AttributeReference[id=7,name=,alias=sum(avg(int_col)),relation=,
+ type=Double NULL]
+[Physical Plan]
+TopLevelPlan
++-plan=Selection
+| +-input=Aggregate
+| | +-input=WindowAggregate
+| | | +-input=Sort[is_ascending=[true,true],nulls_first=[false,false]]
+| | | | +-input=TableReference[relation=Test,alias=test]
+| | | | | +-AttributeReference[id=0,name=int_col,relation=test,type=Int NULL]
+| | | | | +-AttributeReference[id=1,name=long_col,relation=test,type=Long]
+| | | | | +-AttributeReference[id=2,name=float_col,relation=test,type=Float]
+| | | | | +-AttributeReference[id=3,name=double_col,relation=test,
+| | | | | | type=Double NULL]
+| | | | | +-AttributeReference[id=4,name=char_col,relation=test,type=Char(20)]
+| | | | | +-AttributeReference[id=5,name=vchar_col,relation=test,
+| | | | | type=VarChar(20) NULL]
+| | | | +-sort_attributes=
+| | | | +-AttributeReference[id=4,name=char_col,relation=test,type=Char(20)]
+| | | | +-AttributeReference[id=1,name=long_col,relation=test,type=Long]
+| | | +-window_aggregate_expression=Alias[id=6,name=,alias=$window_aggregate0,
+| | | relation=$window_aggregate,type=Double NULL]
+| | | +-WindowAggregateFunction[function=AVG,window_name=w,
+| | | is_ascending=[true],nulls_first=[false],frame_mode=row,
+| | | num_preceding=-1,num_following=0]
+| | | +-arguments=
+| | | | +-AttributeReference[id=0,name=int_col,relation=test,type=Int NULL]
+| | | +-partition_by=
+| | | | +-AttributeReference[id=4,name=char_col,relation=test,type=Char(20)]
+| | | +-order_by=
+| | | +-AttributeReference[id=1,name=long_col,relation=test,type=Long]
+| | +-grouping_expressions=
+| | | +-[]
+| | +-aggregate_expressions=
+| | +-Alias[id=7,name=,alias=$aggregate0,relation=$aggregate,type=Double NULL]
+| | +-AggregateFunction[function=SUM]
+| | +-AttributeReference[id=6,name=,alias=$window_aggregate0,
+| | relation=$window_aggregate,type=Double NULL]
+| +-project_expressions=
+| +-Alias[id=7,name=,alias=sum(avg(int_col)),relation=,type=Double NULL]
+| +-AttributeReference[id=7,name=,alias=$aggregate0,relation=$aggregate,
+| type=Double NULL]
++-output_attributes=
+ +-AttributeReference[id=7,name=,alias=sum(avg(int_col)),relation=,
+ type=Double NULL]
+==
[07/25] incubator-quickstep git commit: Added ExecutionGenerator
support for Window Aggregation.
Posted by zu...@apache.org.
Added ExecutionGenerator support for Window Aggregation.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/367194a1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/367194a1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/367194a1
Branch: refs/heads/dist-exe-test-new
Commit: 367194a1bb3e8f78ae9b6d01a359449b559d0ba6
Parents: 4fb884c
Author: shixuan-fan <sh...@apache.org>
Authored: Wed Jun 29 20:25:18 2016 +0000
Committer: Zuyu Zhang <zu...@twitter.com>
Committed: Fri Jul 29 16:42:10 2016 -0700
----------------------------------------------------------------------
query_execution/CMakeLists.txt | 2 +
query_execution/QueryContext.cpp | 14 ++
query_execution/QueryContext.hpp | 48 +++++
query_execution/QueryContext.proto | 5 +-
query_optimizer/CMakeLists.txt | 2 +
query_optimizer/ExecutionGenerator.cpp | 101 ++++++++++-
query_optimizer/ExecutionGenerator.hpp | 8 +
query_optimizer/cost_model/CMakeLists.txt | 2 +
query_optimizer/cost_model/SimpleCostModel.cpp | 9 +
query_optimizer/cost_model/SimpleCostModel.hpp | 5 +
.../cost_model/StarSchemaSimpleCostModel.cpp | 8 +
.../cost_model/StarSchemaSimpleCostModel.hpp | 4 +
query_optimizer/resolver/Resolver.cpp | 7 +-
.../tests/execution_generator/Select.test | 39 ++++
query_optimizer/tests/resolver/Select.test | 33 ++++
relational_operators/CMakeLists.txt | 14 ++
.../WindowAggregationOperator.cpp | 82 +++++++++
.../WindowAggregationOperator.hpp | 166 +++++++++++++++++
relational_operators/WorkOrder.proto | 9 +
storage/CMakeLists.txt | 45 ++++-
storage/WindowAggregationOperationState.cpp | 179 +++++++++++++++++++
storage/WindowAggregationOperationState.hpp | 177 ++++++++++++++++++
storage/WindowAggregationOperationState.proto | 33 ++++
...WindowAggregationOperationState_unittest.cpp | 92 ++++++++++
24 files changed, 1079 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/367194a1/query_execution/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_execution/CMakeLists.txt b/query_execution/CMakeLists.txt
index 2be451c..e1b1183 100644
--- a/query_execution/CMakeLists.txt
+++ b/query_execution/CMakeLists.txt
@@ -118,6 +118,7 @@ target_link_libraries(quickstep_queryexecution_QueryContext
quickstep_storage_HashTableFactory
quickstep_storage_InsertDestination
quickstep_storage_InsertDestination_proto
+ quickstep_storage_WindowAggregationOperationState
quickstep_types_TypedValue
quickstep_types_containers_Tuple
quickstep_utility_BloomFilter
@@ -130,6 +131,7 @@ target_link_libraries(quickstep_queryexecution_QueryContext_proto
quickstep_storage_AggregationOperationState_proto
quickstep_storage_HashTable_proto
quickstep_storage_InsertDestination_proto
+ quickstep_storage_WindowAggregationOperationState_proto
quickstep_types_containers_Tuple_proto
quickstep_utility_SortConfiguration_proto
${PROTOBUF_LIBRARY})
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/367194a1/query_execution/QueryContext.cpp
----------------------------------------------------------------------
diff --git a/query_execution/QueryContext.cpp b/query_execution/QueryContext.cpp
index 54dd557..7019b6a 100644
--- a/query_execution/QueryContext.cpp
+++ b/query_execution/QueryContext.cpp
@@ -140,6 +140,13 @@ QueryContext::QueryContext(const serialization::QueryContext &proto,
update_groups_.push_back(move(update_group));
}
+
+ for (int i = 0; i < proto.window_aggregation_states_size(); ++i) {
+ window_aggregation_states_.emplace_back(
+ WindowAggregationOperationState::ReconstructFromProto(proto.window_aggregation_states(i),
+ database,
+ storage_manager));
+ }
}
bool QueryContext::ProtoIsValid(const serialization::QueryContext &proto,
@@ -231,6 +238,13 @@ bool QueryContext::ProtoIsValid(const serialization::QueryContext &proto,
}
}
+ for (int i = 0; i < proto.window_aggregation_states_size(); ++i) {
+ if (!WindowAggregationOperationState::ProtoIsValid(proto.window_aggregation_states(i),
+ database)) {
+ return false;
+ }
+ }
+
return proto.IsInitialized();
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/367194a1/query_execution/QueryContext.hpp
----------------------------------------------------------------------
diff --git a/query_execution/QueryContext.hpp b/query_execution/QueryContext.hpp
index 7d5628d..9171250 100644
--- a/query_execution/QueryContext.hpp
+++ b/query_execution/QueryContext.hpp
@@ -33,6 +33,7 @@
#include "storage/AggregationOperationState.hpp"
#include "storage/HashTable.hpp"
#include "storage/InsertDestination.hpp"
+#include "storage/WindowAggregationOperationState.hpp"
#include "types/containers/Tuple.hpp"
#include "utility/BloomFilter.hpp"
#include "utility/Macros.hpp"
@@ -120,6 +121,11 @@ class QueryContext {
typedef std::uint32_t update_group_id;
/**
+ * @brief A unique identifier for a window aggregation state.
+ **/
+ typedef std::uint32_t window_aggregation_state_id;
+
+ /**
* @brief Constructor.
*
* @param proto A serialized Protocol Buffer representation of a
@@ -460,6 +466,47 @@ class QueryContext {
return update_groups_[id];
}
+ /**
+ * @brief Whether the given WindowAggregationOperationState id is valid.
+ *
+ * @param id The WindowAggregationOperationState id.
+ *
+ * @return True if valid, otherwise false.
+ **/
+ bool isValidWindowAggregationStateId(const window_aggregation_state_id id) const {
+ return id < window_aggregation_states_.size();
+ }
+
+ /**
+ * @brief Get the WindowAggregationOperationState.
+ *
+ * @param id The WindowAggregationOperationState id in the query.
+ *
+ * @return The WindowAggregationOperationState, already created in the
+ * constructor.
+ **/
+ inline WindowAggregationOperationState* getWindowAggregationState(
+ const window_aggregation_state_id id) {
+ DCHECK_LT(id, window_aggregation_states_.size());
+ DCHECK(window_aggregation_states_[id]);
+ return window_aggregation_states_[id].get();
+ }
+
+ /**
+ * @brief Release the given WindowAggregationOperationState.
+ *
+ * @param id The id of the WindowAggregationOperationState to destroy.
+ *
+ * @return The WindowAggregationOperationState, already created in the
+ * constructor.
+ **/
+ inline WindowAggregationOperationState* releaseWindowAggregationState(
+ const window_aggregation_state_id id) {
+ DCHECK_LT(id, window_aggregation_states_.size());
+ DCHECK(window_aggregation_states_[id]);
+ return window_aggregation_states_[id].release();
+ }
+
private:
std::vector<std::unique_ptr<AggregationOperationState>> aggregation_states_;
std::vector<std::unique_ptr<BloomFilter>> bloom_filters_;
@@ -471,6 +518,7 @@ class QueryContext {
std::vector<std::unique_ptr<const SortConfiguration>> sort_configs_;
std::vector<std::unique_ptr<Tuple>> tuples_;
std::vector<std::unordered_map<attribute_id, std::unique_ptr<const Scalar>>> update_groups_;
+ std::vector<std::unique_ptr<WindowAggregationOperationState>> window_aggregation_states_;
DISALLOW_COPY_AND_ASSIGN(QueryContext);
};
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/367194a1/query_execution/QueryContext.proto
----------------------------------------------------------------------
diff --git a/query_execution/QueryContext.proto b/query_execution/QueryContext.proto
index 98cd0b6..ddf8326 100644
--- a/query_execution/QueryContext.proto
+++ b/query_execution/QueryContext.proto
@@ -22,6 +22,7 @@ import "expressions/table_generator/GeneratorFunction.proto";
import "storage/AggregationOperationState.proto";
import "storage/HashTable.proto";
import "storage/InsertDestination.proto";
+import "storage/WindowAggregationOperationState.proto";
import "types/containers/Tuple.proto";
import "utility/BloomFilter.proto";
import "utility/SortConfiguration.proto";
@@ -55,5 +56,7 @@ message QueryContext {
// NOTE(zuyu): For UpdateWorkOrder only.
repeated UpdateGroup update_groups = 10;
- required uint64 query_id = 11;
+ repeated WindowAggregationOperationState window_aggregation_states = 11;
+
+ required uint64 query_id = 12;
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/367194a1/query_optimizer/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_optimizer/CMakeLists.txt b/query_optimizer/CMakeLists.txt
index 8912414..7e53b9d 100644
--- a/query_optimizer/CMakeLists.txt
+++ b/query_optimizer/CMakeLists.txt
@@ -88,6 +88,7 @@ target_link_libraries(quickstep_queryoptimizer_ExecutionGenerator
quickstep_queryoptimizer_expressions_Predicate
quickstep_queryoptimizer_expressions_Scalar
quickstep_queryoptimizer_expressions_ScalarLiteral
+ quickstep_queryoptimizer_expressions_WindowAggregateFunction
quickstep_queryoptimizer_physical_Aggregate
quickstep_queryoptimizer_physical_CopyFrom
quickstep_queryoptimizer_physical_CreateIndex
@@ -130,6 +131,7 @@ target_link_libraries(quickstep_queryoptimizer_ExecutionGenerator
quickstep_relationaloperators_TableGeneratorOperator
quickstep_relationaloperators_TextScanOperator
quickstep_relationaloperators_UpdateOperator
+ quickstep_relationaloperators_WindowAggregationOperator
quickstep_storage_AggregationOperationState_proto
quickstep_storage_HashTableFactory
quickstep_storage_HashTable_proto
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/367194a1/query_optimizer/ExecutionGenerator.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/ExecutionGenerator.cpp b/query_optimizer/ExecutionGenerator.cpp
index 45f5f78..43d63f9 100644
--- a/query_optimizer/ExecutionGenerator.cpp
+++ b/query_optimizer/ExecutionGenerator.cpp
@@ -63,6 +63,7 @@
#include "query_optimizer/expressions/PatternMatcher.hpp"
#include "query_optimizer/expressions/Scalar.hpp"
#include "query_optimizer/expressions/ScalarLiteral.hpp"
+#include "query_optimizer/expressions/WindowAggregateFunction.hpp"
#include "query_optimizer/physical/CopyFrom.hpp"
#include "query_optimizer/physical/CreateIndex.hpp"
#include "query_optimizer/physical/CreateTable.hpp"
@@ -104,6 +105,7 @@
#include "relational_operators/TableGeneratorOperator.hpp"
#include "relational_operators/TextScanOperator.hpp"
#include "relational_operators/UpdateOperator.hpp"
+#include "relational_operators/WindowAggregationOperator.hpp"
#include "storage/AggregationOperationState.pb.h"
#include "storage/HashTable.pb.h"
#include "storage/HashTableFactory.hpp"
@@ -284,8 +286,8 @@ void ExecutionGenerator::generatePlanInternal(
return convertUpdateTable(
std::static_pointer_cast<const P::UpdateTable>(physical_plan));
case P::PhysicalType::kWindowAggregate:
- THROW_SQL_ERROR()
- << "Window aggregate function is not supported yet :(";
+ return convertWindowAggregate(
+ std::static_pointer_cast<const P::WindowAggregate>(physical_plan));
default:
LOG(FATAL) << "Unknown physical plan node "
<< physical_plan->getShortString();
@@ -1639,5 +1641,100 @@ void ExecutionGenerator::convertTableGenerator(
temporary_relation_info_vec_.emplace_back(tablegen_index, output_relation);
}
+void ExecutionGenerator::convertWindowAggregate(
+ const P::WindowAggregatePtr &physical_plan) {
+ // Create window_aggregation_operation_state proto.
+ const QueryContext::window_aggregation_state_id window_aggr_state_index =
+ query_context_proto_->window_aggregation_states_size();
+ S::WindowAggregationOperationState *window_aggr_state_proto =
+ query_context_proto_->add_window_aggregation_states();
+
+ // Get input.
+ const CatalogRelationInfo *input_relation_info =
+ findRelationInfoOutputByPhysical(physical_plan->input());
+ window_aggr_state_proto->set_relation_id(input_relation_info->relation->getID());
+
+ // Get window aggregate function expression.
+ const E::AliasPtr &named_window_aggregate_expression =
+ physical_plan->window_aggregate_expression();
+ const E::WindowAggregateFunctionPtr &window_aggregate_function =
+ std::static_pointer_cast<const E::WindowAggregateFunction>(
+ named_window_aggregate_expression->expression());
+
+ // Set the AggregateFunction.
+ window_aggr_state_proto->mutable_function()->MergeFrom(
+ window_aggregate_function->window_aggregate().getProto());
+
+ // Set the arguments.
+ for (const E::ScalarPtr &argument : window_aggregate_function->arguments()) {
+ unique_ptr<const Scalar> concretized_argument(argument->concretize(attribute_substitution_map_));
+ window_aggr_state_proto->add_arguments()->MergeFrom(concretized_argument->getProto());
+ }
+
+ // Set partition keys.
+ const E::WindowInfo &window_info = window_aggregate_function->window_info();
+ for (const E::ScalarPtr &partition_by_attribute
+ : window_info.partition_by_attributes) {
+ unique_ptr<const Scalar> concretized_partition_by_attribute(
+ partition_by_attribute->concretize(attribute_substitution_map_));
+ window_aggr_state_proto->add_partition_by_attributes()
+ ->MergeFrom(concretized_partition_by_attribute->getProto());
+ }
+
+ // Set window frame info.
+ if (window_info.frame_info == nullptr) {
+ // If the frame is not specified, use the default setting:
+ // 1. If ORDER BY key is specified, use cumulative aggregation:
+ // ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW.
+ // 2. If ORDER BY key is not specified either, use the whole partition:
+ // ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING.
+ window_aggr_state_proto->set_is_row(true); // frame mode: ROWS.
+ window_aggr_state_proto->set_num_preceding(-1); // UNBOUNDED PRECEDING.
+ window_aggr_state_proto->set_num_following(
+ window_info.order_by_attributes.empty()
+ ? -1 // UNBOUNDED FOLLOWING.
+ : 0); // CURRENT ROW.
+ } else {
+ const E::WindowFrameInfo *window_frame_info = window_info.frame_info;
+ window_aggr_state_proto->set_is_row(window_frame_info->is_row);
+ window_aggr_state_proto->set_num_preceding(window_frame_info->num_preceding);
+ window_aggr_state_proto->set_num_following(window_frame_info->num_following);
+ }
+
+ // Create InsertDestination proto.
+ const CatalogRelation *output_relation = nullptr;
+ const QueryContext::insert_destination_id insert_destination_index =
+ query_context_proto_->insert_destinations_size();
+ S::InsertDestination *insert_destination_proto = query_context_proto_->add_insert_destinations();
+ createTemporaryCatalogRelation(physical_plan,
+ &output_relation,
+ insert_destination_proto);
+
+ const QueryPlan::DAGNodeIndex window_aggregation_operator_index =
+ execution_plan_->addRelationalOperator(
+ new WindowAggregationOperator(query_handle_->query_id(),
+ *output_relation,
+ window_aggr_state_index,
+ insert_destination_index));
+
+ // TODO(Shixuan): Once parallelism is introduced, the is_pipeline_breaker
+ // could be set to false.
+ if (!input_relation_info->isStoredRelation()) {
+ execution_plan_->addDirectDependency(window_aggregation_operator_index,
+ input_relation_info->producer_operator_index,
+ true /* is_pipeline_breaker */);
+ }
+
+ insert_destination_proto->set_relational_op_index(window_aggregation_operator_index);
+
+ // Add to map and temp_relation_info_vec.
+ physical_to_output_relation_map_.emplace(
+ std::piecewise_construct,
+ std::forward_as_tuple(physical_plan),
+ std::forward_as_tuple(window_aggregation_operator_index, output_relation));
+ temporary_relation_info_vec_.emplace_back(window_aggregation_operator_index,
+ output_relation);
+}
+
} // namespace optimizer
} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/367194a1/query_optimizer/ExecutionGenerator.hpp
----------------------------------------------------------------------
diff --git a/query_optimizer/ExecutionGenerator.hpp b/query_optimizer/ExecutionGenerator.hpp
index c7fd018..9186707 100644
--- a/query_optimizer/ExecutionGenerator.hpp
+++ b/query_optimizer/ExecutionGenerator.hpp
@@ -60,6 +60,7 @@
#include "query_optimizer/physical/TableReference.hpp"
#include "query_optimizer/physical/TopLevelPlan.hpp"
#include "query_optimizer/physical/UpdateTable.hpp"
+#include "query_optimizer/physical/WindowAggregate.hpp"
#include "utility/Macros.hpp"
#include "glog/logging.h"
@@ -347,6 +348,13 @@ class ExecutionGenerator {
void convertTableGenerator(const physical::TableGeneratorPtr &physical_plan);
/**
+ * @brief Converts a physical WindowAggregate to a WindowAggregation operator.
+ *
+ * @param physical_plan The WindowAggregate to be converted.
+ */
+ void convertWindowAggregate(const physical::WindowAggregatePtr &physical_plan);
+
+ /**
* @brief Converts a list of NamedExpressions in the optimizer expression
* system to scalars proto in QueryContext proto.
*
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/367194a1/query_optimizer/cost_model/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_optimizer/cost_model/CMakeLists.txt b/query_optimizer/cost_model/CMakeLists.txt
index 6bf5240..5d5b596 100644
--- a/query_optimizer/cost_model/CMakeLists.txt
+++ b/query_optimizer/cost_model/CMakeLists.txt
@@ -40,6 +40,7 @@ target_link_libraries(quickstep_queryoptimizer_costmodel_SimpleCostModel
quickstep_queryoptimizer_physical_TableGenerator
quickstep_queryoptimizer_physical_TableReference
quickstep_queryoptimizer_physical_TopLevelPlan
+ quickstep_queryoptimizer_physical_WindowAggregate
quickstep_utility_Macros)
target_link_libraries(quickstep_queryoptimizer_costmodel_StarSchemaSimpleCostModel
glog
@@ -64,6 +65,7 @@ target_link_libraries(quickstep_queryoptimizer_costmodel_StarSchemaSimpleCostMod
quickstep_queryoptimizer_physical_TableGenerator
quickstep_queryoptimizer_physical_TableReference
quickstep_queryoptimizer_physical_TopLevelPlan
+ quickstep_queryoptimizer_physical_WindowAggregate
quickstep_utility_Macros)
# Module all-in-one library:
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/367194a1/query_optimizer/cost_model/SimpleCostModel.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/cost_model/SimpleCostModel.cpp b/query_optimizer/cost_model/SimpleCostModel.cpp
index 48f76fa..e5222ff 100644
--- a/query_optimizer/cost_model/SimpleCostModel.cpp
+++ b/query_optimizer/cost_model/SimpleCostModel.cpp
@@ -33,6 +33,7 @@
#include "query_optimizer/physical/TableGenerator.hpp"
#include "query_optimizer/physical/TableReference.hpp"
#include "query_optimizer/physical/TopLevelPlan.hpp"
+#include "query_optimizer/physical/WindowAggregate.hpp"
#include "glog/logging.h"
@@ -72,6 +73,9 @@ std::size_t SimpleCostModel::estimateCardinality(
return estimateCardinality(
shared_subplans_[shared_subplan_reference->subplan_id()]);
}
+ case P::PhysicalType::kWindowAggregate:
+ return estimateCardinalityForWindowAggregate(
+ std::static_pointer_cast<const P::WindowAggregate>(physical_plan));
default:
LOG(FATAL) << "Unsupported physical plan:" << physical_plan->toString();
}
@@ -118,6 +122,11 @@ std::size_t SimpleCostModel::estimateCardinalityForAggregate(
estimateCardinality(physical_plan->input()) / 10);
}
+std::size_t SimpleCostModel::estimateCardinalityForWindowAggregate(
+ const physical::WindowAggregatePtr &physical_plan) {
+ return estimateCardinality(physical_plan->input());
+}
+
} // namespace cost
} // namespace optimizer
} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/367194a1/query_optimizer/cost_model/SimpleCostModel.hpp
----------------------------------------------------------------------
diff --git a/query_optimizer/cost_model/SimpleCostModel.hpp b/query_optimizer/cost_model/SimpleCostModel.hpp
index 9862198..9837039 100644
--- a/query_optimizer/cost_model/SimpleCostModel.hpp
+++ b/query_optimizer/cost_model/SimpleCostModel.hpp
@@ -32,6 +32,7 @@
#include "query_optimizer/physical/TableGenerator.hpp"
#include "query_optimizer/physical/TableReference.hpp"
#include "query_optimizer/physical/TopLevelPlan.hpp"
+#include "query_optimizer/physical/WindowAggregate.hpp"
#include "utility/Macros.hpp"
namespace quickstep {
@@ -88,6 +89,10 @@ class SimpleCostModel : public CostModel {
std::size_t estimateCardinalityForAggregate(
const physical::AggregatePtr &physical_plan);
+ // Return the estimated cardinality of the input plan.
+ std::size_t estimateCardinalityForWindowAggregate(
+ const physical::WindowAggregatePtr &physical_plan);
+
const std::vector<physical::PhysicalPtr> &shared_subplans_;
DISALLOW_COPY_AND_ASSIGN(SimpleCostModel);
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/367194a1/query_optimizer/cost_model/StarSchemaSimpleCostModel.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/cost_model/StarSchemaSimpleCostModel.cpp b/query_optimizer/cost_model/StarSchemaSimpleCostModel.cpp
index eb9fcc1..badfeb1 100644
--- a/query_optimizer/cost_model/StarSchemaSimpleCostModel.cpp
+++ b/query_optimizer/cost_model/StarSchemaSimpleCostModel.cpp
@@ -85,6 +85,9 @@ std::size_t StarSchemaSimpleCostModel::estimateCardinality(
case P::PhysicalType::kSort:
return estimateCardinality(
std::static_pointer_cast<const P::Sort>(physical_plan)->input());
+ case P::PhysicalType::kWindowAggregate:
+ return estimateCardinalityForWindowAggregate(
+ std::static_pointer_cast<const P::WindowAggregate>(physical_plan));
default:
LOG(FATAL) << "Unsupported physical plan:" << physical_plan->toString();
}
@@ -141,6 +144,11 @@ std::size_t StarSchemaSimpleCostModel::estimateCardinalityForAggregate(
estimateCardinality(physical_plan->input()) / 10);
}
+std::size_t StarSchemaSimpleCostModel::estimateCardinalityForWindowAggregate(
+ const P::WindowAggregatePtr &physical_plan) {
+ return estimateCardinality(physical_plan->input());
+}
+
double StarSchemaSimpleCostModel::estimateSelectivity(
const physical::PhysicalPtr &physical_plan) {
switch (physical_plan->getPhysicalType()) {
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/367194a1/query_optimizer/cost_model/StarSchemaSimpleCostModel.hpp
----------------------------------------------------------------------
diff --git a/query_optimizer/cost_model/StarSchemaSimpleCostModel.hpp b/query_optimizer/cost_model/StarSchemaSimpleCostModel.hpp
index c63e55a..83032cf 100644
--- a/query_optimizer/cost_model/StarSchemaSimpleCostModel.hpp
+++ b/query_optimizer/cost_model/StarSchemaSimpleCostModel.hpp
@@ -33,6 +33,7 @@
#include "query_optimizer/physical/TableGenerator.hpp"
#include "query_optimizer/physical/TableReference.hpp"
#include "query_optimizer/physical/TopLevelPlan.hpp"
+#include "query_optimizer/physical/WindowAggregate.hpp"
#include "utility/Macros.hpp"
namespace quickstep {
@@ -94,6 +95,9 @@ class StarSchemaSimpleCostModel : public CostModel {
std::size_t estimateCardinalityForAggregate(
const physical::AggregatePtr &physical_plan);
+ std::size_t estimateCardinalityForWindowAggregate(
+ const physical::WindowAggregatePtr &physical_plan);
+
double estimateSelectivityForSelection(
const physical::SelectionPtr &physical_plan);
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/367194a1/query_optimizer/resolver/Resolver.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/resolver/Resolver.cpp b/query_optimizer/resolver/Resolver.cpp
index c07751a..f10378b 100644
--- a/query_optimizer/resolver/Resolver.cpp
+++ b/query_optimizer/resolver/Resolver.cpp
@@ -1860,12 +1860,17 @@ L::LogicalPtr Resolver::resolveJoinedTableReference(
L::LogicalPtr Resolver::resolveSortInWindow(
const L::LogicalPtr &logical_plan,
const E::WindowInfo &window_info) {
- // Sort the table by (p_key, o_key)
+ // Sort the table by (p_key, o_key).
std::vector<E::AttributeReferencePtr> sort_attributes(window_info.partition_by_attributes);
sort_attributes.insert(sort_attributes.end(),
window_info.order_by_attributes.begin(),
window_info.order_by_attributes.end());
+ // If (p_key, o_key) is empty, no sort is needed.
+ if (sort_attributes.empty()) {
+ return logical_plan;
+ }
+
std::vector<bool> sort_directions(
window_info.partition_by_attributes.size(), true);
sort_directions.insert(sort_directions.end(),
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/367194a1/query_optimizer/tests/execution_generator/Select.test
----------------------------------------------------------------------
diff --git a/query_optimizer/tests/execution_generator/Select.test b/query_optimizer/tests/execution_generator/Select.test
index 05f7108..16127cc 100644
--- a/query_optimizer/tests/execution_generator/Select.test
+++ b/query_optimizer/tests/execution_generator/Select.test
@@ -950,3 +950,42 @@ WHERE double_col < 0
+--------------------+
| 5|
+--------------------+
+==
+
+# Window Aggregation Test.
+# Currently this is not supported, an empty table will be returned.
+SELECT avg(int_col) OVER w FROM test
+WINDOW w AS
+(PARTITION BY char_col
+ ORDER BY long_col DESC NULLS LAST
+ ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW);
+--
++------------------------+
+|avg(int_col) |
++------------------------+
++------------------------+
+==
+
+SELECT int_col, sum(float_col) OVER
+(PARTITION BY char_col, long_col
+ ORDER BY double_col DESC NULLS LAST, int_col ASC NULLS FIRST
+ RANGE BETWEEN 3 PRECEDING AND 3 FOLLOWING)
+FROM test;
+--
++-----------+------------------------+
+|int_col |sum(float_col) |
++-----------+------------------------+
++-----------+------------------------+
+==
+
+SELECT sum(avg(int_col) OVER w) FROM test
+WINDOW w AS
+(PARTITION BY char_col
+ ORDER BY long_col
+ ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW);
+--
++------------------------+
+|sum(avg(int_col)) |
++------------------------+
+| NULL|
++------------------------+
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/367194a1/query_optimizer/tests/resolver/Select.test
----------------------------------------------------------------------
diff --git a/query_optimizer/tests/resolver/Select.test b/query_optimizer/tests/resolver/Select.test
index 89ab84d..5e11ac0 100644
--- a/query_optimizer/tests/resolver/Select.test
+++ b/query_optimizer/tests/resolver/Select.test
@@ -3269,6 +3269,39 @@ TopLevelPlan
type=Double NULL]
==
+SELECT avg(int_col) OVER w FROM test
+WINDOW w AS
+(ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING);
+--
+TopLevelPlan
++-plan=Project
+| +-input=WindowAggregate
+| | +-input=TableReference[relation_name=Test,relation_alias=test]
+| | | +-AttributeReference[id=0,name=int_col,relation=test,type=Int NULL]
+| | | +-AttributeReference[id=1,name=long_col,relation=test,type=Long]
+| | | +-AttributeReference[id=2,name=float_col,relation=test,type=Float]
+| | | +-AttributeReference[id=3,name=double_col,relation=test,type=Double NULL]
+| | | +-AttributeReference[id=4,name=char_col,relation=test,type=Char(20)]
+| | | +-AttributeReference[id=5,name=vchar_col,relation=test,
+| | | type=VarChar(20) NULL]
+| | +-window_aggregate_expression=Alias[id=6,name=,alias=$window_aggregate0,
+| | relation=$window_aggregate,type=Double NULL]
+| | +-WindowAggregateFunction[function=AVG,window_name=w,is_ascending=[],
+| | nulls_first=[],frame_mode=row,num_preceding=-1,num_following=-1]
+| | +-arguments=
+| | | +-AttributeReference[id=0,name=int_col,relation=test,type=Int NULL]
+| | +-partition_by=
+| | | +-[]
+| | +-order_by=
+| | +-[]
+| +-project_list=
+| +-Alias[id=6,name=,alias=avg(int_col),relation=,type=Double NULL]
+| +-AttributeReference[id=6,name=,alias=$window_aggregate0,
+| relation=$window_aggregate,type=Double NULL]
++-output_attributes=
+ +-AttributeReference[id=6,name=,alias=avg(int_col),relation=,type=Double NULL]
+==
+
SELECT int_col, sum(float_col) OVER w1 FROM test
WINDOW w2 AS
(PARTITION BY vchar_col, long_col
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/367194a1/relational_operators/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/relational_operators/CMakeLists.txt b/relational_operators/CMakeLists.txt
index 91d1097..249441d 100644
--- a/relational_operators/CMakeLists.txt
+++ b/relational_operators/CMakeLists.txt
@@ -61,6 +61,7 @@ add_library(quickstep_relationaloperators_SortRunGenerationOperator SortRunGener
add_library(quickstep_relationaloperators_TableGeneratorOperator TableGeneratorOperator.cpp TableGeneratorOperator.hpp)
add_library(quickstep_relationaloperators_TextScanOperator TextScanOperator.cpp TextScanOperator.hpp)
add_library(quickstep_relationaloperators_UpdateOperator UpdateOperator.cpp UpdateOperator.hpp)
+add_library(quickstep_relationaloperators_WindowAggregationOperator WindowAggregationOperator.cpp WindowAggregationOperator.hpp)
add_library(quickstep_relationaloperators_WorkOrder ../empty_src.cpp WorkOrder.hpp)
add_library(quickstep_relationaloperators_WorkOrderFactory WorkOrderFactory.cpp WorkOrderFactory.hpp)
add_library(quickstep_relationaloperators_WorkOrder_proto
@@ -423,6 +424,18 @@ target_link_libraries(quickstep_relationaloperators_UpdateOperator
quickstep_threading_ThreadIDBasedMap
quickstep_utility_Macros
tmb)
+target_link_libraries(quickstep_relationaloperators_WindowAggregationOperator
+ glog
+ quickstep_catalog_CatalogRelation
+ quickstep_queryexecution_QueryContext
+ quickstep_queryexecution_WorkOrderProtosContainer
+ quickstep_queryexecution_WorkOrdersContainer
+ quickstep_relationaloperators_RelationalOperator
+ quickstep_relationaloperators_WorkOrder
+ quickstep_relationaloperators_WorkOrder_proto
+ quickstep_storage_StorageBlockInfo
+ quickstep_utility_Macros
+ tmb)
target_link_libraries(quickstep_relationaloperators_WorkOrder
glog
quickstep_queryexecution_QueryExecutionTypedefs
@@ -487,6 +500,7 @@ target_link_libraries(quickstep_relationaloperators
quickstep_relationaloperators_TableGeneratorOperator
quickstep_relationaloperators_TextScanOperator
quickstep_relationaloperators_UpdateOperator
+ quickstep_relationaloperators_WindowAggregationOperator
quickstep_relationaloperators_WorkOrder
quickstep_relationaloperators_WorkOrderFactory
quickstep_relationaloperators_WorkOrder_proto)
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/367194a1/relational_operators/WindowAggregationOperator.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/WindowAggregationOperator.cpp b/relational_operators/WindowAggregationOperator.cpp
new file mode 100644
index 0000000..93cb9d4
--- /dev/null
+++ b/relational_operators/WindowAggregationOperator.cpp
@@ -0,0 +1,82 @@
+/**
+ * Copyright 2011-2015 Quickstep Technologies LLC.
+ * Copyright 2015-2016 Pivotal Software, Inc.
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#include "relational_operators/WindowAggregationOperator.hpp"
+
+#include <vector>
+
+#include "query_execution/QueryContext.hpp"
+#include "query_execution/WorkOrderProtosContainer.hpp"
+#include "query_execution/WorkOrdersContainer.hpp"
+#include "relational_operators/WorkOrder.pb.h"
+#include "storage/StorageBlockInfo.hpp"
+
+#include "tmb/id_typedefs.h"
+
+namespace quickstep {
+
+bool WindowAggregationOperator::getAllWorkOrders(
+ WorkOrdersContainer *container,
+ QueryContext *query_context,
+ StorageManager *storage_manager,
+ const tmb::client_id scheduler_client_id,
+ tmb::MessageBus *bus) {
+ DCHECK(query_context != nullptr);
+
+ if (blocking_dependencies_met_ && !generated_) {
+ container->addNormalWorkOrder(
+ new WindowAggregationWorkOrder(
+ query_id_,
+ query_context->releaseWindowAggregationState(window_aggregation_state_index_),
+ query_context->getInsertDestination(output_destination_index_)),
+ op_index_);
+ generated_ = true;
+ }
+
+ return generated_;
+}
+
+bool WindowAggregationOperator::getAllWorkOrderProtos(WorkOrderProtosContainer *container) {
+ if (blocking_dependencies_met_ && !generated_) {
+ container->addWorkOrderProto(createWorkOrderProto(), op_index_);
+ generated_ = true;
+ }
+
+ return generated_;
+}
+
+serialization::WorkOrder* WindowAggregationOperator::createWorkOrderProto() {
+ serialization::WorkOrder *proto = new serialization::WorkOrder;
+ proto->set_work_order_type(serialization::WINDOW_AGGREGATION);
+ proto->set_query_id(query_id_);
+ proto->SetExtension(serialization::WindowAggregationWorkOrder::window_aggr_state_index,
+ window_aggregation_state_index_);
+ proto->SetExtension(serialization::WindowAggregationWorkOrder::insert_destination_index,
+ output_destination_index_);
+
+ return proto;
+}
+
+
+void WindowAggregationWorkOrder::execute() {
+ std::cout << "Window aggregation is not supported yet.\n"
+ << "An empty table is returned\n";
+}
+
+} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/367194a1/relational_operators/WindowAggregationOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/WindowAggregationOperator.hpp b/relational_operators/WindowAggregationOperator.hpp
new file mode 100644
index 0000000..f3dfd14
--- /dev/null
+++ b/relational_operators/WindowAggregationOperator.hpp
@@ -0,0 +1,166 @@
+/**
+ * Copyright 2011-2015 Quickstep Technologies LLC.
+ * Copyright 2015-2016 Pivotal Software, Inc.
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#ifndef QUICKSTEP_RELATIONAL_OPERATORS_WINDOW_AGGREGATION_OPERATOR_HPP_
+#define QUICKSTEP_RELATIONAL_OPERATORS_WINDOW_AGGREGATION_OPERATOR_HPP_
+
+#include <vector>
+
+#include "catalog/CatalogRelation.hpp"
+#include "query_execution/QueryContext.hpp"
+#include "relational_operators/RelationalOperator.hpp"
+#include "relational_operators/WorkOrder.hpp"
+#include "storage/StorageBlockInfo.hpp"
+#include "utility/Macros.hpp"
+
+#include "glog/logging.h"
+
+#include "tmb/id_typedefs.h"
+
+namespace tmb { class MessageBus; }
+
+namespace quickstep {
+
+class StorageManager;
+class WindowAggregationOperationState;
+class WorkOrderProtosContainer;
+class WorkOrdersContainer;
+
+namespace serialization { class WorkOrder; }
+
+/** \addtogroup RelationalOperators
+ * @{
+ */
+
+/**
+ * @brief An operator which performs window aggregation over a relation.
+ **/
+class WindowAggregationOperator : public RelationalOperator {
+ public:
+ /**
+ * @brief Constructor.
+ *
+ * @param query_id The ID of this query.
+ * @param input_relation The relation to perform aggregation over.
+ * @param window_aggregation_state_index The index of WindowAggregationState
+ * in QueryContext.
+ * @param output_destination_index The index of InsertDestination in
+ * QueryContext for the output.
+ **/
+ WindowAggregationOperator(const std::size_t query_id,
+ const CatalogRelation &output_relation,
+ const QueryContext::window_aggregation_state_id window_aggregation_state_index,
+ const QueryContext::insert_destination_id output_destination_index)
+ : RelationalOperator(query_id),
+ output_relation_(output_relation),
+ window_aggregation_state_index_(window_aggregation_state_index),
+ output_destination_index_(output_destination_index),
+ generated_(false) {}
+
+ ~WindowAggregationOperator() override {}
+
+ bool getAllWorkOrders(WorkOrdersContainer *container,
+ QueryContext *query_context,
+ StorageManager *storage_manager,
+ const tmb::client_id scheduler_client_id,
+ tmb::MessageBus *bus) override;
+
+ bool getAllWorkOrderProtos(WorkOrderProtosContainer *container) override;
+
+ const relation_id getOutputRelationID() const override {
+ return output_relation_.getID();
+ }
+
+ QueryContext::insert_destination_id getInsertDestinationID() const override {
+ return output_destination_index_;
+ }
+
+ private:
+ /**
+ * @brief Create Work Order proto.
+ *
+ * @return A window aggregation work order.
+ **/
+ serialization::WorkOrder* createWorkOrderProto();
+
+ const CatalogRelation &output_relation_;
+ const QueryContext::window_aggregation_state_id window_aggregation_state_index_;
+ const QueryContext::insert_destination_id output_destination_index_;
+ bool generated_;
+
+ DISALLOW_COPY_AND_ASSIGN(WindowAggregationOperator);
+};
+
+/**
+ * @brief A WorkOrder produced by WindowAggregationOperator.
+ **/
+class WindowAggregationWorkOrder : public WorkOrder {
+ public:
+ /**
+ * @brief Constructor
+ *
+ * @param query_id The ID of this query.
+ * @param state The WindowAggregationOperatorState to use.
+ * @param output_destination The InsertDestination for output.
+ **/
+ WindowAggregationWorkOrder(const std::size_t query_id,
+ WindowAggregationOperationState *state,
+ InsertDestination *output_destination)
+ : WorkOrder(query_id),
+ state_(state),
+ output_destination_(output_destination) {}
+
+ ~WindowAggregationWorkOrder() override {}
+
+ /**
+ * @brief Get the pointer to WindowAggregationOperationState.
+ * @note This is a quickfix for "unused variable". After the window aggregate
+ * functions are built, these methods might be dropped.
+ *
+ * @return A pointer to the window aggregation operation state.
+ **/
+ WindowAggregationOperationState* state() {
+ return state_;
+ }
+
+ /**
+ * @brief Get the pointer to output destination.
+ * @note This is a quickfix for "unused variable". After the window aggregate
+ * functions are built, these methods might be dropped.
+ *
+ * @return A pointer to the output destination.
+ **/
+ InsertDestination* output_destination() {
+ return output_destination_;
+ }
+
+ void execute() override;
+
+ private:
+ WindowAggregationOperationState *state_;
+ InsertDestination *output_destination_;
+
+ DISALLOW_COPY_AND_ASSIGN(WindowAggregationWorkOrder);
+};
+
+/** @} */
+
+} // namespace quickstep
+
+#endif // QUICKSTEP_RELATIONAL_OPERATORS_WINDOW_AGGREGATION_OPERATOR_HPP_
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/367194a1/relational_operators/WorkOrder.proto
----------------------------------------------------------------------
diff --git a/relational_operators/WorkOrder.proto b/relational_operators/WorkOrder.proto
index 3ed065a..69dee1b 100644
--- a/relational_operators/WorkOrder.proto
+++ b/relational_operators/WorkOrder.proto
@@ -41,6 +41,7 @@ enum WorkOrderType {
TABLE_GENERATOR = 17;
TEXT_SCAN = 18;
UPDATE = 19;
+ WINDOW_AGGREGATION = 20;
}
message WorkOrder {
@@ -243,3 +244,11 @@ message UpdateWorkOrder {
optional fixed64 block_id = 325;
}
}
+
+message WindowAggregationWorkOrder {
+ extend WorkOrder {
+ // All required
+ optional uint32 window_aggr_state_index = 336;
+ optional int32 insert_destination_index = 337;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/367194a1/storage/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/storage/CMakeLists.txt b/storage/CMakeLists.txt
index b536411..9df66e1 100644
--- a/storage/CMakeLists.txt
+++ b/storage/CMakeLists.txt
@@ -131,6 +131,9 @@ QS_PROTOBUF_GENERATE_CPP(storage_InsertDestination_proto_srcs
QS_PROTOBUF_GENERATE_CPP(storage_StorageBlockLayout_proto_srcs
storage_StorageBlockLayout_proto_hdrs
StorageBlockLayout.proto)
+QS_PROTOBUF_GENERATE_CPP(storage_WindowAggregationOperationState_proto_srcs
+ storage_WindowAggregationOperationState_proto_hdrs
+ WindowAggregationOperationState.proto)
if (ENABLE_DISTRIBUTED)
GRPC_GENERATE_CPP(storage_DataExchange_proto_srcs
@@ -254,6 +257,11 @@ add_library(quickstep_storage_TupleReference ../empty_src.cpp TupleReference.hpp
add_library(quickstep_storage_TupleStorageSubBlock TupleStorageSubBlock.cpp TupleStorageSubBlock.hpp)
add_library(quickstep_storage_ValueAccessor ../empty_src.cpp ValueAccessor.hpp)
add_library(quickstep_storage_ValueAccessorUtil ../empty_src.cpp ValueAccessorUtil.hpp)
+add_library(quickstep_storage_WindowAggregationOperationState
+ WindowAggregationOperationState.hpp
+ WindowAggregationOperationState.cpp)
+add_library(quickstep_storage_WindowAggregationOperationState_proto ${storage_WindowAggregationOperationState_proto_srcs})
+
# Link dependencies:
target_link_libraries(quickstep_storage_AggregationOperationState
@@ -1038,6 +1046,27 @@ target_link_libraries(quickstep_storage_ValueAccessorUtil
quickstep_storage_ValueAccessor
quickstep_types_containers_ColumnVectorsValueAccessor
quickstep_utility_Macros)
+target_link_libraries(quickstep_storage_WindowAggregationOperationState
+ glog
+ quickstep_catalog_CatalogDatabaseLite
+ quickstep_catalog_CatalogRelationSchema
+ quickstep_catalog_CatalogTypedefs
+ quickstep_expressions_ExpressionFactories
+ quickstep_expressions_Expressions_proto
+ quickstep_expressions_aggregation_AggregateFunction
+ quickstep_expressions_aggregation_AggregateFunctionFactory
+ quickstep_expressions_aggregation_AggregationHandle
+ quickstep_expressions_aggregation_AggregationID
+ quickstep_expressions_scalar_Scalar
+ quickstep_expressions_scalar_ScalarAttribute
+ quickstep_storage_StorageBlockInfo
+ quickstep_storage_StorageManager
+ quickstep_storage_WindowAggregationOperationState_proto
+ quickstep_utility_Macros)
+target_link_libraries(quickstep_storage_WindowAggregationOperationState_proto
+ quickstep_expressions_aggregation_AggregateFunction_proto
+ quickstep_expressions_Expressions_proto
+ ${PROTOBUF_LIBRARY})
# Module all-in-one library:
add_library(quickstep_storage ../empty_src.cpp StorageModule.hpp)
@@ -1096,7 +1125,9 @@ target_link_libraries(quickstep_storage
quickstep_storage_TupleReference
quickstep_storage_TupleStorageSubBlock
quickstep_storage_ValueAccessor
- quickstep_storage_ValueAccessorUtil)
+ quickstep_storage_ValueAccessorUtil
+ quickstep_storage_WindowAggregationOperationState
+ quickstep_storage_WindowAggregationOperationState_proto)
if (QUICKSTEP_HAVE_FILE_MANAGER_HDFS)
target_link_libraries(quickstep_storage
quickstep_storage_FileManagerHdfs)
@@ -1636,6 +1667,18 @@ target_link_libraries(StorageManager_unittest
quickstep_storage_StorageBlockInfo
quickstep_storage_StorageManager
quickstep_utility_ShardedLockManager)
+
+add_executable(WindowAggregationOperationState_unittest
+ "${CMAKE_CURRENT_SOURCE_DIR}/tests/WindowAggregationOperationState_unittest.cpp")
+target_link_libraries(WindowAggregationOperationState_unittest
+ gtest
+ gtest_main
+ quickstep_catalog_CatalogDatabase
+ quickstep_catalog_CatalogRelation
+ quickstep_catalog_CatalogTypedefs
+ quickstep_storage_WindowAggregationOperationState
+ quickstep_storage_WindowAggregationOperationState_proto
+ ${LIBS})
if (QUICKSTEP_HAVE_LIBNUMA)
target_link_libraries(StorageManager_unittest
${LIBNUMA_LIBRARY})
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/367194a1/storage/WindowAggregationOperationState.cpp
----------------------------------------------------------------------
diff --git a/storage/WindowAggregationOperationState.cpp b/storage/WindowAggregationOperationState.cpp
new file mode 100644
index 0000000..a0bcc37
--- /dev/null
+++ b/storage/WindowAggregationOperationState.cpp
@@ -0,0 +1,179 @@
+/**
+ * Copyright 2011-2015 Quickstep Technologies LLC.
+ * Copyright 2015-2016 Pivotal Software, Inc.
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#include "storage/WindowAggregationOperationState.hpp"
+
+#include <cstddef>
+#include <cstdio>
+#include <memory>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "catalog/CatalogDatabaseLite.hpp"
+#include "catalog/CatalogRelationSchema.hpp"
+#include "catalog/CatalogTypedefs.hpp"
+#include "expressions/ExpressionFactories.hpp"
+#include "expressions/Expressions.pb.h"
+#include "expressions/aggregation/AggregateFunction.hpp"
+#include "expressions/aggregation/AggregateFunctionFactory.hpp"
+#include "expressions/aggregation/AggregationHandle.hpp"
+#include "expressions/aggregation/AggregationID.hpp"
+#include "expressions/scalar/Scalar.hpp"
+#include "expressions/scalar/ScalarAttribute.hpp"
+#include "storage/StorageManager.hpp"
+#include "storage/WindowAggregationOperationState.pb.h"
+
+#include "glog/logging.h"
+
+namespace quickstep {
+
+WindowAggregationOperationState::WindowAggregationOperationState(
+ const CatalogRelationSchema &input_relation,
+ const AggregateFunction *window_aggregate_function,
+ std::vector<std::unique_ptr<const Scalar>> &&arguments,
+ std::vector<std::unique_ptr<const Scalar>> &&partition_by_attributes,
+ const bool is_row,
+ const std::int64_t num_preceding,
+ const std::int64_t num_following,
+ StorageManager *storage_manager)
+ : input_relation_(input_relation),
+ arguments_(std::move(arguments)),
+ partition_by_attributes_(std::move(partition_by_attributes)),
+ is_row_(is_row),
+ num_preceding_(num_preceding),
+ num_following_(num_following),
+ storage_manager_(storage_manager) {
+ // Get the Types of this window aggregate's arguments so that we can create an
+ // AggregationHandle.
+ // TODO(Shixuan): Next step: New handles for window aggregation function.
+ std::vector<const Type*> argument_types;
+ for (const std::unique_ptr<const Scalar> &argument : arguments_) {
+ argument_types.emplace_back(&argument->getType());
+ }
+
+ // Check if window aggregate function could apply to the arguments.
+ DCHECK(window_aggregate_function->canApplyToTypes(argument_types));
+
+ // Create the handle and initial state.
+ window_aggregation_handle_.reset(
+ window_aggregate_function->createHandle(argument_types));
+ window_aggregation_state_.reset(
+ window_aggregation_handle_->createInitialState());
+
+#ifdef QUICKSTEP_ENABLE_VECTOR_COPY_ELISION_SELECTION
+ // See if all of this window aggregate's arguments are attributes in the input
+ // relation. If so, remember the attribute IDs so that we can do copy elision
+ // when actually performing the window aggregation.
+ arguments_as_attributes_.reserve(arguments_.size());
+ for (const std::unique_ptr<const Scalar> &argument : arguments_) {
+ const attribute_id argument_id = argument->getAttributeIdForValueAccessor();
+ if (argument_id == -1) {
+ arguments_as_attributes_.clear();
+ break;
+ } else {
+ DCHECK_EQ(input_relation_.getID(), argument->getRelationIdForValueAccessor());
+ arguments_as_attributes_.push_back(argument_id);
+ }
+ }
+#endif
+}
+
+WindowAggregationOperationState* WindowAggregationOperationState::ReconstructFromProto(
+ const serialization::WindowAggregationOperationState &proto,
+ const CatalogDatabaseLite &database,
+ StorageManager *storage_manager) {
+ DCHECK(ProtoIsValid(proto, database));
+
+ // Rebuild contructor arguments from their representation in 'proto'.
+ const AggregateFunction *aggregate_function
+ = &AggregateFunctionFactory::ReconstructFromProto(proto.function());
+
+ std::vector<std::unique_ptr<const Scalar>> arguments;
+ arguments.reserve(proto.arguments_size());
+ for (int argument_idx = 0; argument_idx < proto.arguments_size(); ++argument_idx) {
+ arguments.emplace_back(ScalarFactory::ReconstructFromProto(
+ proto.arguments(argument_idx),
+ database));
+ }
+
+ std::vector<std::unique_ptr<const Scalar>> partition_by_attributes;
+ for (int attribute_idx = 0;
+ attribute_idx < proto.partition_by_attributes_size();
+ ++attribute_idx) {
+ partition_by_attributes.emplace_back(ScalarFactory::ReconstructFromProto(
+ proto.partition_by_attributes(attribute_idx),
+ database));
+ }
+
+ const bool is_row = proto.is_row();
+ const std::int64_t num_preceding = proto.num_preceding();
+ const std::int64_t num_following = proto.num_following();
+
+ return new WindowAggregationOperationState(database.getRelationSchemaById(proto.relation_id()),
+ aggregate_function,
+ std::move(arguments),
+ std::move(partition_by_attributes),
+ is_row,
+ num_preceding,
+ num_following,
+ storage_manager);
+}
+
+bool WindowAggregationOperationState::ProtoIsValid(const serialization::WindowAggregationOperationState &proto,
+ const CatalogDatabaseLite &database) {
+ if (!proto.IsInitialized() ||
+ !database.hasRelationWithId(proto.relation_id())) {
+ return false;
+ }
+
+ if (!AggregateFunctionFactory::ProtoIsValid(proto.function())) {
+ return false;
+ }
+
+ // TODO(chasseur): We may also want to check that the specified
+ // AggregateFunction is applicable to the specified arguments, but that
+ // requires partial deserialization and may be too heavyweight for this
+ // method.
+ // TODO(Shixuan): The TODO for AggregateFunction could also be applied here.
+ for (int argument_idx = 0;
+ argument_idx < proto.arguments_size();
+ ++argument_idx) {
+ if (!ScalarFactory::ProtoIsValid(proto.arguments(argument_idx), database)) {
+ return false;
+ }
+ }
+
+ for (int attribute_idx = 0;
+ attribute_idx < proto.partition_by_attributes_size();
+ ++attribute_idx) {
+ if (!ScalarFactory::ProtoIsValid(proto.partition_by_attributes(attribute_idx),
+ database)) {
+ return false;
+ }
+ }
+
+ if (proto.num_preceding() < -1 || proto.num_following() < -1) {
+ return false;
+ }
+
+ return true;
+}
+
+} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/367194a1/storage/WindowAggregationOperationState.hpp
----------------------------------------------------------------------
diff --git a/storage/WindowAggregationOperationState.hpp b/storage/WindowAggregationOperationState.hpp
new file mode 100644
index 0000000..d7b3e6a
--- /dev/null
+++ b/storage/WindowAggregationOperationState.hpp
@@ -0,0 +1,177 @@
+/**
+ * Copyright 2011-2015 Quickstep Technologies LLC.
+ * Copyright 2015-2016 Pivotal Software, Inc.
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#ifndef QUICKSTEP_STORAGE_WINDOW_AGGREGATION_OPERATION_STATE_HPP_
+#define QUICKSTEP_STORAGE_WINDOW_AGGREGATION_OPERATION_STATE_HPP_
+
+#include <cstddef>
+#include <memory>
+#include <vector>
+
+#include "catalog/CatalogTypedefs.hpp"
+#include "expressions/aggregation/AggregationHandle.hpp"
+#include "expressions/scalar/Scalar.hpp"
+#include "expressions/scalar/ScalarAttribute.hpp"
+#include "storage/StorageBlockInfo.hpp"
+#include "storage/WindowAggregationOperationState.pb.h"
+#include "utility/Macros.hpp"
+
+namespace quickstep {
+
+class AggregateFunction;
+class CatalogDatabaseLite;
+class CatalogRelationSchema;
+class InsertDestination;
+class StorageManager;
+
+/** \addtogroup Storage
+ * @{
+ */
+
+/**
+ * @brief Helper class for maintaining the state of window aggregation.
+ **/
+class WindowAggregationOperationState {
+ public:
+ /**
+ * @brief Constructor for window aggregation operation state.
+ *
+ * @param input_relation Input relation on which window aggregation is computed.
+ * @param window_aggregate_functions The window aggregate function to be
+ * computed.
+ * @param arguments A list of argument expressions to that aggregate.
+ * @param partition_by_attributes A list of window partition key.
+ * @param is_row True if the window frame is calculated by ROW, false if it is
+ * calculated by RANGE.
+ * @param num_preceding The number of rows/range for the tuples preceding the
+ * current row. -1 means UNBOUNDED PRECEDING.
+ * @param num_following The number of rows/range for the tuples following the
+ * current row. -1 means UNBOUNDED FOLLOWING.
+ * @param storage_manager The StorageManager to use for allocating hash
+ * tables.
+ */
+ WindowAggregationOperationState(const CatalogRelationSchema &input_relation,
+ const AggregateFunction *window_aggregate_function,
+ std::vector<std::unique_ptr<const Scalar>> &&arguments,
+ std::vector<std::unique_ptr<const Scalar>> &&partition_by_attributes,
+ const bool is_row,
+ const std::int64_t num_preceding,
+ const std::int64_t num_following,
+ StorageManager *storage_manager);
+
+ ~WindowAggregationOperationState() {}
+
+ /**
+ * @brief Generate the window aggregation operation state from the serialized
+ * Protocol Buffer representation.
+ *
+ * @param proto A serialized protocol buffer representation of a
+ * WindowAggregationOperationState, originally generated by the
+ * optimizer.
+ * @param database The database for resolving relation and attribute
+ * references.
+ * @param storage_manager The StorageManager to use.
+ **/
+ static WindowAggregationOperationState* ReconstructFromProto(
+ const serialization::WindowAggregationOperationState &proto,
+ const CatalogDatabaseLite &database,
+ StorageManager *storage_manager);
+
+ /**
+ * @brief Check whether a serialization::AggregationOperationState is
+ * fully-formed and all parts are valid.
+ *
+ * @param proto A serialized Protocol Buffer representation of an
+ * AggregationOperationState, originally generated by the optimizer.
+ * @param database The Database to resolve relation and attribute references
+ * in.
+ * @return Whether proto is fully-formed and valid.
+ **/
+ static bool ProtoIsValid(const serialization::WindowAggregationOperationState &proto,
+ const CatalogDatabaseLite &database);
+
+ /**
+ * @brief Get the is_row info.
+ * @note This is a quickfix for "unused variable". After the window aggregate
+ * functions are built, these methods might be dropped.
+ *
+ * @return True if the frame mode is ROW, false if it is RANGE.
+ **/
+ const bool is_row() const { return is_row_; }
+
+ /**
+ * @brief Get the num_preceding info.
+ * @note This is a quickfix for "unused variable". After the window aggregate
+ * functions are built, these methods might be dropped.
+ *
+ * @return The number of rows/range that precedes the current row.
+ **/
+ const std::int64_t num_preceding() const { return num_preceding_; }
+
+ /**
+ * @brief Get the num_following info.
+ * @note This is a quickfix for "unused variable". After the window aggregate
+ * functions are built, these methods might be dropped.
+ *
+ * @return The number of rows/range that follows the current row.
+ **/
+ const std::int64_t num_following() const { return num_following_; }
+
+ /**
+ * @brief Get the pointer to StorageManager.
+ * @note This is a quickfix for "unused variable". After the window aggregate
+ * functions are built, these methods might be dropped.
+ *
+ * @return A pointer to the storage manager.
+ **/
+ StorageManager *storage_manager() { return storage_manager_; }
+
+ private:
+ const CatalogRelationSchema &input_relation_;
+
+ // TODO(Shixuan): Handle and State for window aggregation will be needed for
+ // actual calculation.
+ std::unique_ptr<AggregationHandle> window_aggregation_handle_;
+ std::unique_ptr<AggregationState> window_aggregation_state_;
+ std::vector<std::unique_ptr<const Scalar>> arguments_;
+
+ // We don't add order_by_attributes here since it is not needed after sorting.
+ std::vector<std::unique_ptr<const Scalar>> partition_by_attributes_;
+
+ // Window framing information.
+ const bool is_row_;
+ const std::int64_t num_preceding_;
+ const std::int64_t num_following_;
+
+ StorageManager *storage_manager_;
+
+#ifdef QUICKSTEP_ENABLE_VECTOR_COPY_ELISION_SELECTION
+ // If all an aggregate's argument expressions are simply attributes in
+ // 'input_relation_', then this caches the attribute IDs of those arguments.
+ std::vector<attribute_id> arguments_as_attributes_;
+#endif
+
+ DISALLOW_COPY_AND_ASSIGN(WindowAggregationOperationState);
+};
+
+/** @} */
+
+} // namespace quickstep
+
+#endif // QUICKSTEP_STORAGE_WINDOW_AGGREGATION_OPERATION_STATE_HPP_
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/367194a1/storage/WindowAggregationOperationState.proto
----------------------------------------------------------------------
diff --git a/storage/WindowAggregationOperationState.proto b/storage/WindowAggregationOperationState.proto
new file mode 100644
index 0000000..c7bd0ef
--- /dev/null
+++ b/storage/WindowAggregationOperationState.proto
@@ -0,0 +1,33 @@
+// Copyright 2011-2015 Quickstep Technologies LLC.
+// Copyright 2015-2016 Pivotal Software, Inc.
+// Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+// University of Wisconsin\u2014Madison.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package quickstep.serialization;
+
+import "expressions/aggregation/AggregateFunction.proto";
+import "expressions/Expressions.proto";
+
+message WindowAggregationOperationState {
+ required int32 relation_id = 1;
+ required AggregateFunction function = 2;
+ repeated Scalar arguments = 3;
+ repeated Scalar partition_by_attributes = 4;
+ required bool is_row = 5;
+ required int64 num_preceding = 6; // -1 means UNBOUNDED PRECEDING.
+ required int64 num_following = 7; // -1 means UNBOUNDED FOLLOWING.
+}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/367194a1/storage/tests/WindowAggregationOperationState_unittest.cpp
----------------------------------------------------------------------
diff --git a/storage/tests/WindowAggregationOperationState_unittest.cpp b/storage/tests/WindowAggregationOperationState_unittest.cpp
new file mode 100644
index 0000000..c572034
--- /dev/null
+++ b/storage/tests/WindowAggregationOperationState_unittest.cpp
@@ -0,0 +1,92 @@
+/**
+ * Copyright 2011-2015 Quickstep Technologies LLC.
+ * Copyright 2015-2016 Pivotal Software, Inc.
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#include <cstddef>
+#include <memory>
+
+#include "catalog/CatalogDatabase.hpp"
+#include "catalog/CatalogRelation.hpp"
+#include "catalog/CatalogTypedefs.hpp"
+#include "expressions/aggregation/AggregateFunction.pb.h"
+#include "storage/WindowAggregationOperationState.hpp"
+#include "storage/WindowAggregationOperationState.pb.h"
+
+#include "gtest/gtest.h"
+
+using std::unique_ptr;
+
+namespace quickstep {
+
+namespace {
+ constexpr relation_id kInvalidTableId = 100;
+ constexpr std::int64_t kInvalidNum = -10;
+ constexpr std::int64_t kValidNum = 10;
+} // namespace
+
+class WindowAggregationOperationStateProtoTest : public ::testing::Test {
+ protected:
+ virtual void SetUp() {
+ database_.reset(new CatalogDatabase(nullptr, "db"));
+ rel_id_ = database_->addRelation(new CatalogRelation(nullptr, "rel"));
+ }
+
+ unique_ptr<CatalogDatabase> database_;
+ relation_id rel_id_;
+};
+
+TEST_F(WindowAggregationOperationStateProtoTest, UninitializationTest) {
+ serialization::WindowAggregationOperationState proto;
+ EXPECT_FALSE(WindowAggregationOperationState::ProtoIsValid(proto, *database_.get()));
+}
+
+TEST_F(WindowAggregationOperationStateProtoTest, InvalidRelationIdTest) {
+ serialization::WindowAggregationOperationState proto;
+ proto.set_relation_id(kInvalidTableId);
+ proto.mutable_function()->set_aggregation_id(serialization::AggregateFunction::AVG);
+ proto.set_is_row(true);
+ proto.set_num_preceding(kValidNum);
+ proto.set_num_following(kValidNum);
+ EXPECT_FALSE(WindowAggregationOperationState::ProtoIsValid(proto, *database_.get()));
+}
+
+TEST_F(WindowAggregationOperationStateProtoTest, InvalidNumTest) {
+ serialization::WindowAggregationOperationState proto;
+ proto.set_relation_id(rel_id_);
+ proto.mutable_function()->set_aggregation_id(serialization::AggregateFunction::AVG);
+ proto.set_is_row(true);
+ proto.set_num_preceding(kInvalidNum);
+ proto.set_num_following(kValidNum);
+ EXPECT_FALSE(WindowAggregationOperationState::ProtoIsValid(proto, *database_.get()));
+
+ proto.set_num_preceding(kValidNum);
+ proto.set_num_following(kInvalidNum);
+ EXPECT_FALSE(WindowAggregationOperationState::ProtoIsValid(proto, *database_.get()));
+}
+
+TEST_F(WindowAggregationOperationStateProtoTest, ValidTest) {
+ serialization::WindowAggregationOperationState proto;
+ proto.set_relation_id(rel_id_);
+ proto.mutable_function()->set_aggregation_id(serialization::AggregateFunction::AVG);
+ proto.set_is_row(true);
+ proto.set_num_preceding(kValidNum);
+ proto.set_num_following(kValidNum);
+ EXPECT_TRUE(WindowAggregationOperationState::ProtoIsValid(proto, *database_.get()));
+}
+
+} // namespace quickstep
[06/25] incubator-quickstep git commit: Introduced QueryManagerBase,
and renamed QueryManagerSingleNode.
Posted by zu...@apache.org.
Introduced QueryManagerBase, and renamed QueryManagerSingleNode.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/65af8c7b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/65af8c7b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/65af8c7b
Branch: refs/heads/dist-exe-test-new
Commit: 65af8c7b047eb64349a0733a3e9eba968958bdb8
Parents: 367194a
Author: Zuyu Zhang <zu...@apache.org>
Authored: Fri Jul 8 11:49:14 2016 -0500
Committer: Zuyu Zhang <zu...@twitter.com>
Committed: Fri Jul 29 16:42:10 2016 -0700
----------------------------------------------------------------------
query_execution/CMakeLists.txt | 40 +-
query_execution/PolicyEnforcer.cpp | 14 +-
query_execution/PolicyEnforcer.hpp | 8 +-
query_execution/QueryManager.cpp | 470 ---------
query_execution/QueryManager.hpp | 374 --------
query_execution/QueryManagerBase.cpp | 305 ++++++
query_execution/QueryManagerBase.hpp | 312 ++++++
query_execution/QueryManagerSingleNode.cpp | 194 ++++
query_execution/QueryManagerSingleNode.hpp | 140 +++
.../tests/QueryManagerSingleNode_unittest.cpp | 942 +++++++++++++++++++
query_execution/tests/QueryManager_unittest.cpp | 940 ------------------
11 files changed, 1931 insertions(+), 1808 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/65af8c7b/query_execution/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_execution/CMakeLists.txt b/query_execution/CMakeLists.txt
index e1b1183..5a9189c 100644
--- a/query_execution/CMakeLists.txt
+++ b/query_execution/CMakeLists.txt
@@ -45,7 +45,8 @@ add_library(quickstep_queryexecution_QueryExecutionMessages_proto
add_library(quickstep_queryexecution_QueryExecutionState ../empty_src.cpp QueryExecutionState.hpp)
add_library(quickstep_queryexecution_QueryExecutionTypedefs ../empty_src.cpp QueryExecutionTypedefs.hpp)
add_library(quickstep_queryexecution_QueryExecutionUtil ../empty_src.cpp QueryExecutionUtil.hpp)
-add_library(quickstep_queryexecution_QueryManager QueryManager.cpp QueryManager.hpp)
+add_library(quickstep_queryexecution_QueryManagerBase QueryManagerBase.cpp QueryManagerBase.hpp)
+add_library(quickstep_queryexecution_QueryManagerSingleNode QueryManagerSingleNode.cpp QueryManagerSingleNode.hpp)
add_library(quickstep_queryexecution_WorkOrderProtosContainer ../empty_src.cpp WorkOrderProtosContainer.hpp)
add_library(quickstep_queryexecution_WorkOrdersContainer WorkOrdersContainer.cpp WorkOrdersContainer.hpp)
add_library(quickstep_queryexecution_Worker Worker.cpp Worker.hpp)
@@ -89,18 +90,19 @@ target_link_libraries(quickstep_queryexecution_ForemanSingleNode
tmb
${GFLAGS_LIB_NAME})
target_link_libraries(quickstep_queryexecution_PolicyEnforcer
- ${GFLAGS_LIB_NAME}
glog
quickstep_catalog_CatalogTypedefs
quickstep_queryexecution_QueryExecutionMessages_proto
quickstep_queryexecution_QueryExecutionTypedefs
- quickstep_queryexecution_QueryManager
+ quickstep_queryexecution_QueryManagerBase
+ quickstep_queryexecution_QueryManagerSingleNode
quickstep_queryexecution_WorkerDirectory
quickstep_queryexecution_WorkerMessage
quickstep_queryoptimizer_QueryHandle
quickstep_relationaloperators_WorkOrder
quickstep_utility_Macros
- tmb)
+ tmb
+ ${GFLAGS_LIB_NAME})
target_link_libraries(quickstep_queryexecution_QueryContext
glog
quickstep_catalog_CatalogDatabaseLite
@@ -149,7 +151,7 @@ target_link_libraries(quickstep_queryexecution_QueryExecutionUtil
quickstep_queryexecution_WorkerMessage
quickstep_utility_Macros
tmb)
-target_link_libraries(quickstep_queryexecution_QueryManager
+target_link_libraries(quickstep_queryexecution_QueryManagerBase
quickstep_catalog_CatalogDatabase
quickstep_catalog_CatalogRelation
quickstep_catalog_CatalogTypedefs
@@ -158,14 +160,25 @@ target_link_libraries(quickstep_queryexecution_QueryManager
quickstep_queryexecution_QueryExecutionMessages_proto
quickstep_queryexecution_QueryExecutionState
quickstep_queryexecution_QueryExecutionTypedefs
+ quickstep_queryoptimizer_QueryHandle
+ quickstep_queryoptimizer_QueryPlan
+ quickstep_relationaloperators_RelationalOperator
+ quickstep_relationaloperators_WorkOrder
+ quickstep_storage_StorageBlockInfo
+ quickstep_utility_DAG
+ quickstep_utility_Macros)
+target_link_libraries(quickstep_queryexecution_QueryManagerSingleNode
+ quickstep_catalog_CatalogTypedefs
+ quickstep_queryexecution_QueryContext
+ quickstep_queryexecution_QueryExecutionState
+ quickstep_queryexecution_QueryManagerBase
quickstep_queryexecution_WorkOrdersContainer
quickstep_queryexecution_WorkerMessage
quickstep_queryoptimizer_QueryHandle
quickstep_relationaloperators_RebuildWorkOrder
quickstep_relationaloperators_RelationalOperator
- quickstep_relationaloperators_WorkOrder
+ quickstep_storage_InsertDestination
quickstep_storage_StorageBlock
- quickstep_storage_StorageBlockInfo
quickstep_utility_DAG
quickstep_utility_Macros
tmb)
@@ -210,7 +223,8 @@ target_link_libraries(quickstep_queryexecution
quickstep_queryexecution_QueryExecutionState
quickstep_queryexecution_QueryExecutionTypedefs
quickstep_queryexecution_QueryExecutionUtil
- quickstep_queryexecution_QueryManager
+ quickstep_queryexecution_QueryManagerBase
+ quickstep_queryexecution_QueryManagerSingleNode
quickstep_queryexecution_WorkOrderProtosContainer
quickstep_queryexecution_WorkOrdersContainer
quickstep_queryexecution_Worker
@@ -248,9 +262,9 @@ if (ENABLE_DISTRIBUTED)
add_test(BlockLocator_unittest BlockLocator_unittest)
endif()
-add_executable(QueryManager_unittest
- "${CMAKE_CURRENT_SOURCE_DIR}/tests/QueryManager_unittest.cpp")
-target_link_libraries(QueryManager_unittest
+add_executable(QueryManagerSingleNode_unittest
+ "${CMAKE_CURRENT_SOURCE_DIR}/tests/QueryManagerSingleNode_unittest.cpp")
+target_link_libraries(QueryManagerSingleNode_unittest
glog
gtest
gtest_main
@@ -262,7 +276,7 @@ target_link_libraries(QueryManager_unittest
quickstep_queryexecution_QueryExecutionMessages_proto
quickstep_queryexecution_QueryExecutionState
quickstep_queryexecution_QueryExecutionTypedefs
- quickstep_queryexecution_QueryManager
+ quickstep_queryexecution_QueryManagerSingleNode
quickstep_queryexecution_WorkOrdersContainer
quickstep_queryexecution_WorkerDirectory
quickstep_queryexecution_WorkerMessage
@@ -278,7 +292,7 @@ target_link_libraries(QueryManager_unittest
quickstep_utility_DAG
quickstep_utility_Macros
tmb)
-add_test(QueryManager_unittest QueryManager_unittest)
+add_test(QueryManagerSingleNode_unittest QueryManagerSingleNode_unittest)
add_executable(WorkOrdersContainer_unittest
"${CMAKE_CURRENT_SOURCE_DIR}/tests/WorkOrdersContainer_unittest.cpp")
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/65af8c7b/query_execution/PolicyEnforcer.cpp
----------------------------------------------------------------------
diff --git a/query_execution/PolicyEnforcer.cpp b/query_execution/PolicyEnforcer.cpp
index 84aa86a..f310ee1 100644
--- a/query_execution/PolicyEnforcer.cpp
+++ b/query_execution/PolicyEnforcer.cpp
@@ -26,7 +26,7 @@
#include "catalog/CatalogTypedefs.hpp"
#include "query_execution/QueryExecutionMessages.pb.h"
-#include "query_execution/QueryManager.hpp"
+#include "query_execution/QueryManagerSingleNode.hpp"
#include "query_execution/WorkerDirectory.hpp"
#include "query_optimizer/QueryHandle.hpp"
#include "relational_operators/WorkOrder.hpp"
@@ -47,8 +47,8 @@ bool PolicyEnforcer::admitQuery(QueryHandle *query_handle) {
if (admitted_queries_.find(query_id) == admitted_queries_.end()) {
// Query with the same ID not present, ok to admit.
admitted_queries_[query_id].reset(
- new QueryManager(foreman_client_id_, num_numa_nodes_, query_handle,
- catalog_database_, storage_manager_, bus_));
+ new QueryManagerSingleNode(foreman_client_id_, num_numa_nodes_, query_handle,
+ catalog_database_, storage_manager_, bus_));
return true;
} else {
LOG(ERROR) << "Query with the same ID " << query_id << " exists";
@@ -124,9 +124,9 @@ void PolicyEnforcer::processMessage(const TaggedMessage &tagged_message) {
LOG(FATAL) << "Unknown message type found in PolicyEnforcer";
}
DCHECK(admitted_queries_.find(query_id) != admitted_queries_.end());
- const QueryManager::QueryStatusCode return_code =
+ const QueryManagerBase::QueryStatusCode return_code =
admitted_queries_[query_id]->processMessage(tagged_message);
- if (return_code == QueryManager::QueryStatusCode::kQueryExecuted) {
+ if (return_code == QueryManagerBase::QueryStatusCode::kQueryExecuted) {
removeQuery(query_id);
if (!waiting_queries_.empty()) {
// Admit the earliest waiting query.
@@ -156,12 +156,12 @@ void PolicyEnforcer::getWorkerMessages(
std::vector<std::size_t> finished_queries_ids;
for (const auto &admitted_query_info : admitted_queries_) {
- QueryManager *curr_query_manager = admitted_query_info.second.get();
+ QueryManagerBase *curr_query_manager = admitted_query_info.second.get();
DCHECK(curr_query_manager != nullptr);
std::size_t messages_collected_curr_query = 0;
while (messages_collected_curr_query < per_query_share) {
WorkerMessage *next_worker_message =
- curr_query_manager->getNextWorkerMessage(0, kAnyNUMANodeID);
+ static_cast<QueryManagerSingleNode*>(curr_query_manager)->getNextWorkerMessage(0, kAnyNUMANodeID);
if (next_worker_message != nullptr) {
++messages_collected_curr_query;
worker_messages->push_back(std::unique_ptr<WorkerMessage>(next_worker_message));
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/65af8c7b/query_execution/PolicyEnforcer.hpp
----------------------------------------------------------------------
diff --git a/query_execution/PolicyEnforcer.hpp b/query_execution/PolicyEnforcer.hpp
index 470ff2a..79e61d1 100644
--- a/query_execution/PolicyEnforcer.hpp
+++ b/query_execution/PolicyEnforcer.hpp
@@ -26,7 +26,7 @@
#include <vector>
#include "query_execution/QueryExecutionTypedefs.hpp"
-#include "query_execution/QueryManager.hpp"
+#include "query_execution/QueryManagerBase.hpp"
#include "query_execution/WorkerMessage.hpp"
#include "utility/Macros.hpp"
@@ -190,8 +190,8 @@ class PolicyEnforcer {
tmb::MessageBus *bus_;
const bool profile_individual_workorders_;
- // Key = query ID, value = QueryManager* for the key query.
- std::unordered_map<std::size_t, std::unique_ptr<QueryManager>> admitted_queries_;
+ // Key = query ID, value = QueryManagerBase* for the key query.
+ std::unordered_map<std::size_t, std::unique_ptr<QueryManagerBase>> admitted_queries_;
// The queries which haven't been admitted yet.
std::queue<QueryHandle*> waiting_queries_;
@@ -214,4 +214,4 @@ class PolicyEnforcer {
} // namespace quickstep
-#endif // QUICKSTEP_QUERY_EXECUTION_QUERY_MANAGER_HPP_
+#endif // QUICKSTEP_QUERY_EXECUTION_POLICY_ENFORCER_HPP_
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/65af8c7b/query_execution/QueryManager.cpp
----------------------------------------------------------------------
diff --git a/query_execution/QueryManager.cpp b/query_execution/QueryManager.cpp
deleted file mode 100644
index d20b592..0000000
--- a/query_execution/QueryManager.cpp
+++ /dev/null
@@ -1,470 +0,0 @@
-/**
- * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
- * University of Wisconsin\u2014Madison.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- **/
-
-#include "query_execution/QueryManager.hpp"
-
-#include <cstddef>
-#include <memory>
-#include <utility>
-#include <vector>
-
-#include "catalog/CatalogDatabase.hpp"
-#include "catalog/CatalogRelation.hpp"
-#include "catalog/CatalogTypedefs.hpp"
-#include "catalog/PartitionScheme.hpp"
-#include "query_execution/QueryExecutionMessages.pb.h"
-#include "query_execution/QueryExecutionTypedefs.hpp"
-#include "query_execution/WorkerMessage.hpp"
-#include "query_optimizer/QueryHandle.hpp"
-#include "relational_operators/RebuildWorkOrder.hpp"
-#include "relational_operators/WorkOrder.hpp"
-#include "storage/StorageBlock.hpp"
-#include "storage/StorageBlockInfo.hpp"
-
-#include "glog/logging.h"
-
-using std::pair;
-
-namespace quickstep {
-class CatalogDatabaseLite;
-class StorageManager;
-}
-
-namespace quickstep {
-
-QueryManager::QueryManager(const tmb::client_id foreman_client_id,
- const std::size_t num_numa_nodes,
- QueryHandle *query_handle,
- CatalogDatabaseLite *catalog_database,
- StorageManager *storage_manager,
- tmb::MessageBus *bus)
- : foreman_client_id_(foreman_client_id),
- query_id_(DCHECK_NOTNULL(query_handle)->query_id()),
- catalog_database_(DCHECK_NOTNULL(catalog_database)),
- storage_manager_(DCHECK_NOTNULL(storage_manager)),
- bus_(DCHECK_NOTNULL(bus)) {
- DCHECK(query_handle->getQueryPlanMutable() != nullptr);
- query_dag_ = query_handle->getQueryPlanMutable()->getQueryPlanDAGMutable();
- DCHECK(query_dag_ != nullptr);
-
- const dag_node_index num_operators_in_dag = query_dag_->size();
-
- output_consumers_.resize(num_operators_in_dag);
- blocking_dependencies_.resize(num_operators_in_dag);
-
- query_exec_state_.reset(new QueryExecutionState(num_operators_in_dag));
- workorders_container_.reset(
- new WorkOrdersContainer(num_operators_in_dag, num_numa_nodes));
-
- query_context_.reset(new QueryContext(query_handle->getQueryContextProto(),
- *catalog_database_,
- storage_manager_,
- foreman_client_id_,
- bus_));
-
- for (dag_node_index node_index = 0;
- node_index < num_operators_in_dag;
- ++node_index) {
- const QueryContext::insert_destination_id insert_destination_index =
- query_dag_->getNodePayload(node_index).getInsertDestinationID();
- if (insert_destination_index != QueryContext::kInvalidInsertDestinationId) {
- // Rebuild is necessary whenever InsertDestination is present.
- query_exec_state_->setRebuildRequired(node_index);
- query_exec_state_->setRebuildStatus(node_index, 0, false);
- }
-
- for (const pair<dag_node_index, bool> &dependent_link :
- query_dag_->getDependents(node_index)) {
- const dag_node_index dependent_op_index = dependent_link.first;
- if (!query_dag_->getLinkMetadata(node_index, dependent_op_index)) {
- // The link is not a pipeline-breaker. Streaming of blocks is possible
- // between these two operators.
- output_consumers_[node_index].push_back(dependent_op_index);
- } else {
- // The link is a pipeline-breaker. Streaming of blocks is not possible
- // between these two operators.
- blocking_dependencies_[dependent_op_index].push_back(node_index);
- }
- }
- }
-
- // Collect all the workorders from all the relational operators in the DAG.
- for (dag_node_index index = 0; index < num_operators_in_dag; ++index) {
- if (checkAllBlockingDependenciesMet(index)) {
- query_dag_->getNodePayloadMutable(index)->informAllBlockingDependenciesMet();
- processOperator(index, false);
- }
- }
-}
-
-WorkerMessage* QueryManager::getNextWorkerMessage(
- const dag_node_index start_operator_index, const int numa_node) {
- // Default policy: Operator with lowest index first.
- WorkOrder *work_order = nullptr;
- size_t num_operators_checked = 0;
- for (dag_node_index index = start_operator_index;
- num_operators_checked < query_dag_->size();
- index = (index + 1) % query_dag_->size(), ++num_operators_checked) {
- if (query_exec_state_->hasExecutionFinished(index)) {
- continue;
- }
- if (numa_node != -1) {
- // First try to get a normal WorkOrder from the specified NUMA node.
- work_order = workorders_container_->getNormalWorkOrderForNUMANode(index, numa_node);
- if (work_order != nullptr) {
- // A WorkOrder found on the given NUMA node.
- query_exec_state_->incrementNumQueuedWorkOrders(index);
- return WorkerMessage::WorkOrderMessage(work_order, index);
- } else {
- // Normal workorder not found on this node. Look for a rebuild workorder
- // on this NUMA node.
- work_order = workorders_container_->getRebuildWorkOrderForNUMANode(index, numa_node);
- if (work_order != nullptr) {
- return WorkerMessage::RebuildWorkOrderMessage(work_order, index);
- }
- }
- }
- // Either no workorder found on the given NUMA node, or numa_node is -1.
- // Try to get a normal WorkOrder from other NUMA nodes.
- work_order = workorders_container_->getNormalWorkOrder(index);
- if (work_order != nullptr) {
- query_exec_state_->incrementNumQueuedWorkOrders(index);
- return WorkerMessage::WorkOrderMessage(work_order, index);
- } else {
- // Normal WorkOrder not found, look for a RebuildWorkOrder.
- work_order = workorders_container_->getRebuildWorkOrder(index);
- if (work_order != nullptr) {
- return WorkerMessage::RebuildWorkOrderMessage(work_order, index);
- }
- }
- }
- // No WorkOrders available right now.
- return nullptr;
-}
-
-QueryManager::QueryStatusCode QueryManager::processMessage(
- const TaggedMessage &tagged_message) {
- dag_node_index op_index;
- switch (tagged_message.message_type()) {
- case kWorkOrderCompleteMessage: {
- serialization::NormalWorkOrderCompletionMessage proto;
- CHECK(proto.ParseFromArray(tagged_message.message(),
- tagged_message.message_bytes()));
-
- op_index = proto.operator_index();
- processWorkOrderCompleteMessage(proto.operator_index());
- break;
- }
- case kRebuildWorkOrderCompleteMessage: {
- serialization::RebuildWorkOrderCompletionMessage proto;
- CHECK(proto.ParseFromArray(tagged_message.message(),
- tagged_message.message_bytes()));
-
- op_index = proto.operator_index();
- processRebuildWorkOrderCompleteMessage(proto.operator_index());
- break;
- }
- case kCatalogRelationNewBlockMessage: {
- serialization::CatalogRelationNewBlockMessage proto;
- CHECK(proto.ParseFromArray(tagged_message.message(),
- tagged_message.message_bytes()));
-
- const block_id block = proto.block_id();
-
- CatalogRelation *relation =
- static_cast<CatalogDatabase*>(catalog_database_)->getRelationByIdMutable(proto.relation_id());
- relation->addBlock(block);
-
- if (proto.has_partition_id()) {
- relation->getPartitionSchemeMutable()->addBlockToPartition(
- proto.partition_id(), block);
- }
- return QueryStatusCode::kNone;
- }
- case kDataPipelineMessage: {
- // Possible message senders include InsertDestinations and some
- // operators which modify existing blocks.
- serialization::DataPipelineMessage proto;
- CHECK(proto.ParseFromArray(tagged_message.message(),
- tagged_message.message_bytes()));
-
- op_index = proto.operator_index();
- processDataPipelineMessage(proto.operator_index(),
- proto.block_id(),
- proto.relation_id());
- break;
- }
- case kWorkOrdersAvailableMessage: {
- serialization::WorkOrdersAvailableMessage proto;
- CHECK(proto.ParseFromArray(tagged_message.message(),
- tagged_message.message_bytes()));
-
- op_index = proto.operator_index();
-
- // Check if new work orders are available.
- fetchNormalWorkOrders(op_index);
-
- // Dispatch the WorkerMessages to the workers. We prefer to start the search
- // for the schedulable WorkOrders beginning from 'op_index'. The first
- // candidate worker to receive the next WorkOrder is the one that sent the
- // response message to Foreman.
- // TODO(zuyu): Improve the data locality for the next WorkOrder.
- break;
- }
- case kWorkOrderFeedbackMessage: {
- WorkOrder::FeedbackMessage msg(
- const_cast<void *>(tagged_message.message()),
- tagged_message.message_bytes());
-
- op_index = msg.header().rel_op_index;
- processFeedbackMessage(msg);
- break;
- }
- default:
- LOG(FATAL) << "Unknown message type found in QueryManager";
- }
-
- if (query_exec_state_->hasExecutionFinished(op_index)) {
- return QueryStatusCode::kOperatorExecuted;
- }
-
- // As kQueryExecuted takes precedence over kOperatorExecuted, we check again.
- if (query_exec_state_->hasQueryExecutionFinished()) {
- return QueryStatusCode::kQueryExecuted;
- }
-
- return QueryStatusCode::kNone;
-}
-
-void QueryManager::processFeedbackMessage(
- const WorkOrder::FeedbackMessage &msg) {
- RelationalOperator *op =
- query_dag_->getNodePayloadMutable(msg.header().rel_op_index);
- op->receiveFeedbackMessage(msg);
-}
-
-void QueryManager::processWorkOrderCompleteMessage(
- const dag_node_index op_index) {
- query_exec_state_->decrementNumQueuedWorkOrders(op_index);
-
- // Check if new work orders are available and fetch them if so.
- fetchNormalWorkOrders(op_index);
-
- if (checkRebuildRequired(op_index)) {
- if (checkNormalExecutionOver(op_index)) {
- if (!checkRebuildInitiated(op_index)) {
- if (initiateRebuild(op_index)) {
- // Rebuild initiated and completed right away.
- markOperatorFinished(op_index);
- } else {
- // Rebuild under progress.
- }
- } else if (checkRebuildOver(op_index)) {
- // Rebuild was under progress and now it is over.
- markOperatorFinished(op_index);
- }
- } else {
- // Normal execution under progress for this operator.
- }
- } else if (checkOperatorExecutionOver(op_index)) {
- // Rebuild not required for this operator and its normal execution is
- // complete.
- markOperatorFinished(op_index);
- }
-
- for (const pair<dag_node_index, bool> &dependent_link :
- query_dag_->getDependents(op_index)) {
- const dag_node_index dependent_op_index = dependent_link.first;
- if (checkAllBlockingDependenciesMet(dependent_op_index)) {
- // Process the dependent operator (of the operator whose WorkOrder
- // was just executed) for which all the dependencies have been met.
- processOperator(dependent_op_index, true);
- }
- }
-}
-
-void QueryManager::processRebuildWorkOrderCompleteMessage(const dag_node_index op_index) {
- query_exec_state_->decrementNumRebuildWorkOrders(op_index);
-
- if (checkRebuildOver(op_index)) {
- markOperatorFinished(op_index);
-
- for (const pair<dag_node_index, bool> &dependent_link :
- query_dag_->getDependents(op_index)) {
- const dag_node_index dependent_op_index = dependent_link.first;
- if (checkAllBlockingDependenciesMet(dependent_op_index)) {
- processOperator(dependent_op_index, true);
- }
- }
- }
-}
-
-void QueryManager::processOperator(const dag_node_index index,
- const bool recursively_check_dependents) {
- if (fetchNormalWorkOrders(index)) {
- // Fetched work orders. Return to wait for the generated work orders to
- // execute, and skip the execution-finished checks.
- return;
- }
-
- if (checkNormalExecutionOver(index)) {
- if (checkRebuildRequired(index)) {
- if (!checkRebuildInitiated(index)) {
- // Rebuild hasn't started, initiate it.
- if (initiateRebuild(index)) {
- // Rebuild initiated and completed right away.
- markOperatorFinished(index);
- } else {
- // Rebuild WorkOrders have been generated.
- return;
- }
- } else if (checkRebuildOver(index)) {
- // Rebuild had been initiated and it is over.
- markOperatorFinished(index);
- }
- } else {
- // Rebuild is not required and normal execution over, mark finished.
- markOperatorFinished(index);
- }
- // If we reach here, that means the operator has been marked as finished.
- if (recursively_check_dependents) {
- for (const pair<dag_node_index, bool> &dependent_link :
- query_dag_->getDependents(index)) {
- const dag_node_index dependent_op_index = dependent_link.first;
- if (checkAllBlockingDependenciesMet(dependent_op_index)) {
- processOperator(dependent_op_index, true);
- }
- }
- }
- }
-}
-
-void QueryManager::processDataPipelineMessage(const dag_node_index op_index,
- const block_id block,
- const relation_id rel_id) {
- for (const dag_node_index consumer_index :
- output_consumers_[op_index]) {
- // Feed the streamed block to the consumer. Note that 'output_consumers_'
- // only contain those dependents of operator with index = op_index which are
- // eligible to receive streamed input.
- query_dag_->getNodePayloadMutable(consumer_index)->feedInputBlock(block, rel_id);
- // Because of the streamed input just fed, check if there are any new
- // WorkOrders available and if so, fetch them.
- fetchNormalWorkOrders(consumer_index);
- }
-}
-
-bool QueryManager::fetchNormalWorkOrders(const dag_node_index index) {
- bool generated_new_workorders = false;
- if (!query_exec_state_->hasDoneGenerationWorkOrders(index)) {
- // Do not fetch any work units until all blocking dependencies are met.
- // The releational operator is not aware of blocking dependencies for
- // uncorrelated scalar queries.
- if (!checkAllBlockingDependenciesMet(index)) {
- return false;
- }
- const size_t num_pending_workorders_before =
- workorders_container_->getNumNormalWorkOrders(index);
- const bool done_generation =
- query_dag_->getNodePayloadMutable(index)->getAllWorkOrders(workorders_container_.get(),
- query_context_.get(),
- storage_manager_,
- foreman_client_id_,
- bus_);
- if (done_generation) {
- query_exec_state_->setDoneGenerationWorkOrders(index);
- }
-
- // TODO(shoban): It would be a good check to see if operator is making
- // useful progress, i.e., the operator either generates work orders to
- // execute or still has pending work orders executing. However, this will not
- // work if Foreman polls operators without feeding data. This check can be
- // enabled, if Foreman is refactored to call getAllWorkOrders() only when
- // pending work orders are completed or new input blocks feed.
-
- generated_new_workorders =
- (num_pending_workorders_before <
- workorders_container_->getNumNormalWorkOrders(index));
- }
- return generated_new_workorders;
-}
-
-void QueryManager::markOperatorFinished(const dag_node_index index) {
- query_exec_state_->setExecutionFinished(index);
-
- RelationalOperator *op = query_dag_->getNodePayloadMutable(index);
- op->updateCatalogOnCompletion();
-
- const relation_id output_rel = op->getOutputRelationID();
- for (const pair<dag_node_index, bool> &dependent_link : query_dag_->getDependents(index)) {
- const dag_node_index dependent_op_index = dependent_link.first;
- RelationalOperator *dependent_op = query_dag_->getNodePayloadMutable(dependent_op_index);
- // Signal dependent operator that current operator is done feeding input blocks.
- if (output_rel >= 0) {
- dependent_op->doneFeedingInputBlocks(output_rel);
- }
- if (checkAllBlockingDependenciesMet(dependent_op_index)) {
- dependent_op->informAllBlockingDependenciesMet();
- }
- }
-}
-
-bool QueryManager::initiateRebuild(const dag_node_index index) {
- DCHECK(!workorders_container_->hasRebuildWorkOrder(index));
- DCHECK(checkRebuildRequired(index));
- DCHECK(!checkRebuildInitiated(index));
-
- getRebuildWorkOrders(index, workorders_container_.get());
-
- query_exec_state_->setRebuildStatus(
- index, workorders_container_->getNumRebuildWorkOrders(index), true);
-
- return (query_exec_state_->getNumRebuildWorkOrders(index) == 0);
-}
-
-void QueryManager::getRebuildWorkOrders(const dag_node_index index,
- WorkOrdersContainer *container) {
- const RelationalOperator &op = query_dag_->getNodePayload(index);
- const QueryContext::insert_destination_id insert_destination_index = op.getInsertDestinationID();
-
- if (insert_destination_index == QueryContext::kInvalidInsertDestinationId) {
- return;
- }
-
- std::vector<MutableBlockReference> partially_filled_block_refs;
-
- DCHECK(query_context_ != nullptr);
- InsertDestination *insert_destination = query_context_->getInsertDestination(insert_destination_index);
- DCHECK(insert_destination != nullptr);
-
- insert_destination->getPartiallyFilledBlocks(&partially_filled_block_refs);
-
- for (std::vector<MutableBlockReference>::size_type i = 0;
- i < partially_filled_block_refs.size();
- ++i) {
- container->addRebuildWorkOrder(
- new RebuildWorkOrder(query_id_,
- std::move(partially_filled_block_refs[i]),
- index,
- op.getOutputRelationID(),
- foreman_client_id_,
- bus_),
- index);
- }
-}
-
-} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/65af8c7b/query_execution/QueryManager.hpp
----------------------------------------------------------------------
diff --git a/query_execution/QueryManager.hpp b/query_execution/QueryManager.hpp
deleted file mode 100644
index b52460f..0000000
--- a/query_execution/QueryManager.hpp
+++ /dev/null
@@ -1,374 +0,0 @@
-/**
- * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
- * University of Wisconsin\u2014Madison.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- **/
-
-#ifndef QUICKSTEP_QUERY_EXECUTION_QUERY_MANAGER_HPP_
-#define QUICKSTEP_QUERY_EXECUTION_QUERY_MANAGER_HPP_
-
-#include <cstddef>
-#include <memory>
-#include <vector>
-
-#include "catalog/CatalogTypedefs.hpp"
-#include "query_execution/QueryContext.hpp"
-#include "query_execution/QueryExecutionState.hpp"
-#include "query_execution/QueryExecutionTypedefs.hpp"
-#include "query_execution/WorkOrdersContainer.hpp"
-#include "relational_operators/RelationalOperator.hpp"
-#include "relational_operators/WorkOrder.hpp"
-#include "storage/StorageBlockInfo.hpp"
-#include "utility/DAG.hpp"
-#include "utility/Macros.hpp"
-
-#include "tmb/id_typedefs.h"
-#include "tmb/message_bus.h"
-#include "tmb/tagged_message.h"
-
-namespace quickstep {
-
-class CatalogDatabaseLite;
-class QueryHandle;
-class StorageManager;
-class WorkerMessage;
-
-/** \addtogroup QueryExecution
- * @{
- */
-
-/**
- * @brief A class that manages the execution of a query including generation
- * of new work orders, keeping track of the query exection state.
- **/
-class QueryManager {
- public:
- typedef DAG<RelationalOperator, bool>::size_type_nodes dag_node_index;
-
- /**
- * @brief Return codes for processMessage() function.
- *
- * @note When both operator and query get executed, kQueryExecuted takes
- * precedence over kOperatorExecuted.
- **/
- enum class QueryStatusCode {
- kOperatorExecuted = 0, // An operator in the query finished execution.
- kQueryExecuted, // The query got executed.
- kNone // None of the above.
- };
-
- /**
- * @brief Constructor.
- *
- * @param foreman_client_id The TMB client ID of the foreman thread.
- * @param num_numa_nodes The number of NUMA nodes used by the system.
- * @param query_handle The QueryHandle object for this query.
- * @param catalog_database The CatalogDatabse used by the query.
- * @param storage_manager The StorageManager used by the query.
- * @param bus The TMB used for communication.
- **/
- QueryManager(const tmb::client_id foreman_client_id,
- const std::size_t num_numa_nodes,
- QueryHandle *query_handle,
- CatalogDatabaseLite *catalog_database,
- StorageManager *storage_manager,
- tmb::MessageBus *bus);
-
- /**
- * @brief Get the next workorder to be excuted, wrapped in a WorkerMessage.
- *
- * @param start_operator_index Begin the search for the schedulable WorkOrder
- * with the operator at this index.
- * @param numa_node The next WorkOrder should preferably have its input(s)
- * from this numa_node. This is a hint and not a binding requirement.
- *
- * @return A pointer to the WorkerMessage. If there's no WorkOrder to be
- * executed, return NULL.
- **/
- WorkerMessage *getNextWorkerMessage(
- const dag_node_index start_operator_index,
- const numa_node_id node_id = -1);
-
- /**
- * @brief Process a message sent to the QueryManager.
- *
- * @param tagged_message TaggedMessage sent to the QueryManager.
- *
- * @return QueryStatusCode as determined after the message is processed.
- **/
- QueryStatusCode processMessage(const TaggedMessage &tagged_message);
-
- /**
- * @brief Get the QueryExecutionState for this query.
- **/
- inline const QueryExecutionState& getQueryExecutionState() const {
- return *query_exec_state_;
- }
-
- /**
- * @brief Get a pointer to the QueryContext.
- **/
- inline QueryContext* getQueryContextMutable() {
- return query_context_.get();
- }
-
- private:
- /**
- * @brief Process the received WorkOrder complete message.
- *
- * @param node_index The index of the specified operator node in the query DAG
- * for the completed WorkOrder.
- **/
- void processWorkOrderCompleteMessage(const dag_node_index op_index);
-
- /**
- * @brief Process the received RebuildWorkOrder complete message.
- *
- * @param node_index The index of the specified operator node in the query DAG
- * for the completed RebuildWorkOrder.
- **/
- void processRebuildWorkOrderCompleteMessage(const dag_node_index op_index);
-
- /**
- * @brief Process a current relational operator: Get its workorders and store
- * them in the WorkOrdersContainer for this query. If the operator can
- * be marked as done, do so.
- *
- * @param index The index of the relational operator to be processed in the
- * query plan DAG.
- * @param recursively_check_dependents If an operator is done, should we
- * call processOperator on its dependents recursively.
- **/
- void processOperator(const dag_node_index index,
- const bool recursively_check_dependents);
-
- /**
- * @brief Process the received data pipeline message.
- *
- * @param node_index The index of the specified operator node in the query DAG
- * for the pipelining block.
- * @param block The block id.
- * @param rel_id The ID of the relation that produced 'block'.
- **/
- void processDataPipelineMessage(const dag_node_index op_index,
- const block_id block,
- const relation_id rel_id);
-
- /**
- * @brief Process the received work order feedback message and notify
- * relational operator.
- *
- * @param message Feedback message from work order.
- **/
- void processFeedbackMessage(const WorkOrder::FeedbackMessage &message);
-
- /**
- * @brief Fetch all work orders currently available in relational operator and
- * store them internally.
- *
- * @param index The index of the relational operator to be processed in the
- * query plan DAG.
- *
- * @return Whether any work order was generated by op.
- **/
- bool fetchNormalWorkOrders(const dag_node_index index);
-
- /**
- * @brief This function does the following things:
- * 1. Mark the given relational operator as "done".
- * 2. For all the dependents of this operator, check if all of their
- * blocking dependencies are met. If so inform them that the blocking
- * dependencies are met.
- * 3. Check if the given operator is done producing output. If it's
- * done, inform the dependents that they won't receive input anymore
- * from the given operator.
- *
- * @param index The index of the given relational operator in the DAG.
- **/
- void markOperatorFinished(const dag_node_index index);
-
- /**
- * @brief Check if all the dependencies of the node at specified index have
- * finished their execution.
- *
- * @note This function's true return value is a pre-requisite for calling
- * getRebuildWorkOrders()
- *
- * @param node_index The index of the specified node in the query DAG.
- *
- * @return True if all the dependencies have finished their execution. False
- * otherwise.
- **/
- inline bool checkAllDependenciesMet(const dag_node_index node_index) const {
- for (const dag_node_index dependency_index :
- query_dag_->getDependencies(node_index)) {
- // If at least one of the dependencies is not met, return false.
- if (!query_exec_state_->hasExecutionFinished(dependency_index)) {
- return false;
- }
- }
- return true;
- }
-
- /**
- * @brief Check if all the blocking dependencies of the node at specified
- * index have finished their execution.
- *
- * @note A blocking dependency is the one which is pipeline breaker. Output of
- * a dependency can't be streamed to its dependent if the link between
- * them is pipeline breaker.
- *
- * @param node_index The index of the specified node in the query DAG.
- *
- * @return True if all the blocking dependencies have finished their
- * execution. False otherwise.
- **/
- inline bool checkAllBlockingDependenciesMet(
- const dag_node_index node_index) const {
- for (const dag_node_index blocking_dependency_index :
- blocking_dependencies_[node_index]) {
- if (!query_exec_state_->hasExecutionFinished(
- blocking_dependency_index)) {
- return false;
- }
- }
- return true;
- }
-
- /**
- * @brief Check if the execution of the given operator is over.
- *
- * @param index The index of the given operator in the DAG.
- *
- * @return True if the execution of the given operator is over, false
- * otherwise.
- **/
- inline bool checkOperatorExecutionOver(const dag_node_index index) const {
- if (checkRebuildRequired(index)) {
- return (checkNormalExecutionOver(index) && checkRebuildOver(index));
- } else {
- return checkNormalExecutionOver(index);
- }
- }
-
- /**
- * @brief Check if the given operator's normal execution is over.
- *
- * @note The conditions for a given operator's normal execution to get over:
- * 1. All of its normal (i.e. non rebuild) WorkOrders have finished
- * execution.
- * 2. The operator is done generating work orders.
- * 3. All of the dependencies of the given operator have been met.
- *
- * @param index The index of the given operator in the DAG.
- *
- * @return True if the normal execution of the given operator is over, false
- * otherwise.
- **/
- inline bool checkNormalExecutionOver(const dag_node_index index) const {
- return (checkAllDependenciesMet(index) &&
- !workorders_container_->hasNormalWorkOrder(index) &&
- query_exec_state_->getNumQueuedWorkOrders(index) == 0 &&
- query_exec_state_->hasDoneGenerationWorkOrders(index));
- }
-
- /**
- * @brief Check if the rebuild operation is required for a given operator.
- *
- * @param index The index of the given operator in the DAG.
- *
- * @return True if the rebuild operation is required, false otherwise.
- **/
- inline bool checkRebuildRequired(const dag_node_index index) const {
- return query_exec_state_->isRebuildRequired(index);
- }
-
- /**
- * @brief Check if the rebuild operation for a given operator is over.
- *
- * @param index The index of the given operator in the DAG.
- *
- * @return True if the rebuild operation is over, false otherwise.
- **/
- inline bool checkRebuildOver(const dag_node_index index) const {
- return query_exec_state_->hasRebuildInitiated(index) &&
- !workorders_container_->hasRebuildWorkOrder(index) &&
- (query_exec_state_->getNumRebuildWorkOrders(index) == 0);
- }
-
- /**
- * @brief Check if the rebuild operation for a given operator has been
- * initiated.
- *
- * @param index The index of the given operator in the DAG.
- *
- * @return True if the rebuild operation has been initiated, false otherwise.
- **/
- inline bool checkRebuildInitiated(const dag_node_index index) const {
- return query_exec_state_->hasRebuildInitiated(index);
- }
-
- /**
- * @brief Initiate the rebuild process for partially filled blocks generated
- * during the execution of the given operator.
- *
- * @param index The index of the given operator in the DAG.
- *
- * @return True if the rebuild is over immediately, i.e. the operator didn't
- * generate any rebuild WorkOrders, false otherwise.
- **/
- bool initiateRebuild(const dag_node_index index);
-
- /**
- * @brief Get the rebuild WorkOrders for an operator.
- *
- * @note This function should be called only once, when all the normal
- * WorkOrders generated by an operator finish their execution.
- *
- * @param index The index of the operator in the query plan DAG.
- * @param container A pointer to a WorkOrdersContainer to be used to store the
- * generated WorkOrders.
- **/
- void getRebuildWorkOrders(const dag_node_index index,
- WorkOrdersContainer *container);
-
- const tmb::client_id foreman_client_id_;
- const std::size_t query_id_;
-
- CatalogDatabaseLite *catalog_database_;
- StorageManager *storage_manager_;
- tmb::MessageBus *bus_;
-
- DAG<RelationalOperator, bool> *query_dag_;
-
- std::unique_ptr<QueryContext> query_context_;
-
- // For all nodes, store their receiving dependents.
- std::vector<std::vector<dag_node_index>> output_consumers_;
-
- // For all nodes, store their pipeline breaking dependencies (if any).
- std::vector<std::vector<dag_node_index>> blocking_dependencies_;
-
- std::unique_ptr<QueryExecutionState> query_exec_state_;
-
- std::unique_ptr<WorkOrdersContainer> workorders_container_;
-
- DISALLOW_COPY_AND_ASSIGN(QueryManager);
-};
-
-/** @} */
-
-} // namespace quickstep
-
-#endif // QUICKSTEP_QUERY_EXECUTION_QUERY_MANAGER_HPP_
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/65af8c7b/query_execution/QueryManagerBase.cpp
----------------------------------------------------------------------
diff --git a/query_execution/QueryManagerBase.cpp b/query_execution/QueryManagerBase.cpp
new file mode 100644
index 0000000..f7e183f
--- /dev/null
+++ b/query_execution/QueryManagerBase.cpp
@@ -0,0 +1,305 @@
+/**
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#include "query_execution/QueryManagerBase.hpp"
+
+#include <memory>
+#include <utility>
+#include <vector>
+
+#include "catalog/CatalogDatabase.hpp"
+#include "catalog/CatalogRelation.hpp"
+#include "catalog/CatalogTypedefs.hpp"
+#include "catalog/PartitionScheme.hpp"
+#include "query_execution/QueryContext.hpp"
+#include "query_execution/QueryExecutionMessages.pb.h"
+#include "query_execution/QueryExecutionTypedefs.hpp"
+#include "query_optimizer/QueryHandle.hpp"
+#include "query_optimizer/QueryPlan.hpp"
+#include "relational_operators/WorkOrder.hpp"
+#include "storage/StorageBlockInfo.hpp"
+
+#include "glog/logging.h"
+
+using std::pair;
+
+namespace quickstep {
+
+QueryManagerBase::QueryManagerBase(QueryHandle *query_handle,
+ CatalogDatabaseLite *catalog_database)
+ : query_id_(DCHECK_NOTNULL(query_handle)->query_id()),
+ catalog_database_(DCHECK_NOTNULL(catalog_database)),
+ query_dag_(DCHECK_NOTNULL(
+ DCHECK_NOTNULL(query_handle->getQueryPlanMutable())->getQueryPlanDAGMutable())),
+ num_operators_in_dag_(query_dag_->size()),
+ output_consumers_(num_operators_in_dag_),
+ blocking_dependencies_(num_operators_in_dag_),
+ query_exec_state_(new QueryExecutionState(num_operators_in_dag_)) {
+ for (dag_node_index node_index = 0;
+ node_index < num_operators_in_dag_;
+ ++node_index) {
+ const QueryContext::insert_destination_id insert_destination_index =
+ query_dag_->getNodePayload(node_index).getInsertDestinationID();
+ if (insert_destination_index != QueryContext::kInvalidInsertDestinationId) {
+ // Rebuild is necessary whenever InsertDestination is present.
+ query_exec_state_->setRebuildRequired(node_index);
+ query_exec_state_->setRebuildStatus(node_index, 0, false);
+ }
+
+ for (const pair<dag_node_index, bool> &dependent_link :
+ query_dag_->getDependents(node_index)) {
+ const dag_node_index dependent_op_index = dependent_link.first;
+ if (!query_dag_->getLinkMetadata(node_index, dependent_op_index)) {
+ // The link is not a pipeline-breaker. Streaming of blocks is possible
+ // between these two operators.
+ output_consumers_[node_index].push_back(dependent_op_index);
+ } else {
+ // The link is a pipeline-breaker. Streaming of blocks is not possible
+ // between these two operators.
+ blocking_dependencies_[dependent_op_index].push_back(node_index);
+ }
+ }
+ }
+}
+
+QueryManagerBase::QueryStatusCode QueryManagerBase::processMessage(
+ const TaggedMessage &tagged_message) {
+ dag_node_index op_index;
+ switch (tagged_message.message_type()) {
+ case kWorkOrderCompleteMessage: {
+ serialization::NormalWorkOrderCompletionMessage proto;
+ CHECK(proto.ParseFromArray(tagged_message.message(),
+ tagged_message.message_bytes()));
+
+ op_index = proto.operator_index();
+ processWorkOrderCompleteMessage(proto.operator_index());
+ break;
+ }
+ case kRebuildWorkOrderCompleteMessage: {
+ serialization::RebuildWorkOrderCompletionMessage proto;
+ CHECK(proto.ParseFromArray(tagged_message.message(),
+ tagged_message.message_bytes()));
+
+ op_index = proto.operator_index();
+ processRebuildWorkOrderCompleteMessage(proto.operator_index());
+ break;
+ }
+ case kCatalogRelationNewBlockMessage: {
+ serialization::CatalogRelationNewBlockMessage proto;
+ CHECK(proto.ParseFromArray(tagged_message.message(),
+ tagged_message.message_bytes()));
+
+ const block_id block = proto.block_id();
+
+ CatalogRelation *relation =
+ static_cast<CatalogDatabase*>(catalog_database_)->getRelationByIdMutable(proto.relation_id());
+ relation->addBlock(block);
+
+ if (proto.has_partition_id()) {
+ relation->getPartitionSchemeMutable()->addBlockToPartition(
+ proto.partition_id(), block);
+ }
+ return QueryStatusCode::kNone;
+ }
+ case kDataPipelineMessage: {
+ // Possible message senders include InsertDestinations and some
+ // operators which modify existing blocks.
+ serialization::DataPipelineMessage proto;
+ CHECK(proto.ParseFromArray(tagged_message.message(),
+ tagged_message.message_bytes()));
+
+ op_index = proto.operator_index();
+ processDataPipelineMessage(proto.operator_index(),
+ proto.block_id(),
+ proto.relation_id());
+ break;
+ }
+ case kWorkOrdersAvailableMessage: {
+ serialization::WorkOrdersAvailableMessage proto;
+ CHECK(proto.ParseFromArray(tagged_message.message(),
+ tagged_message.message_bytes()));
+
+ op_index = proto.operator_index();
+
+ // Check if new work orders are available.
+ fetchNormalWorkOrders(op_index);
+ break;
+ }
+ case kWorkOrderFeedbackMessage: {
+ WorkOrder::FeedbackMessage msg(
+ const_cast<void *>(tagged_message.message()),
+ tagged_message.message_bytes());
+
+ op_index = msg.header().rel_op_index;
+ processFeedbackMessage(msg);
+ break;
+ }
+ default:
+ LOG(FATAL) << "Unknown message type found in QueryManager";
+ }
+
+ if (query_exec_state_->hasExecutionFinished(op_index)) {
+ return QueryStatusCode::kOperatorExecuted;
+ }
+
+ // As kQueryExecuted takes precedence over kOperatorExecuted, we check again.
+ if (query_exec_state_->hasQueryExecutionFinished()) {
+ return QueryStatusCode::kQueryExecuted;
+ }
+
+ return QueryStatusCode::kNone;
+}
+
+void QueryManagerBase::processFeedbackMessage(
+ const WorkOrder::FeedbackMessage &msg) {
+ RelationalOperator *op =
+ query_dag_->getNodePayloadMutable(msg.header().rel_op_index);
+ op->receiveFeedbackMessage(msg);
+}
+
+void QueryManagerBase::processWorkOrderCompleteMessage(
+ const dag_node_index op_index) {
+ query_exec_state_->decrementNumQueuedWorkOrders(op_index);
+
+ // Check if new work orders are available and fetch them if so.
+ fetchNormalWorkOrders(op_index);
+
+ if (checkRebuildRequired(op_index)) {
+ if (checkNormalExecutionOver(op_index)) {
+ if (!checkRebuildInitiated(op_index)) {
+ if (initiateRebuild(op_index)) {
+ // Rebuild initiated and completed right away.
+ markOperatorFinished(op_index);
+ } else {
+ // Rebuild under progress.
+ }
+ } else if (checkRebuildOver(op_index)) {
+ // Rebuild was under progress and now it is over.
+ markOperatorFinished(op_index);
+ }
+ } else {
+ // Normal execution under progress for this operator.
+ }
+ } else if (checkOperatorExecutionOver(op_index)) {
+ // Rebuild not required for this operator and its normal execution is
+ // complete.
+ markOperatorFinished(op_index);
+ }
+
+ for (const pair<dag_node_index, bool> &dependent_link :
+ query_dag_->getDependents(op_index)) {
+ const dag_node_index dependent_op_index = dependent_link.first;
+ if (checkAllBlockingDependenciesMet(dependent_op_index)) {
+ // Process the dependent operator (of the operator whose WorkOrder
+ // was just executed) for which all the dependencies have been met.
+ processOperator(dependent_op_index, true);
+ }
+ }
+}
+
+void QueryManagerBase::processRebuildWorkOrderCompleteMessage(const dag_node_index op_index) {
+ query_exec_state_->decrementNumRebuildWorkOrders(op_index);
+
+ if (checkRebuildOver(op_index)) {
+ markOperatorFinished(op_index);
+
+ for (const pair<dag_node_index, bool> &dependent_link :
+ query_dag_->getDependents(op_index)) {
+ const dag_node_index dependent_op_index = dependent_link.first;
+ if (checkAllBlockingDependenciesMet(dependent_op_index)) {
+ processOperator(dependent_op_index, true);
+ }
+ }
+ }
+}
+
+void QueryManagerBase::processOperator(const dag_node_index index,
+ const bool recursively_check_dependents) {
+ if (fetchNormalWorkOrders(index)) {
+ // Fetched work orders. Return to wait for the generated work orders to
+ // execute, and skip the execution-finished checks.
+ return;
+ }
+
+ if (checkNormalExecutionOver(index)) {
+ if (checkRebuildRequired(index)) {
+ if (!checkRebuildInitiated(index)) {
+ // Rebuild hasn't started, initiate it.
+ if (initiateRebuild(index)) {
+ // Rebuild initiated and completed right away.
+ markOperatorFinished(index);
+ } else {
+ // Rebuild WorkOrders have been generated.
+ return;
+ }
+ } else if (checkRebuildOver(index)) {
+ // Rebuild had been initiated and it is over.
+ markOperatorFinished(index);
+ }
+ } else {
+ // Rebuild is not required and normal execution over, mark finished.
+ markOperatorFinished(index);
+ }
+ // If we reach here, that means the operator has been marked as finished.
+ if (recursively_check_dependents) {
+ for (const pair<dag_node_index, bool> &dependent_link :
+ query_dag_->getDependents(index)) {
+ const dag_node_index dependent_op_index = dependent_link.first;
+ if (checkAllBlockingDependenciesMet(dependent_op_index)) {
+ processOperator(dependent_op_index, true);
+ }
+ }
+ }
+ }
+}
+
+void QueryManagerBase::processDataPipelineMessage(const dag_node_index op_index,
+ const block_id block,
+ const relation_id rel_id) {
+ for (const dag_node_index consumer_index :
+ output_consumers_[op_index]) {
+ // Feed the streamed block to the consumer. Note that 'output_consumers_'
+ // only contain those dependents of operator with index = op_index which are
+ // eligible to receive streamed input.
+ query_dag_->getNodePayloadMutable(consumer_index)->feedInputBlock(block, rel_id);
+ // Because of the streamed input just fed, check if there are any new
+ // WorkOrders available and if so, fetch them.
+ fetchNormalWorkOrders(consumer_index);
+ }
+}
+
+void QueryManagerBase::markOperatorFinished(const dag_node_index index) {
+ query_exec_state_->setExecutionFinished(index);
+
+ RelationalOperator *op = query_dag_->getNodePayloadMutable(index);
+ op->updateCatalogOnCompletion();
+
+ const relation_id output_rel = op->getOutputRelationID();
+ for (const pair<dag_node_index, bool> &dependent_link : query_dag_->getDependents(index)) {
+ const dag_node_index dependent_op_index = dependent_link.first;
+ RelationalOperator *dependent_op = query_dag_->getNodePayloadMutable(dependent_op_index);
+ // Signal dependent operator that current operator is done feeding input blocks.
+ if (output_rel >= 0) {
+ dependent_op->doneFeedingInputBlocks(output_rel);
+ }
+ if (checkAllBlockingDependenciesMet(dependent_op_index)) {
+ dependent_op->informAllBlockingDependenciesMet();
+ }
+ }
+}
+
+} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/65af8c7b/query_execution/QueryManagerBase.hpp
----------------------------------------------------------------------
diff --git a/query_execution/QueryManagerBase.hpp b/query_execution/QueryManagerBase.hpp
new file mode 100644
index 0000000..9e192c8
--- /dev/null
+++ b/query_execution/QueryManagerBase.hpp
@@ -0,0 +1,312 @@
+/**
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#ifndef QUICKSTEP_QUERY_EXECUTION_QUERY_MANAGER_BASE_HPP_
+#define QUICKSTEP_QUERY_EXECUTION_QUERY_MANAGER_BASE_HPP_
+
+#include <cstddef>
+#include <memory>
+#include <vector>
+
+#include "catalog/CatalogTypedefs.hpp"
+#include "query_execution/QueryExecutionState.hpp"
+#include "query_execution/QueryExecutionTypedefs.hpp"
+#include "relational_operators/RelationalOperator.hpp"
+#include "relational_operators/WorkOrder.hpp"
+#include "storage/StorageBlockInfo.hpp"
+#include "utility/DAG.hpp"
+#include "utility/Macros.hpp"
+
+namespace quickstep {
+
+class CatalogDatabaseLite;
+class QueryHandle;
+
+/** \addtogroup QueryExecution
+ * @{
+ */
+
+/**
+ * @brief A base class that manages the execution of a query including
+ * generation of new work orders, and keeping track of the query
+ * exection state.
+ **/
+class QueryManagerBase {
+ public:
+ typedef DAG<RelationalOperator, bool>::size_type_nodes dag_node_index;
+
+ /**
+ * @brief Return codes for processMessage() function.
+ *
+ * @note When both operator and query get executed, kQueryExecuted takes
+ * precedence over kOperatorExecuted.
+ **/
+ enum class QueryStatusCode {
+ kOperatorExecuted = 0, // An operator in the query finished execution.
+ kQueryExecuted, // The query got executed.
+ kNone // None of the above.
+ };
+
+ /**
+ * @brief Constructor.
+ *
+ * @param query_handle The QueryHandle object for this query.
+ * @param catalog_database The CatalogDatabse used by the query.
+ **/
+ QueryManagerBase(QueryHandle *query_handle,
+ CatalogDatabaseLite *catalog_database);
+
+ /**
+ * @brief Virtual destructor.
+ **/
+ virtual ~QueryManagerBase() {}
+
+ /**
+ * @brief Process a message sent to the QueryManager.
+ *
+ * @param tagged_message TaggedMessage sent to the QueryManager.
+ *
+ * @return QueryStatusCode as determined after the message is processed.
+ **/
+ QueryStatusCode processMessage(const TaggedMessage &tagged_message);
+
+ /**
+ * @brief Get the QueryExecutionState for this query.
+ **/
+ inline const QueryExecutionState& getQueryExecutionState() const {
+ return *query_exec_state_;
+ }
+
+ protected:
+ /**
+ * @brief Process the received WorkOrder complete message.
+ *
+ * @param node_index The index of the specified operator node in the query DAG
+ * for the completed WorkOrder.
+ **/
+ void processWorkOrderCompleteMessage(const dag_node_index op_index);
+
+ /**
+ * @brief Process the received RebuildWorkOrder complete message.
+ *
+ * @param node_index The index of the specified operator node in the query DAG
+ * for the completed RebuildWorkOrder.
+ **/
+ void processRebuildWorkOrderCompleteMessage(const dag_node_index op_index);
+
+ /**
+ * @brief Process a current relational operator: Get its workorders and store
+ * them in the WorkOrdersContainer for this query. If the operator can
+ * be marked as done, do so.
+ *
+ * @param index The index of the relational operator to be processed in the
+ * query plan DAG.
+ * @param recursively_check_dependents If an operator is done, should we
+ * call processOperator on its dependents recursively.
+ **/
+ void processOperator(const dag_node_index index,
+ const bool recursively_check_dependents);
+
+ /**
+ * @brief Process the received data pipeline message.
+ *
+ * @param node_index The index of the specified operator node in the query DAG
+ * for the pipelining block.
+ * @param block The block id.
+ * @param rel_id The ID of the relation that produced 'block'.
+ **/
+ void processDataPipelineMessage(const dag_node_index op_index,
+ const block_id block,
+ const relation_id rel_id);
+
+ /**
+ * @brief Process the received work order feedback message and notify
+ * relational operator.
+ *
+ * @param message Feedback message from work order.
+ **/
+ void processFeedbackMessage(const WorkOrder::FeedbackMessage &message);
+
+ /**
+ * @brief This function does the following things:
+ * 1. Mark the given relational operator as "done".
+ * 2. For all the dependents of this operator, check if all of their
+ * blocking dependencies are met. If so inform them that the blocking
+ * dependencies are met.
+ * 3. Check if the given operator is done producing output. If it's
+ * done, inform the dependents that they won't receive input anymore
+ * from the given operator.
+ *
+ * @param index The index of the given relational operator in the DAG.
+ **/
+ void markOperatorFinished(const dag_node_index index);
+
+ /**
+ * @brief Check if all the dependencies of the node at specified index have
+ * finished their execution.
+ *
+ * @note This function's true return value is a pre-requisite for calling
+ * getRebuildWorkOrders()
+ *
+ * @param node_index The index of the specified node in the query DAG.
+ *
+ * @return True if all the dependencies have finished their execution. False
+ * otherwise.
+ **/
+ inline bool checkAllDependenciesMet(const dag_node_index node_index) const {
+ for (const dag_node_index dependency_index :
+ query_dag_->getDependencies(node_index)) {
+ // If at least one of the dependencies is not met, return false.
+ if (!query_exec_state_->hasExecutionFinished(dependency_index)) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ /**
+ * @brief Check if all the blocking dependencies of the node at specified
+ * index have finished their execution.
+ *
+ * @note A blocking dependency is the one which is pipeline breaker. Output of
+ * a dependency can't be streamed to its dependent if the link between
+ * them is pipeline breaker.
+ *
+ * @param node_index The index of the specified node in the query DAG.
+ *
+ * @return True if all the blocking dependencies have finished their
+ * execution. False otherwise.
+ **/
+ inline bool checkAllBlockingDependenciesMet(
+ const dag_node_index node_index) const {
+ for (const dag_node_index blocking_dependency_index :
+ blocking_dependencies_[node_index]) {
+ if (!query_exec_state_->hasExecutionFinished(
+ blocking_dependency_index)) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ /**
+ * @brief Check if the execution of the given operator is over.
+ *
+ * @param index The index of the given operator in the DAG.
+ *
+ * @return True if the execution of the given operator is over, false
+ * otherwise.
+ **/
+ inline bool checkOperatorExecutionOver(const dag_node_index index) const {
+ return this->checkNormalExecutionOver(index) &&
+ (!checkRebuildRequired(index) || this->checkRebuildOver(index));
+ }
+
+ /**
+ * @brief Check if the rebuild operation is required for a given operator.
+ *
+ * @param index The index of the given operator in the DAG.
+ *
+ * @return True if the rebuild operation is required, false otherwise.
+ **/
+ inline bool checkRebuildRequired(const dag_node_index index) const {
+ return query_exec_state_->isRebuildRequired(index);
+ }
+
+ /**
+ * @brief Check if the rebuild operation for a given operator has been
+ * initiated.
+ *
+ * @param index The index of the given operator in the DAG.
+ *
+ * @return True if the rebuild operation has been initiated, false otherwise.
+ **/
+ inline bool checkRebuildInitiated(const dag_node_index index) const {
+ return query_exec_state_->hasRebuildInitiated(index);
+ }
+
+ const std::size_t query_id_;
+
+ CatalogDatabaseLite *catalog_database_;
+
+ DAG<RelationalOperator, bool> *query_dag_;
+ const dag_node_index num_operators_in_dag_;
+
+ // For all nodes, store their receiving dependents.
+ std::vector<std::vector<dag_node_index>> output_consumers_;
+
+ // For all nodes, store their pipeline breaking dependencies (if any).
+ std::vector<std::vector<dag_node_index>> blocking_dependencies_;
+
+ std::unique_ptr<QueryExecutionState> query_exec_state_;
+
+ private:
+ /**
+ * @brief Fetch all work orders currently available in relational operator and
+ * store them internally.
+ *
+ * @param index The index of the relational operator to be processed in the
+ * query plan DAG.
+ *
+ * @return Whether any work order was generated by op.
+ **/
+ virtual bool fetchNormalWorkOrders(const dag_node_index index) = 0;
+
+ /**
+ * @brief Check if the given operator's normal execution is over.
+ *
+ * @note The conditions for a given operator's normal execution to get over:
+ * 1. All of its normal (i.e. non rebuild) WorkOrders have finished
+ * execution.
+ * 2. The operator is done generating work orders.
+ * 3. All of the dependencies of the given operator have been met.
+ *
+ * @param index The index of the given operator in the DAG.
+ *
+ * @return True if the normal execution of the given operator is over, false
+ * otherwise.
+ **/
+ virtual bool checkNormalExecutionOver(const dag_node_index index) const = 0;
+
+ /**
+ * @brief Initiate the rebuild process for partially filled blocks generated
+ * during the execution of the given operator.
+ *
+ * @param index The index of the given operator in the DAG.
+ *
+ * @return True if the rebuild is over immediately, i.e. the operator didn't
+ * generate any rebuild WorkOrders, false otherwise.
+ **/
+ virtual bool initiateRebuild(const dag_node_index index) = 0;
+
+ /**
+ * @brief Check if the rebuild operation for a given operator is over.
+ *
+ * @param index The index of the given operator in the DAG.
+ *
+ * @return True if the rebuild operation is over, false otherwise.
+ **/
+ virtual bool checkRebuildOver(const dag_node_index index) const = 0;
+
+ DISALLOW_COPY_AND_ASSIGN(QueryManagerBase);
+};
+
+/** @} */
+
+} // namespace quickstep
+
+#endif // QUICKSTEP_QUERY_EXECUTION_QUERY_MANAGER_BASE_HPP_
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/65af8c7b/query_execution/QueryManagerSingleNode.cpp
----------------------------------------------------------------------
diff --git a/query_execution/QueryManagerSingleNode.cpp b/query_execution/QueryManagerSingleNode.cpp
new file mode 100644
index 0000000..193b188
--- /dev/null
+++ b/query_execution/QueryManagerSingleNode.cpp
@@ -0,0 +1,194 @@
+/**
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#include "query_execution/QueryManagerSingleNode.hpp"
+
+#include <cstddef>
+#include <memory>
+#include <utility>
+#include <vector>
+
+#include "catalog/CatalogTypedefs.hpp"
+#include "query_execution/WorkerMessage.hpp"
+#include "query_optimizer/QueryHandle.hpp"
+#include "relational_operators/RebuildWorkOrder.hpp"
+#include "relational_operators/RelationalOperator.hpp"
+#include "storage/InsertDestination.hpp"
+#include "storage/StorageBlock.hpp"
+#include "utility/DAG.hpp"
+
+#include "glog/logging.h"
+
+#include "tmb/id_typedefs.h"
+
+namespace quickstep {
+
+class WorkOrder;
+
+QueryManagerSingleNode::QueryManagerSingleNode(
+ const tmb::client_id foreman_client_id,
+ const std::size_t num_numa_nodes,
+ QueryHandle *query_handle,
+ CatalogDatabaseLite *catalog_database,
+ StorageManager *storage_manager,
+ tmb::MessageBus *bus)
+ : QueryManagerBase(query_handle, catalog_database),
+ foreman_client_id_(foreman_client_id),
+ storage_manager_(DCHECK_NOTNULL(storage_manager)),
+ bus_(DCHECK_NOTNULL(bus)),
+ query_context_(new QueryContext(query_handle->getQueryContextProto(),
+ *catalog_database_,
+ storage_manager_,
+ foreman_client_id_,
+ bus_)),
+ workorders_container_(
+ new WorkOrdersContainer(num_operators_in_dag_, num_numa_nodes)) {
+ // Collect all the workorders from all the relational operators in the DAG.
+ for (dag_node_index index = 0; index < num_operators_in_dag_; ++index) {
+ if (checkAllBlockingDependenciesMet(index)) {
+ query_dag_->getNodePayloadMutable(index)->informAllBlockingDependenciesMet();
+ processOperator(index, false);
+ }
+ }
+}
+
+WorkerMessage* QueryManagerSingleNode::getNextWorkerMessage(
+ const dag_node_index start_operator_index, const numa_node_id numa_node) {
+ // Default policy: Operator with lowest index first.
+ WorkOrder *work_order = nullptr;
+ size_t num_operators_checked = 0;
+ for (dag_node_index index = start_operator_index;
+ num_operators_checked < num_operators_in_dag_;
+ index = (index + 1) % num_operators_in_dag_, ++num_operators_checked) {
+ if (query_exec_state_->hasExecutionFinished(index)) {
+ continue;
+ }
+ if (numa_node != kAnyNUMANodeID) {
+ // First try to get a normal WorkOrder from the specified NUMA node.
+ work_order = workorders_container_->getNormalWorkOrderForNUMANode(index, numa_node);
+ if (work_order != nullptr) {
+ // A WorkOrder found on the given NUMA node.
+ query_exec_state_->incrementNumQueuedWorkOrders(index);
+ return WorkerMessage::WorkOrderMessage(work_order, index);
+ } else {
+ // Normal workorder not found on this node. Look for a rebuild workorder
+ // on this NUMA node.
+ work_order = workorders_container_->getRebuildWorkOrderForNUMANode(index, numa_node);
+ if (work_order != nullptr) {
+ return WorkerMessage::RebuildWorkOrderMessage(work_order, index);
+ }
+ }
+ }
+ // Either no workorder found on the given NUMA node, or numa_node is
+ // 'kAnyNUMANodeID'.
+ // Try to get a normal WorkOrder from other NUMA nodes.
+ work_order = workorders_container_->getNormalWorkOrder(index);
+ if (work_order != nullptr) {
+ query_exec_state_->incrementNumQueuedWorkOrders(index);
+ return WorkerMessage::WorkOrderMessage(work_order, index);
+ } else {
+ // Normal WorkOrder not found, look for a RebuildWorkOrder.
+ work_order = workorders_container_->getRebuildWorkOrder(index);
+ if (work_order != nullptr) {
+ return WorkerMessage::RebuildWorkOrderMessage(work_order, index);
+ }
+ }
+ }
+ // No WorkOrders available right now.
+ return nullptr;
+}
+
+bool QueryManagerSingleNode::fetchNormalWorkOrders(const dag_node_index index) {
+ bool generated_new_workorders = false;
+ if (!query_exec_state_->hasDoneGenerationWorkOrders(index)) {
+ // Do not fetch any work units until all blocking dependencies are met.
+ // The releational operator is not aware of blocking dependencies for
+ // uncorrelated scalar queries.
+ if (!checkAllBlockingDependenciesMet(index)) {
+ return false;
+ }
+ const size_t num_pending_workorders_before =
+ workorders_container_->getNumNormalWorkOrders(index);
+ const bool done_generation =
+ query_dag_->getNodePayloadMutable(index)->getAllWorkOrders(workorders_container_.get(),
+ query_context_.get(),
+ storage_manager_,
+ foreman_client_id_,
+ bus_);
+ if (done_generation) {
+ query_exec_state_->setDoneGenerationWorkOrders(index);
+ }
+
+ // TODO(shoban): It would be a good check to see if operator is making
+ // useful progress, i.e., the operator either generates work orders to
+ // execute or still has pending work orders executing. However, this will not
+ // work if Foreman polls operators without feeding data. This check can be
+ // enabled, if Foreman is refactored to call getAllWorkOrders() only when
+ // pending work orders are completed or new input blocks feed.
+
+ generated_new_workorders =
+ (num_pending_workorders_before <
+ workorders_container_->getNumNormalWorkOrders(index));
+ }
+ return generated_new_workorders;
+}
+
+bool QueryManagerSingleNode::initiateRebuild(const dag_node_index index) {
+ DCHECK(!workorders_container_->hasRebuildWorkOrder(index));
+ DCHECK(checkRebuildRequired(index));
+ DCHECK(!checkRebuildInitiated(index));
+
+ getRebuildWorkOrders(index, workorders_container_.get());
+
+ query_exec_state_->setRebuildStatus(
+ index, workorders_container_->getNumRebuildWorkOrders(index), true);
+
+ return (query_exec_state_->getNumRebuildWorkOrders(index) == 0);
+}
+
+void QueryManagerSingleNode::getRebuildWorkOrders(const dag_node_index index,
+ WorkOrdersContainer *container) {
+ const RelationalOperator &op = query_dag_->getNodePayload(index);
+ const QueryContext::insert_destination_id insert_destination_index = op.getInsertDestinationID();
+
+ if (insert_destination_index == QueryContext::kInvalidInsertDestinationId) {
+ return;
+ }
+
+ std::vector<MutableBlockReference> partially_filled_block_refs;
+
+ DCHECK(query_context_ != nullptr);
+ InsertDestination *insert_destination = query_context_->getInsertDestination(insert_destination_index);
+ DCHECK(insert_destination != nullptr);
+
+ insert_destination->getPartiallyFilledBlocks(&partially_filled_block_refs);
+
+ for (std::vector<MutableBlockReference>::size_type i = 0;
+ i < partially_filled_block_refs.size();
+ ++i) {
+ container->addRebuildWorkOrder(
+ new RebuildWorkOrder(query_id_,
+ std::move(partially_filled_block_refs[i]),
+ index,
+ op.getOutputRelationID(),
+ foreman_client_id_,
+ bus_),
+ index);
+ }
+}
+
+} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/65af8c7b/query_execution/QueryManagerSingleNode.hpp
----------------------------------------------------------------------
diff --git a/query_execution/QueryManagerSingleNode.hpp b/query_execution/QueryManagerSingleNode.hpp
new file mode 100644
index 0000000..5533f06
--- /dev/null
+++ b/query_execution/QueryManagerSingleNode.hpp
@@ -0,0 +1,140 @@
+/**
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#ifndef QUICKSTEP_QUERY_EXECUTION_QUERY_MANAGER_SINGLE_NODE_HPP_
+#define QUICKSTEP_QUERY_EXECUTION_QUERY_MANAGER_SINGLE_NODE_HPP_
+
+#include <cstddef>
+#include <memory>
+
+#include "catalog/CatalogTypedefs.hpp"
+#include "query_execution/QueryContext.hpp"
+#include "query_execution/QueryExecutionState.hpp"
+#include "query_execution/QueryManagerBase.hpp"
+#include "query_execution/WorkOrdersContainer.hpp"
+#include "utility/Macros.hpp"
+
+#include "tmb/id_typedefs.h"
+
+namespace tmb { class MessageBus; }
+
+namespace quickstep {
+
+class CatalogDatabaseLite;
+class QueryHandle;
+class StorageManager;
+class WorkerMessage;
+
+/** \addtogroup QueryExecution
+ * @{
+ */
+
+/**
+ * @brief A class that manages the execution of a query including generation
+ * of new work orders, keeping track of the query exection state.
+ **/
+class QueryManagerSingleNode final : public QueryManagerBase {
+ public:
+ /**
+ * @brief Constructor.
+ *
+ * @param foreman_client_id The TMB client ID of the foreman thread.
+ * @param num_numa_nodes The number of NUMA nodes used by the system.
+ * @param query_handle The QueryHandle object for this query.
+ * @param catalog_database The CatalogDatabse used by the query.
+ * @param storage_manager The StorageManager used by the query.
+ * @param bus The TMB used for communication.
+ **/
+ QueryManagerSingleNode(const tmb::client_id foreman_client_id,
+ const std::size_t num_numa_nodes,
+ QueryHandle *query_handle,
+ CatalogDatabaseLite *catalog_database,
+ StorageManager *storage_manager,
+ tmb::MessageBus *bus);
+
+ ~QueryManagerSingleNode() override {}
+
+ /**
+ * @brief Get the next workorder to be excuted, wrapped in a WorkerMessage.
+ *
+ * @param start_operator_index Begin the search for the schedulable WorkOrder
+ * with the operator at this index.
+ * @param numa_node The next WorkOrder should preferably have its input(s)
+ * from this numa_node. This is a hint and not a binding requirement.
+ *
+ * @return A pointer to the WorkerMessage. If there's no WorkOrder to be
+ * executed, return NULL.
+ **/
+ WorkerMessage* getNextWorkerMessage(
+ const dag_node_index start_operator_index,
+ const numa_node_id node_id = kAnyNUMANodeID);
+
+ /**
+ * @brief Get a pointer to the QueryContext.
+ **/
+ inline QueryContext* getQueryContextMutable() {
+ return query_context_.get();
+ }
+
+ private:
+ bool fetchNormalWorkOrders(const dag_node_index index) override;
+
+ bool checkNormalExecutionOver(const dag_node_index index) const override {
+ return (checkAllDependenciesMet(index) &&
+ !workorders_container_->hasNormalWorkOrder(index) &&
+ query_exec_state_->getNumQueuedWorkOrders(index) == 0 &&
+ query_exec_state_->hasDoneGenerationWorkOrders(index));
+ }
+
+ bool initiateRebuild(const dag_node_index index) override;
+
+ bool checkRebuildOver(const dag_node_index index) const override {
+ return query_exec_state_->hasRebuildInitiated(index) &&
+ !workorders_container_->hasRebuildWorkOrder(index) &&
+ (query_exec_state_->getNumRebuildWorkOrders(index) == 0);
+ }
+
+ /**
+ * @brief Get the rebuild WorkOrders for an operator.
+ *
+ * @note This function should be called only once, when all the normal
+ * WorkOrders generated by an operator finish their execution.
+ *
+ * @param index The index of the operator in the query plan DAG.
+ * @param container A pointer to a WorkOrdersContainer to be used to store the
+ * generated WorkOrders.
+ **/
+ void getRebuildWorkOrders(const dag_node_index index,
+ WorkOrdersContainer *container);
+
+ const tmb::client_id foreman_client_id_;
+
+ StorageManager *storage_manager_;
+ tmb::MessageBus *bus_;
+
+ std::unique_ptr<QueryContext> query_context_;
+
+ std::unique_ptr<WorkOrdersContainer> workorders_container_;
+
+ DISALLOW_COPY_AND_ASSIGN(QueryManagerSingleNode);
+};
+
+/** @} */
+
+} // namespace quickstep
+
+#endif // QUICKSTEP_QUERY_EXECUTION_QUERY_MANAGER_SINGLE_NODE_HPP_
[17/25] incubator-quickstep git commit: Added QueryManagerDistributed.
Posted by zu...@apache.org.
Added QueryManagerDistributed.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/3bb5ca8d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/3bb5ca8d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/3bb5ca8d
Branch: refs/heads/dist-exe-test-new
Commit: 3bb5ca8deae3bb6f7c037d91167c7c6294e36d67
Parents: 9bdce7d
Author: Zuyu Zhang <zu...@apache.org>
Authored: Tue Jul 12 22:04:54 2016 -0700
Committer: Zuyu Zhang <zu...@twitter.com>
Committed: Fri Jul 29 16:43:19 2016 -0700
----------------------------------------------------------------------
CMakeLists.txt | 12 +-
query_execution/CMakeLists.txt | 34 +++-
query_execution/QueryExecutionMessages.proto | 34 +++-
query_execution/QueryExecutionTypedefs.hpp | 3 +
query_execution/QueryManagerDistributed.cpp | 159 +++++++++++++++++++
query_execution/QueryManagerDistributed.hpp | 105 +++++++++++++
query_execution/ShiftbossDirectory.hpp | 181 ++++++++++++++++++++++
7 files changed, 515 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3bb5ca8d/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 042c050..0bbde61 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -281,11 +281,13 @@ else()
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror")
endif()
- # Clang reports such warning when using Protoc 3.0 beta.
- if(${CMAKE_CXX_COMPILER_ID} MATCHES "Clang")
- CHECK_CXX_COMPILER_FLAG("-Wno-extended-offsetof" COMPILER_HAS_WNO_EXTENDED_OFFSETOF)
- if (COMPILER_HAS_WNO_EXTENDED_OFFSETOF)
- set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-extended-offsetof")
+ if (ENABLE_DISTRIBUTED)
+ # Clang reports such warning when using Protoc 3.0 beta.
+ if(${CMAKE_CXX_COMPILER_ID} MATCHES "Clang")
+ CHECK_CXX_COMPILER_FLAG("-Wno-extended-offsetof" COMPILER_HAS_WNO_EXTENDED_OFFSETOF)
+ if (COMPILER_HAS_WNO_EXTENDED_OFFSETOF)
+ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-extended-offsetof")
+ endif()
endif()
endif()
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3bb5ca8d/query_execution/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_execution/CMakeLists.txt b/query_execution/CMakeLists.txt
index 028531d..8c12a5d 100644
--- a/query_execution/CMakeLists.txt
+++ b/query_execution/CMakeLists.txt
@@ -28,10 +28,10 @@ else()
endif()
# Declare micro-libs:
+add_library(quickstep_queryexecution_AdmitRequestMessage ../empty_src.cpp AdmitRequestMessage.hpp)
if (ENABLE_DISTRIBUTED)
add_library(quickstep_queryexecution_BlockLocator BlockLocator.cpp BlockLocator.hpp)
endif()
-add_library(quickstep_queryexecution_AdmitRequestMessage ../empty_src.cpp AdmitRequestMessage.hpp)
add_library(quickstep_queryexecution_ForemanBase ../empty_src.cpp ForemanBase.hpp)
add_library(quickstep_queryexecution_ForemanSingleNode ForemanSingleNode.cpp ForemanSingleNode.hpp)
add_library(quickstep_queryexecution_PolicyEnforcer PolicyEnforcer.cpp PolicyEnforcer.hpp)
@@ -46,7 +46,13 @@ add_library(quickstep_queryexecution_QueryExecutionState ../empty_src.cpp QueryE
add_library(quickstep_queryexecution_QueryExecutionTypedefs ../empty_src.cpp QueryExecutionTypedefs.hpp)
add_library(quickstep_queryexecution_QueryExecutionUtil ../empty_src.cpp QueryExecutionUtil.hpp)
add_library(quickstep_queryexecution_QueryManagerBase QueryManagerBase.cpp QueryManagerBase.hpp)
+if (ENABLE_DISTRIBUTED)
+ add_library(quickstep_queryexecution_QueryManagerDistributed QueryManagerDistributed.cpp QueryManagerDistributed.hpp)
+endif()
add_library(quickstep_queryexecution_QueryManagerSingleNode QueryManagerSingleNode.cpp QueryManagerSingleNode.hpp)
+if (ENABLE_DISTRIBUTED)
+ add_library(quickstep_queryexecution_ShiftbossDirectory ../empty_src.cpp ShiftbossDirectory.hpp)
+endif()
add_library(quickstep_queryexecution_WorkOrderProtosContainer ../empty_src.cpp WorkOrderProtosContainer.hpp)
add_library(quickstep_queryexecution_WorkOrdersContainer WorkOrdersContainer.cpp WorkOrdersContainer.hpp)
add_library(quickstep_queryexecution_Worker Worker.cpp Worker.hpp)
@@ -143,6 +149,7 @@ target_link_libraries(quickstep_queryexecution_QueryContext_proto
quickstep_utility_SortConfiguration_proto
${PROTOBUF_LIBRARY})
target_link_libraries(quickstep_queryexecution_QueryExecutionMessages_proto
+ quickstep_relationaloperators_WorkOrder_proto
${PROTOBUF_LIBRARY})
target_link_libraries(quickstep_queryexecution_QueryExecutionState
glog
@@ -167,6 +174,22 @@ target_link_libraries(quickstep_queryexecution_QueryManagerBase
quickstep_storage_StorageBlockInfo
quickstep_utility_DAG
quickstep_utility_Macros)
+if (ENABLE_DISTRIBUTED)
+ target_link_libraries(quickstep_queryexecution_QueryManagerDistributed
+ quickstep_queryexecution_QueryContext
+ quickstep_queryexecution_QueryExecutionMessages_proto
+ quickstep_queryexecution_QueryExecutionState
+ quickstep_queryexecution_QueryExecutionTypedefs
+ quickstep_queryexecution_QueryExecutionUtil
+ quickstep_queryexecution_QueryManagerBase
+ quickstep_queryexecution_ShiftbossDirectory
+ quickstep_queryexecution_WorkOrderProtosContainer
+ quickstep_relationaloperators_RelationalOperator
+ quickstep_relationaloperators_WorkOrder_proto
+ quickstep_utility_DAG
+ quickstep_utility_Macros
+ tmb)
+endif()
target_link_libraries(quickstep_queryexecution_QueryManagerSingleNode
quickstep_catalog_CatalogTypedefs
quickstep_queryexecution_QueryContext
@@ -182,6 +205,11 @@ target_link_libraries(quickstep_queryexecution_QueryManagerSingleNode
quickstep_utility_DAG
quickstep_utility_Macros
tmb)
+if (ENABLE_DISTRIBUTED)
+ target_link_libraries(quickstep_queryexecution_ShiftbossDirectory
+ quickstep_utility_Macros
+ tmb)
+endif()
target_link_libraries(quickstep_queryexecution_WorkOrderProtosContainer
glog
quickstep_relationaloperators_WorkOrder_proto
@@ -233,7 +261,9 @@ target_link_libraries(quickstep_queryexecution
quickstep_queryexecution_WorkerSelectionPolicy)
if (ENABLE_DISTRIBUTED)
target_link_libraries(quickstep_queryexecution
- quickstep_queryexecution_BlockLocator)
+ quickstep_queryexecution_BlockLocator
+ quickstep_queryexecution_QueryManagerDistributed
+ quickstep_queryexecution_ShiftbossDirectory)
endif()
# Tests:
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3bb5ca8d/query_execution/QueryExecutionMessages.proto
----------------------------------------------------------------------
diff --git a/query_execution/QueryExecutionMessages.proto b/query_execution/QueryExecutionMessages.proto
index 65a8946..fa20993 100644
--- a/query_execution/QueryExecutionMessages.proto
+++ b/query_execution/QueryExecutionMessages.proto
@@ -16,22 +16,24 @@ syntax = "proto2";
package quickstep.serialization;
+import "relational_operators/WorkOrder.proto";
+
// Used for any messages that do not carry payloads.
message EmptyMessage {
}
// Note: There are different types of completion messages for normal work orders
// rebuild work orders. This can be potentially helpful when we want to collect
-// different statistics for executing different types of work orders.
-// e.g. In select normal work order completion message, we could be interested
-// in the selectivity of the block whose work order got execute. In rebuild work
-// order completion message, we may be interested in adding the compression
+// different statistics for executing different types of work orders.
+// e.g. In select normal work order completion message, we could be interested
+// in the selectivity of the block whose work order got execute. In rebuild work
+// order completion message, we may be interested in adding the compression
// ratio or dictionary size of the rebuilt block.
-// TODO(harshad) : If there are different fields in the two message types below,
+// TODO(harshad) : If there are different fields in the two message types below,
// create a base message class called WorkOrderCompletionMessage and make the
// two classes below extend the base class. All the common fields in both the
-// classes can be moved to the base class.
+// classes can be moved to the base class.
// A message sent upon completion of a normal (not rebuild) WorkOrder execution.
message NormalWorkOrderCompletionMessage {
@@ -70,6 +72,26 @@ message WorkOrdersAvailableMessage {
required uint64 query_id = 2;
}
+// Distributed version related messages.
+message WorkOrderMessage {
+ required uint64 query_id = 1;
+ required uint64 operator_index = 2;
+ required WorkOrder work_order = 3;
+}
+
+message InitiateRebuildMessage {
+ required uint64 query_id = 1;
+ required uint64 operator_index = 2;
+ required uint64 insert_destination_index = 3;
+ required int32 relation_id = 4;
+}
+
+message InitiateRebuildResponseMessage {
+ required uint64 query_id = 1;
+ required uint64 operator_index = 2;
+ required uint64 num_rebuild_work_orders = 3;
+}
+
// BlockLocator related messages.
message BlockDomainRegistrationMessage {
// Format IP:Port, i.e., "0.0.0.0:0".
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3bb5ca8d/query_execution/QueryExecutionTypedefs.hpp
----------------------------------------------------------------------
diff --git a/query_execution/QueryExecutionTypedefs.hpp b/query_execution/QueryExecutionTypedefs.hpp
index 9d1060f..61e76d7 100644
--- a/query_execution/QueryExecutionTypedefs.hpp
+++ b/query_execution/QueryExecutionTypedefs.hpp
@@ -73,6 +73,9 @@ enum QueryExecutionMessageType : message_type_id {
kPoisonMessage, // From the main thread to Foreman and Workers.
#ifdef QUICKSTEP_DISTRIBUTED
+ kInitiateRebuildMessage, // From Foreman to Shiftboss.
+ kInitiateRebuildResponseMessage, // From Shiftboss to Foreman.
+
// BlockLocator related messages, sorted in a life cycle of StorageManager
// with a unique block domain.
kBlockDomainRegistrationMessage, // From Worker to BlockLocator.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3bb5ca8d/query_execution/QueryManagerDistributed.cpp
----------------------------------------------------------------------
diff --git a/query_execution/QueryManagerDistributed.cpp b/query_execution/QueryManagerDistributed.cpp
new file mode 100644
index 0000000..e906fa5
--- /dev/null
+++ b/query_execution/QueryManagerDistributed.cpp
@@ -0,0 +1,159 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#include "query_execution/QueryManagerDistributed.hpp"
+
+#include <cstddef>
+#include <cstdlib>
+#include <memory>
+#include <utility>
+
+#include "query_execution/QueryContext.hpp"
+#include "query_execution/QueryExecutionMessages.pb.h"
+#include "query_execution/QueryExecutionTypedefs.hpp"
+#include "query_execution/QueryExecutionUtil.hpp"
+#include "query_execution/ShiftbossDirectory.hpp"
+#include "query_execution/WorkOrderProtosContainer.hpp"
+#include "relational_operators/RelationalOperator.hpp"
+#include "relational_operators/WorkOrder.pb.h"
+#include "utility/DAG.hpp"
+
+#include "glog/logging.h"
+
+#include "tmb/id_typedefs.h"
+
+using std::free;
+using std::malloc;
+using std::move;
+using std::size_t;
+using std::unique_ptr;
+
+namespace quickstep {
+
+QueryManagerDistributed::QueryManagerDistributed(QueryHandle *query_handle,
+ ShiftbossDirectory *shiftbosses,
+ const tmb::client_id foreman_client_id,
+ tmb::MessageBus *bus)
+ : QueryManagerBase(query_handle),
+ shiftbosses_(shiftbosses),
+ foreman_client_id_(foreman_client_id),
+ bus_(bus),
+ normal_workorder_protos_container_(
+ new WorkOrderProtosContainer(num_operators_in_dag_)) {
+ // Collect all the workorders from all the relational operators in the DAG.
+ for (dag_node_index index = 0; index < num_operators_in_dag_; ++index) {
+ if (checkAllBlockingDependenciesMet(index)) {
+ query_dag_->getNodePayloadMutable(index)->informAllBlockingDependenciesMet();
+ processOperator(index, false);
+ }
+ }
+}
+
+serialization::WorkOrderMessage* QueryManagerDistributed::getNextWorkOrderMessage(
+ const dag_node_index start_operator_index) {
+ // Default policy: Operator with lowest index first.
+ size_t num_operators_checked = 0;
+ for (dag_node_index index = start_operator_index;
+ num_operators_checked < num_operators_in_dag_;
+ index = (index + 1) % num_operators_in_dag_, ++num_operators_checked) {
+ if (query_exec_state_->hasExecutionFinished(index)) {
+ continue;
+ }
+ unique_ptr<serialization::WorkOrder> work_order_proto(
+ normal_workorder_protos_container_->getWorkOrderProto(index));
+ if (work_order_proto != nullptr) {
+ query_exec_state_->incrementNumQueuedWorkOrders(index);
+
+ unique_ptr<serialization::WorkOrderMessage> message_proto(new serialization::WorkOrderMessage);
+ message_proto->set_query_id(query_id_);
+ message_proto->set_operator_index(index);
+ message_proto->mutable_work_order()->MergeFrom(*work_order_proto);
+
+ return message_proto.release();
+ }
+ }
+ // No normal WorkOrder protos available right now.
+ return nullptr;
+}
+
+bool QueryManagerDistributed::fetchNormalWorkOrders(const dag_node_index index) {
+ bool generated_new_workorder_protos = false;
+ if (!query_exec_state_->hasDoneGenerationWorkOrders(index)) {
+ // Do not fetch any work units until all blocking dependencies are met.
+ // The releational operator is not aware of blocking dependencies for
+ // uncorrelated scalar queries.
+ if (!checkAllBlockingDependenciesMet(index)) {
+ return false;
+ }
+ const size_t num_pending_workorder_protos_before =
+ normal_workorder_protos_container_->getNumWorkOrderProtos(index);
+ const bool done_generation =
+ query_dag_->getNodePayloadMutable(index)
+ ->getAllWorkOrderProtos(normal_workorder_protos_container_.get());
+ if (done_generation) {
+ query_exec_state_->setDoneGenerationWorkOrders(index);
+ }
+
+ // TODO(shoban): It would be a good check to see if operator is making
+ // useful progress, i.e., the operator either generates work orders to
+ // execute or still has pending work orders executing. However, this will not
+ // work if Foreman polls operators without feeding data. This check can be
+ // enabled, if Foreman is refactored to call getAllWorkOrders() only when
+ // pending work orders are completed or new input blocks feed.
+
+ generated_new_workorder_protos =
+ (num_pending_workorder_protos_before <
+ normal_workorder_protos_container_->getNumWorkOrderProtos(index));
+ }
+ return generated_new_workorder_protos;
+}
+
+bool QueryManagerDistributed::initiateRebuild(const dag_node_index index) {
+ DCHECK(checkRebuildRequired(index));
+ DCHECK(!checkRebuildInitiated(index));
+
+ const RelationalOperator &op = query_dag_->getNodePayload(index);
+ DCHECK_NE(op.getInsertDestinationID(), QueryContext::kInvalidInsertDestinationId);
+
+ serialization::InitiateRebuildMessage proto;
+ proto.set_operator_index(index);
+ proto.set_insert_destination_index(op.getInsertDestinationID());
+ proto.set_relation_id(op.getOutputRelationID());
+
+ const size_t proto_length = proto.ByteSize();
+ char *proto_bytes = static_cast<char*>(malloc(proto_length));
+ CHECK(proto.SerializeToArray(proto_bytes, proto_length));
+
+ TaggedMessage tagged_msg(static_cast<const void *>(proto_bytes),
+ proto_length,
+ kInitiateRebuildMessage);
+ free(proto_bytes);
+
+ LOG(INFO) << "ForemanDistributed sent InitiateRebuildMessage (typed '" << kInitiateRebuildMessage
+ << "') to Shiftboss";
+ // TODO(zuyu): Multiple workers support.
+ QueryExecutionUtil::SendTMBMessage(bus_,
+ foreman_client_id_,
+ shiftbosses_->getClientId(0),
+ move(tagged_msg));
+
+ // The negative value indicates that the number of rebuild work orders is to be
+ // determined.
+ query_exec_state_->setRebuildStatus(index, -1, true);
+
+ // Wait for Shiftbosses to report the number of rebuild work orders.
+ return false;
+}
+
+} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3bb5ca8d/query_execution/QueryManagerDistributed.hpp
----------------------------------------------------------------------
diff --git a/query_execution/QueryManagerDistributed.hpp b/query_execution/QueryManagerDistributed.hpp
new file mode 100644
index 0000000..8641c22
--- /dev/null
+++ b/query_execution/QueryManagerDistributed.hpp
@@ -0,0 +1,105 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#ifndef QUICKSTEP_QUERY_EXECUTION_QUERY_MANAGER_DISTRIBUTED_HPP_
+#define QUICKSTEP_QUERY_EXECUTION_QUERY_MANAGER_DISTRIBUTED_HPP_
+
+#include <memory>
+
+#include "query_execution/QueryExecutionState.hpp"
+#include "query_execution/QueryManagerBase.hpp"
+#include "query_execution/WorkOrderProtosContainer.hpp"
+#include "utility/Macros.hpp"
+
+#include "tmb/id_typedefs.h"
+
+namespace tmb { class MessageBus; }
+
+namespace quickstep {
+
+class QueryHandle;
+class ShiftbossDirectory;
+
+namespace serialization { class WorkOrderMessage; }
+
+/** \addtogroup QueryExecution
+ * @{
+ */
+
+/**
+ * @brief A class that manages the execution of a query including generation
+ * of new work orders, keeping track of the query exection state.
+ **/
+class QueryManagerDistributed final : public QueryManagerBase {
+ public:
+ /**
+ * @brief Constructor.
+ *
+ * @param query_handle The QueryHandle object for this query.
+ * @param shiftbosses The ShiftbossDirectory to use.
+ * @param foreman_client_id The TMB client ID of the foreman thread.
+ * @param bus The TMB used for communication.
+ **/
+ QueryManagerDistributed(QueryHandle *query_handle,
+ ShiftbossDirectory *shiftbosses,
+ const tmb::client_id foreman_client_id,
+ tmb::MessageBus *bus);
+
+ ~QueryManagerDistributed() override {}
+
+ bool fetchNormalWorkOrders(const dag_node_index index) override;
+
+ /**
+ * @brief Get the next normal workorder to be excuted, wrapped in a
+ * WorkOrderMessage proto.
+ *
+ * @param start_operator_index Begin the search for the schedulable WorkOrder
+ * with the operator at this index.
+ *
+ * @return A pointer to the WorkOrderMessage proto. If there is no WorkOrder
+ * to be executed, return NULL.
+ **/
+ serialization::WorkOrderMessage* getNextWorkOrderMessage(
+ const dag_node_index start_operator_index);
+
+ private:
+ bool checkNormalExecutionOver(const dag_node_index index) const override {
+ return (checkAllDependenciesMet(index) &&
+ !normal_workorder_protos_container_->hasWorkOrderProto(index) &&
+ query_exec_state_->getNumQueuedWorkOrders(index) == 0 &&
+ query_exec_state_->hasDoneGenerationWorkOrders(index));
+ }
+
+ bool initiateRebuild(const dag_node_index index) override;
+
+ bool checkRebuildOver(const dag_node_index index) const override {
+ return query_exec_state_->hasRebuildInitiated(index) &&
+ (query_exec_state_->getNumRebuildWorkOrders(index) == 0);
+ }
+
+ ShiftbossDirectory *shiftbosses_;
+
+ const tmb::client_id foreman_client_id_;
+ tmb::MessageBus *bus_;
+
+ std::unique_ptr<WorkOrderProtosContainer> normal_workorder_protos_container_;
+
+ DISALLOW_COPY_AND_ASSIGN(QueryManagerDistributed);
+};
+
+/** @} */
+
+} // namespace quickstep
+
+#endif // QUICKSTEP_QUERY_EXECUTION_QUERY_MANAGER_DISTRIBUTED_HPP_
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3bb5ca8d/query_execution/ShiftbossDirectory.hpp
----------------------------------------------------------------------
diff --git a/query_execution/ShiftbossDirectory.hpp b/query_execution/ShiftbossDirectory.hpp
new file mode 100644
index 0000000..abb5f32
--- /dev/null
+++ b/query_execution/ShiftbossDirectory.hpp
@@ -0,0 +1,181 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#ifndef QUICKSTEP_QUERY_EXECUTION_SHIFTBOSS_DIRECTORY_HPP_
+#define QUICKSTEP_QUERY_EXECUTION_SHIFTBOSS_DIRECTORY_HPP_
+
+#include <cstddef>
+#include <unordered_map>
+#include <utility>
+#include <vector>
+
+#include "utility/Macros.hpp"
+
+#include "glog/logging.h"
+
+#include "tmb/id_typedefs.h"
+
+namespace quickstep {
+
+/** \addtogroup QueryExecution
+ * @{
+ */
+
+/**
+ * @brief A class which keeps the metadata about the shiftbosses.
+ *
+ * @note This class is intended to be used only by ForemanDistributed thread.
+ * Therefore, none of the methods in this class are thread-safe.
+ **/
+class ShiftbossDirectory {
+ public:
+ /**
+ * @brief Constructor.
+ **/
+ ShiftbossDirectory() = default;
+
+ /**
+ * @brief Add the Shiftboss.
+ *
+ * @param shiftboss_id the TMB client ID of Shiftboss thread.
+ * @param capacity The Work Order processing capacity of Shiftboss.
+ **/
+ void addShiftboss(const tmb::client_id shiftboss_id,
+ const std::size_t capacity) {
+ client_id_indices_.emplace(shiftboss_id, client_ids_.size());
+ client_ids_.push_back(shiftboss_id);
+ work_order_capacities_.push_back(capacity);
+ num_queued_work_orders_.push_back(0u);
+ }
+
+ /**
+ * @brief Whether the ShiftbossDirectory has any Shiftboss.
+ *
+ * @return True if no Shiftboss in ShiftbossDirectory. Otherwise false.
+ **/
+ bool empty() const {
+ DCHECK_EQ(client_ids_.empty(), client_id_indices_.empty());
+ DCHECK_EQ(client_ids_.empty(), work_order_capacities_.empty());
+ DCHECK_EQ(client_ids_.empty(), num_queued_work_orders_.empty());
+
+ return client_ids_.empty();
+ }
+
+ /**
+ * @brief Get the number of Shiftboss in ShiftbossDirectory.
+ *
+ * @return The number of Shiftboss in ShiftbossDirectory.
+ **/
+ std::size_t size() const {
+ DCHECK_EQ(client_ids_.size(), client_id_indices_.size());
+ DCHECK_EQ(client_ids_.size(), work_order_capacities_.size());
+ DCHECK_EQ(client_ids_.size(), num_queued_work_orders_.size());
+
+ return client_ids_.size();
+ }
+
+ /**
+ * @brief Get the TMB client ID of the specified Shiftboss.
+ *
+ * @param shiftboss_index The index of Shiftboss.
+ *
+ * @return The TMB client ID of the given Shiftboss.
+ **/
+ tmb::client_id getClientId(const std::size_t shiftboss_index) const {
+ DCHECK_LT(shiftboss_index, size());
+ return client_ids_[shiftboss_index];
+ }
+
+ /**
+ * @brief Get the Shiftboss index from the specified client id.
+ *
+ * @param shiftboss_id The TMB client ID of Shiftboss.
+ *
+ * @return The index of the given Shiftboss.
+ **/
+ std::size_t getShiftbossIndex(const tmb::client_id shiftboss_id) const {
+ const auto it = client_id_indices_.find(shiftboss_id);
+ DCHECK(it != client_id_indices_.end());
+
+ return it->second;
+ }
+
+ /**
+ * @brief Whether the given Shiftboss has reached its capacity.
+ *
+ * @param shiftboss_index The index of Shiftboss.
+ *
+ * @return True if reached the capacity. Otherwise false.
+ **/
+ bool hasReachedCapacity(const std::size_t shiftboss_index) const {
+ DCHECK_LT(shiftboss_index, size());
+ return num_queued_work_orders_[shiftboss_index] >= work_order_capacities_[shiftboss_index];
+ }
+
+ /**
+ * @brief Add the number of new work orders for the given Shiftboss.
+ *
+ * @param shiftboss_index The index of Shiftboss.
+ * @param num_new_work_orders The number of the new work orders will be
+ * executed on Shiftboss indexed by 'shiftboss_index'.
+ **/
+ void addNumQueuedWorkOrders(const std::size_t shiftboss_index,
+ const std::size_t num_new_work_orders) {
+ num_queued_work_orders_[shiftboss_index] += num_new_work_orders;
+ }
+
+ /**
+ * @brief Increase the number of queued workorders for the given Shiftboss by 1.
+ *
+ * @param shiftboss_index The index of Shiftboss.
+ **/
+ void incrementNumQueuedWorkOrders(const std::size_t shiftboss_index) {
+ DCHECK_LT(shiftboss_index, size());
+ ++num_queued_work_orders_[shiftboss_index];
+ }
+
+ /**
+ * @brief Decrease the number of queued workorders for the given Shiftboss by 1.
+ *
+ * @param shiftboss_index The index of Shiftboss.
+ **/
+ void decrementNumQueuedWorkOrders(const std::size_t shiftboss_index) {
+ DCHECK_LT(shiftboss_index, size());
+ DCHECK_GE(num_queued_work_orders_[shiftboss_index], 1u);
+ --num_queued_work_orders_[shiftboss_index];
+ }
+
+ private:
+ // The TMB client IDs of Shiftbosses.
+ // TODO(zuyu): Support deletions, as Shiftbosses go down.
+ std::vector<tmb::client_id> client_ids_;
+
+ // The map from the TMB client ID of Shiftboss to its index in 'client_ids_'.
+ std::unordered_map<tmb::client_id, std::size_t> client_id_indices_;
+
+ // The max number of WorkOrders per Shiftboss.
+ std::vector<std::size_t> work_order_capacities_;
+
+ // The number of WorkOrders queued for execution per Shiftboss, and the value
+ // should be not greater than that of work_order_capacities_.
+ std::vector<std::size_t> num_queued_work_orders_;
+
+ DISALLOW_COPY_AND_ASSIGN(ShiftbossDirectory);
+};
+
+/** @} */
+
+} // namespace quickstep
+
+#endif // QUICKSTEP_QUERY_EXECUTION_SHIFTBOSS_DIRECTORY_HPP_
[15/25] incubator-quickstep git commit: Added the Date Type.
Posted by zu...@apache.org.
Added the Date Type.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/32bee124
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/32bee124
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/32bee124
Branch: refs/heads/dist-exe-test-new
Commit: 32bee1243865330398f2a0cb7c37ecb6fb7463c6
Parents: f35e690
Author: Zuyu Zhang <zu...@twitter.com>
Authored: Fri Jul 15 10:50:08 2016 -0700
Committer: Zuyu Zhang <zu...@twitter.com>
Committed: Fri Jul 29 16:42:11 2016 -0700
----------------------------------------------------------------------
third_party/cpplint/cpplint.py | 3 +-
third_party/iwyu/iwyu_helper.py | 1 +
types/CMakeLists.txt | 27 ++++++
types/DateType.cpp | 148 +++++++++++++++++++++++++++++++++
types/DateType.hpp | 132 +++++++++++++++++++++++++++++
types/DatetimeLit.hpp | 61 ++++++++++++++
types/Type.cpp | 3 +
types/Type.hpp | 5 +-
types/Type.proto | 1 +
types/TypeFactory.cpp | 7 ++
types/TypeFactory.hpp | 1 +
types/TypeID.cpp | 1 +
types/TypeID.hpp | 1 +
types/TypedValue.cpp | 18 ++++
types/TypedValue.hpp | 22 +++++
types/TypedValue.proto | 8 ++
types/tests/DateType_unittest.cpp | 146 ++++++++++++++++++++++++++++++++
types/tests/TypeTest_common.hpp | 8 +-
18 files changed, 587 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/32bee124/third_party/cpplint/cpplint.py
----------------------------------------------------------------------
diff --git a/third_party/cpplint/cpplint.py b/third_party/cpplint/cpplint.py
index 28b8afa..469283f 100755
--- a/third_party/cpplint/cpplint.py
+++ b/third_party/cpplint/cpplint.py
@@ -3,6 +3,7 @@
# Original version: svn revision 141
#
# This file modified for quickstep as follows:
+# - Allow no copyright message at the top of the file.
# - Allow line length up to 120 characters by default.
# - Recognize .hpp files as C++ source.
# - Allow use of C++11 <chrono> header.
@@ -5982,7 +5983,7 @@ def ProcessFileData(filename, file_extension, lines, error,
ResetNolintSuppressions()
- CheckForCopyright(filename, lines, error)
+ # CheckForCopyright(filename, lines, error)
RemoveMultiLineComments(filename, lines, error)
clean_lines = CleansedLines(lines)
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/32bee124/third_party/iwyu/iwyu_helper.py
----------------------------------------------------------------------
diff --git a/third_party/iwyu/iwyu_helper.py b/third_party/iwyu/iwyu_helper.py
index 13697be..dff4d55 100755
--- a/third_party/iwyu/iwyu_helper.py
+++ b/third_party/iwyu/iwyu_helper.py
@@ -25,6 +25,7 @@ QUICKSTEP_INCLUDES = [ '.',
'./third_party/benchmark/include',
'./third_party/glog/src',
'./third_party/googletest/googletest/include',
+ './third_party/protobuf/src',
'./third_party/re2',
'./third_party/tmb/include']
QUICKSTEP_DEFINES = [ '-DQUICKSTEP_DEBUG',
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/32bee124/types/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/types/CMakeLists.txt b/types/CMakeLists.txt
index 0ccdfd7..420891a 100644
--- a/types/CMakeLists.txt
+++ b/types/CMakeLists.txt
@@ -32,6 +32,7 @@ QS_PROTOBUF_GENERATE_CPP(types_Type_proto_srcs types_Type_proto_hdrs Type.proto)
# Declare micro-libs:
add_library(quickstep_types_CharType CharType.cpp CharType.hpp)
add_library(quickstep_types_DateOperatorOverloads ../empty_src.cpp DateOperatorOverloads.hpp)
+add_library(quickstep_types_DateType DateType.cpp DateType.hpp)
add_library(quickstep_types_DatetimeIntervalType DatetimeIntervalType.cpp DatetimeIntervalType.hpp)
add_library(quickstep_types_DatetimeLit ../empty_src.cpp DatetimeLit.hpp)
add_library(quickstep_types_DatetimeType DatetimeType.cpp DatetimeType.hpp)
@@ -71,6 +72,15 @@ target_link_libraries(quickstep_types_DateOperatorOverloads
quickstep_types_IntervalLit
quickstep_types_port_gmtime_r
quickstep_types_port_timegm)
+target_link_libraries(quickstep_types_DateType
+ glog
+ quickstep_types_DatetimeLit
+ quickstep_types_NullCoercibilityCheckMacro
+ quickstep_types_Type
+ quickstep_types_TypeID
+ quickstep_types_TypedValue
+ quickstep_utility_CheckSnprintf
+ quickstep_utility_Macros)
target_link_libraries(quickstep_types_DatetimeIntervalType
glog
quickstep_types_IntervalLit
@@ -156,6 +166,7 @@ target_link_libraries(quickstep_types_Type
target_link_libraries(quickstep_types_TypeFactory
glog
quickstep_types_CharType
+ quickstep_types_DateType
quickstep_types_DatetimeIntervalType
quickstep_types_DatetimeType
quickstep_types_DoubleType
@@ -210,6 +221,7 @@ add_library(quickstep_types ../empty_src.cpp TypesModule.hpp)
target_link_libraries(quickstep_types
quickstep_types_CharType
quickstep_types_DateOperatorOverloads
+ quickstep_types_DateType
quickstep_types_DatetimeIntervalType
quickstep_types_DatetimeLit
quickstep_types_DatetimeType
@@ -264,6 +276,21 @@ target_link_libraries(CharType_unittest
quickstep_utility_MemStream)
add_test(CharType_unittest CharType_unittest)
+add_executable(DateType_unittest "${CMAKE_CURRENT_SOURCE_DIR}/tests/DateType_unittest.cpp")
+target_link_libraries(DateType_unittest
+ gtest
+ gtest_main
+ quickstep_types_DatetimeLit
+ quickstep_types_DateType
+ quickstep_types_NullType
+ quickstep_types_Type
+ quickstep_types_TypeFactory
+ quickstep_types_TypeID
+ quickstep_types_TypedValue
+ quickstep_types_tests_TypeTest_common
+ quickstep_utility_MemStream)
+add_test(DateType_unittest DateType_unittest)
+
add_executable(DatetimeIntervalType_unittest
"${CMAKE_CURRENT_SOURCE_DIR}/tests/DatetimeIntervalType_unittest.cpp")
target_link_libraries(DatetimeIntervalType_unittest
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/32bee124/types/DateType.cpp
----------------------------------------------------------------------
diff --git a/types/DateType.cpp b/types/DateType.cpp
new file mode 100644
index 0000000..9637acc
--- /dev/null
+++ b/types/DateType.cpp
@@ -0,0 +1,148 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#ifndef __STDC_FORMAT_MACROS
+#define __STDC_FORMAT_MACROS
+#endif
+
+#include "types/DateType.hpp"
+
+#include <cinttypes>
+#include <cstddef>
+#include <cstdint>
+#include <cstdio>
+#include <string>
+
+#include "types/DatetimeLit.hpp"
+#include "types/NullCoercibilityCheckMacro.hpp"
+#include "types/Type.hpp"
+#include "types/TypeID.hpp"
+#include "types/TypedValue.hpp"
+#include "utility/CheckSnprintf.hpp"
+
+#include "glog/logging.h"
+
+// NetBSD's libc has snprintf, but it doesn't show up in the std namespace for
+// C++.
+#ifndef __NetBSD__
+using std::snprintf;
+#endif
+
+namespace quickstep {
+
+bool DateType::isCoercibleFrom(const Type &original_type) const {
+ QUICKSTEP_NULL_COERCIBILITY_CHECK();
+ return (original_type.getTypeID() == kDate);
+}
+
+bool DateType::isSafelyCoercibleFrom(const Type &original_type) const {
+ QUICKSTEP_NULL_COERCIBILITY_CHECK();
+ return (original_type.getTypeID() == kDate);
+}
+
+std::string DateType::printValueToString(const TypedValue &value) const {
+ DCHECK(!value.isNull());
+
+ const DateLit literal = value.getLiteral<DateLit>();
+ const std::int32_t year = literal.year;
+
+ char datebuf[DateLit::kIsoChars + 1];
+ std::size_t chars_written = 0;
+ int snprintf_result = 0;
+
+ // ISO 8601 requires that "expanded" year ranges (> 4 digits or before year
+ // 0) are prefixed with a plus or minus.
+ if ((year > 9999) || (year < 0)) {
+ snprintf_result = snprintf(datebuf, sizeof(datebuf), "%+05" PRId32 "-", year);
+ } else {
+ snprintf_result = snprintf(datebuf, sizeof(datebuf), "%04" PRId32 "-", year);
+ }
+ CheckSnprintf(snprintf_result, sizeof(datebuf), &chars_written);
+
+ // All the rest of the ISO 8601 date/time parts.
+ snprintf_result = snprintf(datebuf + chars_written,
+ sizeof(datebuf) - chars_written,
+ "%02d-%02d",
+ literal.month,
+ literal.day);
+ CheckSnprintf(snprintf_result, sizeof(datebuf), &chars_written);
+
+ return std::string(datebuf);
+}
+
+void DateType::printValueToFile(const TypedValue &value,
+ FILE *file,
+ const int padding) const {
+ // We simply re-use the logic from printValueToString(), as trying to do
+ // padding on-the fly with so many different fields is too much of a hassle.
+ std::fprintf(file, "%*s", padding, printValueToString(value).c_str());
+}
+
+bool DateType::parseValueFromString(const std::string &value_string,
+ TypedValue *value) const {
+ std::int32_t year;
+ std::uint32_t month, day;
+
+ int date_chars = 0;
+
+ const int matched = std::sscanf(value_string.c_str(),
+ "%d-%2u-%2u%n",
+ &year, &month, &day, &date_chars);
+
+ // Check that the string is date, then check that there is no unmatched
+ // garbage at the end of the string.
+ if (matched != 3 ||
+ (static_cast<std::string::size_type>(date_chars) != value_string.length())) {
+ return false;
+ }
+
+ // Validate month.
+ if ((month == 0) || (month > 12)) {
+ return false;
+ }
+
+ // Validate day-of-month.
+ if (day == 0) {
+ return false;
+ }
+ switch (month) {
+ case 2: {
+ const std::uint32_t days_in_february
+ = ((year & 3) == 0 && ((year % 25) != 0 || (year & 15) == 0))
+ ? 29 : 28;
+ if (day > days_in_february) {
+ return false;
+ }
+ break;
+ }
+ case 4: // Fall-through for 30-day months.
+ case 6:
+ case 9:
+ case 11:
+ if (day > 30) {
+ return false;
+ }
+ break;
+ default:
+ if (day > 31) {
+ return false;
+ }
+ break;
+ }
+
+ *value = TypedValue(DateLit::Create(year, month, day));
+ return true;
+}
+
+} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/32bee124/types/DateType.hpp
----------------------------------------------------------------------
diff --git a/types/DateType.hpp b/types/DateType.hpp
new file mode 100644
index 0000000..2a13be1
--- /dev/null
+++ b/types/DateType.hpp
@@ -0,0 +1,132 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#ifndef QUICKSTEP_TYPES_DATE_TYPE_HPP_
+#define QUICKSTEP_TYPES_DATE_TYPE_HPP_
+
+#include <cstddef>
+#include <cstdio>
+#include <string>
+
+#include "types/DatetimeLit.hpp"
+#include "types/Type.hpp"
+#include "types/TypeID.hpp"
+#include "utility/Macros.hpp"
+
+namespace quickstep {
+
+class TypedValue;
+
+/** \addtogroup Types
+ * @{
+ */
+
+/**
+ * @brief A type representing the date.
+ **/
+class DateType : public Type {
+ public:
+ typedef DateLit cpptype;
+
+ static const TypeID kStaticTypeID = kDate;
+
+ /**
+ * @brief Get a reference to the non-nullable singleton instance of this
+ * Type.
+ *
+ * @return A reference to the non-nullable singleton instance of this Type.
+ **/
+ static const DateType& InstanceNonNullable() {
+ static DateType instance(false);
+ return instance;
+ }
+
+ /**
+ * @brief Get a reference to the nullable singleton instance of this Type.
+ *
+ * @return A reference to the nullable singleton instance of this Type.
+ **/
+ static const DateType& InstanceNullable() {
+ static DateType instance(true);
+ return instance;
+ }
+
+ /**
+ * @brief Get a reference to a singleton instance of this Type.
+ *
+ * @param nullable Whether to get the nullable version of this Type.
+ * @return A reference to the desired singleton instance of this Type.
+ **/
+ static const DateType& Instance(const bool nullable) {
+ if (nullable) {
+ return InstanceNullable();
+ } else {
+ return InstanceNonNullable();
+ }
+ }
+
+ const Type& getNullableVersion() const override {
+ return InstanceNullable();
+ }
+
+ const Type& getNonNullableVersion() const override {
+ return InstanceNonNullable();
+ }
+
+ std::size_t estimateAverageByteLength() const override {
+ return sizeof(DateLit);
+ }
+
+ bool isCoercibleFrom(const Type &original_type) const override;
+
+ bool isSafelyCoercibleFrom(const Type &original_type) const override;
+
+ int getPrintWidth() const override {
+ return DateLit::kIsoChars;
+ }
+
+ std::string printValueToString(const TypedValue &value) const override;
+
+ void printValueToFile(const TypedValue &value,
+ FILE *file,
+ const int padding = 0) const override;
+
+ /**
+ * @note value_string is expected to be in (possibly extended) ISO-8601
+ * format. Extended ISO-8601 date format is "YYYY-MM-DD". YYYY is an
+ * integer year (in extended format, this may be more than 4 digits and
+ * include an optional sign prefix +/-). MM is a 2-digit month in the
+ * range 01-12. DD is a 2-digit day of month in the appropriate range
+ * for the month (i.e. 28 days for February in non-leap years, 29 days
+ * for February in leap years, 30 days for April, June, September, and
+ * November, and 31 days for all other months). Note that parsing will
+ * fail if there are any "extra" characters at the end of the string
+ * after a parsable ISO-8601 date.
+ **/
+ bool parseValueFromString(const std::string &value_string,
+ TypedValue *value) const override;
+
+ private:
+ explicit DateType(const bool nullable)
+ : Type(Type::kOther, kDate, nullable, sizeof(DateLit), sizeof(DateLit)) {
+ }
+
+ DISALLOW_COPY_AND_ASSIGN(DateType);
+};
+
+/** @} */
+
+} // namespace quickstep
+
+#endif // QUICKSTEP_TYPES_DATE_TYPE_HPP_
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/32bee124/types/DatetimeLit.hpp
----------------------------------------------------------------------
diff --git a/types/DatetimeLit.hpp b/types/DatetimeLit.hpp
index 017bcce..c87b510 100644
--- a/types/DatetimeLit.hpp
+++ b/types/DatetimeLit.hpp
@@ -31,6 +31,67 @@ namespace quickstep {
*/
/**
+ * @brief A literal representing the date.
+ **/
+struct DateLit {
+ // Note that although there is no year 0 in the Gregorian calendar, ISO 8601
+ // has year 0 equivalent to 1 BCE, year -1 equivalent to 2 BCE, and so on.
+ std::int32_t year;
+ std::uint8_t month, day;
+
+ // The maximum number of characters needed to represent any date in ISO 8601
+ // notation.
+ static constexpr int kIsoChars
+ = 1 // + or - prefix
+ + 5 // Year digits
+ + 1 // -
+ + 2 // Month
+ + 1 // -
+ + 2; // Day
+
+ static DateLit Create(const std::int32_t _year,
+ const std::uint8_t _month,
+ const std::uint8_t _day) {
+ DateLit date;
+ date.year = _year;
+ date.month = _month;
+ date.day = _day;
+
+ return date;
+ }
+
+ inline bool operator< (const DateLit& rhs) const {
+ return (year != rhs.year)
+ ? (year < rhs.year)
+ : ((month != rhs.month) ? (month < rhs.month) : (day < rhs.day));
+ }
+
+ inline bool operator> (const DateLit& rhs) const {
+ return (year != rhs.year)
+ ? (year > rhs.year)
+ : ((month != rhs.month) ? (month > rhs.month) : (day > rhs.day));
+ }
+
+ inline bool operator<=(const DateLit& rhs) const {
+ return !(*this > rhs);
+ }
+
+ inline bool operator>=(const DateLit& rhs) const {
+ return !(*this < rhs);
+ }
+
+ inline bool operator==(const DateLit& rhs) const {
+ return (year == rhs.year) &&
+ (month == rhs.month) &&
+ (day == rhs.day);
+ }
+
+ inline bool operator!=(const DateLit& rhs) const {
+ return !(*this == rhs);
+ }
+};
+
+/**
* @brief A literal representing the datetime.
**/
struct DatetimeLit {
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/32bee124/types/Type.cpp
----------------------------------------------------------------------
diff --git a/types/Type.cpp b/types/Type.cpp
index 8981bc2..b89b2ac 100644
--- a/types/Type.cpp
+++ b/types/Type.cpp
@@ -41,6 +41,9 @@ serialization::Type Type::getProto() const {
case kDouble:
proto.set_type_id(serialization::Type::DOUBLE);
break;
+ case kDate:
+ proto.set_type_id(serialization::Type::DATE);
+ break;
case kDatetime:
proto.set_type_id(serialization::Type::DATETIME);
break;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/32bee124/types/Type.hpp
----------------------------------------------------------------------
diff --git a/types/Type.hpp b/types/Type.hpp
index fc6f4e2..84a0de0 100644
--- a/types/Type.hpp
+++ b/types/Type.hpp
@@ -32,6 +32,7 @@
namespace quickstep {
+struct DateLit;
struct DatetimeIntervalLit;
struct DatetimeLit;
struct YearMonthIntervalLit;
@@ -95,7 +96,7 @@ class Type {
enum SuperTypeID {
kNumeric = 0, // Fixed-length numeric types (Int, Long, Float, Double)
kAsciiString, // ASCII strings (Char, VarChar)
- kOther // Others (Datetime, DatetimeInterval, YearMonthInterval)
+ kOther // Others (Date, Datetime, DatetimeInterval, YearMonthInterval)
};
/**
@@ -370,6 +371,8 @@ class Type {
return TypedValue(*static_cast<const float*>(value_ptr));
case kDouble:
return TypedValue(*static_cast<const double*>(value_ptr));
+ case kDate:
+ return TypedValue(*static_cast<const DateLit*>(value_ptr));
case kDatetime:
return TypedValue(*static_cast<const DatetimeLit*>(value_ptr));
case kDatetimeInterval:
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/32bee124/types/Type.proto
----------------------------------------------------------------------
diff --git a/types/Type.proto b/types/Type.proto
index dbf248e..cfcb713 100644
--- a/types/Type.proto
+++ b/types/Type.proto
@@ -29,6 +29,7 @@ message Type {
DATETIME_INTERVAL = 7;
YEAR_MONTH_INTERVAL = 8;
NULL_TYPE = 9;
+ DATE = 10;
}
required TypeID type_id = 1;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/32bee124/types/TypeFactory.cpp
----------------------------------------------------------------------
diff --git a/types/TypeFactory.cpp b/types/TypeFactory.cpp
index a8f2961..3709a50 100644
--- a/types/TypeFactory.cpp
+++ b/types/TypeFactory.cpp
@@ -18,8 +18,10 @@
#include "types/TypeFactory.hpp"
#include <cstddef>
+#include <string>
#include "types/CharType.hpp"
+#include "types/DateType.hpp"
#include "types/DatetimeIntervalType.hpp"
#include "types/DatetimeType.hpp"
#include "types/DoubleType.hpp"
@@ -49,6 +51,8 @@ const Type& TypeFactory::GetType(const TypeID id,
return FloatType::Instance(nullable);
case kDouble:
return DoubleType::Instance(nullable);
+ case kDate:
+ return DateType::Instance(nullable);
case kDatetime:
return DatetimeType::Instance(nullable);
case kDatetimeInterval:
@@ -90,6 +94,7 @@ bool TypeFactory::ProtoIsValid(const serialization::Type &proto) {
case serialization::Type::LONG:
case serialization::Type::FLOAT:
case serialization::Type::DOUBLE:
+ case serialization::Type::DATE:
case serialization::Type::DATETIME:
case serialization::Type::DATETIME_INTERVAL:
case serialization::Type::YEAR_MONTH_INTERVAL:
@@ -119,6 +124,8 @@ const Type& TypeFactory::ReconstructFromProto(const serialization::Type &proto)
return FloatType::Instance(proto.nullable());
case serialization::Type::DOUBLE:
return DoubleType::Instance(proto.nullable());
+ case serialization::Type::DATE:
+ return DateType::Instance(proto.nullable());
case serialization::Type::DATETIME:
return DatetimeType::Instance(proto.nullable());
case serialization::Type::DATETIME_INTERVAL:
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/32bee124/types/TypeFactory.hpp
----------------------------------------------------------------------
diff --git a/types/TypeFactory.hpp b/types/TypeFactory.hpp
index ed9be5e..e63a613 100644
--- a/types/TypeFactory.hpp
+++ b/types/TypeFactory.hpp
@@ -52,6 +52,7 @@ class TypeFactory {
case kLong:
case kFloat:
case kDouble:
+ case kDate:
case kDatetime:
case kDatetimeInterval:
case kYearMonthInterval:
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/32bee124/types/TypeID.cpp
----------------------------------------------------------------------
diff --git a/types/TypeID.cpp b/types/TypeID.cpp
index 1aeea04..6a5a4c7 100644
--- a/types/TypeID.cpp
+++ b/types/TypeID.cpp
@@ -26,6 +26,7 @@ const char *kTypeNames[] = {
"Double",
"Char",
"VarChar",
+ "Date",
"Datetime",
"DatetimeInterval",
"YearMonthInterval",
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/32bee124/types/TypeID.hpp
----------------------------------------------------------------------
diff --git a/types/TypeID.hpp b/types/TypeID.hpp
index 23d32da..069e9c0 100644
--- a/types/TypeID.hpp
+++ b/types/TypeID.hpp
@@ -34,6 +34,7 @@ enum TypeID {
kDouble,
kChar,
kVarChar,
+ kDate,
kDatetime,
kDatetimeInterval,
kYearMonthInterval,
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/32bee124/types/TypedValue.cpp
----------------------------------------------------------------------
diff --git a/types/TypedValue.cpp b/types/TypedValue.cpp
index bc1ebd9..cef7cb0 100644
--- a/types/TypedValue.cpp
+++ b/types/TypedValue.cpp
@@ -49,6 +49,7 @@ bool TypedValue::isPlausibleInstanceOf(const TypeSignature type) const {
case kLong:
case kFloat:
case kDouble:
+ case kDate:
case kDatetime:
case kDatetimeInterval:
case kYearMonthInterval:
@@ -104,6 +105,15 @@ serialization::TypedValue TypedValue::getProto() const {
proto.set_double_value(getLiteral<double>());
}
break;
+ case kDate:
+ proto.set_type_id(serialization::Type::DATE);
+ if (!isNull()) {
+ serialization::TypedValue::DateLit *literal_date_proto = proto.mutable_date_value();
+ literal_date_proto->set_year(value_union_.date_value.year);
+ literal_date_proto->set_month(value_union_.date_value.month);
+ literal_date_proto->set_day(value_union_.date_value.day);
+ }
+ break;
case kDatetime:
proto.set_type_id(serialization::Type::DATETIME);
if (!isNull()) {
@@ -171,6 +181,14 @@ TypedValue TypedValue::ReconstructFromProto(const serialization::TypedValue &pro
return proto.has_double_value() ?
TypedValue(static_cast<double>(proto.double_value())) :
TypedValue(kDouble);
+ case serialization::Type::DATE:
+ if (proto.has_date_value()) {
+ return TypedValue(DateLit::Create(proto.date_value().year(),
+ proto.date_value().month(),
+ proto.date_value().day()));
+ } else {
+ return TypedValue(kDate);
+ }
case serialization::Type::DATETIME:
if (proto.has_datetime_value()) {
DatetimeLit datetime;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/32bee124/types/TypedValue.hpp
----------------------------------------------------------------------
diff --git a/types/TypedValue.hpp b/types/TypedValue.hpp
index 6e22111..10c3c05 100644
--- a/types/TypedValue.hpp
+++ b/types/TypedValue.hpp
@@ -127,6 +127,14 @@ class TypedValue {
}
/**
+ * @brief Constructor for a literal value of DateType.
+ **/
+ explicit TypedValue(const DateLit literal_date)
+ : value_info_(static_cast<std::uint64_t>(kDate)) {
+ value_union_.date_value = literal_date;
+ }
+
+ /**
* @brief Constructor for a literal value of DatetimeType.
**/
explicit TypedValue(const DatetimeLit literal_datetime)
@@ -276,6 +284,7 @@ class TypedValue {
case kLong:
case kFloat:
case kDouble:
+ case kDate:
case kDatetime:
case kDatetimeInterval:
case kYearMonthInterval:
@@ -307,6 +316,7 @@ class TypedValue {
return sizeof(value_union_.int_value) <= sizeof(std::size_t);
case kLong:
case kDouble:
+ case kDate:
case kDatetime:
case kDatetimeInterval:
case kYearMonthInterval:
@@ -384,6 +394,7 @@ class TypedValue {
return sizeof(int);
case kLong:
case kDouble:
+ case kDate:
case kDatetime:
case kDatetimeInterval:
case kYearMonthInterval:
@@ -469,6 +480,7 @@ class TypedValue {
|| getTypeID() == kLong
|| getTypeID() == kFloat
|| getTypeID() == kDouble
+ || getTypeID() == kDate
|| getTypeID() == kDatetime
|| getTypeID() == kDatetimeInterval
|| getTypeID() == kYearMonthInterval));
@@ -564,6 +576,7 @@ class TypedValue {
case kLong:
case kFloat:
case kDouble:
+ case kDate:
case kDatetime:
case kDatetimeInterval:
case kYearMonthInterval:
@@ -659,6 +672,7 @@ class TypedValue {
case kLong:
case kFloat:
case kDouble:
+ case kDate:
case kDatetime:
case kDatetimeInterval:
case kYearMonthInterval:
@@ -778,6 +792,7 @@ class TypedValue {
float float_value;
double double_value;
const void* out_of_line_data;
+ DateLit date_value;
DatetimeLit datetime_value;
DatetimeIntervalLit datetime_interval_value;
YearMonthIntervalLit year_month_interval_value;
@@ -853,6 +868,13 @@ inline double TypedValue::getLiteral<double>() const {
}
template <>
+inline DateLit TypedValue::getLiteral<DateLit>() const {
+ DCHECK_EQ(kDate, getTypeID());
+ DCHECK(!isNull());
+ return value_union_.date_value;
+}
+
+template <>
inline DatetimeLit TypedValue::getLiteral<DatetimeLit>() const {
DCHECK_EQ(kDatetime, getTypeID());
DCHECK(!isNull());
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/32bee124/types/TypedValue.proto
----------------------------------------------------------------------
diff --git a/types/TypedValue.proto b/types/TypedValue.proto
index 78a38cb..c265c73 100644
--- a/types/TypedValue.proto
+++ b/types/TypedValue.proto
@@ -31,4 +31,12 @@ message TypedValue {
optional int64 datetime_value = 7;
optional int64 datetime_interval_value = 8;
optional int64 year_month_interval_value = 9;
+
+ message DateLit {
+ required int32 year = 1;
+ required uint32 month = 2;
+ required uint32 day = 3;
+ }
+
+ optional DateLit date_value = 10;
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/32bee124/types/tests/DateType_unittest.cpp
----------------------------------------------------------------------
diff --git a/types/tests/DateType_unittest.cpp b/types/tests/DateType_unittest.cpp
new file mode 100644
index 0000000..637f369
--- /dev/null
+++ b/types/tests/DateType_unittest.cpp
@@ -0,0 +1,146 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#include <cstddef>
+#include <string>
+
+#include "types/DateType.hpp"
+#include "types/DatetimeLit.hpp"
+#include "types/NullType.hpp"
+#include "types/Type.hpp"
+#include "types/TypeFactory.hpp"
+#include "types/TypeID.hpp"
+#include "types/TypedValue.hpp"
+#include "types/tests/TypeTest_common.hpp"
+#include "utility/MemStream.hpp"
+
+#include "gtest/gtest.h"
+
+namespace quickstep {
+
+TEST(DateTypeTest, PrintValueTest) {
+ const Type &date_type = TypeFactory::GetType(kDate);
+
+ // Skynet becomes self-aware at 2:14 AM on August 29, 1997.
+ TypedValue judgement_day_value(DateLit::Create(1997, 8, 29));
+ EXPECT_EQ(std::string("1997-08-29"),
+ date_type.printValueToString(judgement_day_value));
+
+ // Check extended formatting for years far in the past.
+ TypedValue dawn_age_value(DateLit::Create(-18017, 4, 13));
+ EXPECT_EQ(std::string("-18017-04-13"),
+ date_type.printValueToString(dawn_age_value));
+}
+
+namespace {
+
+static const std::size_t kPaddedLength = 200;
+
+void CheckPrintValueToFile(const DateLit literal_value,
+ const std::string &expected_string) {
+ // Test writing to FILE* with a MemStream.
+ MemStream memstream;
+
+ const Type &date_type = TypeFactory::GetType(kDate);
+ const TypedValue date_value = date_type.makeValue(&literal_value);
+
+ // First, check printing without any padding
+ date_type.printValueToFile(date_value, memstream.file());
+ EXPECT_STREQ(expected_string.c_str(), memstream.str());
+ memstream.reset();
+
+ // Specifying padding less than the length needed to print the value should
+ // have no effect.
+ date_type.printValueToFile(date_value, memstream.file(), 1);
+ EXPECT_STREQ(expected_string.c_str(), memstream.str());
+ memstream.reset();
+
+ // Test padding up to a specified length.
+ std::string padded(kPaddedLength - expected_string.size(), ' ');
+ padded.append(expected_string);
+ date_type.printValueToFile(date_value, memstream.file(), kPaddedLength);
+ EXPECT_STREQ(padded.c_str(), memstream.str());
+ memstream.reset();
+}
+
+} // namespace
+
+TEST(DateTypeTest, PrintValueToFileTest) {
+ CheckPrintValueToFile(DateLit::Create(1997, 8, 29), "1997-08-29");
+
+ CheckPrintValueToFile(DateLit::Create(-18017, 4, 13), "-18017-04-13");
+}
+
+TEST(DateTypeTest, ParseValueFromStringTest) {
+ // Parse the same value in a bunch of different valid format variations.
+ type_test::CheckSuccessfulParseLiteralValueFromString<DateType>(
+ DateLit::Create(1997, 8, 29),
+ "1997-08-29");
+ type_test::CheckSuccessfulParseLiteralValueFromString<DateType>(
+ DateLit::Create(1997, 8, 29),
+ "+1997-08-29");
+ type_test::CheckSuccessfulParseLiteralValueFromString<DateType>(
+ DateLit::Create(1997, 8, 29),
+ "+001997-08-29");
+
+ type_test::CheckSuccessfulParseLiteralValueFromString<DateType>(
+ DateLit::Create(1871, 3, 18),
+ "1871-03-18");
+ type_test::CheckSuccessfulParseLiteralValueFromString<DateType>(
+ DateLit::Create(1871, 3, 18),
+ "+1871-03-18");
+ type_test::CheckSuccessfulParseLiteralValueFromString<DateType>(
+ DateLit::Create(1871, 3, 18),
+ "+001871-03-18");
+
+ // Parse extended format with negative year.
+ type_test::CheckSuccessfulParseLiteralValueFromString<DateType>(
+ DateLit::Create(-18017, 4, 13),
+ "-18017-04-13");
+
+ // Test some parses that we expect to fail.
+ const Type &date_type = TypeFactory::GetType(kDate);
+ TypedValue value;
+ EXPECT_FALSE(date_type.parseValueFromString("foo", &value));
+ EXPECT_FALSE(date_type.parseValueFromString("42", &value));
+ EXPECT_FALSE(date_type.parseValueFromString("1997-08-29 ", &value));
+ EXPECT_FALSE(date_type.parseValueFromString(" 1997-08-29 ", &value));
+ EXPECT_FALSE(date_type.parseValueFromString("1997-08-29.", &value));
+ EXPECT_FALSE(date_type.parseValueFromString("1997-08-29q", &value));
+ EXPECT_FALSE(date_type.parseValueFromString("1997-08-29.q", &value));
+ EXPECT_FALSE(date_type.parseValueFromString("1e3-08-29", &value));
+ EXPECT_FALSE(date_type.parseValueFromString("--18017-04-13", &value));
+ EXPECT_FALSE(date_type.parseValueFromString("1997-08--29 ", &value));
+ EXPECT_FALSE(date_type.parseValueFromString("1997-08--2 ", &value));
+}
+
+TEST(DateTypeTest, IsCoercibleFromTest) {
+ type_test::CheckIsCoercibleFrom(DateType::InstanceNonNullable(),
+ {kDate});
+}
+
+TEST(DateTypeTest, IsSafelyCoercibleFromTest) {
+ type_test::CheckIsSafelyCoercibleFrom(DateType::InstanceNonNullable(),
+ {kDate});
+}
+
+TEST(DateTypeTest, CoerceValueTest) {
+ // The epoch.
+ type_test::CheckLiteralCoerce<DateType, DateType>(DateLit::Create(1970, 1, 1));
+
+ type_test::CheckNullLiteralCoerce<DateType, DateType>();
+ type_test::CheckNullLiteralCoerce<DateType, NullType>();
+}
+
+} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/32bee124/types/tests/TypeTest_common.hpp
----------------------------------------------------------------------
diff --git a/types/tests/TypeTest_common.hpp b/types/tests/TypeTest_common.hpp
index 7cc9cc2..d185405 100644
--- a/types/tests/TypeTest_common.hpp
+++ b/types/tests/TypeTest_common.hpp
@@ -101,7 +101,7 @@ void CheckIsCoercibleFrom(
// Can't coerce from other types.
for (const TypeID original_type_id
: {kInt, kLong, kFloat, kDouble, kChar, kVarChar,
- kDatetime, kDatetimeInterval, kYearMonthInterval}) {
+ kDate, kDatetime, kDatetimeInterval, kYearMonthInterval}) {
if (std::find(expected_coercible.begin(), expected_coercible.end(), original_type_id)
== expected_coercible.end()) {
const Type &original_type = TypeFactory::TypeRequiresLengthParameter(original_type_id)
@@ -183,7 +183,7 @@ void CheckStringTypeIsCoercibleFrom(const TypeID target_type_id,
// Other types are not coercible to strings.
for (const TypeID original_type_id
- : {kInt, kLong, kFloat, kDouble, kDatetime, kDatetimeInterval, kYearMonthInterval}) {
+ : {kInt, kLong, kFloat, kDouble, kDate, kDatetime, kDatetimeInterval, kYearMonthInterval}) {
const Type &original_type = TypeFactory::TypeRequiresLengthParameter(original_type_id)
? TypeFactory::GetType(original_type_id, 10, false)
: TypeFactory::GetType(original_type_id, false);
@@ -232,7 +232,7 @@ void CheckIsSafelyCoercibleFrom(
// Can't coerce from other types.
for (const TypeID original_type_id
: {kInt, kLong, kFloat, kDouble, kChar, kVarChar,
- kDatetime, kDatetimeInterval, kYearMonthInterval}) {
+ kDate, kDatetime, kDatetimeInterval, kYearMonthInterval}) {
if (std::find(expected_coercible.begin(), expected_coercible.end(), original_type_id)
== expected_coercible.end()) {
const Type &original_type = TypeFactory::TypeRequiresLengthParameter(original_type_id)
@@ -314,7 +314,7 @@ void CheckStringTypeIsSafelyCoercibleFrom(const TypeID target_type_id,
// Other types are not coercible to strings.
for (const TypeID original_type_id
- : {kInt, kLong, kFloat, kDouble, kDatetime, kDatetimeInterval, kYearMonthInterval}) {
+ : {kInt, kLong, kFloat, kDouble, kDate, kDatetime, kDatetimeInterval, kYearMonthInterval}) {
const Type &original_type = TypeFactory::TypeRequiresLengthParameter(original_type_id)
? TypeFactory::GetType(original_type_id, 10, false)
: TypeFactory::GetType(original_type_id, false);
[02/25] incubator-quickstep git commit: Remove unused vector_based
HashJoin collector type
Posted by zu...@apache.org.
Remove unused vector_based HashJoin collector type
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/ba2e5bf8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/ba2e5bf8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/ba2e5bf8
Branch: refs/heads/dist-exe-test-new
Commit: ba2e5bf875373be188ddc84c4e90fbc1b1d676c5
Parents: 50edb49
Author: Navneet Potti <na...@gmail.com>
Authored: Mon Jun 27 11:00:04 2016 -0500
Committer: Zuyu Zhang <zu...@twitter.com>
Committed: Fri Jul 29 16:42:09 2016 -0700
----------------------------------------------------------------------
relational_operators/HashJoinOperator.cpp | 116 +------------------------
relational_operators/HashJoinOperator.hpp | 3 -
2 files changed, 2 insertions(+), 117 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/ba2e5bf8/relational_operators/HashJoinOperator.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/HashJoinOperator.cpp b/relational_operators/HashJoinOperator.cpp
index 5a47b50..667df1e 100644
--- a/relational_operators/HashJoinOperator.cpp
+++ b/relational_operators/HashJoinOperator.cpp
@@ -61,25 +61,9 @@ namespace quickstep {
namespace {
-DEFINE_bool(vector_based_joined_tuple_collector, false,
- "If true, use simple vector-based joined tuple collector in "
- "hash join, with a final sort pass to group joined tuple pairs "
- "by inner block. If false, use unordered_map based collector that "
- "keeps joined pairs grouped by inner block as they are found "
- "(this latter option has exhibited performance/scaling problems, "
- "particularly in NUMA contexts).");
-
// Functor passed to HashTable::getAllFromValueAccessor() to collect matching
-// tuples from the inner relation. This version stores matching tuple ID pairs
+// tuples from the inner relation. It stores matching tuple ID pairs
// in an unordered_map keyed by inner block ID.
-//
-// NOTE(chasseur): Performance testing has shown that this particular
-// implementation has problems scaling in a multisocket NUMA machine.
-// Additional benchmarking revealed problems using the STL unordered_map class
-// in a NUMA system (at least for the implementation in GNU libstdc++), even
-// though instances of this class and the internal unordered_map are private to
-// a single thread. Because of this, VectorBasedJoinedTupleCollector is used by
-// default instead.
class MapBasedJoinedTupleCollector {
public:
MapBasedJoinedTupleCollector() {
@@ -91,13 +75,6 @@ class MapBasedJoinedTupleCollector {
joined_tuples_[tref.block].emplace_back(tref.tuple, accessor.getCurrentPosition());
}
- // Consolidation is a no-op for this version, but we provide this trivial
- // call so that MapBasedJoinedTupleCollector and
- // VectorBasedJoinedTupleCollector have the same interface and can both be
- // used in the templated HashInnerJoinWorkOrder::executeWithCollectorType() method.
- inline void consolidate() const {
- }
-
// Get a mutable pointer to the collected map of joined tuple ID pairs. The
// key is inner block_id, values are vectors of joined tuple ID pairs with
// tuple ID from the inner block on the left and the outer block on the
@@ -116,82 +93,6 @@ class MapBasedJoinedTupleCollector {
std::unordered_map<block_id, std::vector<std::pair<tuple_id, tuple_id>>> joined_tuples_;
};
-// Compare std::pair instances based on their first element only.
-template <typename PairT>
-inline bool CompareFirst(const PairT &left, const PairT &right) {
- return left.first < right.first;
-}
-
-// Functor passed to HashTable::getAllFromValueAccessor() to collect matching
-// tuples from the inner relation. This version stores inner block ID and pairs
-// of joined tuple IDs in an unsorted vector, which should then be sorted with
-// a call to consolidate() before materializing join output.
-//
-// NOTE(chasseur): Because of NUMA scaling issues for
-// MapBasedJoinedTupleCollector noted above, this implementation is the
-// default.
-class VectorBasedJoinedTupleCollector {
- public:
- VectorBasedJoinedTupleCollector() {
- }
-
- template <typename ValueAccessorT>
- inline void operator()(const ValueAccessorT &accessor,
- const TupleReference &tref) {
- joined_tuples_.emplace_back(tref.block,
- std::make_pair(tref.tuple, accessor.getCurrentPosition()));
- }
-
- // Sorts joined tuple pairs by inner block ID. Must be called before
- // getJoinedTuples().
- void consolidate() {
- if (joined_tuples_.empty()) {
- return;
- }
-
- // Sort joined tuple_id pairs by inner block_id.
- std::sort(joined_tuples_.begin(),
- joined_tuples_.end(),
- CompareFirst<std::pair<block_id, std::pair<tuple_id, tuple_id>>>);
-
- // Make a single vector of joined block_id pairs for each inner block for
- // compatibility with other join-related APIs.
- consolidated_joined_tuples_.emplace_back(joined_tuples_.front().first,
- std::vector<std::pair<tuple_id, tuple_id>>());
-
- for (const std::pair<block_id, std::pair<tuple_id, tuple_id>> &match_entry
- : joined_tuples_) {
- if (match_entry.first == consolidated_joined_tuples_.back().first) {
- consolidated_joined_tuples_.back().second.emplace_back(match_entry.second);
- } else {
- consolidated_joined_tuples_.emplace_back(
- match_entry.first,
- std::vector<std::pair<tuple_id, tuple_id>>(1, match_entry.second));
- }
- }
- }
-
- // Get a mutable pointer to the collected joined tuple ID pairs. The returned
- // vector has a single entry for each inner block where there are matching
- // joined tuples (the inner block's ID is the first element of the pair). The
- // second element of each pair is another vector consisting of pairs of
- // joined tuple IDs (tuple ID from inner block on the left, from outer block
- // on the right).
- inline std::vector<std::pair<const block_id, std::vector<std::pair<tuple_id, tuple_id>>>>*
- getJoinedTuples() {
- return &consolidated_joined_tuples_;
- }
-
- private:
- // Unsorted vector of join matches that is appended to by call operator().
- std::vector<std::pair<block_id, std::pair<tuple_id, tuple_id>>> joined_tuples_;
-
- // Joined tuples sorted by inner block_id. consolidate() populates this from
- // 'joined_tuples_'.
- std::vector<std::pair<const block_id, std::vector<std::pair<tuple_id, tuple_id>>>>
- consolidated_joined_tuples_;
-};
-
class SemiAntiJoinTupleCollector {
public:
explicit SemiAntiJoinTupleCollector(const TupleStorageSubBlock &tuple_store) {
@@ -516,21 +417,12 @@ serialization::WorkOrder* HashJoinOperator::createOuterJoinWorkOrderProto(const
void HashInnerJoinWorkOrder::execute() {
- if (FLAGS_vector_based_joined_tuple_collector) {
- executeWithCollectorType<VectorBasedJoinedTupleCollector>();
- } else {
- executeWithCollectorType<MapBasedJoinedTupleCollector>();
- }
-}
-
-template <typename CollectorT>
-void HashInnerJoinWorkOrder::executeWithCollectorType() {
BlockReference probe_block(
storage_manager_->getBlock(block_id_, probe_relation_));
const TupleStorageSubBlock &probe_store = probe_block->getTupleStorageSubBlock();
std::unique_ptr<ValueAccessor> probe_accessor(probe_store.createValueAccessor());
- CollectorT collector;
+ MapBasedJoinedTupleCollector collector;
if (join_key_attributes_.size() == 1) {
hash_table_.getAllFromValueAccessor(
probe_accessor.get(),
@@ -544,7 +436,6 @@ void HashInnerJoinWorkOrder::executeWithCollectorType() {
any_join_key_attributes_nullable_,
&collector);
}
- collector.consolidate();
const relation_id build_relation_id = build_relation_.getID();
const relation_id probe_relation_id = probe_relation_.getID();
@@ -637,8 +528,6 @@ void HashSemiJoinWorkOrder::executeWithResidualPredicate() {
std::unique_ptr<ValueAccessor> probe_accessor(probe_store.createValueAccessor());
- // TODO(harshad) - Make this function work with both types of collectors.
-
// We collect all the matching probe relation tuples, as there's a residual
// preidcate that needs to be applied after collecting these matches.
MapBasedJoinedTupleCollector collector;
@@ -810,7 +699,6 @@ void HashAntiJoinWorkOrder::executeWithResidualPredicate() {
const TupleStorageSubBlock &probe_store = probe_block->getTupleStorageSubBlock();
std::unique_ptr<ValueAccessor> probe_accessor(probe_store.createValueAccessor());
- // TODO(harshad) - Make the following code work with both types of collectors.
MapBasedJoinedTupleCollector collector;
// We probe the hash table and get all the matches. Unlike
// executeWithoutResidualPredicate(), we have to collect all the matching
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/ba2e5bf8/relational_operators/HashJoinOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/HashJoinOperator.hpp b/relational_operators/HashJoinOperator.hpp
index 9762f04..5d3d7da 100644
--- a/relational_operators/HashJoinOperator.hpp
+++ b/relational_operators/HashJoinOperator.hpp
@@ -356,9 +356,6 @@ class HashInnerJoinWorkOrder : public WorkOrder {
void execute() override;
private:
- template <typename CollectorT>
- void executeWithCollectorType();
-
const CatalogRelationSchema &build_relation_;
const CatalogRelationSchema &probe_relation_;
const std::vector<attribute_id> join_key_attributes_;
[14/25] incubator-quickstep git commit: Add an example to the
quickstart guide
Posted by zu...@apache.org.
Add an example to the quickstart guide
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/f35e6901
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/f35e6901
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/f35e6901
Branch: refs/heads/dist-exe-test-new
Commit: f35e69017c4c3fd6374957a03aa9dd85a60ab6c4
Parents: ca09ec4
Author: Jignesh Patel <jm...@hotmail.com>
Authored: Thu Jul 14 01:27:05 2016 -0500
Committer: Zuyu Zhang <zu...@twitter.com>
Committed: Fri Jul 29 16:42:11 2016 -0700
----------------------------------------------------------------------
README.md | 75 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++--
1 file changed, 73 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f35e6901/README.md
----------------------------------------------------------------------
diff --git a/README.md b/README.md
index bdfb7fc..bf9ed8d 100644
--- a/README.md
+++ b/README.md
@@ -40,8 +40,79 @@ And, it is **open source!**
4. Checkout the dependencies: ```git submodule update```
5. Go into the build directory: ```cd build```
6. Create the Makefile: ```cmake -D CMAKE_BUILD_TYPE=Release ..```
-7. Build: ```make -j4```. Note you may replace the 4 with the number of cores on your machine.
-8. Start quickstep: ```./quickstep_cli_shell --initialize_db=true```. You can now fire SQL queries. To quit, you can type in ```quit;``` Your data is stored in the directory ```qsstor```
+7. Build: ```make -j4```. Note you may replace the 4 with the number of cores
+ on your machine.
+8. Start quickstep: ```./quickstep_cli_shell --initialize_db=true```. You can
+ now fire SQL queries. To quit, you can type in ```quit;``` Your data is
+ stored in the directory ```qsstor```. Note the next time you start Quickstep,
+ you can omit the ``` --initialize_db``` flag (as the database has already
+ been initialized), and simply start Quickstep as: ```./quickstep_cli_shell```.
+ There are also a number of optional flags that you can specify, and to see
+ the full list, you can type in: ```./quickstep_cli_shell --help```
+9. Next let us load some data and fire some queries. A few points to note:
+The SQL surface of Quickstep is small (it will grow over time). The
+traditional SQL CREATE TABLE and SELECT statements work. The data types
+that are supported include INTEGER, FLOAT, DOUBLE, VARCHAR, CHAR, DATE,
+and DATETIME. Quickstep also does not have support for NULLS or keys (yet).
+Let create two tables by typing into the Quickstep shell (which you opened
+in the step above.
+
+```
+CREATE TABLE Weather (cid INTEGER, recordDate DATE, highTemperature FLOAT, lowTemperature FLOAT);
+```
+
+and then,
+
+```
+CREATE TABLE City (cid Integer, name VARCHAR(80), state CHAR(2));
+```
+
+10. Next, let us insert some tuples in these two tables.
+ ```
+ INSERT INTO City VALUES (1, 'Madison', 'WI');
+ INSERT INTO City VALUES (2, 'Palo Alto', 'CA');
+ INSERT INTO Weather VALUES (1, '2015-11-1', 50, 30);
+ INSERT INTO Weather VALUES (1, '2015-11-2', 51, 32);
+ INSERT INTO Weather VALUES (2, '2015-11-1', 60, 50);
+ ```
+
+11. Now we can issue SQL queries such as:
+ a. Find all weather records for California:
+ ```
+ SELECT * FROM WEATHER W, City C WHERE C.cid = W.cid AND C.state = 'CA';
+ ```
+
+ b. Find the min and max temperature for each city, printing the ```cid```:
+ ```
+ SELECT cid, MIN(lowTemperature), MAX(highTemperature) FROM Weather GROUP BY cid;
+ ```
+
+ c. Find the min and max temperature for each city using a nested query, and
+ printing thie city name:
+ ```
+ SELECT * FROM City C, (SELECT cid, MIN(lowTemperature), MAX(highTemperature) FROM Weather GROUP BY cid) AS T WHERE C.cid = T.cid;
+ ```
+
+12. Quickstep also supports a COPY TABLE command. If you want to try that, then
+ from a separate shell file type in the following:
+
+ ```
+ echo "3|2015-11-3|49|29" > /tmp/tmp.tbl
+ echo "3|2015-11-4|48|28" >> /tmp/tmp.tbl
+ echo "3|2015-11-5|47|27" >> /tmp/tmp.tbl
+ ```
+
+ Then, load this new data by typing the following SQL in the Quickstep shell:
+
+ ```
+ COPY Weather FROM '/tmp/tmp.tbl' WITH (DELIMITER '|');
+ ```
+
+ Now, you have loaded three more tuples into the Weather table, and you can
+ fire the SQL queries above again against this modified database.
+
+ Remember, to quit Quickstep, you can type in ```quit;``` into the Quickstep
+ shell.
## Additional pointers
[11/25] incubator-quickstep git commit: Minor changes in profiling
work order output.
Posted by zu...@apache.org.
Minor changes in profiling work order output.
- Now prints query ID along with each work order entry.
- Removed spaces between two columns.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/a37bf265
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/a37bf265
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/a37bf265
Branch: refs/heads/dist-exe-test-new
Commit: a37bf26526c0ceb5419f1c80a9ef1933e597f824
Parents: e5224a1
Author: Harshad Deshmukh <hb...@apache.org>
Authored: Wed Jul 6 11:38:49 2016 -0500
Committer: Zuyu Zhang <zu...@twitter.com>
Committed: Fri Jul 29 16:42:10 2016 -0700
----------------------------------------------------------------------
query_execution/Foreman.cpp | 9 +++++----
1 file changed, 5 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/a37bf265/query_execution/Foreman.cpp
----------------------------------------------------------------------
diff --git a/query_execution/Foreman.cpp b/query_execution/Foreman.cpp
index f9f2e7a..98146e2 100644
--- a/query_execution/Foreman.cpp
+++ b/query_execution/Foreman.cpp
@@ -238,16 +238,17 @@ void Foreman::printWorkOrderProfilingResults(const std::size_t query_id,
const std::vector<
std::tuple<std::size_t, std::size_t, std::size_t>>
&recorded_times = policy_enforcer_->getProfilingResults(query_id);
- fputs("Worker ID, NUMA Socket, Operator ID, Time (microseconds)\n", out);
+ fputs("Query ID,Worker ID,NUMA Socket,Operator ID,Time (microseconds)\n", out);
for (auto workorder_entry : recorded_times) {
// Note: Index of the "worker thread index" in the tuple is 0.
const std::size_t worker_id = std::get<0>(workorder_entry);
fprintf(out,
- "%lu, %d, %lu, %lu\n",
+ "%lu,%lu,%d,%lu,%lu\n",
+ query_id,
worker_id,
worker_directory_->getNUMANode(worker_id),
- std::get<1>(workorder_entry),
- std::get<2>(workorder_entry));
+ std::get<1>(workorder_entry), // Operator ID.
+ std::get<2>(workorder_entry)); // Time.
}
}
[19/25] incubator-quickstep git commit: Minor bug fix in QueryManager
base.
Posted by zu...@apache.org.
Minor bug fix in QueryManager base.
- Modified the order in which we check the completion of query and
completion of an operator in the queryStatus function.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/18b9a1f7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/18b9a1f7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/18b9a1f7
Branch: refs/heads/dist-exe-test-new
Commit: 18b9a1f7d3829270e58098230d783045968a1021
Parents: f5bf353
Author: Harshad Deshmukh <hb...@apache.org>
Authored: Mon Jul 25 12:23:34 2016 -0500
Committer: Zuyu Zhang <zu...@twitter.com>
Committed: Fri Jul 29 16:43:24 2016 -0700
----------------------------------------------------------------------
query_execution/QueryManagerBase.cpp | 11 ++++++-----
1 file changed, 6 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/18b9a1f7/query_execution/QueryManagerBase.cpp
----------------------------------------------------------------------
diff --git a/query_execution/QueryManagerBase.cpp b/query_execution/QueryManagerBase.cpp
index 37beb02..d2a3341 100644
--- a/query_execution/QueryManagerBase.cpp
+++ b/query_execution/QueryManagerBase.cpp
@@ -71,15 +71,16 @@ QueryManagerBase::QueryManagerBase(QueryHandle *query_handle)
QueryManagerBase::QueryStatusCode QueryManagerBase::queryStatus(
const dag_node_index op_index) {
- if (query_exec_state_->hasExecutionFinished(op_index)) {
- return QueryStatusCode::kOperatorExecuted;
- }
-
- // As kQueryExecuted takes precedence over kOperatorExecuted, we check again.
+ // As kQueryExecuted takes precedence over kOperatorExecuted, we first check
+ // whether the query has finished its execution.
if (query_exec_state_->hasQueryExecutionFinished()) {
return QueryStatusCode::kQueryExecuted;
}
+ if (query_exec_state_->hasExecutionFinished(op_index)) {
+ return QueryStatusCode::kOperatorExecuted;
+ }
+
return QueryStatusCode::kNone;
}
[13/25] incubator-quickstep git commit: Refactored messages
processing in both PolicyEnforcer and QueryManager.
Posted by zu...@apache.org.
Refactored messages processing in both PolicyEnforcer and QueryManager.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/9bdce7d8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/9bdce7d8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/9bdce7d8
Branch: refs/heads/dist-exe-test-new
Commit: 9bdce7d88c0e6053d15cf850a78bfbd35c034537
Parents: cde7bcb
Author: Zuyu Zhang <zu...@apache.org>
Authored: Sat Jul 9 14:54:05 2016 -0700
Committer: Zuyu Zhang <zu...@twitter.com>
Committed: Fri Jul 29 16:42:11 2016 -0700
----------------------------------------------------------------------
query_execution/CMakeLists.txt | 10 +--
query_execution/PolicyEnforcer.cpp | 61 ++++++++++---
query_execution/PolicyEnforcer.hpp | 10 ++-
query_execution/QueryManagerBase.cpp | 91 ++------------------
query_execution/QueryManagerBase.hpp | 90 ++++++++++---------
query_execution/QueryManagerSingleNode.cpp | 4 +-
query_execution/QueryManagerSingleNode.hpp | 4 +-
.../tests/QueryManagerSingleNode_unittest.cpp | 79 +++--------------
8 files changed, 126 insertions(+), 223 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/9bdce7d8/query_execution/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_execution/CMakeLists.txt b/query_execution/CMakeLists.txt
index 5a9189c..028531d 100644
--- a/query_execution/CMakeLists.txt
+++ b/query_execution/CMakeLists.txt
@@ -91,8 +91,12 @@ target_link_libraries(quickstep_queryexecution_ForemanSingleNode
${GFLAGS_LIB_NAME})
target_link_libraries(quickstep_queryexecution_PolicyEnforcer
glog
+ quickstep_catalog_CatalogDatabase
+ quickstep_catalog_CatalogRelation
quickstep_catalog_CatalogTypedefs
+ quickstep_catalog_PartitionScheme
quickstep_queryexecution_QueryExecutionMessages_proto
+ quickstep_queryexecution_QueryExecutionState
quickstep_queryexecution_QueryExecutionTypedefs
quickstep_queryexecution_QueryManagerBase
quickstep_queryexecution_QueryManagerSingleNode
@@ -100,6 +104,7 @@ target_link_libraries(quickstep_queryexecution_PolicyEnforcer
quickstep_queryexecution_WorkerMessage
quickstep_queryoptimizer_QueryHandle
quickstep_relationaloperators_WorkOrder
+ quickstep_storage_StorageBlockInfo
quickstep_utility_Macros
tmb
${GFLAGS_LIB_NAME})
@@ -152,14 +157,9 @@ target_link_libraries(quickstep_queryexecution_QueryExecutionUtil
quickstep_utility_Macros
tmb)
target_link_libraries(quickstep_queryexecution_QueryManagerBase
- quickstep_catalog_CatalogDatabase
- quickstep_catalog_CatalogRelation
quickstep_catalog_CatalogTypedefs
- quickstep_catalog_PartitionScheme
quickstep_queryexecution_QueryContext
- quickstep_queryexecution_QueryExecutionMessages_proto
quickstep_queryexecution_QueryExecutionState
- quickstep_queryexecution_QueryExecutionTypedefs
quickstep_queryoptimizer_QueryHandle
quickstep_queryoptimizer_QueryPlan
quickstep_relationaloperators_RelationalOperator
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/9bdce7d8/query_execution/PolicyEnforcer.cpp
----------------------------------------------------------------------
diff --git a/query_execution/PolicyEnforcer.cpp b/query_execution/PolicyEnforcer.cpp
index f310ee1..4cba8c5 100644
--- a/query_execution/PolicyEnforcer.cpp
+++ b/query_execution/PolicyEnforcer.cpp
@@ -24,12 +24,19 @@
#include <unordered_map>
#include <vector>
+#include "catalog/CatalogDatabase.hpp"
+#include "catalog/CatalogRelation.hpp"
#include "catalog/CatalogTypedefs.hpp"
+#include "catalog/PartitionScheme.hpp"
#include "query_execution/QueryExecutionMessages.pb.h"
+#include "query_execution/QueryExecutionState.hpp"
+#include "query_execution/QueryManagerBase.hpp"
#include "query_execution/QueryManagerSingleNode.hpp"
#include "query_execution/WorkerDirectory.hpp"
+#include "query_execution/WorkerMessage.hpp"
#include "query_optimizer/QueryHandle.hpp"
#include "relational_operators/WorkOrder.hpp"
+#include "storage/StorageBlockInfo.hpp"
#include "gflags/gflags.h"
#include "glog/logging.h"
@@ -62,10 +69,9 @@ bool PolicyEnforcer::admitQuery(QueryHandle *query_handle) {
}
void PolicyEnforcer::processMessage(const TaggedMessage &tagged_message) {
- // TODO(harshad) : Provide processXMessage() public functions in
- // QueryManager, so that we need to extract message from the
- // TaggedMessage only once.
std::size_t query_id;
+ QueryManagerBase::dag_node_index op_index;
+
switch (tagged_message.message_type()) {
case kWorkOrderCompleteMessage: {
serialization::NormalWorkOrderCompletionMessage proto;
@@ -73,12 +79,17 @@ void PolicyEnforcer::processMessage(const TaggedMessage &tagged_message) {
// WorkOrder. It can be accessed in this scope.
CHECK(proto.ParseFromArray(tagged_message.message(),
tagged_message.message_bytes()));
- query_id = proto.query_id();
worker_directory_->decrementNumQueuedWorkOrders(
proto.worker_thread_index());
if (profile_individual_workorders_) {
recordTimeForWorkOrder(proto);
}
+
+ query_id = proto.query_id();
+ DCHECK(admitted_queries_.find(query_id) != admitted_queries_.end());
+
+ op_index = proto.operator_index();
+ admitted_queries_[query_id]->processWorkOrderCompleteMessage(op_index);
break;
}
case kRebuildWorkOrderCompleteMessage: {
@@ -87,23 +98,43 @@ void PolicyEnforcer::processMessage(const TaggedMessage &tagged_message) {
// rebuild WorkOrder. It can be accessed in this scope.
CHECK(proto.ParseFromArray(tagged_message.message(),
tagged_message.message_bytes()));
- query_id = proto.query_id();
worker_directory_->decrementNumQueuedWorkOrders(
proto.worker_thread_index());
+
+ query_id = proto.query_id();
+ DCHECK(admitted_queries_.find(query_id) != admitted_queries_.end());
+
+ op_index = proto.operator_index();
+ admitted_queries_[query_id]->processRebuildWorkOrderCompleteMessage(op_index);
break;
}
case kCatalogRelationNewBlockMessage: {
serialization::CatalogRelationNewBlockMessage proto;
CHECK(proto.ParseFromArray(tagged_message.message(),
tagged_message.message_bytes()));
- query_id = proto.query_id();
- break;
+
+ const block_id block = proto.block_id();
+
+ CatalogRelation *relation =
+ static_cast<CatalogDatabase*>(catalog_database_)->getRelationByIdMutable(proto.relation_id());
+ relation->addBlock(block);
+
+ if (proto.has_partition_id()) {
+ relation->getPartitionSchemeMutable()->addBlockToPartition(
+ proto.partition_id(), block);
+ }
+ return;
}
case kDataPipelineMessage: {
serialization::DataPipelineMessage proto;
CHECK(proto.ParseFromArray(tagged_message.message(),
tagged_message.message_bytes()));
query_id = proto.query_id();
+ DCHECK(admitted_queries_.find(query_id) != admitted_queries_.end());
+
+ op_index = proto.operator_index();
+ admitted_queries_[query_id]->processDataPipelineMessage(
+ op_index, proto.block_id(), proto.relation_id());
break;
}
case kWorkOrdersAvailableMessage: {
@@ -111,6 +142,12 @@ void PolicyEnforcer::processMessage(const TaggedMessage &tagged_message) {
CHECK(proto.ParseFromArray(tagged_message.message(),
tagged_message.message_bytes()));
query_id = proto.query_id();
+ DCHECK(admitted_queries_.find(query_id) != admitted_queries_.end());
+
+ op_index = proto.operator_index();
+
+ // Check if new work orders are available.
+ admitted_queries_[query_id]->fetchNormalWorkOrders(op_index);
break;
}
case kWorkOrderFeedbackMessage: {
@@ -118,15 +155,17 @@ void PolicyEnforcer::processMessage(const TaggedMessage &tagged_message) {
const_cast<void *>(tagged_message.message()),
tagged_message.message_bytes());
query_id = msg.header().query_id;
+ DCHECK(admitted_queries_.find(query_id) != admitted_queries_.end());
+
+ op_index = msg.header().rel_op_index;
+ admitted_queries_[query_id]->processFeedbackMessage(op_index, msg);
break;
}
default:
LOG(FATAL) << "Unknown message type found in PolicyEnforcer";
}
- DCHECK(admitted_queries_.find(query_id) != admitted_queries_.end());
- const QueryManagerBase::QueryStatusCode return_code =
- admitted_queries_[query_id]->processMessage(tagged_message);
- if (return_code == QueryManagerBase::QueryStatusCode::kQueryExecuted) {
+ if (admitted_queries_[query_id]->queryStatus(op_index) ==
+ QueryManagerBase::QueryStatusCode::kQueryExecuted) {
removeQuery(query_id);
if (!waiting_queries_.empty()) {
// Admit the earliest waiting query.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/9bdce7d8/query_execution/PolicyEnforcer.hpp
----------------------------------------------------------------------
diff --git a/query_execution/PolicyEnforcer.hpp b/query_execution/PolicyEnforcer.hpp
index 79e61d1..8bd6d92 100644
--- a/query_execution/PolicyEnforcer.hpp
+++ b/query_execution/PolicyEnforcer.hpp
@@ -33,8 +33,8 @@
#include "glog/logging.h"
#include "tmb/id_typedefs.h"
-#include "tmb/message_bus.h"
-#include "tmb/tagged_message.h"
+
+namespace tmb { class MessageBus; }
namespace quickstep {
@@ -43,6 +43,12 @@ class QueryHandle;
class StorageManager;
class WorkerDirectory;
+namespace serialization { class NormalWorkOrderCompletionMessage; }
+
+/** \addtogroup QueryExecution
+ * @{
+ */
+
/**
* @brief A class that ensures that a high level policy is maintained
* in sharing resources among concurrent queries.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/9bdce7d8/query_execution/QueryManagerBase.cpp
----------------------------------------------------------------------
diff --git a/query_execution/QueryManagerBase.cpp b/query_execution/QueryManagerBase.cpp
index f7e183f..37beb02 100644
--- a/query_execution/QueryManagerBase.cpp
+++ b/query_execution/QueryManagerBase.cpp
@@ -21,13 +21,8 @@
#include <utility>
#include <vector>
-#include "catalog/CatalogDatabase.hpp"
-#include "catalog/CatalogRelation.hpp"
#include "catalog/CatalogTypedefs.hpp"
-#include "catalog/PartitionScheme.hpp"
#include "query_execution/QueryContext.hpp"
-#include "query_execution/QueryExecutionMessages.pb.h"
-#include "query_execution/QueryExecutionTypedefs.hpp"
#include "query_optimizer/QueryHandle.hpp"
#include "query_optimizer/QueryPlan.hpp"
#include "relational_operators/WorkOrder.hpp"
@@ -39,10 +34,8 @@ using std::pair;
namespace quickstep {
-QueryManagerBase::QueryManagerBase(QueryHandle *query_handle,
- CatalogDatabaseLite *catalog_database)
+QueryManagerBase::QueryManagerBase(QueryHandle *query_handle)
: query_id_(DCHECK_NOTNULL(query_handle)->query_id()),
- catalog_database_(DCHECK_NOTNULL(catalog_database)),
query_dag_(DCHECK_NOTNULL(
DCHECK_NOTNULL(query_handle->getQueryPlanMutable())->getQueryPlanDAGMutable())),
num_operators_in_dag_(query_dag_->size()),
@@ -76,82 +69,8 @@ QueryManagerBase::QueryManagerBase(QueryHandle *query_handle,
}
}
-QueryManagerBase::QueryStatusCode QueryManagerBase::processMessage(
- const TaggedMessage &tagged_message) {
- dag_node_index op_index;
- switch (tagged_message.message_type()) {
- case kWorkOrderCompleteMessage: {
- serialization::NormalWorkOrderCompletionMessage proto;
- CHECK(proto.ParseFromArray(tagged_message.message(),
- tagged_message.message_bytes()));
-
- op_index = proto.operator_index();
- processWorkOrderCompleteMessage(proto.operator_index());
- break;
- }
- case kRebuildWorkOrderCompleteMessage: {
- serialization::RebuildWorkOrderCompletionMessage proto;
- CHECK(proto.ParseFromArray(tagged_message.message(),
- tagged_message.message_bytes()));
-
- op_index = proto.operator_index();
- processRebuildWorkOrderCompleteMessage(proto.operator_index());
- break;
- }
- case kCatalogRelationNewBlockMessage: {
- serialization::CatalogRelationNewBlockMessage proto;
- CHECK(proto.ParseFromArray(tagged_message.message(),
- tagged_message.message_bytes()));
-
- const block_id block = proto.block_id();
-
- CatalogRelation *relation =
- static_cast<CatalogDatabase*>(catalog_database_)->getRelationByIdMutable(proto.relation_id());
- relation->addBlock(block);
-
- if (proto.has_partition_id()) {
- relation->getPartitionSchemeMutable()->addBlockToPartition(
- proto.partition_id(), block);
- }
- return QueryStatusCode::kNone;
- }
- case kDataPipelineMessage: {
- // Possible message senders include InsertDestinations and some
- // operators which modify existing blocks.
- serialization::DataPipelineMessage proto;
- CHECK(proto.ParseFromArray(tagged_message.message(),
- tagged_message.message_bytes()));
-
- op_index = proto.operator_index();
- processDataPipelineMessage(proto.operator_index(),
- proto.block_id(),
- proto.relation_id());
- break;
- }
- case kWorkOrdersAvailableMessage: {
- serialization::WorkOrdersAvailableMessage proto;
- CHECK(proto.ParseFromArray(tagged_message.message(),
- tagged_message.message_bytes()));
-
- op_index = proto.operator_index();
-
- // Check if new work orders are available.
- fetchNormalWorkOrders(op_index);
- break;
- }
- case kWorkOrderFeedbackMessage: {
- WorkOrder::FeedbackMessage msg(
- const_cast<void *>(tagged_message.message()),
- tagged_message.message_bytes());
-
- op_index = msg.header().rel_op_index;
- processFeedbackMessage(msg);
- break;
- }
- default:
- LOG(FATAL) << "Unknown message type found in QueryManager";
- }
-
+QueryManagerBase::QueryStatusCode QueryManagerBase::queryStatus(
+ const dag_node_index op_index) {
if (query_exec_state_->hasExecutionFinished(op_index)) {
return QueryStatusCode::kOperatorExecuted;
}
@@ -165,9 +84,9 @@ QueryManagerBase::QueryStatusCode QueryManagerBase::processMessage(
}
void QueryManagerBase::processFeedbackMessage(
- const WorkOrder::FeedbackMessage &msg) {
+ const dag_node_index op_index, const WorkOrder::FeedbackMessage &msg) {
RelationalOperator *op =
- query_dag_->getNodePayloadMutable(msg.header().rel_op_index);
+ query_dag_->getNodePayloadMutable(op_index);
op->receiveFeedbackMessage(msg);
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/9bdce7d8/query_execution/QueryManagerBase.hpp
----------------------------------------------------------------------
diff --git a/query_execution/QueryManagerBase.hpp b/query_execution/QueryManagerBase.hpp
index 9e192c8..6edfd5c 100644
--- a/query_execution/QueryManagerBase.hpp
+++ b/query_execution/QueryManagerBase.hpp
@@ -24,7 +24,6 @@
#include "catalog/CatalogTypedefs.hpp"
#include "query_execution/QueryExecutionState.hpp"
-#include "query_execution/QueryExecutionTypedefs.hpp"
#include "relational_operators/RelationalOperator.hpp"
#include "relational_operators/WorkOrder.hpp"
#include "storage/StorageBlockInfo.hpp"
@@ -33,7 +32,6 @@
namespace quickstep {
-class CatalogDatabaseLite;
class QueryHandle;
/** \addtogroup QueryExecution
@@ -50,7 +48,7 @@ class QueryManagerBase {
typedef DAG<RelationalOperator, bool>::size_type_nodes dag_node_index;
/**
- * @brief Return codes for processMessage() function.
+ * @brief Return codes for queryStatus() function.
*
* @note When both operator and query get executed, kQueryExecuted takes
* precedence over kOperatorExecuted.
@@ -65,10 +63,8 @@ class QueryManagerBase {
* @brief Constructor.
*
* @param query_handle The QueryHandle object for this query.
- * @param catalog_database The CatalogDatabse used by the query.
**/
- QueryManagerBase(QueryHandle *query_handle,
- CatalogDatabaseLite *catalog_database);
+ explicit QueryManagerBase(QueryHandle *query_handle);
/**
* @brief Virtual destructor.
@@ -76,26 +72,16 @@ class QueryManagerBase {
virtual ~QueryManagerBase() {}
/**
- * @brief Process a message sent to the QueryManager.
- *
- * @param tagged_message TaggedMessage sent to the QueryManager.
- *
- * @return QueryStatusCode as determined after the message is processed.
- **/
- QueryStatusCode processMessage(const TaggedMessage &tagged_message);
-
- /**
* @brief Get the QueryExecutionState for this query.
**/
inline const QueryExecutionState& getQueryExecutionState() const {
return *query_exec_state_;
}
- protected:
/**
* @brief Process the received WorkOrder complete message.
*
- * @param node_index The index of the specified operator node in the query DAG
+ * @param op_index The index of the specified operator node in the query DAG
* for the completed WorkOrder.
**/
void processWorkOrderCompleteMessage(const dag_node_index op_index);
@@ -103,28 +89,15 @@ class QueryManagerBase {
/**
* @brief Process the received RebuildWorkOrder complete message.
*
- * @param node_index The index of the specified operator node in the query DAG
+ * @param op_index The index of the specified operator node in the query DAG
* for the completed RebuildWorkOrder.
**/
void processRebuildWorkOrderCompleteMessage(const dag_node_index op_index);
/**
- * @brief Process a current relational operator: Get its workorders and store
- * them in the WorkOrdersContainer for this query. If the operator can
- * be marked as done, do so.
- *
- * @param index The index of the relational operator to be processed in the
- * query plan DAG.
- * @param recursively_check_dependents If an operator is done, should we
- * call processOperator on its dependents recursively.
- **/
- void processOperator(const dag_node_index index,
- const bool recursively_check_dependents);
-
- /**
* @brief Process the received data pipeline message.
*
- * @param node_index The index of the specified operator node in the query DAG
+ * @param op_index The index of the specified operator node in the query DAG
* for the pipelining block.
* @param block The block id.
* @param rel_id The ID of the relation that produced 'block'.
@@ -134,12 +107,50 @@ class QueryManagerBase {
const relation_id rel_id);
/**
+ * @brief Fetch all work orders currently available in relational operator and
+ * store them internally.
+ *
+ * @param index The index of the relational operator to be processed in the
+ * query plan DAG.
+ *
+ * @return Whether any work order was generated by op.
+ **/
+ virtual bool fetchNormalWorkOrders(const dag_node_index index) = 0;
+
+ /**
* @brief Process the received work order feedback message and notify
* relational operator.
*
+ * @param op_index The index of the specified operator node in the query DAG
+ * for the feedback message.
* @param message Feedback message from work order.
**/
- void processFeedbackMessage(const WorkOrder::FeedbackMessage &message);
+ void processFeedbackMessage(const dag_node_index op_index,
+ const WorkOrder::FeedbackMessage &message);
+
+ /**
+ * @brief Get the query status after processing an incoming message.
+ *
+ * @param op_index The index of the specified operator node in the query DAG
+ * for the incoming message.
+ *
+ * @return QueryStatusCode as determined after the message is processed.
+ **/
+ QueryStatusCode queryStatus(const dag_node_index op_index);
+
+ protected:
+ /**
+ * @brief Process a current relational operator: Get its workorders and store
+ * them in the WorkOrdersContainer for this query. If the operator can
+ * be marked as done, do so.
+ *
+ * @param index The index of the relational operator to be processed in the
+ * query plan DAG.
+ * @param recursively_check_dependents If an operator is done, should we
+ * call processOperator on its dependents recursively.
+ **/
+ void processOperator(const dag_node_index index,
+ const bool recursively_check_dependents);
/**
* @brief This function does the following things:
@@ -241,8 +252,6 @@ class QueryManagerBase {
const std::size_t query_id_;
- CatalogDatabaseLite *catalog_database_;
-
DAG<RelationalOperator, bool> *query_dag_;
const dag_node_index num_operators_in_dag_;
@@ -256,17 +265,6 @@ class QueryManagerBase {
private:
/**
- * @brief Fetch all work orders currently available in relational operator and
- * store them internally.
- *
- * @param index The index of the relational operator to be processed in the
- * query plan DAG.
- *
- * @return Whether any work order was generated by op.
- **/
- virtual bool fetchNormalWorkOrders(const dag_node_index index) = 0;
-
- /**
* @brief Check if the given operator's normal execution is over.
*
* @note The conditions for a given operator's normal execution to get over:
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/9bdce7d8/query_execution/QueryManagerSingleNode.cpp
----------------------------------------------------------------------
diff --git a/query_execution/QueryManagerSingleNode.cpp b/query_execution/QueryManagerSingleNode.cpp
index 193b188..12f8ff5 100644
--- a/query_execution/QueryManagerSingleNode.cpp
+++ b/query_execution/QueryManagerSingleNode.cpp
@@ -46,12 +46,12 @@ QueryManagerSingleNode::QueryManagerSingleNode(
CatalogDatabaseLite *catalog_database,
StorageManager *storage_manager,
tmb::MessageBus *bus)
- : QueryManagerBase(query_handle, catalog_database),
+ : QueryManagerBase(query_handle),
foreman_client_id_(foreman_client_id),
storage_manager_(DCHECK_NOTNULL(storage_manager)),
bus_(DCHECK_NOTNULL(bus)),
query_context_(new QueryContext(query_handle->getQueryContextProto(),
- *catalog_database_,
+ *catalog_database,
storage_manager_,
foreman_client_id_,
bus_)),
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/9bdce7d8/query_execution/QueryManagerSingleNode.hpp
----------------------------------------------------------------------
diff --git a/query_execution/QueryManagerSingleNode.hpp b/query_execution/QueryManagerSingleNode.hpp
index 5533f06..b9bad2e 100644
--- a/query_execution/QueryManagerSingleNode.hpp
+++ b/query_execution/QueryManagerSingleNode.hpp
@@ -68,6 +68,8 @@ class QueryManagerSingleNode final : public QueryManagerBase {
~QueryManagerSingleNode() override {}
+ bool fetchNormalWorkOrders(const dag_node_index index) override;
+
/**
* @brief Get the next workorder to be excuted, wrapped in a WorkerMessage.
*
@@ -91,8 +93,6 @@ class QueryManagerSingleNode final : public QueryManagerBase {
}
private:
- bool fetchNormalWorkOrders(const dag_node_index index) override;
-
bool checkNormalExecutionOver(const dag_node_index index) const override {
return (checkAllDependenciesMet(index) &&
!workorders_container_->hasNormalWorkOrder(index) &&
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/9bdce7d8/query_execution/tests/QueryManagerSingleNode_unittest.cpp
----------------------------------------------------------------------
diff --git a/query_execution/tests/QueryManagerSingleNode_unittest.cpp b/query_execution/tests/QueryManagerSingleNode_unittest.cpp
index 52cee20..39ca58c 100644
--- a/query_execution/tests/QueryManagerSingleNode_unittest.cpp
+++ b/query_execution/tests/QueryManagerSingleNode_unittest.cpp
@@ -16,7 +16,6 @@
**/
#include <climits>
-#include <cstdlib>
#include <memory>
#include <utility>
#include <vector>
@@ -26,7 +25,6 @@
#include "catalog/CatalogTypedefs.hpp"
#include "query_execution/QueryContext.hpp"
#include "query_execution/QueryContext.pb.h"
-#include "query_execution/QueryExecutionMessages.pb.h"
#include "query_execution/QueryExecutionState.hpp"
#include "query_execution/QueryExecutionTypedefs.hpp"
#include "query_execution/QueryManagerSingleNode.hpp"
@@ -49,7 +47,6 @@
#include "gtest/gtest.h"
#include "tmb/id_typedefs.h"
-#include "tmb/tagged_message.h"
namespace tmb { class MessageBus; }
@@ -254,89 +251,33 @@ class QueryManagerTest : public ::testing::Test {
inline bool placeDataPipelineMessage(const QueryPlan::DAGNodeIndex source_operator_index) {
VLOG(3) << "Place DataPipeline message for Op[" << source_operator_index << "]";
- serialization::DataPipelineMessage proto;
- proto.set_operator_index(source_operator_index);
-
- proto.set_block_id(0); // dummy block ID
- proto.set_relation_id(0); // dummy relation ID.
- proto.set_query_id(0); // dummy query ID.
-
- // NOTE(zuyu): Using the heap memory to serialize proto as a c-like string.
- const std::size_t proto_length = proto.ByteSize();
- char *proto_bytes = static_cast<char*>(std::malloc(proto_length));
- CHECK(proto.SerializeToArray(proto_bytes, proto_length));
-
- tmb::TaggedMessage tagged_message(static_cast<const void *>(proto_bytes),
- proto_length,
- kDataPipelineMessage);
- std::free(proto_bytes);
- query_manager_->processMessage(tagged_message);
+
+ query_manager_->processDataPipelineMessage(source_operator_index,
+ 0 /* dummy block ID */,
+ 0 /* dummy relation ID */);
return query_manager_->getQueryExecutionState().hasQueryExecutionFinished();
}
inline bool placeWorkOrderCompleteMessage(const QueryPlan::DAGNodeIndex index) {
VLOG(3) << "Place WorkOrderComplete message for Op[" << index << "]";
- TaggedMessage tagged_message;
- serialization::NormalWorkOrderCompletionMessage proto;
- proto.set_operator_index(index);
- proto.set_worker_thread_index(1); // dummy worker ID.
- proto.set_query_id(0); // dummy query ID.
-
- // NOTE(zuyu): Using the heap memory to serialize proto as a c-like string.
- const size_t proto_length = proto.ByteSize();
- char *proto_bytes = static_cast<char*>(std::malloc(proto_length));
- CHECK(proto.SerializeToArray(proto_bytes, proto_length));
-
- TaggedMessage message(static_cast<const void*>(proto_bytes),
- proto_length,
- kWorkOrderCompleteMessage);
- std::free(proto_bytes);
- query_manager_->processMessage(message);
+ query_manager_->processWorkOrderCompleteMessage(index);
return query_manager_->getQueryExecutionState().hasQueryExecutionFinished();
}
inline bool placeRebuildWorkOrderCompleteMessage(const QueryPlan::DAGNodeIndex index) {
VLOG(3) << "Place RebuildWorkOrderComplete message for Op[" << index << "]";
- serialization::RebuildWorkOrderCompletionMessage proto;
- proto.set_operator_index(index);
- proto.set_worker_thread_index(1); // dummy worker thread ID.
- proto.set_query_id(0); // dummy query ID.
-
- // NOTE(zuyu): Using the heap memory to serialize proto as a c-like string.
- const size_t proto_length = proto.ByteSize();
- char *proto_bytes = static_cast<char*>(std::malloc(proto_length));
- CHECK(proto.SerializeToArray(proto_bytes, proto_length));
-
- TaggedMessage message(static_cast<const void*>(proto_bytes),
- proto_length,
- kRebuildWorkOrderCompleteMessage);
-
- std::free(proto_bytes);
- query_manager_->processMessage(message);
+ query_manager_->processRebuildWorkOrderCompleteMessage(index);
return query_manager_->getQueryExecutionState().hasQueryExecutionFinished();
}
inline bool placeOutputBlockMessage(const QueryPlan::DAGNodeIndex index) {
VLOG(3) << "Place OutputBlock message for Op[" << index << "]";
- serialization::DataPipelineMessage proto;
- proto.set_operator_index(index);
-
- proto.set_block_id(0); // dummy block ID
- proto.set_relation_id(0); // dummy relation ID.
- proto.set_query_id(0); // dummy query ID.
-
- // NOTE(zuyu): Using the heap memory to serialize proto as a c-like string.
- const std::size_t proto_length = proto.ByteSize();
- char *proto_bytes = static_cast<char*>(std::malloc(proto_length));
- CHECK(proto.SerializeToArray(proto_bytes, proto_length));
-
- tmb::TaggedMessage tagged_message(static_cast<const void *>(proto_bytes),
- proto_length,
- kDataPipelineMessage);
- std::free(proto_bytes);
- query_manager_->processMessage(tagged_message);
+
+ query_manager_->processDataPipelineMessage(index,
+ 0 /* dummy block ID */,
+ 0 /* dummy relation ID */);
return query_manager_->getQueryExecutionState().hasQueryExecutionFinished();
}
[08/25] incubator-quickstep git commit: Fixed the time measurement
from milli to microseconds.
Posted by zu...@apache.org.
Fixed the time measurement from milli to microseconds.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/6bbdb4db
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/6bbdb4db
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/6bbdb4db
Branch: refs/heads/dist-exe-test-new
Commit: 6bbdb4db84c9e617291e11e72a53878eaaa44404
Parents: 8af55df
Author: Harshad Deshmukh <hb...@apache.org>
Authored: Sat Jul 2 15:58:52 2016 -0500
Committer: Zuyu Zhang <zu...@twitter.com>
Committed: Fri Jul 29 16:42:10 2016 -0700
----------------------------------------------------------------------
query_execution/Worker.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/6bbdb4db/query_execution/Worker.cpp
----------------------------------------------------------------------
diff --git a/query_execution/Worker.cpp b/query_execution/Worker.cpp
index ae889c7..6ba27f1 100644
--- a/query_execution/Worker.cpp
+++ b/query_execution/Worker.cpp
@@ -121,7 +121,7 @@ void Worker::executeWorkOrderHelper(const TaggedMessage &tagged_message,
end = std::chrono::steady_clock::now();
delete worker_message.getWorkOrder();
const uint64_t execution_time_microseconds =
- std::chrono::duration_cast<std::chrono::milliseconds>(end - start)
+ std::chrono::duration_cast<std::chrono::microseconds>(end - start)
.count();
// Construct the proto message.
proto->set_operator_index(worker_message.getRelationalOpIndex());
[23/25] incubator-quickstep git commit: Renamed a
QueryExecutionMessage.
Posted by zu...@apache.org.
Renamed a QueryExecutionMessage.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/784b1edf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/784b1edf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/784b1edf
Branch: refs/heads/dist-exe-test-new
Commit: 784b1edf75c5e8a55149b14ed1c3f13bee2994ed
Parents: 614abab
Author: Zuyu Zhang <zu...@twitter.com>
Authored: Thu Jul 28 11:11:33 2016 -0700
Committer: Zuyu Zhang <zu...@twitter.com>
Committed: Fri Jul 29 16:43:24 2016 -0700
----------------------------------------------------------------------
query_execution/QueryExecutionMessages.proto | 4 +-
query_execution/QueryExecutionTypedefs.hpp | 4 +-
query_execution/Shiftboss.cpp | 45 +++++++++++------------
query_execution/Shiftboss.hpp | 4 +-
4 files changed, 28 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/784b1edf/query_execution/QueryExecutionMessages.proto
----------------------------------------------------------------------
diff --git a/query_execution/QueryExecutionMessages.proto b/query_execution/QueryExecutionMessages.proto
index 591ca6c..308d736 100644
--- a/query_execution/QueryExecutionMessages.proto
+++ b/query_execution/QueryExecutionMessages.proto
@@ -113,12 +113,12 @@ message InitiateRebuildResponseMessage {
required uint64 num_rebuild_work_orders = 3;
}
-message QueryResultRelationMessage {
+message SaveQueryResultMessage {
required int32 relation_id = 1;
repeated fixed64 blocks = 2 [packed=true];
}
-message QueryResultRelationResponseMessage {
+message SaveQueryResultResponseMessage {
required int32 relation_id = 1;
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/784b1edf/query_execution/QueryExecutionTypedefs.hpp
----------------------------------------------------------------------
diff --git a/query_execution/QueryExecutionTypedefs.hpp b/query_execution/QueryExecutionTypedefs.hpp
index d73d4ee..b67209f 100644
--- a/query_execution/QueryExecutionTypedefs.hpp
+++ b/query_execution/QueryExecutionTypedefs.hpp
@@ -81,8 +81,8 @@ enum QueryExecutionMessageType : message_type_id {
kInitiateRebuildMessage, // From Foreman to Shiftboss.
kInitiateRebuildResponseMessage, // From Shiftboss to Foreman.
- kQueryResultRelationMessage, // From Foreman to Shiftboss.
- kQueryResultRelationResponseMessage, // From Shiftboss to Foreman.
+ kSaveQueryResultMessage, // From Foreman to Shiftboss.
+ kSaveQueryResultResponseMessage, // From Shiftboss to Foreman.
// BlockLocator related messages, sorted in a life cycle of StorageManager
// with a unique block domain.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/784b1edf/query_execution/Shiftboss.cpp
----------------------------------------------------------------------
diff --git a/query_execution/Shiftboss.cpp b/query_execution/Shiftboss.cpp
index af56306..7f655c6 100644
--- a/query_execution/Shiftboss.cpp
+++ b/query_execution/Shiftboss.cpp
@@ -149,11 +149,10 @@ void Shiftboss::run() {
move(annotated_message.tagged_message));
break;
}
- case kQueryResultRelationMessage: {
- // TODO(zuyu): Rename to kSaveQueryResultMessage.
+ case kSaveQueryResultMessage: {
const TaggedMessage &tagged_message = annotated_message.tagged_message;
- serialization::QueryResultRelationMessage proto;
+ serialization::SaveQueryResultMessage proto;
CHECK(proto.ParseFromArray(tagged_message.message(), tagged_message.message_bytes()));
for (int i = 0; i < proto.blocks_size(); ++i) {
@@ -168,25 +167,25 @@ void Shiftboss::run() {
}
}
- serialization::QueryResultRelationResponseMessage ack_proto;
- ack_proto.set_relation_id(proto.relation_id());
+ serialization::SaveQueryResultResponseMessage proto_response;
+ proto_response.set_relation_id(proto.relation_id());
- const size_t ack_proto_length = ack_proto.ByteSize();
- char *ack_proto_bytes = static_cast<char*>(malloc(ack_proto_length));
- CHECK(ack_proto.SerializeToArray(ack_proto_bytes, ack_proto_length));
+ const size_t proto_response_length = proto_response.ByteSize();
+ char *proto_response_bytes = static_cast<char*>(malloc(proto_response_length));
+ CHECK(proto_response.SerializeToArray(proto_response_bytes, proto_response_length));
- TaggedMessage ack_message(static_cast<const void*>(ack_proto_bytes),
- ack_proto_length,
- kQueryResultRelationResponseMessage);
- free(ack_proto_bytes);
+ TaggedMessage message_response(static_cast<const void*>(proto_response_bytes),
+ proto_response_length,
+ kSaveQueryResultResponseMessage);
+ free(proto_response_bytes);
LOG(INFO) << "Shiftboss (id '" << shiftboss_client_id_
- << "') sent QueryResultRelationResponseMessage (typed '" << kQueryResultRelationResponseMessage
- << ") to Foreman";
+ << "') sent SaveQueryResultResponseMessage (typed '" << kSaveQueryResultResponseMessage
+ << "') to Foreman";
QueryExecutionUtil::SendTMBMessage(bus_,
shiftboss_client_id_,
foreman_client_id_,
- move(ack_message));
+ move(message_response));
break;
}
case kPoisonMessage: {
@@ -280,15 +279,15 @@ void Shiftboss::processQueryInitiateMessage(
char *proto_bytes = static_cast<char*>(malloc(proto_length));
CHECK(proto.SerializeToArray(proto_bytes, proto_length));
- TaggedMessage ack_message(static_cast<const void*>(proto_bytes),
- proto_length,
- kQueryInitiateResponseMessage);
+ TaggedMessage message_response(static_cast<const void*>(proto_bytes),
+ proto_length,
+ kQueryInitiateResponseMessage);
free(proto_bytes);
QueryExecutionUtil::SendTMBMessage(bus_,
shiftboss_client_id_,
foreman_client_id_,
- move(ack_message));
+ move(message_response));
}
void Shiftboss::processInitiateRebuildMessage(const std::size_t query_id,
@@ -317,15 +316,15 @@ void Shiftboss::processInitiateRebuildMessage(const std::size_t query_id,
char *proto_bytes = static_cast<char*>(malloc(proto_length));
CHECK(proto.SerializeToArray(proto_bytes, proto_length));
- TaggedMessage ack_message(static_cast<const void*>(proto_bytes),
- proto_length,
- kInitiateRebuildResponseMessage);
+ TaggedMessage message_response(static_cast<const void*>(proto_bytes),
+ proto_length,
+ kInitiateRebuildResponseMessage);
free(proto_bytes);
QueryExecutionUtil::SendTMBMessage(bus_,
shiftboss_client_id_,
foreman_client_id_,
- move(ack_message));
+ move(message_response));
for (size_t i = 0; i < partially_filled_block_refs.size(); ++i) {
// NOTE(zuyu): Worker releases the memory after the execution of
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/784b1edf/query_execution/Shiftboss.hpp
----------------------------------------------------------------------
diff --git a/query_execution/Shiftboss.hpp b/query_execution/Shiftboss.hpp
index 096ab74..9e24d62 100644
--- a/query_execution/Shiftboss.hpp
+++ b/query_execution/Shiftboss.hpp
@@ -112,8 +112,8 @@ class Shiftboss : public Thread {
bus_->RegisterClientAsReceiver(shiftboss_client_id_, kRebuildWorkOrderCompleteMessage);
bus_->RegisterClientAsSender(shiftboss_client_id_, kRebuildWorkOrderCompleteMessage);
- bus_->RegisterClientAsReceiver(shiftboss_client_id_, kQueryResultRelationMessage);
- bus_->RegisterClientAsSender(shiftboss_client_id_, kQueryResultRelationResponseMessage);
+ bus_->RegisterClientAsReceiver(shiftboss_client_id_, kSaveQueryResultMessage);
+ bus_->RegisterClientAsSender(shiftboss_client_id_, kSaveQueryResultResponseMessage);
// Stop itself.
bus_->RegisterClientAsReceiver(shiftboss_client_id_, kPoisonMessage);
[16/25] incubator-quickstep git commit: Added Comparisons for the
Date Type.
Posted by zu...@apache.org.
Added Comparisons for the Date Type.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/cde7bcb4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/cde7bcb4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/cde7bcb4
Branch: refs/heads/dist-exe-test-new
Commit: cde7bcb448bf8783fb0af06989b1657486d7c58e
Parents: 32bee12
Author: Zuyu Zhang <zu...@twitter.com>
Authored: Fri Jul 15 16:56:14 2016 -0700
Committer: Zuyu Zhang <zu...@twitter.com>
Committed: Fri Jul 29 16:42:11 2016 -0700
----------------------------------------------------------------------
.../operations/comparisons/BasicComparison.cpp | 6 ++-
.../operations/comparisons/BasicComparison.hpp | 29 ++++++++++--
types/operations/comparisons/CMakeLists.txt | 2 -
types/operations/comparisons/ComparisonUtil.hpp | 34 +++++++++++++++
.../comparisons/LiteralComparators.hpp | 24 +++++-----
.../comparisons/tests/Comparison_unittest.cpp | 46 +++++++++++++++++---
6 files changed, 115 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/cde7bcb4/types/operations/comparisons/BasicComparison.cpp
----------------------------------------------------------------------
diff --git a/types/operations/comparisons/BasicComparison.cpp b/types/operations/comparisons/BasicComparison.cpp
index 21b92ca..ed0ce62 100644
--- a/types/operations/comparisons/BasicComparison.cpp
+++ b/types/operations/comparisons/BasicComparison.cpp
@@ -23,7 +23,6 @@
#include "types/Type.hpp"
#include "types/TypeID.hpp"
#include "utility/EqualsAnyConstant.hpp"
-#include "utility/Macros.hpp"
#include "glog/logging.h"
@@ -49,6 +48,9 @@ bool BasicComparison::canCompareTypes(const Type &left, const Type &right) const
return false;
}
}
+ case kDate: {
+ return right.getTypeID() == kDate;
+ }
case kDatetime: {
return right.getTypeID() == kDatetime;
}
@@ -89,7 +91,7 @@ bool BasicComparison::canComparePartialTypes(const Type *left_type,
const Type *known_type = (left_type != nullptr) ? left_type : right_type;
return QUICKSTEP_EQUALS_ANY_CONSTANT(known_type->getTypeID(),
kInt, kLong, kFloat, kDouble,
- kDatetime, kDatetimeInterval, kYearMonthInterval,
+ kDate, kDatetime, kDatetimeInterval, kYearMonthInterval,
kChar, kVarChar);
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/cde7bcb4/types/operations/comparisons/BasicComparison.hpp
----------------------------------------------------------------------
diff --git a/types/operations/comparisons/BasicComparison.hpp b/types/operations/comparisons/BasicComparison.hpp
index 9741c32..c95f774 100644
--- a/types/operations/comparisons/BasicComparison.hpp
+++ b/types/operations/comparisons/BasicComparison.hpp
@@ -22,8 +22,6 @@
#include <cstdint>
#include <string>
-#include "types/DatetimeLit.hpp"
-#include "types/IntervalLit.hpp"
#include "types/Type.hpp"
#include "types/TypeErrors.hpp"
#include "types/TypeFactory.hpp"
@@ -37,6 +35,11 @@
namespace quickstep {
+struct DateLit;
+struct DatetimeIntervalLit;
+struct DatetimeLit;
+struct YearMonthIntervalLit;
+
/** \addtogroup Types
* @{
*/
@@ -161,6 +164,18 @@ bool BasicComparison::compareTypedValuesCheckedHelper(const TypedValue &left,
}
break;
}
+ case kDate: {
+ if (right.getTypeID() == kDate) {
+ if (left.isNull() || right.isNull()) {
+ return false;
+ }
+
+ ComparisonFunctor<DateLit> comparison_functor;
+ return comparison_functor(left.getLiteral<DateLit>(), right.getLiteral<DateLit>());
+ }
+ LOG(FATAL) << "Comparison " << getName() << " can not be applied to types "
+ << left_type.getName() << " and " << right_type.getName();
+ }
case kDatetime: {
if (right.getTypeID() == kDatetime) {
if (left.isNull() || right.isNull()) {
@@ -269,7 +284,8 @@ UncheckedComparator* BasicComparison::makeUncheckedComparatorForTypesHelper(cons
const Type &right) const {
if (left.getSuperTypeID() == Type::kNumeric && right.getSuperTypeID() == Type::kNumeric) {
return makeNumericComparatorOuterHelper<LiteralComparator>(left, right);
- } else if ((left.getTypeID() == kDatetime && right.getTypeID() == kDatetime) ||
+ } else if ((left.getTypeID() == kDate && right.getTypeID() == kDate) ||
+ (left.getTypeID() == kDatetime && right.getTypeID() == kDatetime) ||
(left.getTypeID() == kDatetimeInterval && right.getTypeID() == kDatetimeInterval) ||
(left.getTypeID() == kYearMonthInterval && right.getTypeID() == kYearMonthInterval)) {
return makeDateComparatorOuterHelper<LiteralComparator>(left, right);
@@ -358,6 +374,12 @@ UncheckedComparator* BasicComparison::makeDateComparatorOuterHelper(
const Type &left,
const Type &right) const {
switch (left.getTypeID()) {
+ case kDate:
+ if (left.isNullable()) {
+ return makeDateComparatorInnerHelper<ComparatorType, DateLit, true>(left, right);
+ } else {
+ return makeDateComparatorInnerHelper<ComparatorType, DateLit, false>(left, right);
+ }
case kDatetime:
if (left.isNullable()) {
return makeDateComparatorInnerHelper<ComparatorType, DatetimeLit, true>(left, right);
@@ -389,6 +411,7 @@ UncheckedComparator* BasicComparison::makeDateComparatorInnerHelper(
const Type &left,
const Type &right) const {
switch (right.getTypeID()) {
+ case kDate:
case kDatetime:
case kDatetimeInterval:
case kYearMonthInterval:
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/cde7bcb4/types/operations/comparisons/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/types/operations/comparisons/CMakeLists.txt b/types/operations/comparisons/CMakeLists.txt
index d2dfecf..d48cdc2 100644
--- a/types/operations/comparisons/CMakeLists.txt
+++ b/types/operations/comparisons/CMakeLists.txt
@@ -64,8 +64,6 @@ target_link_libraries(quickstep_types_operations_comparisons_AsciiStringComparat
quickstep_utility_Macros)
target_link_libraries(quickstep_types_operations_comparisons_BasicComparison
glog
- quickstep_types_DatetimeLit
- quickstep_types_IntervalLit
quickstep_types_Type
quickstep_types_TypeErrors
quickstep_types_TypeFactory
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/cde7bcb4/types/operations/comparisons/ComparisonUtil.hpp
----------------------------------------------------------------------
diff --git a/types/operations/comparisons/ComparisonUtil.hpp b/types/operations/comparisons/ComparisonUtil.hpp
index d7da3fd..d589192 100644
--- a/types/operations/comparisons/ComparisonUtil.hpp
+++ b/types/operations/comparisons/ComparisonUtil.hpp
@@ -163,6 +163,11 @@ auto InvokeOnLessComparatorForTypeIgnoreNullability(const Type &type,
comp(0, 0);
return functor(comp);
}
+ case kDate: {
+ LessLiteralUncheckedComparator<DateLit, false,
+ DateLit, false> comp;
+ return functor(comp);
+ }
case kDatetime: {
LessLiteralUncheckedComparator<DatetimeLit, false,
DatetimeLit, false> comp;
@@ -429,6 +434,18 @@ auto InvokeOnLessComparatorForDifferentTypesIgnoreNullability(
left_type, right_type, functor);
}
}
+ case kDate: {
+ switch (right_type.getTypeID()) {
+ case kDate: {
+ LessLiteralUncheckedComparator<DateLit, false,
+ DateLit, false> comp;
+ return functor(comp);
+ }
+ default:
+ return comparison_util_detail::InvokeOnLessComparatorForDifferentTypesFallback(
+ left_type, right_type, functor);
+ }
+ }
case kDatetime: {
switch (right_type.getTypeID()) {
case kDatetime: {
@@ -766,6 +783,18 @@ auto InvokeOnBothLessComparatorsForDifferentTypesIgnoreNullability(
left_type, right_type, functor);
}
}
+ case kDate: {
+ switch (right_type.getTypeID()) {
+ case kDate: {
+ LessLiteralUncheckedComparator<DateLit, false,
+ DateLit, false> comp;
+ return functor(comp, comp);
+ }
+ default:
+ return comparison_util_detail::InvokeOnBothLessComparatorsForDifferentTypesFallback(
+ left_type, right_type, functor);
+ }
+ }
case kDatetime: {
switch (right_type.getTypeID()) {
case kDatetime: {
@@ -963,6 +992,8 @@ inline bool CheckUntypedValuesEqual(const Type &type, const void *left, const vo
return STLCharEqual(static_cast<const AsciiStringSuperType&>(type).getStringLength())(left, right);
case kVarChar:
return STLVarCharEqual()(left, right);
+ case kDate:
+ return STLLiteralEqual<DateLit>()(left, right);
case kDatetime:
return STLLiteralEqual<DatetimeLit>()(left, right);
case kDatetimeInterval:
@@ -1220,6 +1251,9 @@ inline TypedValue GetLastValueForType(const Type &type) {
return TypedValue(kChar, kLastString, 2);
case kVarChar:
return TypedValue(kVarChar, kLastString, 2);
+ case kDate: {
+ return TypedValue(DateLit::Create(99999, 12, 31));
+ }
case kDatetime: {
DatetimeLit lit;
lit.ticks = std::numeric_limits<std::int64_t>::max();
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/cde7bcb4/types/operations/comparisons/LiteralComparators.hpp
----------------------------------------------------------------------
diff --git a/types/operations/comparisons/LiteralComparators.hpp b/types/operations/comparisons/LiteralComparators.hpp
index 3bf6b00..406ae40 100644
--- a/types/operations/comparisons/LiteralComparators.hpp
+++ b/types/operations/comparisons/LiteralComparators.hpp
@@ -257,8 +257,8 @@ class LiteralUncheckedComparator : public UncheckedComparator {
};
/**
- * @brief The equal UncheckedComparator for seven literals
- * (int, std::int64_t, float, double, DatetimeLit, DatetimeIntervalLit, and YearMonthIntervalLit).
+ * @brief The equal UncheckedComparator for the following literals
+ * (int, std::int64_t, float, double, DateLit, DatetimeLit, DatetimeIntervalLit, and YearMonthIntervalLit).
**/
template <typename LeftCppType, bool left_nullable,
typename RightCppType, bool right_nullable>
@@ -267,8 +267,8 @@ using EqualLiteralUncheckedComparator = LiteralUncheckedComparator<EqualFunctor,
RightCppType, right_nullable>;
/**
- * @brief The not-equal UncheckedComparator for seven literals
- * (int, std::int64_t, float, double, DatetimeLit, DatetimeIntervalLit, and YearMonthIntervalLit).
+ * @brief The not-equal UncheckedComparator for the following literals
+ * (int, std::int64_t, float, double, DateLit, DatetimeLit, DatetimeIntervalLit, and YearMonthIntervalLit).
**/
template <typename LeftCppType, bool left_nullable,
typename RightCppType, bool right_nullable>
@@ -277,8 +277,8 @@ using NotEqualLiteralUncheckedComparator = LiteralUncheckedComparator<NotEqualFu
RightCppType, right_nullable>;
/**
- * @brief The less-than UncheckedComparator for seven literals
- * (int, std::int64_t, float, double, DatetimeLit, DatetimeIntervalLit, and YearMonthIntervalLit).
+ * @brief The less-than UncheckedComparator for the following literals
+ * (int, std::int64_t, float, double, DateLit, DatetimeLit, DatetimeIntervalLit, and YearMonthIntervalLit).
**/
template <typename LeftCppType, bool left_nullable,
typename RightCppType, bool right_nullable>
@@ -287,8 +287,8 @@ using LessLiteralUncheckedComparator = LiteralUncheckedComparator<LessFunctor,
RightCppType, right_nullable>;
/**
- * @brief The less-than-or-equal UncheckedComparator for seven literals
- * (int, std::int64_t, float, double, DatetimeLit, DatetimeIntervalLit, and YearMonthIntervalLit).
+ * @brief The less-than-or-equal UncheckedComparator for the following literals
+ * (int, std::int64_t, float, double, DateLit, DatetimeLit, DatetimeIntervalLit, and YearMonthIntervalLit).
**/
template <typename LeftCppType, bool left_nullable,
typename RightCppType, bool right_nullable>
@@ -297,8 +297,8 @@ using LessOrEqualLiteralUncheckedComparator = LiteralUncheckedComparator<LessOrE
RightCppType, right_nullable>;
/**
- * @brief The greater-than UncheckedComparator for seven literals
- * (int, std::int64_t, float, double, DatetimeLit, DatetimeIntervalLit, and YearMonthIntervalLit).
+ * @brief The greater-than UncheckedComparator for the following literals
+ * (int, std::int64_t, float, double, DateLit, DatetimeLit, DatetimeIntervalLit, and YearMonthIntervalLit).
**/
template <typename LeftCppType, bool left_nullable,
typename RightCppType, bool right_nullable>
@@ -307,8 +307,8 @@ using GreaterLiteralUncheckedComparator = LiteralUncheckedComparator<GreaterFunc
RightCppType, right_nullable>;
/**
- * @brief The greater-than-or-equal UncheckedComparator for seven literals
- * (int, std::int64_t, float, double, DatetimeLit, DatetimeIntervalLit, and YearMonthIntervalLit).
+ * @brief The greater-than-or-equal UncheckedComparator for the following literals
+ * (int, std::int64_t, float, double, DateLit, DatetimeLit, DatetimeIntervalLit, and YearMonthIntervalLit).
**/
template <typename LeftCppType, bool left_nullable,
typename RightCppType, bool right_nullable>
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/cde7bcb4/types/operations/comparisons/tests/Comparison_unittest.cpp
----------------------------------------------------------------------
diff --git a/types/operations/comparisons/tests/Comparison_unittest.cpp b/types/operations/comparisons/tests/Comparison_unittest.cpp
index 0d5e387..d09aab7 100644
--- a/types/operations/comparisons/tests/Comparison_unittest.cpp
+++ b/types/operations/comparisons/tests/Comparison_unittest.cpp
@@ -25,8 +25,6 @@
#include <utility>
#include <vector>
-#include "gtest/gtest.h"
-
#include "types/DatetimeLit.hpp"
#include "types/IntervalLit.hpp"
#include "types/Type.hpp"
@@ -40,6 +38,8 @@
#include "utility/Macros.hpp"
#include "utility/ScopedBuffer.hpp"
+#include "gtest/gtest.h"
+
using std::int64_t;
using std::min;
using std::numeric_limits;
@@ -147,6 +147,23 @@ class ComparisonTest : public ::testing::Test {
numeric_typed_values_.emplace_back(double_min_.get(),
&TypeFactory::GetType(kDouble, false));
+ date_null_.reset(new TypedValue(kDate));
+ date_positive_.reset(new TypedValue(DateLit::Create(2016, 7, 15)));
+ date_negative_.reset(new TypedValue(DateLit::Create(-18017, 4, 13)));
+ date_max_.reset(new TypedValue(DateLit::Create(99999, 12, 31)));
+ date_min_.reset(new TypedValue(DateLit::Create(-99999, 1, 1)));
+
+ date_typed_values_.emplace_back(date_null_.get(),
+ &TypeFactory::GetType(kDate, true));
+ date_typed_values_.emplace_back(date_positive_.get(),
+ &TypeFactory::GetType(kDate, false));
+ date_typed_values_.emplace_back(date_negative_.get(),
+ &TypeFactory::GetType(kDate, false));
+ date_typed_values_.emplace_back(date_max_.get(),
+ &TypeFactory::GetType(kDate, false));
+ date_typed_values_.emplace_back(date_min_.get(),
+ &TypeFactory::GetType(kDate, false));
+
datetime_null_.reset(new TypedValue(kDatetime));
DatetimeLit datetime;
datetime.ticks = 0;
@@ -347,6 +364,7 @@ class ComparisonTest : public ::testing::Test {
void checkComparison(const Comparison &comparison) {
checkNumericComparison(comparison);
checkStringComparison(comparison);
+ checkDateComparison(comparison);
checkDatetimeComparison(comparison);
checkDatetimeIntervalComparison(comparison);
checkYearMonthIntervalComparison(comparison);
@@ -524,6 +542,19 @@ class ComparisonTest : public ::testing::Test {
}
}
+ void checkDateComparison(const Comparison &comparison) {
+ for (const std::pair<const TypedValue*, const Type*> &left_item : date_typed_values_) {
+ for (const std::pair<const TypedValue*, const Type*> &right_item : date_typed_values_) {
+ checkDateComparisonChecked<DateLit>(comparison,
+ *left_item.first, *left_item.second,
+ *right_item.first, *right_item.second);
+ checkDateComparisonUnchecked<DateLit>(comparison,
+ *left_item.first, *left_item.second,
+ *right_item.first, *right_item.second);
+ }
+ }
+ }
+
void checkDatetimeComparison(const Comparison &comparison) {
for (const std::pair<const TypedValue*, const Type*> &left_item : datetime_typed_values_) {
for (const std::pair<const TypedValue*, const Type*> &right_item : datetime_typed_values_) {
@@ -719,6 +750,7 @@ class ComparisonTest : public ::testing::Test {
char_ref_extended_short_, char_lit_extended_short_,
char_ref_extended_long_, char_lit_extended_long_,
varchar_null_, varchar_ref_short_, varchar_lit_short_, varchar_ref_long_, varchar_lit_long_,
+ date_null_, date_positive_, date_negative_, date_max_, date_min_,
datetime_null_, datetime_zero_, datetime_positive_, datetime_negative_, datetime_max_, datetime_min_,
datetime_interval_null_, datetime_interval_zero_, datetime_interval_positive_, datetime_interval_negative_,
year_month_interval_null_, year_month_interval_zero_,
@@ -727,7 +759,7 @@ class ComparisonTest : public ::testing::Test {
unique_ptr<ScopedBuffer> extended_short_char_buffer_, extended_long_char_buffer_;
vector<pair<const TypedValue*, const Type*>>
- numeric_typed_values_, datetime_typed_values_, datetime_interval_typed_values_,
+ numeric_typed_values_, date_typed_values_, datetime_typed_values_, datetime_interval_typed_values_,
year_month_interval_typed_values_, string_typed_values_;
};
@@ -803,7 +835,7 @@ void CheckBasicComparisonApplicableToPartialTypes(const Comparison &comparison)
// If only one argument is known, any orderable Type is OK.
for (const TypeID type_id
- : {kInt, kLong, kFloat, kDouble, kDatetime, kDatetimeInterval, kYearMonthInterval}) {
+ : {kInt, kLong, kFloat, kDouble, kDate, kDatetime, kDatetimeInterval, kYearMonthInterval}) {
EXPECT_TRUE(comparison.canComparePartialTypes(
&TypeFactory::GetType(type_id, false), nullptr));
EXPECT_TRUE(comparison.canComparePartialTypes(
@@ -827,12 +859,12 @@ void CheckBasicComparisonApplicableToPartialTypes(const Comparison &comparison)
// If both types are known, expect the same thing as canCompareTypes().
for (const TypeID first_type_id
- : {kInt, kLong, kFloat, kDouble, kDatetime, kDatetimeInterval, kYearMonthInterval}) {
+ : {kInt, kLong, kFloat, kDouble, kDate, kDatetime, kDatetimeInterval, kYearMonthInterval}) {
const Type &first_type = TypeFactory::GetType(first_type_id, false);
const Type &first_type_nullable = TypeFactory::GetType(first_type_id, true);
for (const TypeID second_type_id
- : {kInt, kLong, kFloat, kDouble, kDatetime, kDatetimeInterval, kYearMonthInterval}) {
+ : {kInt, kLong, kFloat, kDouble, kDate, kDatetime, kDatetimeInterval, kYearMonthInterval}) {
const Type &second_type = TypeFactory::GetType(second_type_id, false);
const Type &second_type_nullable = TypeFactory::GetType(second_type_id, true);
EXPECT_EQ(comparison.canCompareTypes(first_type, second_type),
@@ -864,7 +896,7 @@ void CheckBasicComparisonApplicableToPartialTypes(const Comparison &comparison)
const Type &first_type_nullable = TypeFactory::GetType(first_type_id, 10, true);
for (const TypeID second_type_id
- : {kInt, kLong, kFloat, kDouble, kDatetime, kDatetimeInterval, kYearMonthInterval}) {
+ : {kInt, kLong, kFloat, kDouble, kDate, kDatetime, kDatetimeInterval, kYearMonthInterval}) {
const Type &second_type = TypeFactory::GetType(second_type_id, false);
const Type &second_type_nullable = TypeFactory::GetType(second_type_id, true);
EXPECT_EQ(comparison.canCompareTypes(first_type, second_type),
[21/25] incubator-quickstep git commit: - Supported ROWS mode for AVG
window aggregation. - Created WindowAggregateFunctions in
expressions/window_aggregation. - Created WindowAggregationHandle for AVG to
actually do the calculation. - Other functions wi
Posted by zu...@apache.org.
- Supported ROWS mode for AVG window aggregation.
- Created WindowAggregateFunctions in expressions/window_aggregation.
- Created WindowAggregationHandle for AVG to actually do the calculation.
- Other functions will be supported in future PRs.
- RANGE mode is not supported yet.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/f5bf3532
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/f5bf3532
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/f5bf3532
Branch: refs/heads/dist-exe-test-new
Commit: f5bf35323db678fa457b21b42a308a95e6baa70a
Parents: 95c451a
Author: shixuan-fan <sh...@apache.org>
Authored: Fri Jul 8 18:23:47 2016 +0000
Committer: Zuyu Zhang <zu...@twitter.com>
Committed: Fri Jul 29 16:43:24 2016 -0700
----------------------------------------------------------------------
expressions/CMakeLists.txt | 1 +
expressions/window_aggregation/CMakeLists.txt | 206 ++++++++++
.../WindowAggregateFunction.cpp | 58 +++
.../WindowAggregateFunction.hpp | 149 +++++++
.../WindowAggregateFunction.proto | 32 ++
.../WindowAggregateFunctionAvg.cpp | 85 ++++
.../WindowAggregateFunctionAvg.hpp | 75 ++++
.../WindowAggregateFunctionCount.cpp | 59 +++
.../WindowAggregateFunctionCount.hpp | 75 ++++
.../WindowAggregateFunctionFactory.cpp | 106 +++++
.../WindowAggregateFunctionFactory.hpp | 96 +++++
.../WindowAggregateFunctionMax.cpp | 67 ++++
.../WindowAggregateFunctionMax.hpp | 75 ++++
.../WindowAggregateFunctionMin.cpp | 66 ++++
.../WindowAggregateFunctionMin.hpp | 75 ++++
.../WindowAggregateFunctionSum.cpp | 82 ++++
.../WindowAggregateFunctionSum.hpp | 75 ++++
.../WindowAggregationHandle.hpp | 137 +++++++
.../WindowAggregationHandleAvg.cpp | 241 ++++++++++++
.../WindowAggregationHandleAvg.hpp | 101 +++++
.../window_aggregation/WindowAggregationID.hpp | 44 +++
.../WindowAggregationHandleAvg_unittest.cpp | 387 +++++++++++++++++++
query_optimizer/CMakeLists.txt | 2 +
query_optimizer/ExecutionGenerator.cpp | 5 +-
query_optimizer/expressions/CMakeLists.txt | 2 +-
.../expressions/WindowAggregateFunction.cpp | 4 +-
.../expressions/WindowAggregateFunction.hpp | 10 +-
query_optimizer/resolver/CMakeLists.txt | 2 +
query_optimizer/resolver/Resolver.cpp | 40 +-
query_optimizer/resolver/Resolver.hpp | 5 +-
.../tests/execution_generator/Select.test | 78 +++-
relational_operators/CMakeLists.txt | 1 +
.../WindowAggregationOperator.cpp | 15 +-
.../WindowAggregationOperator.hpp | 30 +-
relational_operators/WorkOrder.proto | 3 +-
storage/CMakeLists.txt | 17 +-
storage/WindowAggregationOperationState.cpp | 183 +++++++--
storage/WindowAggregationOperationState.hpp | 68 +---
storage/WindowAggregationOperationState.proto | 6 +-
...WindowAggregationOperationState_unittest.cpp | 14 +-
40 files changed, 2613 insertions(+), 164 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f5bf3532/expressions/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/expressions/CMakeLists.txt b/expressions/CMakeLists.txt
index 53ad5d4..6ef3c24 100644
--- a/expressions/CMakeLists.txt
+++ b/expressions/CMakeLists.txt
@@ -17,6 +17,7 @@ add_subdirectory(aggregation)
add_subdirectory(predicate)
add_subdirectory(scalar)
add_subdirectory(table_generator)
+add_subdirectory(window_aggregation)
QS_PROTOBUF_GENERATE_CPP(expressions_Expressions_proto_srcs
expressions_Expressions_proto_hdrs
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f5bf3532/expressions/window_aggregation/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/expressions/window_aggregation/CMakeLists.txt b/expressions/window_aggregation/CMakeLists.txt
new file mode 100644
index 0000000..6a16fcc
--- /dev/null
+++ b/expressions/window_aggregation/CMakeLists.txt
@@ -0,0 +1,206 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+QS_PROTOBUF_GENERATE_CPP(expressions_windowaggregation_WindowAggregateFunction_proto_srcs
+ expressions_windowaggregation_WindowAggregateFunction_proto_hdrs
+ WindowAggregateFunction.proto)
+
+# Declare micro-libs:
+add_library(quickstep_expressions_windowaggregation_WindowAggregateFunction
+ WindowAggregateFunction.cpp
+ WindowAggregateFunction.hpp)
+add_library(quickstep_expressions_windowaggregation_WindowAggregateFunction_proto
+ ${expressions_windowaggregation_WindowAggregateFunction_proto_srcs})
+add_library(quickstep_expressions_windowaggregation_WindowAggregateFunctionAvg
+ WindowAggregateFunctionAvg.cpp
+ WindowAggregateFunctionAvg.hpp)
+add_library(quickstep_expressions_windowaggregation_WindowAggregateFunctionCount
+ WindowAggregateFunctionCount.cpp
+ WindowAggregateFunctionCount.hpp)
+add_library(quickstep_expressions_windowaggregation_WindowAggregateFunctionFactory
+ WindowAggregateFunctionFactory.cpp
+ WindowAggregateFunctionFactory.hpp)
+add_library(quickstep_expressions_windowaggregation_WindowAggregateFunctionMax
+ WindowAggregateFunctionMax.cpp
+ WindowAggregateFunctionMax.hpp)
+add_library(quickstep_expressions_windowaggregation_WindowAggregateFunctionMin
+ WindowAggregateFunctionMin.cpp
+ WindowAggregateFunctionMin.hpp)
+add_library(quickstep_expressions_windowaggregation_WindowAggregateFunctionSum
+ WindowAggregateFunctionSum.cpp
+ WindowAggregateFunctionSum.hpp)
+add_library(quickstep_expressions_windowaggregation_WindowAggregationHandle
+ ../../empty_src.cpp
+ WindowAggregationHandle.hpp)
+add_library(quickstep_expressions_windowaggregation_WindowAggregationHandleAvg
+ WindowAggregationHandleAvg.cpp
+ WindowAggregationHandleAvg.hpp)
+add_library(quickstep_expressions_windowaggregation_WindowAggregationID
+ ../../empty_src.cpp
+ WindowAggregationID.hpp)
+
+# Link dependencies:
+target_link_libraries(quickstep_expressions_windowaggregation_WindowAggregateFunction
+ glog
+ quickstep_catalog_CatalogTypedefs
+ quickstep_expressions_windowaggregation_WindowAggregateFunction_proto
+ quickstep_expressions_windowaggregation_WindowAggregationID
+ quickstep_storage_StorageBlockInfo
+ quickstep_utility_Macros)
+target_link_libraries(quickstep_expressions_windowaggregation_WindowAggregateFunction_proto
+ ${PROTOBUF_LIBRARY})
+target_link_libraries(quickstep_expressions_windowaggregation_WindowAggregateFunctionAvg
+ glog
+ quickstep_expressions_windowaggregation_WindowAggregateFunction
+ quickstep_expressions_windowaggregation_WindowAggregationHandleAvg
+ quickstep_expressions_windowaggregation_WindowAggregationID
+ quickstep_types_Type
+ quickstep_types_TypeFactory
+ quickstep_types_TypeID
+ quickstep_types_operations_binaryoperations_BinaryOperation
+ quickstep_types_operations_binaryoperations_BinaryOperationFactory
+ quickstep_types_operations_binaryoperations_BinaryOperationID
+ quickstep_utility_Macros)
+target_link_libraries(quickstep_expressions_windowaggregation_WindowAggregateFunctionCount
+ glog
+ quickstep_expressions_windowaggregation_WindowAggregateFunction
+ quickstep_expressions_windowaggregation_WindowAggregationHandle
+ quickstep_expressions_windowaggregation_WindowAggregationID
+ quickstep_types_Type
+ quickstep_types_TypeFactory
+ quickstep_types_TypeID
+ quickstep_utility_Macros)
+target_link_libraries(quickstep_expressions_windowaggregation_WindowAggregateFunctionFactory
+ glog
+ quickstep_expressions_windowaggregation_WindowAggregateFunctionAvg
+ quickstep_expressions_windowaggregation_WindowAggregateFunctionCount
+ quickstep_expressions_windowaggregation_WindowAggregateFunctionMax
+ quickstep_expressions_windowaggregation_WindowAggregateFunctionMin
+ quickstep_expressions_windowaggregation_WindowAggregateFunctionSum
+ quickstep_expressions_windowaggregation_WindowAggregateFunction_proto
+ quickstep_expressions_windowaggregation_WindowAggregationID
+ quickstep_utility_Macros)
+target_link_libraries(quickstep_expressions_windowaggregation_WindowAggregateFunctionMax
+ glog
+ quickstep_expressions_windowaggregation_WindowAggregateFunction
+ quickstep_expressions_windowaggregation_WindowAggregationHandle
+ quickstep_expressions_windowaggregation_WindowAggregationID
+ quickstep_types_Type
+ quickstep_types_operations_comparisons_Comparison
+ quickstep_types_operations_comparisons_ComparisonFactory
+ quickstep_types_operations_comparisons_ComparisonID
+ quickstep_utility_Macros)
+target_link_libraries(quickstep_expressions_windowaggregation_WindowAggregateFunctionMin
+ glog
+ quickstep_expressions_windowaggregation_WindowAggregateFunction
+ quickstep_expressions_windowaggregation_WindowAggregationHandle
+ quickstep_expressions_windowaggregation_WindowAggregationID
+ quickstep_types_Type
+ quickstep_types_operations_comparisons_Comparison
+ quickstep_types_operations_comparisons_ComparisonFactory
+ quickstep_types_operations_comparisons_ComparisonID
+ quickstep_utility_Macros)
+target_link_libraries(quickstep_expressions_windowaggregation_WindowAggregateFunctionSum
+ glog
+ quickstep_expressions_windowaggregation_WindowAggregateFunction
+ quickstep_expressions_windowaggregation_WindowAggregationHandle
+ quickstep_expressions_windowaggregation_WindowAggregationID
+ quickstep_types_Type
+ quickstep_types_TypeFactory
+ quickstep_types_TypeID
+ quickstep_types_operations_binaryoperations_BinaryOperation
+ quickstep_types_operations_binaryoperations_BinaryOperationFactory
+ quickstep_types_operations_binaryoperations_BinaryOperationID
+ quickstep_utility_Macros)
+target_link_libraries(quickstep_expressions_windowaggregation_WindowAggregationHandle
+ glog
+ quickstep_catalog_CatalogRelationSchema
+ quickstep_catalog_CatalogTypedefs
+ quickstep_storage_StorageBlockInfo
+ quickstep_types_TypedValue
+ quickstep_types_containers_ColumnVector
+ quickstep_types_containers_ColumnVectorsValueAccessor
+ quickstep_types_operations_comparisons_Comparison
+ quickstep_types_operations_comparisons_ComparisonFactory
+ quickstep_types_operations_comparisons_ComparisonID
+ quickstep_utility_Macros)
+target_link_libraries(quickstep_expressions_windowaggregation_WindowAggregationHandleAvg
+ glog
+ quickstep_catalog_CatalogTypedefs
+ quickstep_expressions_scalar_Scalar
+ quickstep_expressions_scalar_ScalarAttribute
+ quickstep_expressions_windowaggregation_WindowAggregationHandle
+ quickstep_storage_ValueAccessor
+ quickstep_types_Type
+ quickstep_types_TypeFactory
+ quickstep_types_TypeID
+ quickstep_types_TypedValue
+ quickstep_types_containers_ColumnVectorsValueAccessor
+ quickstep_types_operations_binaryoperations_BinaryOperation
+ quickstep_types_operations_binaryoperations_BinaryOperationFactory
+ quickstep_types_operations_binaryoperations_BinaryOperationID
+ quickstep_types_operations_comparisons_Comparison
+ quickstep_utility_Macros)
+
+# Submodule all-in-one library:
+add_library(quickstep_expressions_windowaggregation ../../empty_src.cpp)
+target_link_libraries(quickstep_expressions_windowaggregation
+ quickstep_expressions_windowaggregation_WindowAggregateFunction
+ quickstep_expressions_windowaggregation_WindowAggregateFunction_proto
+ quickstep_expressions_windowaggregation_WindowAggregateFunctionAvg
+ quickstep_expressions_windowaggregation_WindowAggregateFunctionCount
+ quickstep_expressions_windowaggregation_WindowAggregateFunctionFactory
+ quickstep_expressions_windowaggregation_WindowAggregateFunctionMax
+ quickstep_expressions_windowaggregation_WindowAggregateFunctionMin
+ quickstep_expressions_windowaggregation_WindowAggregateFunctionSum
+ quickstep_expressions_windowaggregation_WindowAggregationHandle
+ quickstep_expressions_windowaggregation_WindowAggregationHandleAvg
+ quickstep_expressions_windowaggregation_WindowAggregationID)
+
+# Tests:
+
+# Unified executable to ammortize cost of linking.
+add_executable(WindowAggregationHandle_tests
+ "${CMAKE_CURRENT_SOURCE_DIR}/tests/WindowAggregationHandleAvg_unittest.cpp")
+target_link_libraries(WindowAggregationHandle_tests
+ gtest
+ gtest_main
+ quickstep_catalog_CatalogTypedefs
+ quickstep_expressions_windowaggregation_WindowAggregateFunction
+ quickstep_expressions_windowaggregation_WindowAggregateFunctionFactory
+ quickstep_expressions_windowaggregation_WindowAggregationHandle
+ quickstep_expressions_windowaggregation_WindowAggregationHandleAvg
+ quickstep_expressions_windowaggregation_WindowAggregationID
+ quickstep_storage_ValueAccessor
+ quickstep_types_CharType
+ quickstep_types_DateOperatorOverloads
+ quickstep_types_DatetimeIntervalType
+ quickstep_types_DatetimeType
+ quickstep_types_DoubleType
+ quickstep_types_FloatType
+ quickstep_types_IntType
+ quickstep_types_IntervalLit
+ quickstep_types_LongType
+ quickstep_types_Type
+ quickstep_types_TypeFactory
+ quickstep_types_TypeID
+ quickstep_types_TypedValue
+ quickstep_types_VarCharType
+ quickstep_types_YearMonthIntervalType
+ quickstep_types_containers_ColumnVector
+ quickstep_types_containers_ColumnVectorsValueAccessor)
+add_test(WindowAggregationHandle_tests WindowAggregationHandle_tests)
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f5bf3532/expressions/window_aggregation/WindowAggregateFunction.cpp
----------------------------------------------------------------------
diff --git a/expressions/window_aggregation/WindowAggregateFunction.cpp b/expressions/window_aggregation/WindowAggregateFunction.cpp
new file mode 100644
index 0000000..3911e1c
--- /dev/null
+++ b/expressions/window_aggregation/WindowAggregateFunction.cpp
@@ -0,0 +1,58 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ **/
+
+#include "expressions/window_aggregation/WindowAggregateFunction.hpp"
+
+#include <type_traits>
+
+#include "expressions/window_aggregation/WindowAggregateFunction.pb.h"
+#include "expressions/window_aggregation/WindowAggregationID.hpp"
+
+#include "glog/logging.h"
+
+namespace quickstep {
+
+serialization::WindowAggregateFunction WindowAggregateFunction::getProto() const {
+ serialization::WindowAggregateFunction proto;
+ switch (win_agg_id_) {
+ case WindowAggregationID::kAvg:
+ proto.set_window_aggregation_id(serialization::WindowAggregateFunction::AVG);
+ break;
+ case WindowAggregationID::kCount:
+ proto.set_window_aggregation_id(serialization::WindowAggregateFunction::COUNT);
+ break;
+ case WindowAggregationID::kMax:
+ proto.set_window_aggregation_id(serialization::WindowAggregateFunction::MAX);
+ break;
+ case WindowAggregationID::kMin:
+ proto.set_window_aggregation_id(serialization::WindowAggregateFunction::MIN);
+ break;
+ case WindowAggregationID::kSum:
+ proto.set_window_aggregation_id(serialization::WindowAggregateFunction::SUM);
+ break;
+ default: {
+ LOG(FATAL) << "Unrecognized WindowAggregationID: "
+ << static_cast<std::underlying_type<WindowAggregationID>::type>(win_agg_id_);
+ }
+ }
+
+ return proto;
+}
+
+} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f5bf3532/expressions/window_aggregation/WindowAggregateFunction.hpp
----------------------------------------------------------------------
diff --git a/expressions/window_aggregation/WindowAggregateFunction.hpp b/expressions/window_aggregation/WindowAggregateFunction.hpp
new file mode 100644
index 0000000..e40479b
--- /dev/null
+++ b/expressions/window_aggregation/WindowAggregateFunction.hpp
@@ -0,0 +1,149 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ **/
+
+#ifndef QUICKSTEP_EXPRESSIONS_WINDOW_AGGREGATION_WINDOW_AGGREGATE_FUNCTION_HPP_
+#define QUICKSTEP_EXPRESSIONS_WINDOW_AGGREGATION_WINDOW_AGGREGATE_FUNCTION_HPP_
+
+#include <string>
+#include <vector>
+
+#include "catalog/CatalogTypedefs.hpp"
+#include "expressions/window_aggregation/WindowAggregateFunction.pb.h"
+#include "expressions/window_aggregation/WindowAggregationID.hpp"
+#include "storage/StorageBlockInfo.hpp"
+#include "utility/Macros.hpp"
+
+namespace quickstep {
+
+class CatalogRelationSchema;
+class Type;
+class WindowAggregationHandle;
+
+/** \addtogroup Expressions
+ * @{
+ */
+
+/**
+ * @brief A class representing a particular window aggregate function in the
+ * abstract sense. Each named aggregate function is represented by a
+ * singleton subclass of WindowAggregateFunction.
+ *
+ * WindowAggregateFunction provides informational methods about the
+ * applicability of a particular window aggregate function to particular
+ * argument Type(s). The actual implementation of the window aggregate
+ * functions' logic is in the WindowAggregationHandle class hierarchy, and can
+ * be different depending on the particular argument Type(s) given to the window
+ * aggregate. To perform a window aggregation, a caller should first call
+ * WindowAggregateFunction::createHandle() to instantiate an
+ * WindowAggregationHandle object, then use the methods of
+ * WindowAggregationHandle to do the actual window aggregation. Finally, delete
+ * the WindowAggregationHandle after finished.
+ *
+ * See WindowAggregationHandle for more detailed information about how
+ * window aggregates are actually computed.
+ **/
+class WindowAggregateFunction {
+ public:
+ /**
+ * @brief Get the ID of this window aggregate (i.e. its unique ID amongst all
+ * the WindowAggregateFunctions).
+ *
+ * @return The WindowAggregationID of this WindowAggregateFunction.
+ **/
+ inline WindowAggregationID getWindowAggregationID() const {
+ return win_agg_id_;
+ }
+
+ /**
+ * @brief Get the human-readable name of this WindowAggregateFunction.
+ *
+ * @return The human-readable name of this WindowAggregateFunction.
+ **/
+ virtual std::string getName() const = 0;
+
+ /**
+ * @brief Get the serialized protocol buffer representation of this
+ * WindowAggregateFunction.
+ *
+ * @return A serialized protocol buffer representation of this
+ * WindowAggregateFunction.
+ **/
+ virtual serialization::WindowAggregateFunction getProto() const;
+
+ /**
+ * @brief Determine if this WindowAggregateFunction can be applied to
+ * arguments of particular Type(s).
+ *
+ * @param argument_types A list of zero or more Types (in order) for
+ * arguments to this WindowAggregateFunction.
+ * @return Whether this WindowAggregateFunction is applicable to the given
+ * argument_types.
+ **/
+ virtual bool canApplyToTypes(
+ const std::vector<const Type*> &argument_types) const = 0;
+
+ /**
+ * @brief Determine the result Type for this WindowAggregateFunction given
+ * arguments of particular Type(s).
+ *
+ * @param argument_types A list of zero or more Types (in order) for
+ * arguments to this WindowAggregateFunction.
+ * @return The result Type for this WindowAggregateFunction applied to the
+ * specified argument_types, or nullptr if this
+ * WindowAggregateFunction is not applicable to the specified Type(s).
+ **/
+ virtual const Type* resultTypeForArgumentTypes(
+ const std::vector<const Type*> &argument_types) const = 0;
+
+ /**
+ * @brief Create a WindowAggregationHandle to compute aggregates.
+ *
+ * @warning It is an error to call this method for argument_types which this
+ * WindowAggregateFunction can not apply to. For safety, check
+ * canApplyToTypes() first.
+ *
+ * @param argument_types A list of zero or more Types (in order) for
+ * arguments to this WindowAggregateFunction.
+ * @param partition_key_types A list or zero or more Types for partition keys
+ * to this WindowAggregateFunction.
+ *
+ * @return A new WindowAggregationHandle that can be used to compute this
+ * WindowAggregateFunction over the specified argument_types. Caller
+ * is responsible for deleting the returned object.
+ **/
+ virtual WindowAggregationHandle* createHandle(
+ const std::vector<const Type*> &argument_types,
+ const std::vector<const Type*> &partition_key_types) const = 0;
+
+ protected:
+ explicit WindowAggregateFunction(const WindowAggregationID win_agg_id)
+ : win_agg_id_(win_agg_id) {
+ }
+
+ private:
+ const WindowAggregationID win_agg_id_;
+
+ DISALLOW_COPY_AND_ASSIGN(WindowAggregateFunction);
+};
+
+/** @} */
+
+} // namespace quickstep
+
+#endif // QUICKSTEP_EXPRESSIONS_WINDOW_AGGREGATION_WINDOW_AGGREGATE_FUNCTION_HPP_
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f5bf3532/expressions/window_aggregation/WindowAggregateFunction.proto
----------------------------------------------------------------------
diff --git a/expressions/window_aggregation/WindowAggregateFunction.proto b/expressions/window_aggregation/WindowAggregateFunction.proto
new file mode 100644
index 0000000..fe8d799
--- /dev/null
+++ b/expressions/window_aggregation/WindowAggregateFunction.proto
@@ -0,0 +1,32 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+syntax = "proto2";
+
+package quickstep.serialization;
+
+message WindowAggregateFunction {
+ enum WindowAggregationID {
+ AVG = 0;
+ COUNT = 1;
+ MAX = 2;
+ MIN = 3;
+ SUM = 4;
+ }
+
+ required WindowAggregationID window_aggregation_id = 1;
+}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f5bf3532/expressions/window_aggregation/WindowAggregateFunctionAvg.cpp
----------------------------------------------------------------------
diff --git a/expressions/window_aggregation/WindowAggregateFunctionAvg.cpp b/expressions/window_aggregation/WindowAggregateFunctionAvg.cpp
new file mode 100644
index 0000000..bc31a53
--- /dev/null
+++ b/expressions/window_aggregation/WindowAggregateFunctionAvg.cpp
@@ -0,0 +1,85 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ **/
+
+#include "expressions/window_aggregation/WindowAggregateFunctionAvg.hpp"
+
+#include <vector>
+
+#include "expressions/window_aggregation/WindowAggregationHandleAvg.hpp"
+#include "types/Type.hpp"
+#include "types/TypeFactory.hpp"
+#include "types/TypeID.hpp"
+#include "types/operations/binary_operations/BinaryOperation.hpp"
+#include "types/operations/binary_operations/BinaryOperationFactory.hpp"
+#include "types/operations/binary_operations/BinaryOperationID.hpp"
+
+#include "glog/logging.h"
+
+namespace quickstep {
+
+bool WindowAggregateFunctionAvg::canApplyToTypes(
+ const std::vector<const Type*> &argument_types) const {
+ // AVG is unary.
+ if (argument_types.size() != 1) {
+ return false;
+ }
+
+ // Argument must be addable and divisible.
+ return BinaryOperationFactory::GetBinaryOperation(BinaryOperationID::kAdd)
+ .canApplyToTypes(*argument_types.front(), *argument_types.front()) &&
+ BinaryOperationFactory::GetBinaryOperation(BinaryOperationID::kDivide)
+ .canApplyToTypes(*argument_types.front(), TypeFactory::GetType(kDouble));
+}
+
+const Type* WindowAggregateFunctionAvg::resultTypeForArgumentTypes(
+ const std::vector<const Type*> &argument_types) const {
+ if (!canApplyToTypes(argument_types)) {
+ return nullptr;
+ }
+
+ // The type used to sum values is nullable, and we automatically widen int to
+ // long and float to double to have more headroom when adding up many values.
+ const Type *sum_type = &(argument_types.front()->getNullableVersion());
+ switch (sum_type->getTypeID()) {
+ case kInt:
+ sum_type = &TypeFactory::GetType(kLong, true);
+ break;
+ case kFloat:
+ sum_type = &TypeFactory::GetType(kDouble, true);
+ break;
+ default:
+ break;
+ }
+
+ return BinaryOperationFactory::GetBinaryOperation(BinaryOperationID::kDivide)
+ .resultTypeForArgumentTypes(*sum_type, TypeFactory::GetType(kDouble));
+}
+
+WindowAggregationHandle* WindowAggregateFunctionAvg::createHandle(
+ const std::vector<const Type*> &argument_types,
+ const std::vector<const Type*> &partition_key_types) const {
+ DCHECK(canApplyToTypes(argument_types))
+ << "Attempted to create an WindowAggregationHandleAvg for argument Type(s)"
+ << " that AVG can not be applied to.";
+
+ return new WindowAggregationHandleAvg(partition_key_types,
+ *argument_types.front());
+}
+
+} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f5bf3532/expressions/window_aggregation/WindowAggregateFunctionAvg.hpp
----------------------------------------------------------------------
diff --git a/expressions/window_aggregation/WindowAggregateFunctionAvg.hpp b/expressions/window_aggregation/WindowAggregateFunctionAvg.hpp
new file mode 100644
index 0000000..32fd9d5
--- /dev/null
+++ b/expressions/window_aggregation/WindowAggregateFunctionAvg.hpp
@@ -0,0 +1,75 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ **/
+
+#ifndef QUICKSTEP_EXPRESSIONS_WINDOW_AGGREGATION_WINDOW_AGGREGATE_FUNCTION_AVG_HPP_
+#define QUICKSTEP_EXPRESSIONS_WINDOW_AGGREGATION_WINDOW_AGGREGATE_FUNCTION_AVG_HPP_
+
+#include <string>
+#include <vector>
+
+#include "expressions/window_aggregation/WindowAggregateFunction.hpp"
+#include "expressions/window_aggregation/WindowAggregationID.hpp"
+#include "utility/Macros.hpp"
+
+namespace quickstep {
+
+class Type;
+class WindowAggregationHandle;
+
+/** \addtogroup Expressions
+ * @{
+ */
+
+/**
+ * @brief WindowAggregateFunction representing SQL AVG() OVER term.
+ **/
+class WindowAggregateFunctionAvg : public WindowAggregateFunction {
+ public:
+ static const WindowAggregateFunctionAvg& Instance() {
+ static WindowAggregateFunctionAvg instance;
+ return instance;
+ }
+
+ std::string getName() const override {
+ return "AVG";
+ }
+
+ bool canApplyToTypes(
+ const std::vector<const Type*> &argument_types) const override;
+
+ const Type* resultTypeForArgumentTypes(
+ const std::vector<const Type*> &argument_types) const override;
+
+ WindowAggregationHandle* createHandle(
+ const std::vector<const Type*> &argument_types,
+ const std::vector<const Type*> &partition_key_types) const override;
+
+ private:
+ WindowAggregateFunctionAvg()
+ : WindowAggregateFunction(WindowAggregationID::kAvg) {
+ }
+
+ DISALLOW_COPY_AND_ASSIGN(WindowAggregateFunctionAvg);
+};
+
+/** @} */
+
+} // namespace quickstep
+
+#endif // QUICKSTEP_EXPRESSIONS_WINDOW_AGGREGATION_WINDOW_AGGREGATE_FUNCTION_AVG_HPP_
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f5bf3532/expressions/window_aggregation/WindowAggregateFunctionCount.cpp
----------------------------------------------------------------------
diff --git a/expressions/window_aggregation/WindowAggregateFunctionCount.cpp b/expressions/window_aggregation/WindowAggregateFunctionCount.cpp
new file mode 100644
index 0000000..504e000
--- /dev/null
+++ b/expressions/window_aggregation/WindowAggregateFunctionCount.cpp
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ **/
+
+#include "expressions/window_aggregation/WindowAggregateFunctionCount.hpp"
+
+#include <vector>
+
+#include "expressions/window_aggregation/WindowAggregationHandle.hpp"
+#include "types/Type.hpp"
+#include "types/TypeFactory.hpp"
+#include "types/TypeID.hpp"
+
+#include "glog/logging.h"
+
+namespace quickstep {
+
+bool WindowAggregateFunctionCount::canApplyToTypes(
+ const std::vector<const Type*> &argument_types) const {
+ // COUNT may be nullary (i.e. COUNT(*)) or unary.
+ return argument_types.size() <= 1;
+}
+
+const Type* WindowAggregateFunctionCount::resultTypeForArgumentTypes(
+ const std::vector<const Type*> &argument_types) const {
+ if (!canApplyToTypes(argument_types)) {
+ return nullptr;
+ }
+
+ return &TypeFactory::GetType(kLong);
+}
+
+WindowAggregationHandle* WindowAggregateFunctionCount::createHandle(
+ const std::vector<const Type*> &argument_types,
+ const std::vector<const Type*> &partition_key_types) const {
+ DCHECK(canApplyToTypes(argument_types))
+ << "Attempted to create a WindowAggregationHandleCount for argument Types "
+ << "that COUNT can not be applied to (> 1 argument).";
+
+ // TODO(Shixuan): Add handle for Count.
+ return nullptr;
+}
+
+} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f5bf3532/expressions/window_aggregation/WindowAggregateFunctionCount.hpp
----------------------------------------------------------------------
diff --git a/expressions/window_aggregation/WindowAggregateFunctionCount.hpp b/expressions/window_aggregation/WindowAggregateFunctionCount.hpp
new file mode 100644
index 0000000..1b40fdd
--- /dev/null
+++ b/expressions/window_aggregation/WindowAggregateFunctionCount.hpp
@@ -0,0 +1,75 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ **/
+
+#ifndef QUICKSTEP_EXPRESSIONS_WINDOW_AGGREGATION_WINDOW_AGGREGATE_FUNCTION_COUNT_HPP_
+#define QUICKSTEP_EXPRESSIONS_WINDOW_AGGREGATION_WINDOW_AGGREGATE_FUNCTION_COUNT_HPP_
+
+#include <string>
+#include <vector>
+
+#include "expressions/window_aggregation/WindowAggregateFunction.hpp"
+#include "expressions/window_aggregation/WindowAggregationID.hpp"
+#include "utility/Macros.hpp"
+
+namespace quickstep {
+
+class Type;
+class WindowAggregationHandle;
+
+/** \addtogroup Expressions
+ * @{
+ */
+
+/**
+ * @brief WindowAggregateFunction representing SQL COUNT() OVER term.
+ **/
+class WindowAggregateFunctionCount : public WindowAggregateFunction {
+ public:
+ static const WindowAggregateFunctionCount& Instance() {
+ static WindowAggregateFunctionCount instance;
+ return instance;
+ }
+
+ std::string getName() const override {
+ return "COUNT";
+ }
+
+ bool canApplyToTypes(
+ const std::vector<const Type*> &argument_types) const override;
+
+ const Type* resultTypeForArgumentTypes(
+ const std::vector<const Type*> &argument_types) const override;
+
+ WindowAggregationHandle* createHandle(
+ const std::vector<const Type*> &argument_types,
+ const std::vector<const Type*> &partition_key_types) const override;
+
+ private:
+ WindowAggregateFunctionCount()
+ : WindowAggregateFunction(WindowAggregationID::kCount) {
+ }
+
+ DISALLOW_COPY_AND_ASSIGN(WindowAggregateFunctionCount);
+};
+
+/** @} */
+
+} // namespace quickstep
+
+#endif // QUICKSTEP_EXPRESSIONS_WINDOW_AGGREGATION_WINDOW_AGGREGATE_FUNCTION_COUNT_HPP_
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f5bf3532/expressions/window_aggregation/WindowAggregateFunctionFactory.cpp
----------------------------------------------------------------------
diff --git a/expressions/window_aggregation/WindowAggregateFunctionFactory.cpp b/expressions/window_aggregation/WindowAggregateFunctionFactory.cpp
new file mode 100644
index 0000000..65247f2
--- /dev/null
+++ b/expressions/window_aggregation/WindowAggregateFunctionFactory.cpp
@@ -0,0 +1,106 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ **/
+
+#include "expressions/window_aggregation/WindowAggregateFunctionFactory.hpp"
+
+#include <string>
+#include <type_traits>
+
+#include "expressions/window_aggregation/WindowAggregateFunction.pb.h"
+#include "expressions/window_aggregation/WindowAggregateFunctionAvg.hpp"
+#include "expressions/window_aggregation/WindowAggregateFunctionCount.hpp"
+#include "expressions/window_aggregation/WindowAggregateFunctionMax.hpp"
+#include "expressions/window_aggregation/WindowAggregateFunctionMin.hpp"
+#include "expressions/window_aggregation/WindowAggregateFunctionSum.hpp"
+#include "expressions/window_aggregation/WindowAggregationID.hpp"
+
+#include "glog/logging.h"
+
+namespace quickstep {
+
+const WindowAggregateFunction& WindowAggregateFunctionFactory::Get(
+ const WindowAggregationID agg_id) {
+ switch (agg_id) {
+ case WindowAggregationID::kAvg:
+ return WindowAggregateFunctionAvg::Instance();
+ case WindowAggregationID::kCount:
+ return WindowAggregateFunctionCount::Instance();
+ case WindowAggregationID::kMax:
+ return WindowAggregateFunctionMax::Instance();
+ case WindowAggregationID::kMin:
+ return WindowAggregateFunctionMin::Instance();
+ case WindowAggregationID::kSum:
+ return WindowAggregateFunctionSum::Instance();
+ default: {
+ LOG(FATAL) << "Unrecognized WindowAggregationID: "
+ << static_cast<std::underlying_type<WindowAggregationID>::type>(agg_id);
+ }
+ }
+}
+
+const WindowAggregateFunction* WindowAggregateFunctionFactory::GetByName(
+ const std::string &name) {
+ if (name == "avg") {
+ return &WindowAggregateFunctionAvg::Instance();
+ } else if (name == "count") {
+ return &WindowAggregateFunctionCount::Instance();
+ } else if (name == "max") {
+ return &WindowAggregateFunctionMax::Instance();
+ } else if (name == "min") {
+ return &WindowAggregateFunctionMin::Instance();
+ } else if (name == "sum") {
+ return &WindowAggregateFunctionSum::Instance();
+ } else {
+ return nullptr;
+ }
+}
+
+bool WindowAggregateFunctionFactory::ProtoIsValid(
+ const serialization::WindowAggregateFunction &proto) {
+ return proto.IsInitialized() &&
+ serialization::WindowAggregateFunction::WindowAggregationID_IsValid(proto.window_aggregation_id());
+}
+
+const WindowAggregateFunction& WindowAggregateFunctionFactory::ReconstructFromProto(
+ const serialization::WindowAggregateFunction &proto) {
+ DCHECK(ProtoIsValid(proto))
+ << "Attempted to reconstruct an WindowAggregateFunction from an invalid proto:\n"
+ << proto.DebugString();
+
+ switch (proto.window_aggregation_id()) {
+ case serialization::WindowAggregateFunction::AVG:
+ return WindowAggregateFunctionAvg::Instance();
+ case serialization::WindowAggregateFunction::COUNT:
+ return WindowAggregateFunctionCount::Instance();
+ case serialization::WindowAggregateFunction::MAX:
+ return WindowAggregateFunctionMax::Instance();
+ case serialization::WindowAggregateFunction::MIN:
+ return WindowAggregateFunctionMin::Instance();
+ case serialization::WindowAggregateFunction::SUM:
+ return WindowAggregateFunctionSum::Instance();
+ default: {
+ LOG(FATAL) << "Unrecognized serialization::WindowAggregateFunction::WindowAggregationID: "
+ << proto.window_aggregation_id()
+ << "\nFull proto debug string:\n"
+ << proto.DebugString();
+ }
+ }
+}
+
+} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f5bf3532/expressions/window_aggregation/WindowAggregateFunctionFactory.hpp
----------------------------------------------------------------------
diff --git a/expressions/window_aggregation/WindowAggregateFunctionFactory.hpp b/expressions/window_aggregation/WindowAggregateFunctionFactory.hpp
new file mode 100644
index 0000000..1d59e93
--- /dev/null
+++ b/expressions/window_aggregation/WindowAggregateFunctionFactory.hpp
@@ -0,0 +1,96 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ **/
+
+#ifndef QUICKSTEP_EXPRESSIONS_WINDOW_AGGREGATION_WINDOW_AGGREGATE_FUNCTION_FACTORY_HPP_
+#define QUICKSTEP_EXPRESSIONS_WINDOW_AGGREGATION_WINDOW_AGGREGATE_FUNCTION_FACTORY_HPP_
+
+#include <string>
+
+#include "expressions/window_aggregation/WindowAggregationID.hpp"
+#include "utility/Macros.hpp"
+
+namespace quickstep {
+
+class WindowAggregateFunction;
+namespace serialization { class WindowAggregateFunction; }
+
+/** \addtogroup Expressions
+ * @{
+ */
+
+/**
+ * @brief All-static factory with methods that provide access to the various
+ * concrete implementations of WindowAggregateFunction.
+ *
+ * WindowAggregateFunctionFactory allows client code to use any
+ * WindowAggregateFunction in Quickstep in a generic way without having to know
+ * about all the specific subclasses of WindowAggregateFunction. In particular,
+ * it is used to deserialize WindowAggregateFunctions used in
+ * WindowAggregationOperationState from their protobuf representations
+ * (originally created by the optimizer) when deserializing a QueryContext.
+ **/
+namespace WindowAggregateFunctionFactory {
+ /**
+ * @brief Get a particular WindowAggregateFunction by its ID.
+ *
+ * @param agg_id The ID of the desired WindowAggregateFunction.
+ * @return A reference to the singleton instance of the
+ * WindowAggregateFunction specified by agg_id.
+ **/
+ const WindowAggregateFunction& Get(const WindowAggregationID agg_id);
+
+ /**
+ * @brief Get a particular WindowAggregateFunction by its name in SQL syntax.
+ *
+ * @param name The name of the desired WindowAggregateFunction in lower case.
+ * @return A pointer to the WindowAggregateFunction specified by name, or NULL
+ * if name does not match any known WindowAggregateFunction.
+ **/
+ const WindowAggregateFunction* GetByName(const std::string &name);
+
+ /**
+ * @brief Determine if a serialized protobuf representation of a
+ * WindowAggregateFunction is fully-formed and valid.
+ *
+ * @param proto A serialized protobuf representation of a
+ * WindowAggregateFunction to check for validity.
+ * @return Whether proto is fully-formed and valid.
+ **/
+ bool ProtoIsValid(const serialization::WindowAggregateFunction &proto);
+
+ /**
+ * @brief Get the WindowAggregateFunction represented by a proto.
+ *
+ * @warning It is an error to call this method with an invalid proto.
+ * ProtoIsValid() should be called first to check.
+ *
+ * @param proto A serialized protobuf representation of a
+ * WindowAggregateFunction.
+ * @return The WindowAggregateFunction represented by proto.
+ **/
+ const WindowAggregateFunction& ReconstructFromProto(
+ const serialization::WindowAggregateFunction &proto);
+
+} // namespace WindowAggregateFunctionFactory
+
+/** @} */
+
+} // namespace quickstep
+
+#endif // QUICKSTEP_EXPRESSIONS_WINDOW_AGGREGATION_WINDOW_AGGREGATE_FUNCTION_FACTORY_HPP_
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f5bf3532/expressions/window_aggregation/WindowAggregateFunctionMax.cpp
----------------------------------------------------------------------
diff --git a/expressions/window_aggregation/WindowAggregateFunctionMax.cpp b/expressions/window_aggregation/WindowAggregateFunctionMax.cpp
new file mode 100644
index 0000000..f3997c7
--- /dev/null
+++ b/expressions/window_aggregation/WindowAggregateFunctionMax.cpp
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ **/
+
+#include "expressions/window_aggregation/WindowAggregateFunctionMax.hpp"
+
+#include <vector>
+
+#include "expressions/window_aggregation/WindowAggregationHandle.hpp"
+#include "types/Type.hpp"
+#include "types/operations/comparisons/Comparison.hpp"
+#include "types/operations/comparisons/ComparisonFactory.hpp"
+#include "types/operations/comparisons/ComparisonID.hpp"
+
+#include "glog/logging.h"
+
+namespace quickstep {
+
+bool WindowAggregateFunctionMax::canApplyToTypes(
+ const std::vector<const Type*> &argument_types) const {
+ // MAX is unary.
+ if (argument_types.size() != 1) {
+ return false;
+ }
+
+ // Argument must be comparable by '>'.
+ return ComparisonFactory::GetComparison(ComparisonID::kGreater).canCompareTypes(
+ *argument_types.front(),
+ *argument_types.front());
+}
+
+const Type* WindowAggregateFunctionMax::resultTypeForArgumentTypes(
+ const std::vector<const Type*> &argument_types) const {
+ if (!canApplyToTypes(argument_types)) {
+ return nullptr;
+ }
+
+ return &(argument_types.front()->getNullableVersion());
+}
+
+WindowAggregationHandle* WindowAggregateFunctionMax::createHandle(
+ const std::vector<const Type*> &argument_types,
+ const std::vector<const Type*> &partition_key_types) const {
+ DCHECK(canApplyToTypes(argument_types))
+ << "Attempted to create a WindowAggregationHandleMax for argument Type(s) "
+ << "that MAX can not be applied to.";
+
+ // TODO(Shixuan): Add handle for Max.
+ return nullptr;
+}
+
+} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f5bf3532/expressions/window_aggregation/WindowAggregateFunctionMax.hpp
----------------------------------------------------------------------
diff --git a/expressions/window_aggregation/WindowAggregateFunctionMax.hpp b/expressions/window_aggregation/WindowAggregateFunctionMax.hpp
new file mode 100644
index 0000000..00c788e
--- /dev/null
+++ b/expressions/window_aggregation/WindowAggregateFunctionMax.hpp
@@ -0,0 +1,75 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ **/
+
+#ifndef QUICKSTEP_EXPRESSIONS_WINDOW_AGGREGATION_WINDOW_AGGREGATE_FUNCTION_MAX_HPP_
+#define QUICKSTEP_EXPRESSIONS_WINDOW_AGGREGATION_WINDOW_AGGREGATE_FUNCTION_MAX_HPP_
+
+#include <string>
+#include <vector>
+
+#include "expressions/window_aggregation/WindowAggregateFunction.hpp"
+#include "expressions/window_aggregation/WindowAggregationID.hpp"
+#include "utility/Macros.hpp"
+
+namespace quickstep {
+
+class Type;
+class WindowAggregationHandle;
+
+/** \addtogroup Expressions
+ * @{
+ */
+
+/**
+ * @brief WindowAggregateFunction representing SQL MAX() OVER term.
+ **/
+class WindowAggregateFunctionMax : public WindowAggregateFunction {
+ public:
+ static const WindowAggregateFunctionMax& Instance() {
+ static WindowAggregateFunctionMax instance;
+ return instance;
+ }
+
+ std::string getName() const override {
+ return "MAX";
+ }
+
+ bool canApplyToTypes(
+ const std::vector<const Type*> &argument_types) const override;
+
+ const Type* resultTypeForArgumentTypes(
+ const std::vector<const Type*> &argument_types) const override;
+
+ WindowAggregationHandle* createHandle(
+ const std::vector<const Type*> &argument_types,
+ const std::vector<const Type*> &partition_key_types) const override;
+
+ private:
+ WindowAggregateFunctionMax()
+ : WindowAggregateFunction(WindowAggregationID::kMax) {
+ }
+
+ DISALLOW_COPY_AND_ASSIGN(WindowAggregateFunctionMax);
+};
+
+/** @} */
+
+} // namespace quickstep
+
+#endif // QUICKSTEP_EXPRESSIONS_WINDOW_AGGREGATION_WINDOW_AGGREGATE_FUNCTION_MAX_HPP_
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f5bf3532/expressions/window_aggregation/WindowAggregateFunctionMin.cpp
----------------------------------------------------------------------
diff --git a/expressions/window_aggregation/WindowAggregateFunctionMin.cpp b/expressions/window_aggregation/WindowAggregateFunctionMin.cpp
new file mode 100644
index 0000000..a13e28e
--- /dev/null
+++ b/expressions/window_aggregation/WindowAggregateFunctionMin.cpp
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ **/
+
+#include "expressions/window_aggregation/WindowAggregateFunctionMin.hpp"
+
+#include <vector>
+
+#include "expressions/window_aggregation/WindowAggregationHandle.hpp"
+#include "types/Type.hpp"
+#include "types/operations/comparisons/Comparison.hpp"
+#include "types/operations/comparisons/ComparisonFactory.hpp"
+#include "types/operations/comparisons/ComparisonID.hpp"
+
+#include "glog/logging.h"
+
+namespace quickstep {
+
+bool WindowAggregateFunctionMin::canApplyToTypes(
+ const std::vector<const Type*> &argument_types) const {
+ // MIN is unary.
+ if (argument_types.size() != 1) {
+ return false;
+ }
+
+ // Argument must be comparable by '<'.
+ return ComparisonFactory::GetComparison(ComparisonID::kLess).canCompareTypes(
+ *argument_types.front(),
+ *argument_types.front());
+}
+
+const Type* WindowAggregateFunctionMin::resultTypeForArgumentTypes(
+ const std::vector<const Type*> &argument_types) const {
+ if (!canApplyToTypes(argument_types)) {
+ return nullptr;
+ }
+
+ return &(argument_types.front()->getNullableVersion());
+}
+
+WindowAggregationHandle* WindowAggregateFunctionMin::createHandle(
+ const std::vector<const Type*> &argument_types,
+ const std::vector<const Type*> &partition_key_types) const {
+ DCHECK(canApplyToTypes(argument_types))
+ << "Attempted to create a WindowAggregationHandleMin for argument Type(s) "
+ << "that MIN can not be applied to.";
+
+ return nullptr;
+}
+
+} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f5bf3532/expressions/window_aggregation/WindowAggregateFunctionMin.hpp
----------------------------------------------------------------------
diff --git a/expressions/window_aggregation/WindowAggregateFunctionMin.hpp b/expressions/window_aggregation/WindowAggregateFunctionMin.hpp
new file mode 100644
index 0000000..aeba539
--- /dev/null
+++ b/expressions/window_aggregation/WindowAggregateFunctionMin.hpp
@@ -0,0 +1,75 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ **/
+
+#ifndef QUICKSTEP_EXPRESSIONS_WINDOW_AGGREGATION_WINDOW_AGGREGATE_FUNCTION_MIN_HPP_
+#define QUICKSTEP_EXPRESSIONS_WINDOW_AGGREGATION_WINDOW_AGGREGATE_FUNCTION_MIN_HPP_
+
+#include <string>
+#include <vector>
+
+#include "expressions/window_aggregation/WindowAggregateFunction.hpp"
+#include "expressions/window_aggregation/WindowAggregationID.hpp"
+#include "utility/Macros.hpp"
+
+namespace quickstep {
+
+class Type;
+class WindowAggregationHandle;
+
+/** \addtogroup Expressions
+ * @{
+ */
+
+/**
+ * @brief WindowAggregateFunction representing SQL MIN() OVER term.
+ **/
+class WindowAggregateFunctionMin : public WindowAggregateFunction {
+ public:
+ static const WindowAggregateFunctionMin& Instance() {
+ static WindowAggregateFunctionMin instance;
+ return instance;
+ }
+
+ std::string getName() const override {
+ return "MIN";
+ }
+
+ bool canApplyToTypes(
+ const std::vector<const Type*> &argument_types) const override;
+
+ const Type* resultTypeForArgumentTypes(
+ const std::vector<const Type*> &argument_types) const override;
+
+ WindowAggregationHandle* createHandle(
+ const std::vector<const Type*> &argument_types,
+ const std::vector<const Type*> &partition_key_types) const override;
+
+ private:
+ WindowAggregateFunctionMin()
+ : WindowAggregateFunction(WindowAggregationID::kMin) {
+ }
+
+ DISALLOW_COPY_AND_ASSIGN(WindowAggregateFunctionMin);
+};
+
+/** @} */
+
+} // namespace quickstep
+
+#endif // QUICKSTEP_EXPRESSIONS_WINDOW_AGGREGATION_WINDOW_AGGREGATE_FUNCTION_MIN_HPP_
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f5bf3532/expressions/window_aggregation/WindowAggregateFunctionSum.cpp
----------------------------------------------------------------------
diff --git a/expressions/window_aggregation/WindowAggregateFunctionSum.cpp b/expressions/window_aggregation/WindowAggregateFunctionSum.cpp
new file mode 100644
index 0000000..636c53a
--- /dev/null
+++ b/expressions/window_aggregation/WindowAggregateFunctionSum.cpp
@@ -0,0 +1,82 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ **/
+
+#include "expressions/window_aggregation/WindowAggregateFunctionSum.hpp"
+
+#include <vector>
+
+#include "expressions/window_aggregation/WindowAggregationHandle.hpp"
+#include "types/Type.hpp"
+#include "types/TypeFactory.hpp"
+#include "types/TypeID.hpp"
+#include "types/operations/binary_operations/BinaryOperation.hpp"
+#include "types/operations/binary_operations/BinaryOperationFactory.hpp"
+#include "types/operations/binary_operations/BinaryOperationID.hpp"
+
+#include "glog/logging.h"
+
+namespace quickstep {
+
+bool WindowAggregateFunctionSum::canApplyToTypes(
+ const std::vector<const Type*> &argument_types) const {
+ // SUM is unary.
+ if (argument_types.size() != 1) {
+ return false;
+ }
+
+ // Argument must be addable.
+ return BinaryOperationFactory::GetBinaryOperation(BinaryOperationID::kAdd)
+ .canApplyToTypes(*argument_types.front(), *argument_types.front());
+}
+
+const Type* WindowAggregateFunctionSum::resultTypeForArgumentTypes(
+ const std::vector<const Type*> &argument_types) const {
+ if (!canApplyToTypes(argument_types)) {
+ return nullptr;
+ }
+
+ // SUM may return NULL if there are no input rows, and we automatically widen
+ // int to long and float to double to have more headroom when adding up many
+ // values.
+ const Type *sum_type = &(argument_types.front()->getNullableVersion());
+ switch (sum_type->getTypeID()) {
+ case kInt:
+ sum_type = &TypeFactory::GetType(kLong, true);
+ break;
+ case kFloat:
+ sum_type = &TypeFactory::GetType(kDouble, true);
+ break;
+ default:
+ break;
+ }
+
+ return sum_type;
+}
+
+WindowAggregationHandle* WindowAggregateFunctionSum::createHandle(
+ const std::vector<const Type*> &argument_types,
+ const std::vector<const Type*> &partition_key_types) const {
+ DCHECK(canApplyToTypes(argument_types))
+ << "Attempted to create a WindowAggregationHandleSum for argument Type(s) "
+ << "that SUM can not be applied to.";
+
+ return nullptr;
+}
+
+} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f5bf3532/expressions/window_aggregation/WindowAggregateFunctionSum.hpp
----------------------------------------------------------------------
diff --git a/expressions/window_aggregation/WindowAggregateFunctionSum.hpp b/expressions/window_aggregation/WindowAggregateFunctionSum.hpp
new file mode 100644
index 0000000..047113c
--- /dev/null
+++ b/expressions/window_aggregation/WindowAggregateFunctionSum.hpp
@@ -0,0 +1,75 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ **/
+
+#ifndef QUICKSTEP_EXPRESSIONS_WINDOW_AGGREGATION_WINDOW_AGGREGATE_FUNCTION_SUM_HPP_
+#define QUICKSTEP_EXPRESSIONS_WINDOW_AGGREGATION_WINDOW_AGGREGATE_FUNCTION_SUM_HPP_
+
+#include <string>
+#include <vector>
+
+#include "expressions/window_aggregation/WindowAggregateFunction.hpp"
+#include "expressions/window_aggregation/WindowAggregationID.hpp"
+#include "utility/Macros.hpp"
+
+namespace quickstep {
+
+class Type;
+class WindowAggregationHandle;
+
+/** \addtogroup Expressions
+ * @{
+ */
+
+/**
+ * @brief WindowAggregateFunction representing SQL SUM() OVER term.
+ **/
+class WindowAggregateFunctionSum : public WindowAggregateFunction {
+ public:
+ static const WindowAggregateFunctionSum& Instance() {
+ static WindowAggregateFunctionSum instance;
+ return instance;
+ }
+
+ std::string getName() const override {
+ return "SUM";
+ }
+
+ bool canApplyToTypes(
+ const std::vector<const Type*> &argument_types) const override;
+
+ const Type* resultTypeForArgumentTypes(
+ const std::vector<const Type*> &argument_types) const override;
+
+ WindowAggregationHandle* createHandle(
+ const std::vector<const Type*> &argument_types,
+ const std::vector<const Type*> &partition_key_types) const override;
+
+ private:
+ WindowAggregateFunctionSum()
+ : WindowAggregateFunction(WindowAggregationID::kSum) {
+ }
+
+ DISALLOW_COPY_AND_ASSIGN(WindowAggregateFunctionSum);
+};
+
+/** @} */
+
+} // namespace quickstep
+
+#endif // QUICKSTEP_EXPRESSIONS_WINDOW_AGGREGATION_WINDOW_AGGREGATE_FUNCTION_SUM_HPP_
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f5bf3532/expressions/window_aggregation/WindowAggregationHandle.hpp
----------------------------------------------------------------------
diff --git a/expressions/window_aggregation/WindowAggregationHandle.hpp b/expressions/window_aggregation/WindowAggregationHandle.hpp
new file mode 100644
index 0000000..65f95d9
--- /dev/null
+++ b/expressions/window_aggregation/WindowAggregationHandle.hpp
@@ -0,0 +1,137 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ **/
+
+#ifndef QUICKSTEP_EXPRESSIONS_WINDOW_AGGREGATION_WINDOW_AGGREGATION_HANDLE_HPP_
+#define QUICKSTEP_EXPRESSIONS_WINDOW_AGGREGATION_WINDOW_AGGREGATION_HANDLE_HPP_
+
+#include <cstddef>
+#include <memory>
+#include <vector>
+
+#include "catalog/CatalogRelationSchema.hpp"
+#include "catalog/CatalogTypedefs.hpp"
+#include "storage/StorageBlockInfo.hpp"
+#include "types/TypedValue.hpp"
+#include "types/containers/ColumnVector.hpp"
+#include "types/containers/ColumnVectorsValueAccessor.hpp"
+#include "types/operations/comparisons/Comparison.hpp"
+#include "types/operations/comparisons/ComparisonFactory.hpp"
+#include "types/operations/comparisons/ComparisonID.hpp"
+#include "utility/Macros.hpp"
+
+namespace quickstep {
+
+class InsertDestinationInterface;
+class Scalar;
+class StorageManager;
+class Type;
+class ValueAccessor;
+
+/** \addtogroup Expressions
+ * @{
+ */
+
+/**
+ * @brief WindowAggregationHandle encapsulates logic for actually computing
+ * window aggregates with particular argument(s).
+ * @note See also WindowAggregateFunction, which represents a SQL aggregate
+ * function in the abstract sense.
+ *
+ * A WindowAggregationHandle is created by calling
+ * WindowAggregateFunction::createHandle(). The WindowAggregationHandle object
+ * provides methods that are used to actually compute the window aggregate,
+ * storing intermediate results in WindowAggregationState objects.
+ *
+ * The work flow for computing a window aggregate is:
+ * 1. Create an initial state by createInitialState().
+ * 2. One thread will handle all the computation, iterating from the first
+ * tuple to the last tuple. Note there will be two modes that could be
+ * used upon different situations:
+ * a. If the window aggregate is defined as accumulative, which are:
+ * i. Functions applied to whole partition, such as rank(), ntile()
+ * and dense_rank().
+ * ii. The window frame is defined as "BETWEEN UNBOUNDED PRECEDING
+ * AND CURRENT ROW" or "BETWEEN CURRENT ROW AND UNBOUNDED
+ * FOLLOWING".
+ * Then, for functions except median, we could store some global
+ * values in the state without keeping all the tuple values around.
+ * b. If the window frame is sliding, such as "BETWEEN 3 PRECEDING AND
+ * 3 FOLLOWING", we have to store all the tuples in the state so that
+ * we could know which values should be dropped as the window slides.
+ * For each computed value, generate a tuple store in the column vector.
+ * 3. Insert the new column into the original relation and return.
+ *
+ * TODO(Shixuan): Currently we don't support parallelization. The basic idea for
+ * parallelization is to calculate the partial result inside each block. Each
+ * block could visit the following blocks as long as the block's last partition
+ * is not finished. WindowAggregationOperationState will be used for handling
+ * the global state of the calculation.
+ **/
+
+class WindowAggregationHandle {
+ public:
+ /**
+ * @brief Destructor.
+ **/
+ virtual ~WindowAggregationHandle() {}
+
+ /**
+ * @brief Calculate the window aggregate result.
+ *
+ * @param block_accessors A pointer to the value accessor of block attributes.
+ * @param arguments The ColumnVectors of arguments
+ * @param partition_by_ids The ids of partition keys.
+ * @param is_row True if the frame mode is ROWS, false if it is RANGE.
+ * @param num_preceding The number of rows/range that precedes the current row.
+ * @param num_following The number of rows/range that follows the current row.
+ *
+ * @return A ColumnVector of the calculated window aggregates.
+ **/
+ virtual ColumnVector* calculate(ColumnVectorsValueAccessor* block_accessors,
+ std::vector<ColumnVector*> &&arguments,
+ const std::vector<attribute_id> &partition_by_ids,
+ const bool is_row,
+ const std::int64_t num_preceding,
+ const std::int64_t num_following) const = 0;
+
+ protected:
+ /**
+ * @brief Constructor.
+ *
+ * @param partition_key_types The Types of the partition key.
+ **/
+ explicit WindowAggregationHandle(
+ const std::vector<const Type*> &partition_key_types) {
+ // Comparison operators for checking if two tuples belong to the same partition.
+ for (const Type *partition_key_type : partition_key_types) {
+ equal_comparators_.emplace_back(
+ ComparisonFactory::GetComparison(ComparisonID::kEqual)
+ .makeUncheckedComparatorForTypes(*partition_key_type, *partition_key_type));
+ }
+ }
+
+ std::vector<std::unique_ptr<UncheckedComparator>> equal_comparators_;
+
+ private:
+ DISALLOW_COPY_AND_ASSIGN(WindowAggregationHandle);
+};
+
+} // namespace quickstep
+
+#endif // QUICKSTEP_EXPRESSIONS_WINDOW_AGGREGATION_WINDOW_AGGREGATION_HANDLE_HPP_
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f5bf3532/expressions/window_aggregation/WindowAggregationHandleAvg.cpp
----------------------------------------------------------------------
diff --git a/expressions/window_aggregation/WindowAggregationHandleAvg.cpp b/expressions/window_aggregation/WindowAggregationHandleAvg.cpp
new file mode 100644
index 0000000..a6a10d4
--- /dev/null
+++ b/expressions/window_aggregation/WindowAggregationHandleAvg.cpp
@@ -0,0 +1,241 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ **/
+
+#include "expressions/window_aggregation/WindowAggregationHandleAvg.hpp"
+
+#include <cstddef>
+#include <memory>
+#include <vector>
+
+#include "catalog/CatalogTypedefs.hpp"
+#include "expressions/scalar/Scalar.hpp"
+#include "expressions/scalar/ScalarAttribute.hpp"
+#include "storage/ValueAccessor.hpp"
+#include "types/Type.hpp"
+#include "types/TypeFactory.hpp"
+#include "types/TypeID.hpp"
+#include "types/TypedValue.hpp"
+#include "types/containers/ColumnVectorsValueAccessor.hpp"
+#include "types/operations/binary_operations/BinaryOperation.hpp"
+#include "types/operations/binary_operations/BinaryOperationFactory.hpp"
+#include "types/operations/binary_operations/BinaryOperationID.hpp"
+#include "types/operations/comparisons/Comparison.hpp"
+
+#include "glog/logging.h"
+
+namespace quickstep {
+
+WindowAggregationHandleAvg::WindowAggregationHandleAvg(
+ const std::vector<const Type*> &partition_key_types,
+ const Type &type)
+ : WindowAggregationHandle(partition_key_types),
+ argument_type_(type) {
+ // We sum Int as Long and Float as Double so that we have more headroom when
+ // adding many values.
+ TypeID type_id;
+ switch (type.getTypeID()) {
+ case kInt:
+ case kLong:
+ type_id = kLong;
+ break;
+ case kFloat:
+ case kDouble:
+ type_id = kDouble;
+ break;
+ default:
+ type_id = type.getTypeID();
+ break;
+ }
+
+ sum_type_ = &(TypeFactory::GetType(type_id));
+
+ // Result is nullable, because AVG() over 0 values (or all NULL values) is
+ // NULL.
+ result_type_
+ = &(BinaryOperationFactory::GetBinaryOperation(BinaryOperationID::kDivide)
+ .resultTypeForArgumentTypes(*sum_type_, TypeFactory::GetType(kDouble))
+ ->getNullableVersion());
+
+ // Make operators to do arithmetic:
+ // Add operator for summing argument values.
+ fast_add_operator_.reset(
+ BinaryOperationFactory::GetBinaryOperation(BinaryOperationID::kAdd)
+ .makeUncheckedBinaryOperatorForTypes(*sum_type_, argument_type_));
+ // Divide operator for dividing sum by count to get final average.
+ divide_operator_.reset(
+ BinaryOperationFactory::GetBinaryOperation(BinaryOperationID::kDivide)
+ .makeUncheckedBinaryOperatorForTypes(*sum_type_, TypeFactory::GetType(kDouble)));
+}
+
+ColumnVector* WindowAggregationHandleAvg::calculate(
+ ColumnVectorsValueAccessor *tuple_accessor,
+ std::vector<ColumnVector*> &&arguments,
+ const std::vector<attribute_id> &partition_by_ids,
+ const bool is_row,
+ const std::int64_t num_preceding,
+ const std::int64_t num_following) const {
+ DCHECK_EQ(1u, arguments.size());
+ DCHECK(arguments[0]->isNative());
+ DCHECK_EQ(static_cast<std::size_t>(tuple_accessor->getNumTuples()),
+ static_cast<const NativeColumnVector*>(arguments[0])->size());
+
+ // Initialize the output column and argument accessor.
+ NativeColumnVector *window_aggregates =
+ new NativeColumnVector(*result_type_, tuple_accessor->getNumTuples());
+ ColumnVectorsValueAccessor* argument_accessor = new ColumnVectorsValueAccessor();
+ argument_accessor->addColumn(arguments[0]);
+
+ // Create a window for each tuple and calculate the window aggregate.
+ tuple_accessor->beginIteration();
+ argument_accessor->beginIteration();
+
+ while (tuple_accessor->next() && argument_accessor->next()) {
+ const TypedValue window_aggregate = this->calculateOneWindow(tuple_accessor,
+ argument_accessor,
+ partition_by_ids,
+ is_row,
+ num_preceding,
+ num_following);
+ window_aggregates->appendTypedValue(window_aggregate);
+ }
+
+ return window_aggregates;
+}
+
+TypedValue WindowAggregationHandleAvg::calculateOneWindow(
+ ColumnVectorsValueAccessor *tuple_accessor,
+ ColumnVectorsValueAccessor *argument_accessor,
+ const std::vector<attribute_id> &partition_by_ids,
+ const bool is_row,
+ const std::int64_t num_preceding,
+ const std::int64_t num_following) const {
+ // Initialize.
+ TypedValue sum = sum_type_->makeZeroValue();
+ TypedValue current_value = argument_accessor->getTypedValue(0);
+ std::uint64_t count = 0;
+
+ // Ignore the value if null.
+ if (!current_value.isNull()) {
+ sum = fast_add_operator_->applyToTypedValues(sum, current_value);
+ count++;
+ }
+
+ // Get the partition key for the current row.
+ std::vector<TypedValue> current_row_partition_key;
+ for (attribute_id partition_by_id : partition_by_ids) {
+ current_row_partition_key.push_back(
+ tuple_accessor->getTypedValue(partition_by_id));
+ }
+
+ // Get current position.
+ tuple_id current_tuple_id = tuple_accessor->getCurrentPositionVirtual();
+
+ // Find preceding tuples.
+ int count_preceding = 0;
+ tuple_id preceding_tuple_id = current_tuple_id;
+ while (num_preceding == -1 || count_preceding < num_preceding) {
+ preceding_tuple_id--;
+
+ // No more preceding tuples.
+ if (preceding_tuple_id < 0) {
+ break;
+ }
+
+ // Get the partition keys and compare. If not the same partition as the
+ // current row, stop searching preceding tuples.
+ if (!samePartition(tuple_accessor,
+ current_row_partition_key,
+ preceding_tuple_id,
+ partition_by_ids)) {
+ break;
+ }
+
+ // Actually count the element and do the calculation.
+ count_preceding++;
+ TypedValue preceding_value =
+ argument_accessor->getTypedValueAtAbsolutePosition(0, preceding_tuple_id);
+
+ // Ignore the value if null.
+ if (!preceding_value.isNull()) {
+ sum = fast_add_operator_->applyToTypedValues(sum, preceding_value);
+ count++;
+ }
+ }
+
+ // Find following tuples.
+ int count_following = 0;
+ tuple_id following_tuple_id = current_tuple_id;
+ while (num_following == -1 || count_following < num_following) {
+ following_tuple_id++;
+
+ // No more following tuples.
+ if (following_tuple_id == tuple_accessor->getNumTuples()) {
+ break;
+ }
+
+ // Get the partition keys and compare. If not the same partition as the
+ // current row, stop searching preceding tuples.
+ if (!samePartition(tuple_accessor,
+ current_row_partition_key,
+ following_tuple_id,
+ partition_by_ids)) {
+ break;
+ }
+
+ // Actually count the element and do the calculation.
+ count_following++;
+ TypedValue following_value =
+ argument_accessor->getTypedValueAtAbsolutePosition(0, following_tuple_id);
+
+ // Ignore the value if null.
+ if (!following_value.isNull()) {
+ sum = fast_add_operator_->applyToTypedValues(sum, following_value);
+ count++;
+ }
+ }
+
+ // If all values are NULLs, return NULL; Otherwise, return the quotient.
+ if (count == 0) {
+ return result_type_->makeNullValue();
+ } else {
+ return divide_operator_->applyToTypedValues(sum,
+ TypedValue(static_cast<double>(count)));
+ }
+}
+
+bool WindowAggregationHandleAvg::samePartition(
+ const ColumnVectorsValueAccessor *tuple_accessor,
+ const std::vector<TypedValue> ¤t_row_partition_key,
+ const tuple_id boundary_tuple_id,
+ const std::vector<attribute_id> &partition_by_ids) const {
+ for (std::size_t partition_by_index = 0;
+ partition_by_index < partition_by_ids.size();
+ ++partition_by_index) {
+ if (!equal_comparators_[partition_by_index]->compareTypedValues(
+ current_row_partition_key[partition_by_index],
+ tuple_accessor->getTypedValueAtAbsolutePosition(
+ partition_by_ids[partition_by_index], boundary_tuple_id))) {
+ return false;
+ }
+ }
+
+ return true;
+}
+
+} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f5bf3532/expressions/window_aggregation/WindowAggregationHandleAvg.hpp
----------------------------------------------------------------------
diff --git a/expressions/window_aggregation/WindowAggregationHandleAvg.hpp b/expressions/window_aggregation/WindowAggregationHandleAvg.hpp
new file mode 100644
index 0000000..5b41779
--- /dev/null
+++ b/expressions/window_aggregation/WindowAggregationHandleAvg.hpp
@@ -0,0 +1,101 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ **/
+
+#ifndef QUICKSTEP_EXPRESSIONS_WINDOW_AGGREGATION_WINDOW_AGGREGATION_HANDLE_AVG_HPP_
+#define QUICKSTEP_EXPRESSIONS_WINDOW_AGGREGATION_WINDOW_AGGREGATION_HANDLE_AVG_HPP_
+
+#include <cstddef>
+#include <cstdint>
+#include <memory>
+#include <queue>
+#include <vector>
+
+#include "catalog/CatalogTypedefs.hpp"
+#include "expressions/window_aggregation/WindowAggregationHandle.hpp"
+#include "types/Type.hpp"
+#include "types/TypedValue.hpp"
+#include "types/operations/binary_operations/BinaryOperation.hpp"
+#include "types/operations/comparisons/Comparison.hpp"
+#include "utility/Macros.hpp"
+
+#include "glog/logging.h"
+
+namespace quickstep {
+
+class ColumnVector;
+class ColumnVectorsValueAccessor;
+class ValueAccessor;
+
+/** \addtogroup Expressions
+ * @{
+ */
+
+/**
+ * @brief A WindowAggregationHandle for average.
+ **/
+class WindowAggregationHandleAvg : public WindowAggregationHandle {
+ public:
+ ~WindowAggregationHandleAvg() override {}
+
+ ColumnVector* calculate(ColumnVectorsValueAccessor* block_accessors,
+ std::vector<ColumnVector*> &&arguments,
+ const std::vector<attribute_id> &partition_by_ids,
+ const bool is_row,
+ const std::int64_t num_preceding,
+ const std::int64_t num_following) const override;
+
+ private:
+ friend class WindowAggregateFunctionAvg;
+
+ /**
+ * @brief Constructor.
+ *
+ * @param partition_key_types The Types of the partition key.
+ * @param type Type of the avg value.
+ **/
+ WindowAggregationHandleAvg(const std::vector<const Type*> &partition_key_types,
+ const Type &type);
+
+ TypedValue calculateOneWindow(
+ ColumnVectorsValueAccessor *tuple_accessor,
+ ColumnVectorsValueAccessor *argument_accessor,
+ const std::vector<attribute_id> &partition_by_ids,
+ const bool is_row,
+ const std::int64_t num_preceding,
+ const std::int64_t num_following) const;
+
+ bool samePartition(const ColumnVectorsValueAccessor *tuple_accessor,
+ const std::vector<TypedValue> ¤t_row_partition_key,
+ const tuple_id boundary_tuple_id,
+ const std::vector<attribute_id> &partition_by_ids) const;
+
+ const Type &argument_type_;
+ const Type *sum_type_;
+ const Type *result_type_;
+ std::unique_ptr<UncheckedBinaryOperator> fast_add_operator_;
+ std::unique_ptr<UncheckedBinaryOperator> divide_operator_;
+
+ DISALLOW_COPY_AND_ASSIGN(WindowAggregationHandleAvg);
+};
+
+/** @} */
+
+} // namespace quickstep
+
+#endif // QUICKSTEP_EXPRESSIONS_WINDOW_AGGREGATION_WINDOW_AGGREGATION_HANDLE_AVG_HPP_
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f5bf3532/expressions/window_aggregation/WindowAggregationID.hpp
----------------------------------------------------------------------
diff --git a/expressions/window_aggregation/WindowAggregationID.hpp b/expressions/window_aggregation/WindowAggregationID.hpp
new file mode 100644
index 0000000..8122df3
--- /dev/null
+++ b/expressions/window_aggregation/WindowAggregationID.hpp
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ **/
+
+#ifndef QUICKSTEP_EXPRESSIONS_WINDOW_AGGREGATION_WINDOW_AGGREGATION_ID_HPP_
+#define QUICKSTEP_EXPRESSIONS_WINDOW_AGGREGATION_WINDOW_AGGREGATION_ID_HPP_
+
+namespace quickstep {
+
+/** \addtogroup Expressions
+ * @{
+ */
+
+/**
+ * @brief The possible types of window aggregations.
+ **/
+enum class WindowAggregationID {
+ kAvg,
+ kCount,
+ kMin,
+ kMax,
+ kSum
+};
+
+/** @} */
+
+} // namespace quickstep
+
+#endif // QUICKSTEP_EXPRESSIONS_WINDOW_AGGREGATION_WINDOW_AGGREGATION_ID_HPP_
[20/25] incubator-quickstep git commit: - Supported ROWS mode for AVG
window aggregation. - Created WindowAggregateFunctions in
expressions/window_aggregation. - Created WindowAggregationHandle for AVG to
actually do the calculation. - Other functions wi
Posted by zu...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f5bf3532/expressions/window_aggregation/tests/WindowAggregationHandleAvg_unittest.cpp
----------------------------------------------------------------------
diff --git a/expressions/window_aggregation/tests/WindowAggregationHandleAvg_unittest.cpp b/expressions/window_aggregation/tests/WindowAggregationHandleAvg_unittest.cpp
new file mode 100644
index 0000000..c044a98
--- /dev/null
+++ b/expressions/window_aggregation/tests/WindowAggregationHandleAvg_unittest.cpp
@@ -0,0 +1,387 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ * limitations under the License.
+ **/
+
+#include <cstddef>
+#include <cstdint>
+#include <memory>
+#include <vector>
+
+#include "catalog/CatalogTypedefs.hpp"
+#include "expressions/window_aggregation/WindowAggregateFunction.hpp"
+#include "expressions/window_aggregation/WindowAggregateFunctionFactory.hpp"
+#include "expressions/window_aggregation/WindowAggregationHandle.hpp"
+#include "expressions/window_aggregation/WindowAggregationHandleAvg.hpp"
+#include "expressions/window_aggregation/WindowAggregationID.hpp"
+#include "storage/ValueAccessor.hpp"
+#include "types/CharType.hpp"
+#include "types/DateOperatorOverloads.hpp"
+#include "types/DatetimeIntervalType.hpp"
+#include "types/DoubleType.hpp"
+#include "types/FloatType.hpp"
+#include "types/IntType.hpp"
+#include "types/IntervalLit.hpp"
+#include "types/LongType.hpp"
+#include "types/Type.hpp"
+#include "types/TypeFactory.hpp"
+#include "types/TypeID.hpp"
+#include "types/TypedValue.hpp"
+#include "types/VarCharType.hpp"
+#include "types/YearMonthIntervalType.hpp"
+#include "types/containers/ColumnVector.hpp"
+#include "types/containers/ColumnVectorsValueAccessor.hpp"
+
+#include "gtest/gtest.h"
+
+namespace quickstep {
+
+namespace {
+
+ constexpr int kNumTuples = 100;
+ constexpr int kNumTuplesPerPartition = 8;
+ constexpr int kNullInterval = 25;
+ constexpr int kNumPreceding = 2;
+ constexpr int kNumFollowing = 2;
+
+} // namespace
+
+// Attribute value could be null if set true.
+class WindowAggregationHandleAvgTest : public::testing::Test {
+ protected:
+ // Handle initialization.
+ void initializeHandle(const Type &argument_type) {
+ const WindowAggregateFunction &function =
+ WindowAggregateFunctionFactory::Get(WindowAggregationID::kAvg);
+ std::vector<const Type*> partition_key_types(1, &TypeFactory::GetType(kInt, false));
+ handle_avg_.reset(function.createHandle(std::vector<const Type*>(1, &argument_type),
+ std::move(partition_key_types)));
+ }
+
+ // Test canApplyToTypes().
+ static bool CanApplyToTypesTest(TypeID typeID) {
+ const Type &type = (typeID == kChar || typeID == kVarChar) ?
+ TypeFactory::GetType(typeID, static_cast<std::size_t>(10)) :
+ TypeFactory::GetType(typeID);
+
+ return WindowAggregateFunctionFactory::Get(WindowAggregationID::kAvg).canApplyToTypes(
+ std::vector<const Type*>(1, &type));
+ }
+
+ // Test resultTypeForArgumentTypes().
+ static bool ResultTypeForArgumentTypesTest(TypeID input_type_id,
+ TypeID output_type_id) {
+ const Type *result_type
+ = WindowAggregateFunctionFactory::Get(WindowAggregationID::kAvg).resultTypeForArgumentTypes(
+ std::vector<const Type*>(1, &TypeFactory::GetType(input_type_id)));
+ return (result_type->getTypeID() == output_type_id);
+ }
+
+ template <typename CppType>
+ static void CheckAvgValues(
+ const std::vector<CppType*> &expected,
+ const ColumnVector *actual) {
+ EXPECT_TRUE(actual->isNative());
+ const NativeColumnVector *native = static_cast<const NativeColumnVector*>(actual);
+
+ EXPECT_EQ(expected.size(), native->size());
+ for (std::size_t i = 0; i < expected.size(); ++i) {
+ if (expected[i] == nullptr) {
+ EXPECT_TRUE(native->getTypedValue(i).isNull());
+ } else {
+ EXPECT_EQ(*expected[i], native->getTypedValue(i).getLiteral<CppType>());
+ }
+ }
+ }
+
+ // Static templated methods for set a meaningful value to data types.
+ template <typename CppType>
+ static void SetDataType(int value, CppType *data) {
+ *data = value;
+ }
+
+ template <typename GenericType, typename OutputType = DoubleType>
+ void checkWindowAggregationAvgGeneric() {
+ const GenericType &type = GenericType::Instance(true);
+ initializeHandle(type);
+
+ // Create argument, partition key and cpptype vectors.
+ std::vector<typename GenericType::cpptype*> argument_cpp_vector;
+ argument_cpp_vector.reserve(kNumTuples);
+ ColumnVector *argument_type_vector =
+ createArgumentGeneric<GenericType>(&argument_cpp_vector);
+ NativeColumnVector *partition_key_vector =
+ new NativeColumnVector(IntType::InstanceNonNullable(), kNumTuples + 2);
+
+ for (int i = 0; i < kNumTuples; ++i) {
+ partition_key_vector->appendTypedValue(TypedValue(i / kNumTuplesPerPartition));
+ }
+
+ // Create tuple ValueAccessor.
+ ColumnVectorsValueAccessor *tuple_accessor = new ColumnVectorsValueAccessor();
+ tuple_accessor->addColumn(partition_key_vector);
+ tuple_accessor->addColumn(argument_type_vector);
+
+ // Test UNBOUNDED PRECEDING AND CURRENT ROW.
+ checkAccumulate<GenericType, OutputType>(tuple_accessor,
+ argument_type_vector,
+ argument_cpp_vector);
+ // Test kNumPreceding PRECEDING AND kNumFollowing FOLLOWING.
+ checkSlidingWindow<GenericType, OutputType>(tuple_accessor,
+ argument_type_vector,
+ argument_cpp_vector);
+ }
+
+ template <typename GenericType>
+ ColumnVector *createArgumentGeneric(
+ std::vector<typename GenericType::cpptype*> *argument_cpp_vector) {
+ const GenericType &type = GenericType::Instance(true);
+ NativeColumnVector *column = new NativeColumnVector(type, kNumTuples);
+
+ for (int i = 0; i < kNumTuples; ++i) {
+ // Insert a NULL every kNullInterval tuples.
+ if (i % kNullInterval == 0) {
+ argument_cpp_vector->push_back(nullptr);
+ column->appendTypedValue(type.makeNullValue());
+ continue;
+ }
+
+ typename GenericType::cpptype *val = new typename GenericType::cpptype;
+
+ if (type.getTypeID() == kInt || type.getTypeID() == kLong) {
+ SetDataType(i - 10, val);
+ } else {
+ SetDataType(static_cast<float>(i - 10) / 10, val);
+ }
+
+ column->appendTypedValue(type.makeValue(val));
+ argument_cpp_vector->push_back(val);
+ }
+
+ return column;
+ }
+
+ template <typename GenericType, typename OutputType>
+ void checkAccumulate(ColumnVectorsValueAccessor *tuple_accessor,
+ ColumnVector *argument_type_vector,
+ const std::vector<typename GenericType::cpptype*> &argument_cpp_vector) {
+ std::vector<ColumnVector*> arguments;
+ arguments.push_back(argument_type_vector);
+ // The partition key index is 0.
+ std::vector<attribute_id> partition_key(1, 0);
+
+ ColumnVector *result =
+ handle_avg_->calculate(tuple_accessor,
+ std::move(arguments),
+ partition_key,
+ true /* is_row */,
+ -1 /* num_preceding: UNBOUNDED PRECEDING */,
+ 0 /* num_following: CURRENT ROW */);
+
+ // Get the cpptype result.
+ std::vector<typename OutputType::cpptype*> result_cpp_vector;
+ typename GenericType::cpptype sum;
+ int count;
+ for (std::size_t i = 0; i < argument_cpp_vector.size(); ++i) {
+ // Start of new partition
+ if (i % kNumTuplesPerPartition == 0) {
+ SetDataType(0, &sum);
+ count = 0;
+ }
+
+ typename GenericType::cpptype *value = argument_cpp_vector[i];
+ if (value != nullptr) {
+ sum += *value;
+ count++;
+ }
+
+ if (count == 0) {
+ result_cpp_vector.push_back(nullptr);
+ } else {
+ typename OutputType::cpptype *result_cpp_value =
+ new typename OutputType::cpptype;
+ *result_cpp_value = static_cast<typename OutputType::cpptype>(sum) / count;
+ result_cpp_vector.push_back(result_cpp_value);
+ }
+ }
+
+ CheckAvgValues(result_cpp_vector, result);
+ }
+
+ template <typename GenericType, typename OutputType>
+ void checkSlidingWindow(ColumnVectorsValueAccessor *tuple_accessor,
+ ColumnVector *argument_type_vector,
+ const std::vector<typename GenericType::cpptype*> &argument_cpp_vector) {
+ std::vector<ColumnVector*> arguments;
+ arguments.push_back(argument_type_vector);
+ // The partition key index is 0.
+ std::vector<attribute_id> partition_key(1, 0);
+
+ ColumnVector *result =
+ handle_avg_->calculate(tuple_accessor,
+ std::move(arguments),
+ partition_key,
+ true /* is_row */,
+ kNumPreceding,
+ kNumFollowing);
+
+ // Get the cpptype result.
+ // For each value, calculate all surrounding values in the window.
+ std::vector<typename OutputType::cpptype*> result_cpp_vector;
+
+ for (std::size_t i = 0; i < argument_cpp_vector.size(); ++i) {
+ typename GenericType::cpptype sum;
+ SetDataType(0, &sum);
+ int count = 0;
+
+ if (argument_cpp_vector[i] != nullptr) {
+ sum += *argument_cpp_vector[i];
+ count++;
+ }
+
+ for (std::size_t precede = 1; precede <= kNumPreceding; ++precede) {
+ // Not the same partition.
+ if (i / kNumTuplesPerPartition != (i - precede) / kNumTuplesPerPartition ||
+ i < precede) {
+ break;
+ }
+
+ if (argument_cpp_vector[i - precede] != nullptr) {
+ sum += *argument_cpp_vector[i - precede];
+ count++;
+ }
+ }
+
+ for (int follow = 1; follow <= kNumPreceding; ++follow) {
+ // Not the same partition.
+ if (i / kNumTuplesPerPartition != (i + follow) / kNumTuplesPerPartition ||
+ i + follow >= kNumTuples) {
+ break;
+ }
+
+ if (argument_cpp_vector[i + follow] != nullptr) {
+ sum += *argument_cpp_vector[i + follow];
+ count++;
+ }
+ }
+
+ if (count == 0) {
+ result_cpp_vector.push_back(nullptr);
+ } else {
+ typename OutputType::cpptype *result_cpp_value =
+ new typename OutputType::cpptype;
+ *result_cpp_value = static_cast<typename OutputType::cpptype>(sum) / count;
+ result_cpp_vector.push_back(result_cpp_value);
+ }
+ }
+
+ CheckAvgValues(result_cpp_vector, result);
+ }
+
+ std::unique_ptr<WindowAggregationHandle> handle_avg_;
+};
+
+template <>
+void WindowAggregationHandleAvgTest::CheckAvgValues<double>(
+ const std::vector<double*> &expected,
+ const ColumnVector *actual) {
+ EXPECT_TRUE(actual->isNative());
+ const NativeColumnVector *native = static_cast<const NativeColumnVector*>(actual);
+
+ EXPECT_EQ(expected.size(), native->size());
+ for (std::size_t i = 0; i < expected.size(); ++i) {
+ if (expected[i] == nullptr) {
+ EXPECT_TRUE(native->getTypedValue(i).isNull());
+ } else {
+ EXPECT_EQ(*expected[i], native->getTypedValue(i).getLiteral<double>());
+ }
+ }
+}
+
+template <>
+void WindowAggregationHandleAvgTest::SetDataType<DatetimeIntervalLit>(
+ int value, DatetimeIntervalLit *data) {
+ data->interval_ticks = value;
+}
+
+template <>
+void WindowAggregationHandleAvgTest::SetDataType<YearMonthIntervalLit>(
+ int value, YearMonthIntervalLit *data) {
+ data->months = value;
+}
+
+typedef WindowAggregationHandleAvgTest WindowAggregationHandleAvgDeathTest;
+
+TEST_F(WindowAggregationHandleAvgTest, IntTypeTest) {
+ checkWindowAggregationAvgGeneric<IntType>();
+}
+
+TEST_F(WindowAggregationHandleAvgTest, LongTypeTest) {
+ checkWindowAggregationAvgGeneric<LongType>();
+}
+
+TEST_F(WindowAggregationHandleAvgTest, FloatTypeTest) {
+ checkWindowAggregationAvgGeneric<FloatType>();
+}
+
+TEST_F(WindowAggregationHandleAvgTest, DoubleTypeTest) {
+ checkWindowAggregationAvgGeneric<DoubleType>();
+}
+
+TEST_F(WindowAggregationHandleAvgTest, DatetimeIntervalTypeTest) {
+ checkWindowAggregationAvgGeneric<DatetimeIntervalType, DatetimeIntervalType>();
+}
+
+TEST_F(WindowAggregationHandleAvgTest, YearMonthIntervalTypeTest) {
+ checkWindowAggregationAvgGeneric<YearMonthIntervalType, YearMonthIntervalType>();
+}
+
+#ifdef QUICKSTEP_DEBUG
+TEST_F(WindowAggregationHandleAvgDeathTest, CharTypeTest) {
+ const Type &type = CharType::Instance(true, 10);
+ EXPECT_DEATH(initializeHandle(type), "");
+}
+
+TEST_F(WindowAggregationHandleAvgDeathTest, VarTypeTest) {
+ const Type &type = VarCharType::Instance(true, 10);
+ EXPECT_DEATH(initializeHandle(type), "");
+}
+#endif
+
+TEST_F(WindowAggregationHandleAvgDeathTest, canApplyToTypeTest) {
+ EXPECT_TRUE(CanApplyToTypesTest(kInt));
+ EXPECT_TRUE(CanApplyToTypesTest(kLong));
+ EXPECT_TRUE(CanApplyToTypesTest(kFloat));
+ EXPECT_TRUE(CanApplyToTypesTest(kDouble));
+ EXPECT_FALSE(CanApplyToTypesTest(kChar));
+ EXPECT_FALSE(CanApplyToTypesTest(kVarChar));
+ EXPECT_FALSE(CanApplyToTypesTest(kDatetime));
+ EXPECT_TRUE(CanApplyToTypesTest(kDatetimeInterval));
+ EXPECT_TRUE(CanApplyToTypesTest(kYearMonthInterval));
+}
+
+TEST_F(WindowAggregationHandleAvgDeathTest, ResultTypeForArgumentTypeTest) {
+ EXPECT_TRUE(ResultTypeForArgumentTypesTest(kInt, kDouble));
+ EXPECT_TRUE(ResultTypeForArgumentTypesTest(kLong, kDouble));
+ EXPECT_TRUE(ResultTypeForArgumentTypesTest(kFloat, kDouble));
+ EXPECT_TRUE(ResultTypeForArgumentTypesTest(kDouble, kDouble));
+ EXPECT_TRUE(ResultTypeForArgumentTypesTest(kDatetimeInterval, kDatetimeInterval));
+ EXPECT_TRUE(ResultTypeForArgumentTypesTest(kYearMonthInterval, kYearMonthInterval));
+}
+
+} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f5bf3532/query_optimizer/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_optimizer/CMakeLists.txt b/query_optimizer/CMakeLists.txt
index 7e53b9d..a56b714 100644
--- a/query_optimizer/CMakeLists.txt
+++ b/query_optimizer/CMakeLists.txt
@@ -69,6 +69,8 @@ target_link_libraries(quickstep_queryoptimizer_ExecutionGenerator
quickstep_expressions_predicate_Predicate
quickstep_expressions_scalar_Scalar
quickstep_expressions_scalar_ScalarAttribute
+ quickstep_expressions_windowaggregation_WindowAggregateFunction
+ quickstep_expressions_windowaggregation_WindowAggregateFunction_proto
quickstep_queryexecution_QueryContext
quickstep_queryexecution_QueryContext_proto
quickstep_queryoptimizer_ExecutionHeuristics
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f5bf3532/query_optimizer/ExecutionGenerator.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/ExecutionGenerator.cpp b/query_optimizer/ExecutionGenerator.cpp
index 43d63f9..ce21ade 100644
--- a/query_optimizer/ExecutionGenerator.cpp
+++ b/query_optimizer/ExecutionGenerator.cpp
@@ -48,6 +48,8 @@
#include "expressions/predicate/Predicate.hpp"
#include "expressions/scalar/Scalar.hpp"
#include "expressions/scalar/ScalarAttribute.hpp"
+#include "expressions/window_aggregation/WindowAggregateFunction.hpp"
+#include "expressions/window_aggregation/WindowAggregateFunction.pb.h"
#include "query_execution/QueryContext.hpp"
#include "query_execution/QueryContext.pb.h"
#include "query_optimizer/ExecutionHeuristics.hpp"
@@ -1652,7 +1654,7 @@ void ExecutionGenerator::convertWindowAggregate(
// Get input.
const CatalogRelationInfo *input_relation_info =
findRelationInfoOutputByPhysical(physical_plan->input());
- window_aggr_state_proto->set_relation_id(input_relation_info->relation->getID());
+ window_aggr_state_proto->set_input_relation_id(input_relation_info->relation->getID());
// Get window aggregate function expression.
const E::AliasPtr &named_window_aggregate_expression =
@@ -1713,6 +1715,7 @@ void ExecutionGenerator::convertWindowAggregate(
const QueryPlan::DAGNodeIndex window_aggregation_operator_index =
execution_plan_->addRelationalOperator(
new WindowAggregationOperator(query_handle_->query_id(),
+ *input_relation_info->relation,
*output_relation,
window_aggr_state_index,
insert_destination_index));
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f5bf3532/query_optimizer/expressions/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_optimizer/expressions/CMakeLists.txt b/query_optimizer/expressions/CMakeLists.txt
index 08d7df5..d12644a 100644
--- a/query_optimizer/expressions/CMakeLists.txt
+++ b/query_optimizer/expressions/CMakeLists.txt
@@ -304,7 +304,7 @@ target_link_libraries(quickstep_queryoptimizer_expressions_UnaryExpression
quickstep_utility_Macros)
target_link_libraries(quickstep_queryoptimizer_expressions_WindowAggregateFunction
glog
- quickstep_expressions_aggregation_AggregateFunction
+ quickstep_expressions_windowaggregation_WindowAggregateFunction
quickstep_queryoptimizer_OptimizerTree
quickstep_queryoptimizer_expressions_AttributeReference
quickstep_queryoptimizer_expressions_Expression
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f5bf3532/query_optimizer/expressions/WindowAggregateFunction.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/expressions/WindowAggregateFunction.cpp b/query_optimizer/expressions/WindowAggregateFunction.cpp
index 7b1f304..be5db59 100644
--- a/query_optimizer/expressions/WindowAggregateFunction.cpp
+++ b/query_optimizer/expressions/WindowAggregateFunction.cpp
@@ -22,7 +22,7 @@
#include <utility>
#include <vector>
-#include "expressions/aggregation/AggregateFunction.hpp"
+#include "expressions/window_aggregation/WindowAggregateFunction.hpp"
#include "query_optimizer/expressions/AttributeReference.hpp"
#include "query_optimizer/expressions/Expression.hpp"
#include "query_optimizer/expressions/PatternMatcher.hpp"
@@ -59,7 +59,7 @@ const Type& WindowAggregateFunction::getValueType() const {
}
WindowAggregateFunctionPtr WindowAggregateFunction::Create(
- const ::quickstep::AggregateFunction &window_aggregate,
+ const ::quickstep::WindowAggregateFunction &window_aggregate,
const std::vector<ScalarPtr> &arguments,
const WindowInfo &window_info,
const std::string &window_name,
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f5bf3532/query_optimizer/expressions/WindowAggregateFunction.hpp
----------------------------------------------------------------------
diff --git a/query_optimizer/expressions/WindowAggregateFunction.hpp b/query_optimizer/expressions/WindowAggregateFunction.hpp
index 0bee28f..0cc80df 100644
--- a/query_optimizer/expressions/WindowAggregateFunction.hpp
+++ b/query_optimizer/expressions/WindowAggregateFunction.hpp
@@ -33,8 +33,8 @@
namespace quickstep {
-class AggregateFunction;
class Type;
+class WindowAggregateFunction;
namespace optimizer {
namespace expressions {
@@ -140,7 +140,7 @@ class WindowAggregateFunction : public Expression {
* @return The WindowAggregateFunction singleton (from the expression system)
* for this node.
**/
- inline const ::quickstep::AggregateFunction& window_aggregate() const {
+ inline const ::quickstep::WindowAggregateFunction& window_aggregate() const {
return window_aggregate_;
}
@@ -185,7 +185,7 @@ class WindowAggregateFunction : public Expression {
* @param is_distinct Whether this is a DISTINCT aggregation.
* @return A new AggregateFunctionPtr.
**/
- static WindowAggregateFunctionPtr Create(const ::quickstep::AggregateFunction &window_aggregate,
+ static WindowAggregateFunctionPtr Create(const ::quickstep::WindowAggregateFunction &window_aggregate,
const std::vector<ScalarPtr> &arguments,
const WindowInfo &window_info,
const std::string &window_name,
@@ -209,7 +209,7 @@ class WindowAggregateFunction : public Expression {
* @param window_info The window info of the window aggregate function.
* @param is_distinct Indicates whether this is a DISTINCT aggregation.
*/
- WindowAggregateFunction(const ::quickstep::AggregateFunction &window_aggregate,
+ WindowAggregateFunction(const ::quickstep::WindowAggregateFunction &window_aggregate,
const std::vector<ScalarPtr> &arguments,
const WindowInfo &window_info,
const std::string &window_name,
@@ -228,7 +228,7 @@ class WindowAggregateFunction : public Expression {
// window_aggregate_. If it really needs to be seperated from the
// AggregationFunction, a new class for WindowAggregationFunction should be
// created as quickstep::WindowAggregateFunction.
- const ::quickstep::AggregateFunction &window_aggregate_;
+ const ::quickstep::WindowAggregateFunction &window_aggregate_;
std::vector<ScalarPtr> arguments_;
const WindowInfo window_info_;
const std::string window_name_;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f5bf3532/query_optimizer/resolver/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_optimizer/resolver/CMakeLists.txt b/query_optimizer/resolver/CMakeLists.txt
index fb75767..9313e51 100644
--- a/query_optimizer/resolver/CMakeLists.txt
+++ b/query_optimizer/resolver/CMakeLists.txt
@@ -39,6 +39,8 @@ target_link_libraries(quickstep_queryoptimizer_resolver_Resolver
quickstep_expressions_tablegenerator_GeneratorFunction
quickstep_expressions_tablegenerator_GeneratorFunctionFactory
quickstep_expressions_tablegenerator_GeneratorFunctionHandle
+ quickstep_expressions_windowaggregation_WindowAggregateFunction
+ quickstep_expressions_windowaggregation_WindowAggregateFunctionFactory
quickstep_parser_ParseAssignment
quickstep_parser_ParseBasicExpressions
quickstep_parser_ParseBlockProperties
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f5bf3532/query_optimizer/resolver/Resolver.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/resolver/Resolver.cpp b/query_optimizer/resolver/Resolver.cpp
index f10378b..c224388 100644
--- a/query_optimizer/resolver/Resolver.cpp
+++ b/query_optimizer/resolver/Resolver.cpp
@@ -35,6 +35,8 @@
#include "expressions/table_generator/GeneratorFunction.hpp"
#include "expressions/table_generator/GeneratorFunctionFactory.hpp"
#include "expressions/table_generator/GeneratorFunctionHandle.hpp"
+#include "expressions/window_aggregation/WindowAggregateFunction.hpp"
+#include "expressions/window_aggregation/WindowAggregateFunctionFactory.hpp"
#include "parser/ParseAssignment.hpp"
#include "parser/ParseBasicExpressions.hpp"
#include "parser/ParseBlockProperties.hpp"
@@ -2624,11 +2626,19 @@ E::ScalarPtr Resolver::resolveFunctionCall(
<< "COUNT aggregate has both star (*) and non-star arguments.";
}
- // Try to look up the AggregateFunction by name using
- // AggregateFunctionFactory.
- const ::quickstep::AggregateFunction *aggregate
- = AggregateFunctionFactory::GetByName(function_name);
- if (aggregate == nullptr) {
+ // Try to look up the AggregateFunction/WindowAggregationFunction by name.
+ // TODO(Shixuan): We might want to create a new abstract class Function to
+ // include both AggregateFunction and WindowAggregateFunction, which will make
+ // this part of code cleaner.
+ const ::quickstep::AggregateFunction *aggregate = nullptr;
+ const ::quickstep::WindowAggregateFunction *window_aggregate = nullptr;
+ if (parse_function_call.isWindow()) {
+ window_aggregate = WindowAggregateFunctionFactory::GetByName(function_name);
+ } else {
+ aggregate = AggregateFunctionFactory::GetByName(function_name);
+ }
+
+ if (aggregate == nullptr && window_aggregate == nullptr) {
THROW_SQL_ERROR_AT(&parse_function_call)
<< "Unrecognized function name \""
<< parse_function_call.name()->value()
@@ -2656,11 +2666,14 @@ E::ScalarPtr Resolver::resolveFunctionCall(
}
// Make sure a naked COUNT() with no arguments wasn't specified.
- if ((aggregate->getAggregationID() == AggregationID::kCount)
- && (resolved_arguments.empty())
- && (!count_star)) {
- THROW_SQL_ERROR_AT(&parse_function_call)
- << "COUNT aggregate requires an argument (either scalar or star (*))";
+ if ((aggregate != nullptr &&
+ aggregate->getAggregationID() == AggregationID::kCount) ||
+ (window_aggregate != nullptr &&
+ window_aggregate->getWindowAggregationID() == WindowAggregationID::kCount)) {
+ if ((resolved_arguments.empty()) && !count_star) {
+ THROW_SQL_ERROR_AT(&parse_function_call)
+ << "COUNT aggregate requires an argument (either scalar or star (*))";
+ }
}
// Resolve each of the Scalar arguments to the aggregate.
@@ -2670,7 +2683,8 @@ E::ScalarPtr Resolver::resolveFunctionCall(
}
// Make sure that the aggregate can apply to the specified argument(s).
- if (!aggregate->canApplyToTypes(argument_types)) {
+ if ((aggregate != nullptr && !aggregate->canApplyToTypes(argument_types))
+ || (window_aggregate != nullptr && !window_aggregate->canApplyToTypes(argument_types))) {
THROW_SQL_ERROR_AT(&parse_function_call)
<< "Aggregate function " << aggregate->getName()
<< " can not apply to the given argument(s).";
@@ -2679,7 +2693,7 @@ E::ScalarPtr Resolver::resolveFunctionCall(
if (parse_function_call.isWindow()) {
return resolveWindowAggregateFunction(parse_function_call,
expression_resolution_info,
- aggregate,
+ window_aggregate,
resolved_arguments);
}
@@ -2705,7 +2719,7 @@ E::ScalarPtr Resolver::resolveFunctionCall(
E::ScalarPtr Resolver::resolveWindowAggregateFunction(
const ParseFunctionCall &parse_function_call,
ExpressionResolutionInfo *expression_resolution_info,
- const ::quickstep::AggregateFunction *window_aggregate,
+ const ::quickstep::WindowAggregateFunction *window_aggregate,
const std::vector<E::ScalarPtr> &resolved_arguments) {
// A window aggregate function might be defined OVER a window name or a window.
E::WindowAggregateFunctionPtr window_aggregate_function;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f5bf3532/query_optimizer/resolver/Resolver.hpp
----------------------------------------------------------------------
diff --git a/query_optimizer/resolver/Resolver.hpp b/query_optimizer/resolver/Resolver.hpp
index f4024e9..373430c 100644
--- a/query_optimizer/resolver/Resolver.hpp
+++ b/query_optimizer/resolver/Resolver.hpp
@@ -23,7 +23,6 @@
#include <unordered_set>
#include <vector>
-#include "query_optimizer/expressions/AggregateFunction.hpp"
#include "query_optimizer/expressions/Alias.hpp"
#include "query_optimizer/expressions/ExprId.hpp"
#include "query_optimizer/expressions/NamedExpression.hpp"
@@ -460,14 +459,14 @@ class Resolver {
* @param expression_resolution_info Resolution info that contains the name
* resolver and info to be updated after
* resolution.
- * @param aggregate The aggregate function.
+ * @param aggregate The window aggregate function.
* @param resolved_arguments The resolved arguments.
* @return An expression in the query optimizer.
*/
expressions::ScalarPtr resolveWindowAggregateFunction(
const ParseFunctionCall &parse_function_call,
ExpressionResolutionInfo *expression_resolution_info,
- const ::quickstep::AggregateFunction *aggregate,
+ const ::quickstep::WindowAggregateFunction *aggregate,
const std::vector<expressions::ScalarPtr> &resolved_arguments);
/**
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f5bf3532/query_optimizer/tests/execution_generator/Select.test
----------------------------------------------------------------------
diff --git a/query_optimizer/tests/execution_generator/Select.test b/query_optimizer/tests/execution_generator/Select.test
index 16127cc..30a3c39 100644
--- a/query_optimizer/tests/execution_generator/Select.test
+++ b/query_optimizer/tests/execution_generator/Select.test
@@ -953,19 +953,79 @@ WHERE double_col < 0
==
# Window Aggregation Test.
-# Currently this is not supported, an empty table will be returned.
-SELECT avg(int_col) OVER w FROM test
+SELECT char_col, long_col, avg(long_col) OVER w FROM test
WINDOW w AS
-(PARTITION BY char_col
- ORDER BY long_col DESC NULLS LAST
+(ORDER BY char_col DESC NULLS LAST
ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW);
--
-+------------------------+
-|avg(int_col) |
-+------------------------+
-+------------------------+
++--------------------+--------------------+------------------------+
+|char_col |long_col |avg(long_col) |
++--------------------+--------------------+------------------------+
+| 8 2.828427| 64| 64|
+| 6 2.449490| 36| 50|
+| 4 2.000000| 16| 38.666666666666664|
+| 24 4.898979| 576| 173|
+| 22 4.690416| 484| 235.19999999999999|
+| 20 4.472136| 400| 262.66666666666669|
+| 2 1.414214| 4| 225.71428571428572|
+| 18 4.242641| 324| 238|
+| 16 4.000000| 256| 240|
+| 14 3.741657| 196| 235.59999999999999|
+| 12 3.464102| 144| 227.27272727272728|
+| 10 3.162278| 100| 216.66666666666666|
+| 0 0.000000| 0| 200|
+| -9 3.000000| 81| 191.5|
+| -7 2.645751| 49| 182|
+| -5 2.236068| 25| 172.1875|
+| -3 1.732051| 9| 162.58823529411765|
+| -23 4.795832| 529| 182.94444444444446|
+| -21 4.582576| 441| 196.52631578947367|
+| -19 4.358899| 361| 204.75|
+| -17 4.123106| 289| 208.76190476190476|
+| -15 3.872983| 225| 209.5|
+| -13 3.605551| 169| 207.7391304347826|
+| -11 3.316625| 121| 204.125|
+| -1 1.000000| 1| 196|
++--------------------+--------------------+------------------------+
==
+SELECT long_col, int_col, avg(int_col) OVER w FROM test
+WINDOW w AS
+(ORDER BY long_col DESC NULLS LAST
+ ROWS BETWEEN 2 PRECEDING AND 2 FOLLOWING);
+--
++--------------------+-----------+------------------------+
+|long_col |int_col |avg(int_col) |
++--------------------+-----------+------------------------+
+| 576| 24| 7.666666666666667|
+| 529| -23| 0.5|
+| 484| 22| 0.5|
+| 441| -21| -10.25|
+| 400| NULL| 0|
+| 361| -19| -9.75|
+| 324| 18| -0.5|
+| 289| -17| -3.3999999999999999|
+| 256| 16| 3.2000000000000002|
+| 225| -15| -3|
+| 196| 14| 2.7999999999999998|
+| 169| -13| -2.6000000000000001|
+| 144| 12| 0.5|
+| 121| -11| -5.25|
+| 100| NULL| 0|
+| 81| -9| -4.75|
+| 64| 8| -0.5|
+| 49| -7| -1.3999999999999999|
+| 36| 6| 1.2|
+| 25| -5| -1|
+| 16| 4| 0.80000000000000004|
+| 9| -3| -0.59999999999999998|
+| 4| 2| 0.5|
+| 1| -1| -0.66666666666666663|
+| 0| NULL| 0.5|
++--------------------+-----------+------------------------+
+==
+
+# Currently this is not supported, an empty table will be returned.
SELECT int_col, sum(float_col) OVER
(PARTITION BY char_col, long_col
ORDER BY double_col DESC NULLS LAST, int_col ASC NULLS FIRST
@@ -987,5 +1047,5 @@ WINDOW w AS
+------------------------+
|sum(avg(int_col)) |
+------------------------+
-| NULL|
+| -18|
+------------------------+
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f5bf3532/relational_operators/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/relational_operators/CMakeLists.txt b/relational_operators/CMakeLists.txt
index 249441d..a51370b 100644
--- a/relational_operators/CMakeLists.txt
+++ b/relational_operators/CMakeLists.txt
@@ -434,6 +434,7 @@ target_link_libraries(quickstep_relationaloperators_WindowAggregationOperator
quickstep_relationaloperators_WorkOrder
quickstep_relationaloperators_WorkOrder_proto
quickstep_storage_StorageBlockInfo
+ quickstep_storage_WindowAggregationOperationState
quickstep_utility_Macros
tmb)
target_link_libraries(quickstep_relationaloperators_WorkOrder
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f5bf3532/relational_operators/WindowAggregationOperator.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/WindowAggregationOperator.cpp b/relational_operators/WindowAggregationOperator.cpp
index 93cb9d4..3149864 100644
--- a/relational_operators/WindowAggregationOperator.cpp
+++ b/relational_operators/WindowAggregationOperator.cpp
@@ -21,11 +21,13 @@
#include <vector>
+#include "catalog/CatalogRelation.hpp"
#include "query_execution/QueryContext.hpp"
#include "query_execution/WorkOrderProtosContainer.hpp"
#include "query_execution/WorkOrdersContainer.hpp"
#include "relational_operators/WorkOrder.pb.h"
#include "storage/StorageBlockInfo.hpp"
+#include "storage/WindowAggregationOperationState.hpp"
#include "tmb/id_typedefs.h"
@@ -40,10 +42,14 @@ bool WindowAggregationOperator::getAllWorkOrders(
DCHECK(query_context != nullptr);
if (blocking_dependencies_met_ && !generated_) {
+ std::vector<block_id> relation_blocks =
+ input_relation_.getBlocksSnapshot();
+
container->addNormalWorkOrder(
new WindowAggregationWorkOrder(
query_id_,
query_context->releaseWindowAggregationState(window_aggregation_state_index_),
+ std::move(relation_blocks),
query_context->getInsertDestination(output_destination_index_)),
op_index_);
generated_ = true;
@@ -67,6 +73,11 @@ serialization::WorkOrder* WindowAggregationOperator::createWorkOrderProto() {
proto->set_query_id(query_id_);
proto->SetExtension(serialization::WindowAggregationWorkOrder::window_aggr_state_index,
window_aggregation_state_index_);
+
+ const std::vector<block_id> relation_blocks = input_relation_.getBlocksSnapshot();
+ for (const block_id bid : relation_blocks) {
+ proto->AddExtension(serialization::WindowAggregationWorkOrder::block_ids, bid);
+ }
proto->SetExtension(serialization::WindowAggregationWorkOrder::insert_destination_index,
output_destination_index_);
@@ -75,8 +86,8 @@ serialization::WorkOrder* WindowAggregationOperator::createWorkOrderProto() {
void WindowAggregationWorkOrder::execute() {
- std::cout << "Window aggregation is not supported yet.\n"
- << "An empty table is returned\n";
+ state_->windowAggregateBlocks(output_destination_,
+ block_ids_);
}
} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f5bf3532/relational_operators/WindowAggregationOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/WindowAggregationOperator.hpp b/relational_operators/WindowAggregationOperator.hpp
index f3dfd14..bd83248 100644
--- a/relational_operators/WindowAggregationOperator.hpp
+++ b/relational_operators/WindowAggregationOperator.hpp
@@ -58,16 +58,19 @@ class WindowAggregationOperator : public RelationalOperator {
*
* @param query_id The ID of this query.
* @param input_relation The relation to perform aggregation over.
+ * @param output_relation The relation for output.
* @param window_aggregation_state_index The index of WindowAggregationState
* in QueryContext.
* @param output_destination_index The index of InsertDestination in
* QueryContext for the output.
**/
WindowAggregationOperator(const std::size_t query_id,
+ const CatalogRelation &input_relation,
const CatalogRelation &output_relation,
const QueryContext::window_aggregation_state_id window_aggregation_state_index,
const QueryContext::insert_destination_id output_destination_index)
: RelationalOperator(query_id),
+ input_relation_(input_relation),
output_relation_(output_relation),
window_aggregation_state_index_(window_aggregation_state_index),
output_destination_index_(output_destination_index),
@@ -99,6 +102,7 @@ class WindowAggregationOperator : public RelationalOperator {
**/
serialization::WorkOrder* createWorkOrderProto();
+ const CatalogRelation &input_relation_;
const CatalogRelation &output_relation_;
const QueryContext::window_aggregation_state_id window_aggregation_state_index_;
const QueryContext::insert_destination_id output_destination_index_;
@@ -117,43 +121,25 @@ class WindowAggregationWorkOrder : public WorkOrder {
*
* @param query_id The ID of this query.
* @param state The WindowAggregationOperatorState to use.
+ * @param block_ids The blocks' id of the input relation.
* @param output_destination The InsertDestination for output.
**/
WindowAggregationWorkOrder(const std::size_t query_id,
WindowAggregationOperationState *state,
+ std::vector<block_id> &&block_ids,
InsertDestination *output_destination)
: WorkOrder(query_id),
state_(state),
+ block_ids_(std::move(block_ids)),
output_destination_(output_destination) {}
~WindowAggregationWorkOrder() override {}
- /**
- * @brief Get the pointer to WindowAggregationOperationState.
- * @note This is a quickfix for "unused variable". After the window aggregate
- * functions are built, these methods might be dropped.
- *
- * @return A pointer to the window aggregation operation state.
- **/
- WindowAggregationOperationState* state() {
- return state_;
- }
-
- /**
- * @brief Get the pointer to output destination.
- * @note This is a quickfix for "unused variable". After the window aggregate
- * functions are built, these methods might be dropped.
- *
- * @return A pointer to the output destination.
- **/
- InsertDestination* output_destination() {
- return output_destination_;
- }
-
void execute() override;
private:
WindowAggregationOperationState *state_;
+ const std::vector<block_id> block_ids_;
InsertDestination *output_destination_;
DISALLOW_COPY_AND_ASSIGN(WindowAggregationWorkOrder);
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f5bf3532/relational_operators/WorkOrder.proto
----------------------------------------------------------------------
diff --git a/relational_operators/WorkOrder.proto b/relational_operators/WorkOrder.proto
index 69dee1b..076735f 100644
--- a/relational_operators/WorkOrder.proto
+++ b/relational_operators/WorkOrder.proto
@@ -249,6 +249,7 @@ message WindowAggregationWorkOrder {
extend WorkOrder {
// All required
optional uint32 window_aggr_state_index = 336;
- optional int32 insert_destination_index = 337;
+ repeated fixed64 block_ids = 337;
+ optional int32 insert_destination_index = 338;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f5bf3532/storage/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/storage/CMakeLists.txt b/storage/CMakeLists.txt
index 9df66e1..582effd 100644
--- a/storage/CMakeLists.txt
+++ b/storage/CMakeLists.txt
@@ -1053,19 +1053,26 @@ target_link_libraries(quickstep_storage_WindowAggregationOperationState
quickstep_catalog_CatalogTypedefs
quickstep_expressions_ExpressionFactories
quickstep_expressions_Expressions_proto
- quickstep_expressions_aggregation_AggregateFunction
- quickstep_expressions_aggregation_AggregateFunctionFactory
- quickstep_expressions_aggregation_AggregationHandle
- quickstep_expressions_aggregation_AggregationID
quickstep_expressions_scalar_Scalar
quickstep_expressions_scalar_ScalarAttribute
+ quickstep_expressions_windowaggregation_WindowAggregateFunction
+ quickstep_expressions_windowaggregation_WindowAggregateFunctionFactory
+ quickstep_expressions_windowaggregation_WindowAggregationHandle
+ quickstep_expressions_windowaggregation_WindowAggregationID
+ quickstep_storage_InsertDestination
quickstep_storage_StorageBlockInfo
quickstep_storage_StorageManager
+ quickstep_storage_SubBlocksReference
+ quickstep_storage_ValueAccessor
+ quickstep_storage_ValueAccessorUtil
quickstep_storage_WindowAggregationOperationState_proto
+ quickstep_types_containers_ColumnVector
+ quickstep_types_containers_ColumnVectorUtil
+ quickstep_types_containers_ColumnVectorsValueAccessor
quickstep_utility_Macros)
target_link_libraries(quickstep_storage_WindowAggregationOperationState_proto
- quickstep_expressions_aggregation_AggregateFunction_proto
quickstep_expressions_Expressions_proto
+ quickstep_expressions_windowaggregation_WindowAggregateFunction_proto
${PROTOBUF_LIBRARY})
# Module all-in-one library:
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f5bf3532/storage/WindowAggregationOperationState.cpp
----------------------------------------------------------------------
diff --git a/storage/WindowAggregationOperationState.cpp b/storage/WindowAggregationOperationState.cpp
index a0bcc37..0cdfc1a 100644
--- a/storage/WindowAggregationOperationState.cpp
+++ b/storage/WindowAggregationOperationState.cpp
@@ -31,14 +31,21 @@
#include "catalog/CatalogTypedefs.hpp"
#include "expressions/ExpressionFactories.hpp"
#include "expressions/Expressions.pb.h"
-#include "expressions/aggregation/AggregateFunction.hpp"
-#include "expressions/aggregation/AggregateFunctionFactory.hpp"
-#include "expressions/aggregation/AggregationHandle.hpp"
-#include "expressions/aggregation/AggregationID.hpp"
#include "expressions/scalar/Scalar.hpp"
#include "expressions/scalar/ScalarAttribute.hpp"
+#include "expressions/window_aggregation/WindowAggregateFunction.hpp"
+#include "expressions/window_aggregation/WindowAggregateFunctionFactory.hpp"
+#include "expressions/window_aggregation/WindowAggregationHandle.hpp"
+#include "expressions/window_aggregation/WindowAggregationID.hpp"
+#include "storage/InsertDestination.hpp"
#include "storage/StorageManager.hpp"
+#include "storage/SubBlocksReference.hpp"
+#include "storage/ValueAccessor.hpp"
+#include "storage/ValueAccessorUtil.hpp"
#include "storage/WindowAggregationOperationState.pb.h"
+#include "types/containers/ColumnVector.hpp"
+#include "types/containers/ColumnVectorsValueAccessor.hpp"
+#include "types/containers/ColumnVectorUtil.hpp"
#include "glog/logging.h"
@@ -46,23 +53,21 @@ namespace quickstep {
WindowAggregationOperationState::WindowAggregationOperationState(
const CatalogRelationSchema &input_relation,
- const AggregateFunction *window_aggregate_function,
+ const WindowAggregateFunction *window_aggregate_function,
std::vector<std::unique_ptr<const Scalar>> &&arguments,
- std::vector<std::unique_ptr<const Scalar>> &&partition_by_attributes,
+ const std::vector<std::unique_ptr<const Scalar>> &partition_by_attributes,
const bool is_row,
const std::int64_t num_preceding,
const std::int64_t num_following,
StorageManager *storage_manager)
: input_relation_(input_relation),
arguments_(std::move(arguments)),
- partition_by_attributes_(std::move(partition_by_attributes)),
is_row_(is_row),
num_preceding_(num_preceding),
num_following_(num_following),
storage_manager_(storage_manager) {
// Get the Types of this window aggregate's arguments so that we can create an
// AggregationHandle.
- // TODO(Shixuan): Next step: New handles for window aggregation function.
std::vector<const Type*> argument_types;
for (const std::unique_ptr<const Scalar> &argument : arguments_) {
argument_types.emplace_back(&argument->getType());
@@ -71,28 +76,18 @@ WindowAggregationOperationState::WindowAggregationOperationState(
// Check if window aggregate function could apply to the arguments.
DCHECK(window_aggregate_function->canApplyToTypes(argument_types));
+ // IDs and types of partition keys.
+ std::vector<const Type*> partition_by_types;
+ for (const std::unique_ptr<const Scalar> &partition_by_attribute : partition_by_attributes) {
+ partition_by_ids_.push_back(
+ partition_by_attribute->getAttributeIdForValueAccessor());
+ partition_by_types.push_back(&partition_by_attribute->getType());
+ }
+
// Create the handle and initial state.
window_aggregation_handle_.reset(
- window_aggregate_function->createHandle(argument_types));
- window_aggregation_state_.reset(
- window_aggregation_handle_->createInitialState());
-
-#ifdef QUICKSTEP_ENABLE_VECTOR_COPY_ELISION_SELECTION
- // See if all of this window aggregate's arguments are attributes in the input
- // relation. If so, remember the attribute IDs so that we can do copy elision
- // when actually performing the window aggregation.
- arguments_as_attributes_.reserve(arguments_.size());
- for (const std::unique_ptr<const Scalar> &argument : arguments_) {
- const attribute_id argument_id = argument->getAttributeIdForValueAccessor();
- if (argument_id == -1) {
- arguments_as_attributes_.clear();
- break;
- } else {
- DCHECK_EQ(input_relation_.getID(), argument->getRelationIdForValueAccessor());
- arguments_as_attributes_.push_back(argument_id);
- }
- }
-#endif
+ window_aggregate_function->createHandle(std::move(argument_types),
+ std::move(partition_by_types)));
}
WindowAggregationOperationState* WindowAggregationOperationState::ReconstructFromProto(
@@ -101,10 +96,6 @@ WindowAggregationOperationState* WindowAggregationOperationState::ReconstructFro
StorageManager *storage_manager) {
DCHECK(ProtoIsValid(proto, database));
- // Rebuild contructor arguments from their representation in 'proto'.
- const AggregateFunction *aggregate_function
- = &AggregateFunctionFactory::ReconstructFromProto(proto.function());
-
std::vector<std::unique_ptr<const Scalar>> arguments;
arguments.reserve(proto.arguments_size());
for (int argument_idx = 0; argument_idx < proto.arguments_size(); ++argument_idx) {
@@ -126,10 +117,10 @@ WindowAggregationOperationState* WindowAggregationOperationState::ReconstructFro
const std::int64_t num_preceding = proto.num_preceding();
const std::int64_t num_following = proto.num_following();
- return new WindowAggregationOperationState(database.getRelationSchemaById(proto.relation_id()),
- aggregate_function,
+ return new WindowAggregationOperationState(database.getRelationSchemaById(proto.input_relation_id()),
+ &WindowAggregateFunctionFactory::ReconstructFromProto(proto.function()),
std::move(arguments),
- std::move(partition_by_attributes),
+ partition_by_attributes,
is_row,
num_preceding,
num_following,
@@ -139,11 +130,11 @@ WindowAggregationOperationState* WindowAggregationOperationState::ReconstructFro
bool WindowAggregationOperationState::ProtoIsValid(const serialization::WindowAggregationOperationState &proto,
const CatalogDatabaseLite &database) {
if (!proto.IsInitialized() ||
- !database.hasRelationWithId(proto.relation_id())) {
+ !database.hasRelationWithId(proto.input_relation_id())) {
return false;
}
- if (!AggregateFunctionFactory::ProtoIsValid(proto.function())) {
+ if (!WindowAggregateFunctionFactory::ProtoIsValid(proto.function())) {
return false;
}
@@ -176,4 +167,122 @@ bool WindowAggregationOperationState::ProtoIsValid(const serialization::WindowAg
return true;
}
+void WindowAggregationOperationState::windowAggregateBlocks(
+ InsertDestination *output_destination,
+ const std::vector<block_id> &block_ids) {
+ // TODO(Shixuan): This is a quick fix for currently unsupported functions in
+ // order to pass the query_optimizer test.
+ if (window_aggregation_handle_.get() == nullptr) {
+ std::cout << "The function will be supported in the near future :)\n";
+ return;
+ }
+
+ // TODO(Shixuan): RANGE frame mode should be supported to make SQL grammar
+ // work. This will need Order Key to be passed so that we know where the
+ // window should start and end.
+ if (!is_row_) {
+ std::cout << "Currently we don't support RANGE frame mode :(\n";
+ return;
+ }
+
+ // Get the total number of tuples.
+ int num_tuples = 0;
+ for (const block_id block_idx : block_ids) {
+ num_tuples +=
+ storage_manager_->getBlock(block_idx, input_relation_)->getNumTuples();
+ }
+
+ // Construct column vectors for attributes.
+ std::vector<ColumnVector*> attribute_vecs;
+ for (std::size_t attr_id = 0; attr_id < input_relation_.size(); ++attr_id) {
+ const CatalogAttribute *attr = input_relation_.getAttributeById(attr_id);
+ const Type &type = attr->getType();
+
+ if (NativeColumnVector::UsableForType(type)) {
+ attribute_vecs.push_back(new NativeColumnVector(type, num_tuples));
+ } else {
+ attribute_vecs.push_back(new IndirectColumnVector(type, num_tuples));
+ }
+ }
+
+ // Construct column vectors for arguments.
+ std::vector<ColumnVector*> argument_vecs;
+ for (const std::unique_ptr<const Scalar> &argument : arguments_) {
+ const Type &type = argument->getType();
+
+ if (NativeColumnVector::UsableForType(type)) {
+ argument_vecs.push_back(new NativeColumnVector(type, num_tuples));
+ } else {
+ argument_vecs.push_back(new IndirectColumnVector(type, num_tuples));
+ }
+ }
+
+ // TODO(Shixuan): Add Support for Vector Copy Elision Selection.
+ // Add tuples and arguments into ColumnVectors.
+ for (const block_id block_idx : block_ids) {
+ BlockReference block = storage_manager_->getBlock(block_idx, input_relation_);
+ const TupleStorageSubBlock &tuple_block = block->getTupleStorageSubBlock();
+ SubBlocksReference sub_block_ref(tuple_block,
+ block->getIndices(),
+ block->getIndicesConsistent());
+ ValueAccessor *tuple_accessor = tuple_block.createValueAccessor();
+ ColumnVectorsValueAccessor *argument_accessor = new ColumnVectorsValueAccessor();
+ for (const std::unique_ptr<const Scalar> &argument : arguments_) {
+ argument_accessor->addColumn(argument->getAllValues(tuple_accessor,
+ &sub_block_ref));
+ }
+
+ InvokeOnAnyValueAccessor(tuple_accessor,
+ [&] (auto *tuple_accessor) -> void { // NOLINT(build/c++11)
+ tuple_accessor->beginIteration();
+ argument_accessor->beginIteration();
+
+ while (tuple_accessor->next() && argument_accessor->next()) {
+ for (std::size_t attr_id = 0; attr_id < attribute_vecs.size(); ++attr_id) {
+ ColumnVector *attr_vec = attribute_vecs[attr_id];
+ if (attr_vec->isNative()) {
+ static_cast<NativeColumnVector*>(attr_vec)->appendTypedValue(
+ tuple_accessor->getTypedValue(attr_id));
+ } else {
+ static_cast<IndirectColumnVector*>(attr_vec)->appendTypedValue(
+ tuple_accessor->getTypedValue(attr_id));
+ }
+ }
+
+ for (std::size_t argument_idx = 0;
+ argument_idx < argument_vecs.size();
+ ++argument_idx) {
+ ColumnVector *argument = argument_vecs[argument_idx];
+ if (argument->isNative()) {
+ static_cast<NativeColumnVector*>(argument)->appendTypedValue(
+ argument_accessor->getTypedValue(argument_idx));
+ } else {
+ static_cast<IndirectColumnVector*>(argument)->appendTypedValue(
+ argument_accessor->getTypedValue(argument_idx));
+ }
+ }
+ }
+ });
+ }
+
+ // Construct the value accessor for tuples in all blocks
+ ColumnVectorsValueAccessor *all_blocks_accessor
+ = new ColumnVectorsValueAccessor();
+ for (ColumnVector *attr_vec : attribute_vecs) {
+ all_blocks_accessor->addColumn(attr_vec);
+ }
+
+ // Do actual calculation in handle.
+ ColumnVector *window_aggregates =
+ window_aggregation_handle_->calculate(all_blocks_accessor,
+ std::move(argument_vecs),
+ partition_by_ids_,
+ is_row_,
+ num_preceding_,
+ num_following_);
+
+ all_blocks_accessor->addColumn(window_aggregates);
+ output_destination->bulkInsertTuples(all_blocks_accessor);
+}
+
} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f5bf3532/storage/WindowAggregationOperationState.hpp
----------------------------------------------------------------------
diff --git a/storage/WindowAggregationOperationState.hpp b/storage/WindowAggregationOperationState.hpp
index d7b3e6a..9792a99 100644
--- a/storage/WindowAggregationOperationState.hpp
+++ b/storage/WindowAggregationOperationState.hpp
@@ -25,20 +25,20 @@
#include <vector>
#include "catalog/CatalogTypedefs.hpp"
-#include "expressions/aggregation/AggregationHandle.hpp"
#include "expressions/scalar/Scalar.hpp"
#include "expressions/scalar/ScalarAttribute.hpp"
+#include "expressions/window_aggregation/WindowAggregationHandle.hpp"
#include "storage/StorageBlockInfo.hpp"
#include "storage/WindowAggregationOperationState.pb.h"
#include "utility/Macros.hpp"
namespace quickstep {
-class AggregateFunction;
class CatalogDatabaseLite;
class CatalogRelationSchema;
class InsertDestination;
class StorageManager;
+class WindowAggregateFunction;
/** \addtogroup Storage
* @{
@@ -63,13 +63,12 @@ class WindowAggregationOperationState {
* current row. -1 means UNBOUNDED PRECEDING.
* @param num_following The number of rows/range for the tuples following the
* current row. -1 means UNBOUNDED FOLLOWING.
- * @param storage_manager The StorageManager to use for allocating hash
- * tables.
+ * @param storage_manager The StorageManager to get block references.
*/
WindowAggregationOperationState(const CatalogRelationSchema &input_relation,
- const AggregateFunction *window_aggregate_function,
+ const WindowAggregateFunction *window_aggregate_function,
std::vector<std::unique_ptr<const Scalar>> &&arguments,
- std::vector<std::unique_ptr<const Scalar>> &&partition_by_attributes,
+ const std::vector<std::unique_ptr<const Scalar>> &partition_by_attributes,
const bool is_row,
const std::int64_t num_preceding,
const std::int64_t num_following,
@@ -107,66 +106,29 @@ class WindowAggregationOperationState {
const CatalogDatabaseLite &database);
/**
- * @brief Get the is_row info.
- * @note This is a quickfix for "unused variable". After the window aggregate
- * functions are built, these methods might be dropped.
- *
- * @return True if the frame mode is ROW, false if it is RANGE.
- **/
- const bool is_row() const { return is_row_; }
-
- /**
- * @brief Get the num_preceding info.
- * @note This is a quickfix for "unused variable". After the window aggregate
- * functions are built, these methods might be dropped.
- *
- * @return The number of rows/range that precedes the current row.
- **/
- const std::int64_t num_preceding() const { return num_preceding_; }
-
- /**
- * @brief Get the num_following info.
- * @note This is a quickfix for "unused variable". After the window aggregate
- * functions are built, these methods might be dropped.
+ * @brief Compute window aggregates on the tuples of the given relation.
*
- * @return The number of rows/range that follows the current row.
+ * @param output_destination The output destination for the computed window
+ * aggregate.
+ * @param block_ids The id of the blocks to be computed.
**/
- const std::int64_t num_following() const { return num_following_; }
-
- /**
- * @brief Get the pointer to StorageManager.
- * @note This is a quickfix for "unused variable". After the window aggregate
- * functions are built, these methods might be dropped.
- *
- * @return A pointer to the storage manager.
- **/
- StorageManager *storage_manager() { return storage_manager_; }
+ void windowAggregateBlocks(InsertDestination *output_destination,
+ const std::vector<block_id> &block_ids);
private:
const CatalogRelationSchema &input_relation_;
-
- // TODO(Shixuan): Handle and State for window aggregation will be needed for
- // actual calculation.
- std::unique_ptr<AggregationHandle> window_aggregation_handle_;
- std::unique_ptr<AggregationState> window_aggregation_state_;
+ const std::vector<block_id> block_ids_;
+ std::unique_ptr<WindowAggregationHandle> window_aggregation_handle_;
std::vector<std::unique_ptr<const Scalar>> arguments_;
+ std::vector<attribute_id> partition_by_ids_;
- // We don't add order_by_attributes here since it is not needed after sorting.
- std::vector<std::unique_ptr<const Scalar>> partition_by_attributes_;
-
- // Window framing information.
+ // Frame info.
const bool is_row_;
const std::int64_t num_preceding_;
const std::int64_t num_following_;
StorageManager *storage_manager_;
-#ifdef QUICKSTEP_ENABLE_VECTOR_COPY_ELISION_SELECTION
- // If all an aggregate's argument expressions are simply attributes in
- // 'input_relation_', then this caches the attribute IDs of those arguments.
- std::vector<attribute_id> arguments_as_attributes_;
-#endif
-
DISALLOW_COPY_AND_ASSIGN(WindowAggregationOperationState);
};
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f5bf3532/storage/WindowAggregationOperationState.proto
----------------------------------------------------------------------
diff --git a/storage/WindowAggregationOperationState.proto b/storage/WindowAggregationOperationState.proto
index c7bd0ef..d888461 100644
--- a/storage/WindowAggregationOperationState.proto
+++ b/storage/WindowAggregationOperationState.proto
@@ -19,12 +19,12 @@ syntax = "proto2";
package quickstep.serialization;
-import "expressions/aggregation/AggregateFunction.proto";
+import "expressions/window_aggregation/WindowAggregateFunction.proto";
import "expressions/Expressions.proto";
message WindowAggregationOperationState {
- required int32 relation_id = 1;
- required AggregateFunction function = 2;
+ required int32 input_relation_id = 1;
+ required WindowAggregateFunction function = 2;
repeated Scalar arguments = 3;
repeated Scalar partition_by_attributes = 4;
required bool is_row = 5;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f5bf3532/storage/tests/WindowAggregationOperationState_unittest.cpp
----------------------------------------------------------------------
diff --git a/storage/tests/WindowAggregationOperationState_unittest.cpp b/storage/tests/WindowAggregationOperationState_unittest.cpp
index c572034..d58f0f5 100644
--- a/storage/tests/WindowAggregationOperationState_unittest.cpp
+++ b/storage/tests/WindowAggregationOperationState_unittest.cpp
@@ -23,7 +23,7 @@
#include "catalog/CatalogDatabase.hpp"
#include "catalog/CatalogRelation.hpp"
#include "catalog/CatalogTypedefs.hpp"
-#include "expressions/aggregation/AggregateFunction.pb.h"
+#include "expressions/window_aggregation/WindowAggregateFunction.pb.h"
#include "storage/WindowAggregationOperationState.hpp"
#include "storage/WindowAggregationOperationState.pb.h"
@@ -57,8 +57,8 @@ TEST_F(WindowAggregationOperationStateProtoTest, UninitializationTest) {
TEST_F(WindowAggregationOperationStateProtoTest, InvalidRelationIdTest) {
serialization::WindowAggregationOperationState proto;
- proto.set_relation_id(kInvalidTableId);
- proto.mutable_function()->set_aggregation_id(serialization::AggregateFunction::AVG);
+ proto.set_input_relation_id(kInvalidTableId);
+ proto.mutable_function()->set_window_aggregation_id(serialization::WindowAggregateFunction::AVG);
proto.set_is_row(true);
proto.set_num_preceding(kValidNum);
proto.set_num_following(kValidNum);
@@ -67,8 +67,8 @@ TEST_F(WindowAggregationOperationStateProtoTest, InvalidRelationIdTest) {
TEST_F(WindowAggregationOperationStateProtoTest, InvalidNumTest) {
serialization::WindowAggregationOperationState proto;
- proto.set_relation_id(rel_id_);
- proto.mutable_function()->set_aggregation_id(serialization::AggregateFunction::AVG);
+ proto.set_input_relation_id(rel_id_);
+ proto.mutable_function()->set_window_aggregation_id(serialization::WindowAggregateFunction::AVG);
proto.set_is_row(true);
proto.set_num_preceding(kInvalidNum);
proto.set_num_following(kValidNum);
@@ -81,8 +81,8 @@ TEST_F(WindowAggregationOperationStateProtoTest, InvalidNumTest) {
TEST_F(WindowAggregationOperationStateProtoTest, ValidTest) {
serialization::WindowAggregationOperationState proto;
- proto.set_relation_id(rel_id_);
- proto.mutable_function()->set_aggregation_id(serialization::AggregateFunction::AVG);
+ proto.set_input_relation_id(rel_id_);
+ proto.mutable_function()->set_window_aggregation_id(serialization::WindowAggregateFunction::AVG);
proto.set_is_row(true);
proto.set_num_preceding(kValidNum);
proto.set_num_following(kValidNum);
[04/25] incubator-quickstep git commit: Added move semantic in the
constructor of window plan.
Posted by zu...@apache.org.
Added move semantic in the constructor of window plan.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/06177e8a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/06177e8a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/06177e8a
Branch: refs/heads/dist-exe-test-new
Commit: 06177e8a999d3edf2f905492de9f1b30bf76b356
Parents: 6b3aa03
Author: shixuan-fan <sh...@apache.org>
Authored: Tue Jun 28 15:42:09 2016 +0000
Committer: Zuyu Zhang <zu...@twitter.com>
Committed: Fri Jul 29 16:42:09 2016 -0700
----------------------------------------------------------------------
query_optimizer/resolver/Resolver.cpp | 14 +++++++-------
1 file changed, 7 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/06177e8a/query_optimizer/resolver/Resolver.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/resolver/Resolver.cpp b/query_optimizer/resolver/Resolver.cpp
index 1eb6d86..c07751a 100644
--- a/query_optimizer/resolver/Resolver.cpp
+++ b/query_optimizer/resolver/Resolver.cpp
@@ -228,13 +228,13 @@ struct Resolver::QueryAggregationInfo {
};
struct Resolver::WindowPlan {
- WindowPlan(const E::WindowInfo &window_info_in,
- const L::LogicalPtr &logical_plan_in)
- : window_info(window_info_in),
- logical_plan(logical_plan_in) {}
+ WindowPlan(const L::LogicalPtr &logical_plan_in,
+ E::WindowInfo &&window_info_in) // NOLINT(whitespace/operators)
+ : logical_plan(logical_plan_in),
+ window_info(std::move(window_info_in)) {}
- const E::WindowInfo window_info;
const L::LogicalPtr logical_plan;
+ const E::WindowInfo window_info;
};
struct Resolver::WindowAggregationInfo {
@@ -1032,9 +1032,9 @@ L::LogicalPtr Resolver::resolveSelect(
L::LogicalPtr sorted_logical_plan = resolveSortInWindow(logical_plan,
resolved_window);
- WindowPlan window_plan(resolved_window, sorted_logical_plan);
+ WindowPlan window_plan(sorted_logical_plan, std::move(resolved_window));
- sorted_window_map.emplace(window.name()->value(), window_plan);
+ sorted_window_map.emplace(window.name()->value(), std::move(window_plan));
}
}
[10/25] incubator-quickstep git commit: Renamed Foreman related
classes.
Posted by zu...@apache.org.
Renamed Foreman related classes.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/4fb884c1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/4fb884c1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/4fb884c1
Branch: refs/heads/dist-exe-test-new
Commit: 4fb884c1bef53b00ec3e0362b8de401b7c0b07f6
Parents: a37bf26
Author: Zuyu Zhang <zu...@apache.org>
Authored: Thu Jul 7 14:13:19 2016 -0500
Committer: Zuyu Zhang <zu...@twitter.com>
Committed: Fri Jul 29 16:42:10 2016 -0700
----------------------------------------------------------------------
CMakeLists.txt | 2 +-
cli/CMakeLists.txt | 1 -
cli/CommandExecutor.cpp | 1 -
cli/CommandExecutor.hpp | 1 -
cli/QuickstepCli.cpp | 21 +-
cli/tests/CMakeLists.txt | 2 +-
cli/tests/CommandExecutorTestRunner.cpp | 2 +-
cli/tests/CommandExecutorTestRunner.hpp | 15 +-
query_execution/CMakeLists.txt | 28 +-
query_execution/Foreman.cpp | 255 ------------------
query_execution/Foreman.hpp | 140 ----------
query_execution/ForemanBase.hpp | 85 ++++++
query_execution/ForemanLite.hpp | 85 ------
query_execution/ForemanSingleNode.cpp | 256 +++++++++++++++++++
query_execution/ForemanSingleNode.hpp | 140 ++++++++++
query_optimizer/tests/CMakeLists.txt | 2 +-
.../tests/ExecutionGeneratorTestRunner.cpp | 2 +-
.../tests/ExecutionGeneratorTestRunner.hpp | 15 +-
18 files changed, 527 insertions(+), 526 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4fb884c1/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/CMakeLists.txt b/CMakeLists.txt
index de6754a..042c050 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -756,7 +756,7 @@ target_link_libraries(quickstep_cli_shell
quickstep_parser_ParseStatement
quickstep_parser_SqlParserWrapper
quickstep_queryexecution_AdmitRequestMessage
- quickstep_queryexecution_Foreman
+ quickstep_queryexecution_ForemanSingleNode
quickstep_queryexecution_QueryContext
quickstep_queryexecution_QueryExecutionTypedefs
quickstep_queryexecution_QueryExecutionUtil
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4fb884c1/cli/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/cli/CMakeLists.txt b/cli/CMakeLists.txt
index 44ec223..9637055 100644
--- a/cli/CMakeLists.txt
+++ b/cli/CMakeLists.txt
@@ -89,7 +89,6 @@ target_link_libraries(quickstep_cli_CommandExecutor
quickstep_cli_PrintToScreen
quickstep_parser_ParseStatement
quickstep_parser_SqlParserWrapper
- quickstep_queryexecution_Foreman
quickstep_queryoptimizer_QueryHandle
quickstep_queryoptimizer_QueryPlan
quickstep_queryoptimizer_QueryProcessor
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4fb884c1/cli/CommandExecutor.cpp
----------------------------------------------------------------------
diff --git a/cli/CommandExecutor.cpp b/cli/CommandExecutor.cpp
index 7083ef5..8acfae8 100644
--- a/cli/CommandExecutor.cpp
+++ b/cli/CommandExecutor.cpp
@@ -34,7 +34,6 @@
#include "parser/ParseStatement.hpp"
#include "parser/ParseString.hpp"
#include "parser/SqlParserWrapper.hpp"
-#include "query_execution/Foreman.hpp"
#include "query_optimizer/QueryHandle.hpp"
#include "query_optimizer/QueryPlan.hpp"
#include "query_optimizer/QueryProcessor.hpp"
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4fb884c1/cli/CommandExecutor.hpp
----------------------------------------------------------------------
diff --git a/cli/CommandExecutor.hpp b/cli/CommandExecutor.hpp
index 3435aeb..19d03e6 100644
--- a/cli/CommandExecutor.hpp
+++ b/cli/CommandExecutor.hpp
@@ -32,7 +32,6 @@ namespace tmb { class MessageBus; }
namespace quickstep {
class CatalogDatabase;
-class Foreman;
class ParseStatement;
class QueryProcessor;
class StorageManager;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4fb884c1/cli/QuickstepCli.cpp
----------------------------------------------------------------------
diff --git a/cli/QuickstepCli.cpp b/cli/QuickstepCli.cpp
index 02a55a0..68a3599 100644
--- a/cli/QuickstepCli.cpp
+++ b/cli/QuickstepCli.cpp
@@ -58,7 +58,7 @@ typedef quickstep::LineReaderDumb LineReaderImpl;
#include "parser/ParseStatement.hpp"
#include "parser/SqlParserWrapper.hpp"
#include "query_execution/AdmitRequestMessage.hpp"
-#include "query_execution/Foreman.hpp"
+#include "query_execution/ForemanSingleNode.hpp"
#include "query_execution/QueryExecutionTypedefs.hpp"
#include "query_execution/QueryExecutionUtil.hpp"
#include "query_execution/Worker.hpp"
@@ -104,7 +104,7 @@ using quickstep::AdmitRequestMessage;
using quickstep::CatalogRelation;
using quickstep::DefaultsConfigurator;
using quickstep::DropRelation;
-using quickstep::Foreman;
+using quickstep::ForemanSingleNode;
using quickstep::InputParserUtil;
using quickstep::MessageBusImpl;
using quickstep::MessageStyle;
@@ -353,14 +353,15 @@ int main(int argc, char* argv[]) {
worker_client_ids,
worker_numa_nodes);
- Foreman foreman(main_thread_client_id,
- &worker_directory,
- &bus,
- query_processor->getDefaultDatabase(),
- query_processor->getStorageManager(),
- -1, // Don't pin the Foreman thread.
- num_numa_nodes_system,
- quickstep::FLAGS_profile_and_report_workorder_perf);
+ ForemanSingleNode foreman(
+ main_thread_client_id,
+ &worker_directory,
+ &bus,
+ query_processor->getDefaultDatabase(),
+ query_processor->getStorageManager(),
+ -1, // Don't pin the Foreman thread.
+ num_numa_nodes_system,
+ quickstep::FLAGS_profile_and_report_workorder_perf);
// Start the worker threads.
for (Worker &worker : workers) {
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4fb884c1/cli/tests/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/cli/tests/CMakeLists.txt b/cli/tests/CMakeLists.txt
index d177d6c..7da56d1 100644
--- a/cli/tests/CMakeLists.txt
+++ b/cli/tests/CMakeLists.txt
@@ -33,7 +33,7 @@ target_link_libraries(quickstep_cli_tests_CommandExecutorTest
quickstep_parser_ParseStatement
quickstep_parser_SqlParserWrapper
quickstep_queryexecution_AdmitRequestMessage
- quickstep_queryexecution_Foreman
+ quickstep_queryexecution_ForemanSingleNode
quickstep_queryexecution_QueryContext
quickstep_queryexecution_QueryExecutionTypedefs
quickstep_queryexecution_QueryExecutionUtil
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4fb884c1/cli/tests/CommandExecutorTestRunner.cpp
----------------------------------------------------------------------
diff --git a/cli/tests/CommandExecutorTestRunner.cpp b/cli/tests/CommandExecutorTestRunner.cpp
index 794f7e1..bd7082f 100644
--- a/cli/tests/CommandExecutorTestRunner.cpp
+++ b/cli/tests/CommandExecutorTestRunner.cpp
@@ -27,7 +27,7 @@
#include "cli/PrintToScreen.hpp"
#include "parser/ParseStatement.hpp"
#include "query_execution/AdmitRequestMessage.hpp"
-#include "query_execution/Foreman.hpp"
+#include "query_execution/ForemanSingleNode.hpp"
#include "query_execution/QueryExecutionTypedefs.hpp"
#include "query_execution/Worker.hpp"
#include "query_optimizer/ExecutionGenerator.hpp"
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4fb884c1/cli/tests/CommandExecutorTestRunner.hpp
----------------------------------------------------------------------
diff --git a/cli/tests/CommandExecutorTestRunner.hpp b/cli/tests/CommandExecutorTestRunner.hpp
index 8fb5b65..69692ae 100644
--- a/cli/tests/CommandExecutorTestRunner.hpp
+++ b/cli/tests/CommandExecutorTestRunner.hpp
@@ -25,7 +25,7 @@
#include <vector>
#include "parser/SqlParserWrapper.hpp"
-#include "query_execution/Foreman.hpp"
+#include "query_execution/ForemanSingleNode.hpp"
#include "query_execution/QueryExecutionTypedefs.hpp"
#include "query_execution/QueryExecutionUtil.hpp"
#include "query_execution/Worker.hpp"
@@ -77,11 +77,12 @@ class CommandExecutorTestRunner : public TextBasedTestRunner {
workers_.reset(new WorkerDirectory(1 /* number of workers */,
worker_client_ids, numa_nodes));
- foreman_.reset(new Foreman(main_thread_client_id_,
- workers_.get(),
- &bus_,
- test_database_loader_.catalog_database(),
- test_database_loader_.storage_manager()));
+ foreman_.reset(
+ new ForemanSingleNode(main_thread_client_id_,
+ workers_.get(),
+ &bus_,
+ test_database_loader_.catalog_database(),
+ test_database_loader_.storage_manager()));
foreman_->start();
worker_->start();
@@ -104,7 +105,7 @@ class CommandExecutorTestRunner : public TextBasedTestRunner {
tmb::client_id main_thread_client_id_;
MessageBusImpl bus_;
- std::unique_ptr<Foreman> foreman_;
+ std::unique_ptr<ForemanSingleNode> foreman_;
std::unique_ptr<Worker> worker_;
std::unique_ptr<WorkerDirectory> workers_;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4fb884c1/query_execution/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_execution/CMakeLists.txt b/query_execution/CMakeLists.txt
index b031a44..2be451c 100644
--- a/query_execution/CMakeLists.txt
+++ b/query_execution/CMakeLists.txt
@@ -1,6 +1,6 @@
# Copyright 2011-2015 Quickstep Technologies LLC.
# Copyright 2015-2016 Pivotal Software, Inc.
-# Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+# Copyright 2016, Quickstep Research Group, Computer Sciences Department,
# University of Wisconsin\u2014Madison.
#
# Licensed under the Apache License, Version 2.0 (the "License");
@@ -32,8 +32,8 @@ if (ENABLE_DISTRIBUTED)
add_library(quickstep_queryexecution_BlockLocator BlockLocator.cpp BlockLocator.hpp)
endif()
add_library(quickstep_queryexecution_AdmitRequestMessage ../empty_src.cpp AdmitRequestMessage.hpp)
-add_library(quickstep_queryexecution_Foreman Foreman.cpp Foreman.hpp)
-add_library(quickstep_queryexecution_ForemanLite ../empty_src.cpp ForemanLite.hpp)
+add_library(quickstep_queryexecution_ForemanBase ../empty_src.cpp ForemanBase.hpp)
+add_library(quickstep_queryexecution_ForemanSingleNode ForemanSingleNode.cpp ForemanSingleNode.hpp)
add_library(quickstep_queryexecution_PolicyEnforcer PolicyEnforcer.cpp PolicyEnforcer.hpp)
add_library(quickstep_queryexecution_QueryContext QueryContext.cpp QueryContext.hpp)
add_library(quickstep_queryexecution_QueryContext_proto
@@ -69,11 +69,15 @@ if (ENABLE_DISTRIBUTED)
quickstep_utility_Macros
tmb)
endif()
-target_link_libraries(quickstep_queryexecution_Foreman
- ${GFLAGS_LIB_NAME}
+target_link_libraries(quickstep_queryexecution_ForemanBase
+ glog
+ quickstep_threading_Thread
+ quickstep_utility_Macros
+ tmb)
+target_link_libraries(quickstep_queryexecution_ForemanSingleNode
glog
quickstep_queryexecution_AdmitRequestMessage
- quickstep_queryexecution_ForemanLite
+ quickstep_queryexecution_ForemanBase
quickstep_queryexecution_PolicyEnforcer
quickstep_queryexecution_QueryExecutionTypedefs
quickstep_queryexecution_QueryExecutionUtil
@@ -82,12 +86,8 @@ target_link_libraries(quickstep_queryexecution_Foreman
quickstep_threading_ThreadUtil
quickstep_utility_EqualsAnyConstant
quickstep_utility_Macros
- tmb)
-target_link_libraries(quickstep_queryexecution_ForemanLite
- glog
- quickstep_threading_Thread
- quickstep_utility_Macros
- tmb)
+ tmb
+ ${GFLAGS_LIB_NAME})
target_link_libraries(quickstep_queryexecution_PolicyEnforcer
${GFLAGS_LIB_NAME}
glog
@@ -199,8 +199,8 @@ target_link_libraries(quickstep_queryexecution_WorkerSelectionPolicy
add_library(quickstep_queryexecution ../empty_src.cpp QueryExecutionModule.hpp)
target_link_libraries(quickstep_queryexecution
quickstep_queryexecution_AdmitRequestMessage
- quickstep_queryexecution_Foreman
- quickstep_queryexecution_ForemanLite
+ quickstep_queryexecution_ForemanBase
+ quickstep_queryexecution_ForemanSingleNode
quickstep_queryexecution_PolicyEnforcer
quickstep_queryexecution_QueryContext
quickstep_queryexecution_QueryContext_proto
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4fb884c1/query_execution/Foreman.cpp
----------------------------------------------------------------------
diff --git a/query_execution/Foreman.cpp b/query_execution/Foreman.cpp
deleted file mode 100644
index 98146e2..0000000
--- a/query_execution/Foreman.cpp
+++ /dev/null
@@ -1,255 +0,0 @@
-/**
- * Copyright 2011-2015 Quickstep Technologies LLC.
- * Copyright 2015-2016 Pivotal Software, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- **/
-
-#include "query_execution/Foreman.hpp"
-
-#include <cstddef>
-#include <cstdio>
-#include <memory>
-#include <tuple>
-#include <utility>
-#include <vector>
-
-#include "query_execution/AdmitRequestMessage.hpp"
-#include "query_execution/QueryExecutionTypedefs.hpp"
-#include "query_execution/QueryExecutionUtil.hpp"
-#include "query_execution/WorkerDirectory.hpp"
-#include "query_execution/WorkerMessage.hpp"
-#include "threading/ThreadUtil.hpp"
-#include "utility/EqualsAnyConstant.hpp"
-#include "utility/Macros.hpp"
-
-#include "gflags/gflags.h"
-#include "glog/logging.h"
-
-#include "tmb/message_bus.h"
-#include "tmb/tagged_message.h"
-
-using std::move;
-using std::size_t;
-using std::unique_ptr;
-using std::vector;
-
-namespace quickstep {
-
-DEFINE_uint64(min_load_per_worker, 2, "The minimum load defined as the number "
- "of pending work orders for the worker. This information is used "
- "by the Foreman to assign work orders to worker threads");
-
-Foreman::Foreman(const tmb::client_id main_thread_client_id,
- WorkerDirectory *worker_directory,
- tmb::MessageBus *bus,
- CatalogDatabaseLite *catalog_database,
- StorageManager *storage_manager,
- const int cpu_id,
- const size_t num_numa_nodes,
- const bool profile_individual_workorders)
- : ForemanLite(bus, cpu_id),
- main_thread_client_id_(main_thread_client_id),
- worker_directory_(DCHECK_NOTNULL(worker_directory)),
- catalog_database_(DCHECK_NOTNULL(catalog_database)),
- storage_manager_(DCHECK_NOTNULL(storage_manager)) {
- const std::vector<QueryExecutionMessageType> sender_message_types{
- kPoisonMessage,
- kRebuildWorkOrderMessage,
- kWorkOrderMessage,
- kWorkloadCompletionMessage};
-
- for (const auto message_type : sender_message_types) {
- bus_->RegisterClientAsSender(foreman_client_id_, message_type);
- }
-
- const std::vector<QueryExecutionMessageType> receiver_message_types{
- kAdmitRequestMessage,
- kCatalogRelationNewBlockMessage,
- kDataPipelineMessage,
- kPoisonMessage,
- kRebuildWorkOrderCompleteMessage,
- kWorkOrderFeedbackMessage,
- kWorkOrdersAvailableMessage,
- kWorkOrderCompleteMessage};
-
- for (const auto message_type : receiver_message_types) {
- bus_->RegisterClientAsReceiver(foreman_client_id_, message_type);
- }
-
- policy_enforcer_.reset(new PolicyEnforcer(
- foreman_client_id_,
- num_numa_nodes,
- catalog_database_,
- storage_manager_,
- worker_directory_,
- bus_,
- profile_individual_workorders));
-}
-
-void Foreman::run() {
- if (cpu_id_ >= 0) {
- // We can pin the foreman thread to a CPU if specified.
- ThreadUtil::BindToCPU(cpu_id_);
- }
-
- // Event loop
- for (;;) {
- // Receive() causes this thread to sleep until next message is received.
- const AnnotatedMessage annotated_msg =
- bus_->Receive(foreman_client_id_, 0, true);
- const TaggedMessage &tagged_message = annotated_msg.tagged_message;
- const tmb::message_type_id message_type = tagged_message.message_type();
- switch (message_type) {
- case kCatalogRelationNewBlockMessage: // Fall through
- case kDataPipelineMessage:
- case kRebuildWorkOrderCompleteMessage:
- case kWorkOrderCompleteMessage:
- case kWorkOrderFeedbackMessage:
- case kWorkOrdersAvailableMessage: {
- policy_enforcer_->processMessage(tagged_message);
- break;
- }
-
- case kAdmitRequestMessage: {
- const AdmitRequestMessage *msg =
- static_cast<const AdmitRequestMessage *>(tagged_message.message());
- const vector<QueryHandle *> &query_handles = msg->getQueryHandles();
-
- DCHECK(!query_handles.empty());
- bool all_queries_admitted = true;
- if (query_handles.size() == 1u) {
- all_queries_admitted =
- policy_enforcer_->admitQuery(query_handles.front());
- } else {
- all_queries_admitted = policy_enforcer_->admitQueries(query_handles);
- }
- if (!all_queries_admitted) {
- LOG(WARNING) << "The scheduler could not admit all the queries";
- // TODO(harshad) - Inform the main thread about the failure.
- }
- break;
- }
- case kPoisonMessage: {
- if (policy_enforcer_->hasQueries()) {
- LOG(WARNING) << "Foreman thread exiting while some queries are "
- "under execution or waiting to be admitted";
- }
- return;
- }
- default:
- LOG(FATAL) << "Unknown message type to Foreman";
- }
-
- if (canCollectNewMessages(message_type)) {
- vector<unique_ptr<WorkerMessage>> new_messages;
- policy_enforcer_->getWorkerMessages(&new_messages);
- dispatchWorkerMessages(new_messages);
- }
-
- // We check again, as some queries may produce zero work orders and finish
- // their execution.
- if (!policy_enforcer_->hasQueries()) {
- // Signal the main thread that there are no queries to be executed.
- // Currently the message doesn't have any real content.
- const int dummy_payload = 0;
- TaggedMessage completion_tagged_message(
- &dummy_payload, sizeof(dummy_payload), kWorkloadCompletionMessage);
- const tmb::MessageBus::SendStatus send_status =
- QueryExecutionUtil::SendTMBMessage(
- bus_,
- foreman_client_id_,
- main_thread_client_id_,
- move(completion_tagged_message));
- CHECK(send_status == tmb::MessageBus::SendStatus::kOK)
- << "Message could not be sent from Foreman with TMB client ID "
- << foreman_client_id_ << " to main thread with TMB client ID"
- << main_thread_client_id_;
- }
- }
-}
-
-bool Foreman::canCollectNewMessages(const tmb::message_type_id message_type) {
- if (QUICKSTEP_EQUALS_ANY_CONSTANT(message_type,
- kCatalogRelationNewBlockMessage,
- kWorkOrderFeedbackMessage)) {
- return false;
- } else if (worker_directory_->getLeastLoadedWorker().second <=
- FLAGS_min_load_per_worker) {
- // If the least loaded worker has only one pending work order, we should
- // collect new messages and dispatch them.
- return true;
- } else {
- return false;
- }
-}
-
-void Foreman::dispatchWorkerMessages(const vector<unique_ptr<WorkerMessage>> &messages) {
- for (const auto &message : messages) {
- DCHECK(message != nullptr);
- const int recipient_worker_thread_index = message->getRecipientHint();
- if (recipient_worker_thread_index != WorkerMessage::kInvalidRecipientIndexHint) {
- sendWorkerMessage(static_cast<size_t>(recipient_worker_thread_index),
- *message);
- worker_directory_->incrementNumQueuedWorkOrders(recipient_worker_thread_index);
- } else {
- const size_t least_loaded_worker_thread_index = worker_directory_->getLeastLoadedWorker().first;
- sendWorkerMessage(least_loaded_worker_thread_index, *message);
- worker_directory_->incrementNumQueuedWorkOrders(least_loaded_worker_thread_index);
- }
- }
-}
-
-void Foreman::sendWorkerMessage(const size_t worker_thread_index,
- const WorkerMessage &message) {
- tmb::message_type_id type;
- if (message.getType() == WorkerMessage::WorkerMessageType::kRebuildWorkOrder) {
- type = kRebuildWorkOrderMessage;
- } else if (message.getType() == WorkerMessage::WorkerMessageType::kWorkOrder) {
- type = kWorkOrderMessage;
- } else {
- FATAL_ERROR("Invalid WorkerMessageType");
- }
- TaggedMessage worker_tagged_message(&message, sizeof(message), type);
-
- const tmb::MessageBus::SendStatus send_status =
- QueryExecutionUtil::SendTMBMessage(bus_,
- foreman_client_id_,
- worker_directory_->getClientID(worker_thread_index),
- move(worker_tagged_message));
- CHECK(send_status == tmb::MessageBus::SendStatus::kOK) <<
- "Message could not be sent from Foreman with TMB client ID "
- << foreman_client_id_ << " to Foreman with TMB client ID "
- << worker_directory_->getClientID(worker_thread_index);
-}
-
-void Foreman::printWorkOrderProfilingResults(const std::size_t query_id,
- std::FILE *out) const {
- const std::vector<
- std::tuple<std::size_t, std::size_t, std::size_t>>
- &recorded_times = policy_enforcer_->getProfilingResults(query_id);
- fputs("Query ID,Worker ID,NUMA Socket,Operator ID,Time (microseconds)\n", out);
- for (auto workorder_entry : recorded_times) {
- // Note: Index of the "worker thread index" in the tuple is 0.
- const std::size_t worker_id = std::get<0>(workorder_entry);
- fprintf(out,
- "%lu,%lu,%d,%lu,%lu\n",
- query_id,
- worker_id,
- worker_directory_->getNUMANode(worker_id),
- std::get<1>(workorder_entry), // Operator ID.
- std::get<2>(workorder_entry)); // Time.
- }
-}
-
-} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4fb884c1/query_execution/Foreman.hpp
----------------------------------------------------------------------
diff --git a/query_execution/Foreman.hpp b/query_execution/Foreman.hpp
deleted file mode 100644
index 7be57e7..0000000
--- a/query_execution/Foreman.hpp
+++ /dev/null
@@ -1,140 +0,0 @@
-/**
- * Copyright 2011-2015 Quickstep Technologies LLC.
- * Copyright 2015-2016 Pivotal Software, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- **/
-
-#ifndef QUICKSTEP_QUERY_EXECUTION_FOREMAN_HPP_
-#define QUICKSTEP_QUERY_EXECUTION_FOREMAN_HPP_
-
-#include <cstddef>
-#include <cstdio>
-#include <memory>
-#include <vector>
-
-#include "query_execution/ForemanLite.hpp"
-#include "query_execution/PolicyEnforcer.hpp"
-#include "utility/Macros.hpp"
-
-#include "tmb/id_typedefs.h"
-#include "tmb/message_bus.h"
-
-namespace quickstep {
-
-class CatalogDatabaseLite;
-class StorageManager;
-class WorkerDirectory;
-class WorkerMessage;
-
-/** \addtogroup QueryExecution
- * @{
- */
-
-/**
- * @brief The Foreman receives queries from the main thread, messages from the
- * policy enforcer and dispatches the work to worker threads. It also
- * receives work completion messages from workers.
- **/
-class Foreman final : public ForemanLite {
- public:
- /**
- * @brief Constructor.
- *
- * @param main_thread_client_id The TMB client ID of the main thread.
- * @param worker_directory The worker directory.
- * @param bus A pointer to the TMB.
- * @param catalog_database The catalog database where this query is executed.
- * @param storage_manager The StorageManager to use.
- * @param cpu_id The ID of the CPU to which the Foreman thread can be pinned.
- * @param num_numa_nodes The number of NUMA nodes in the system.
- * @param profile_individual_workorders Whether every workorder's execution
- * be profiled or not.
- *
- * @note If cpu_id is not specified, Foreman thread can be possibly moved
- * around on different CPUs by the OS.
- **/
- Foreman(const tmb::client_id main_thread_client_id,
- WorkerDirectory *worker_directory,
- tmb::MessageBus *bus,
- CatalogDatabaseLite *catalog_database,
- StorageManager *storage_manager,
- const int cpu_id = -1,
- const std::size_t num_numa_nodes = 1,
- const bool profile_individual_workorders = false);
-
- ~Foreman() override {}
-
- /**
- * @brief Print the results of profiling individual work orders for a given
- * query.
- *
- * TODO(harshad) - Add the name of the operator to the output.
- * TODO(harshad) - Add the CPU core ID of the operator to the output. This
- * will require modifying the WorkerDirectory to remember worker affinities.
- * Until then, the users can refer to the worker_affinities provided to the
- * cli to infer the CPU core ID where a given worker is pinned.
- *
- * @param query_id The ID of the query for which the results are to be printed.
- * @param out The file stream.
- **/
- void printWorkOrderProfilingResults(const std::size_t query_id,
- std::FILE *out) const;
-
- protected:
- void run() override;
-
- private:
- /**
- * @brief Dispatch schedulable WorkOrders, wrapped in WorkerMessages to the
- * worker threads.
- *
- * @param messages The messages to be dispatched.
- **/
- void dispatchWorkerMessages(
- const std::vector<std::unique_ptr<WorkerMessage>> &messages);
-
- /**
- * @brief Send the given message to the specified worker.
- *
- * @param worker_thread_index The logical index of the recipient worker thread
- * in WorkerDirectory.
- * @param message The WorkerMessage to be sent.
- **/
- void sendWorkerMessage(const std::size_t worker_thread_index,
- const WorkerMessage &message);
-
- /**
- * @brief Check if we can collect new messages from the PolicyEnforcer.
- *
- * @param message_type The type of the last received message.
- **/
- bool canCollectNewMessages(const tmb::message_type_id message_type);
-
- const tmb::client_id main_thread_client_id_;
-
- WorkerDirectory *worker_directory_;
-
- CatalogDatabaseLite *catalog_database_;
- StorageManager *storage_manager_;
-
- std::unique_ptr<PolicyEnforcer> policy_enforcer_;
-
- DISALLOW_COPY_AND_ASSIGN(Foreman);
-};
-
-/** @} */
-
-} // namespace quickstep
-
-#endif // QUICKSTEP_QUERY_EXECUTION_FOREMAN_HPP_
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4fb884c1/query_execution/ForemanBase.hpp
----------------------------------------------------------------------
diff --git a/query_execution/ForemanBase.hpp b/query_execution/ForemanBase.hpp
new file mode 100644
index 0000000..274b8fc
--- /dev/null
+++ b/query_execution/ForemanBase.hpp
@@ -0,0 +1,85 @@
+/**
+ * Copyright 2016 Pivotal Software, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#ifndef QUICKSTEP_QUERY_EXECUTION_FOREMAN_BASE_HPP_
+#define QUICKSTEP_QUERY_EXECUTION_FOREMAN_BASE_HPP_
+
+#include "threading/Thread.hpp"
+#include "utility/Macros.hpp"
+
+#include "glog/logging.h"
+
+#include "tmb/id_typedefs.h"
+#include "tmb/message_bus.h"
+
+namespace quickstep {
+
+/** \addtogroup QueryExecution
+ * @{
+ */
+
+/**
+ * @brief A base class that Foreman implements. This class is used to derive
+ * for implementations for both the single-node and distributed versions.
+ **/
+class ForemanBase : public Thread {
+ public:
+ /**
+ * @brief Constructor.
+ *
+ * @param bus A pointer to the TMB.
+ * @param cpu_id The ID of the CPU to which the Foreman thread can be pinned.
+ *
+ * @note If cpu_id is not specified, Foreman thread can be possibly moved
+ * around on different CPUs by the OS.
+ **/
+ ForemanBase(tmb::MessageBus *bus,
+ const int cpu_id)
+ : bus_(DCHECK_NOTNULL(bus)),
+ cpu_id_(cpu_id) {
+ foreman_client_id_ = bus_->Connect();
+ }
+
+ ~ForemanBase() override {}
+
+ /**
+ * @brief Get the TMB client ID of Foreman thread.
+ *
+ * @return TMB client ID of foreman thread.
+ **/
+ tmb::client_id getBusClientID() const {
+ return foreman_client_id_;
+ }
+
+ protected:
+ void run() override = 0;
+
+ tmb::MessageBus *bus_;
+
+ tmb::client_id foreman_client_id_;
+
+ // The ID of the CPU that the Foreman thread can optionally be pinned to.
+ const int cpu_id_;
+
+ private:
+ DISALLOW_COPY_AND_ASSIGN(ForemanBase);
+};
+
+/** @} */
+
+} // namespace quickstep
+
+#endif // QUICKSTEP_QUERY_EXECUTION_FOREMAN_BASE_HPP_
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4fb884c1/query_execution/ForemanLite.hpp
----------------------------------------------------------------------
diff --git a/query_execution/ForemanLite.hpp b/query_execution/ForemanLite.hpp
deleted file mode 100644
index cb6cdf3..0000000
--- a/query_execution/ForemanLite.hpp
+++ /dev/null
@@ -1,85 +0,0 @@
-/**
- * Copyright 2016 Pivotal Software, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- **/
-
-#ifndef QUICKSTEP_QUERY_EXECUTION_FOREMAN_LITE_HPP_
-#define QUICKSTEP_QUERY_EXECUTION_FOREMAN_LITE_HPP_
-
-#include "threading/Thread.hpp"
-#include "utility/Macros.hpp"
-
-#include "glog/logging.h"
-
-#include "tmb/id_typedefs.h"
-#include "tmb/message_bus.h"
-
-namespace quickstep {
-
-/** \addtogroup QueryExecution
- * @{
- */
-
-/**
- * @brief A base class that Foreman implements. This class is used to derive
- * for implementations for both the single-node and distributed versions.
- **/
-class ForemanLite : public Thread {
- public:
- /**
- * @brief Constructor.
- *
- * @param bus A pointer to the TMB.
- * @param cpu_id The ID of the CPU to which the Foreman thread can be pinned.
- *
- * @note If cpu_id is not specified, Foreman thread can be possibly moved
- * around on different CPUs by the OS.
- **/
- ForemanLite(tmb::MessageBus *bus,
- const int cpu_id)
- : bus_(DCHECK_NOTNULL(bus)),
- cpu_id_(cpu_id) {
- foreman_client_id_ = bus_->Connect();
- }
-
- ~ForemanLite() override {}
-
- /**
- * @brief Get the TMB client ID of Foreman thread.
- *
- * @return TMB client ID of foreman thread.
- **/
- tmb::client_id getBusClientID() const {
- return foreman_client_id_;
- }
-
- protected:
- void run() override = 0;
-
- tmb::MessageBus *bus_;
-
- tmb::client_id foreman_client_id_;
-
- // The ID of the CPU that the Foreman thread can optionally be pinned to.
- const int cpu_id_;
-
- private:
- DISALLOW_COPY_AND_ASSIGN(ForemanLite);
-};
-
-/** @} */
-
-} // namespace quickstep
-
-#endif // QUICKSTEP_QUERY_EXECUTION_FOREMAN_LITE_HPP_
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4fb884c1/query_execution/ForemanSingleNode.cpp
----------------------------------------------------------------------
diff --git a/query_execution/ForemanSingleNode.cpp b/query_execution/ForemanSingleNode.cpp
new file mode 100644
index 0000000..3aa1f0b
--- /dev/null
+++ b/query_execution/ForemanSingleNode.cpp
@@ -0,0 +1,256 @@
+/**
+ * Copyright 2011-2015 Quickstep Technologies LLC.
+ * Copyright 2015-2016 Pivotal Software, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#include "query_execution/ForemanSingleNode.hpp"
+
+#include <cstddef>
+#include <cstdio>
+#include <memory>
+#include <tuple>
+#include <utility>
+#include <vector>
+
+#include "query_execution/AdmitRequestMessage.hpp"
+#include "query_execution/QueryExecutionTypedefs.hpp"
+#include "query_execution/QueryExecutionUtil.hpp"
+#include "query_execution/WorkerDirectory.hpp"
+#include "query_execution/WorkerMessage.hpp"
+#include "threading/ThreadUtil.hpp"
+#include "utility/EqualsAnyConstant.hpp"
+#include "utility/Macros.hpp"
+
+#include "gflags/gflags.h"
+#include "glog/logging.h"
+
+#include "tmb/message_bus.h"
+#include "tmb/tagged_message.h"
+
+using std::move;
+using std::size_t;
+using std::unique_ptr;
+using std::vector;
+
+namespace quickstep {
+
+DEFINE_uint64(min_load_per_worker, 2, "The minimum load defined as the number "
+ "of pending work orders for the worker. This information is used "
+ "by the Foreman to assign work orders to worker threads");
+
+ForemanSingleNode::ForemanSingleNode(
+ const tmb::client_id main_thread_client_id,
+ WorkerDirectory *worker_directory,
+ tmb::MessageBus *bus,
+ CatalogDatabaseLite *catalog_database,
+ StorageManager *storage_manager,
+ const int cpu_id,
+ const size_t num_numa_nodes,
+ const bool profile_individual_workorders)
+ : ForemanBase(bus, cpu_id),
+ main_thread_client_id_(main_thread_client_id),
+ worker_directory_(DCHECK_NOTNULL(worker_directory)),
+ catalog_database_(DCHECK_NOTNULL(catalog_database)),
+ storage_manager_(DCHECK_NOTNULL(storage_manager)) {
+ const std::vector<QueryExecutionMessageType> sender_message_types{
+ kPoisonMessage,
+ kRebuildWorkOrderMessage,
+ kWorkOrderMessage,
+ kWorkloadCompletionMessage};
+
+ for (const auto message_type : sender_message_types) {
+ bus_->RegisterClientAsSender(foreman_client_id_, message_type);
+ }
+
+ const std::vector<QueryExecutionMessageType> receiver_message_types{
+ kAdmitRequestMessage,
+ kCatalogRelationNewBlockMessage,
+ kDataPipelineMessage,
+ kPoisonMessage,
+ kRebuildWorkOrderCompleteMessage,
+ kWorkOrderFeedbackMessage,
+ kWorkOrdersAvailableMessage,
+ kWorkOrderCompleteMessage};
+
+ for (const auto message_type : receiver_message_types) {
+ bus_->RegisterClientAsReceiver(foreman_client_id_, message_type);
+ }
+
+ policy_enforcer_.reset(new PolicyEnforcer(
+ foreman_client_id_,
+ num_numa_nodes,
+ catalog_database_,
+ storage_manager_,
+ worker_directory_,
+ bus_,
+ profile_individual_workorders));
+}
+
+void ForemanSingleNode::run() {
+ if (cpu_id_ >= 0) {
+ // We can pin the foreman thread to a CPU if specified.
+ ThreadUtil::BindToCPU(cpu_id_);
+ }
+
+ // Event loop
+ for (;;) {
+ // Receive() causes this thread to sleep until next message is received.
+ const AnnotatedMessage annotated_msg =
+ bus_->Receive(foreman_client_id_, 0, true);
+ const TaggedMessage &tagged_message = annotated_msg.tagged_message;
+ const tmb::message_type_id message_type = tagged_message.message_type();
+ switch (message_type) {
+ case kCatalogRelationNewBlockMessage: // Fall through
+ case kDataPipelineMessage:
+ case kRebuildWorkOrderCompleteMessage:
+ case kWorkOrderCompleteMessage:
+ case kWorkOrderFeedbackMessage:
+ case kWorkOrdersAvailableMessage: {
+ policy_enforcer_->processMessage(tagged_message);
+ break;
+ }
+
+ case kAdmitRequestMessage: {
+ const AdmitRequestMessage *msg =
+ static_cast<const AdmitRequestMessage *>(tagged_message.message());
+ const vector<QueryHandle *> &query_handles = msg->getQueryHandles();
+
+ DCHECK(!query_handles.empty());
+ bool all_queries_admitted = true;
+ if (query_handles.size() == 1u) {
+ all_queries_admitted =
+ policy_enforcer_->admitQuery(query_handles.front());
+ } else {
+ all_queries_admitted = policy_enforcer_->admitQueries(query_handles);
+ }
+ if (!all_queries_admitted) {
+ LOG(WARNING) << "The scheduler could not admit all the queries";
+ // TODO(harshad) - Inform the main thread about the failure.
+ }
+ break;
+ }
+ case kPoisonMessage: {
+ if (policy_enforcer_->hasQueries()) {
+ LOG(WARNING) << "Foreman thread exiting while some queries are "
+ "under execution or waiting to be admitted";
+ }
+ return;
+ }
+ default:
+ LOG(FATAL) << "Unknown message type to Foreman";
+ }
+
+ if (canCollectNewMessages(message_type)) {
+ vector<unique_ptr<WorkerMessage>> new_messages;
+ policy_enforcer_->getWorkerMessages(&new_messages);
+ dispatchWorkerMessages(new_messages);
+ }
+
+ // We check again, as some queries may produce zero work orders and finish
+ // their execution.
+ if (!policy_enforcer_->hasQueries()) {
+ // Signal the main thread that there are no queries to be executed.
+ // Currently the message doesn't have any real content.
+ const int dummy_payload = 0;
+ TaggedMessage completion_tagged_message(
+ &dummy_payload, sizeof(dummy_payload), kWorkloadCompletionMessage);
+ const tmb::MessageBus::SendStatus send_status =
+ QueryExecutionUtil::SendTMBMessage(
+ bus_,
+ foreman_client_id_,
+ main_thread_client_id_,
+ move(completion_tagged_message));
+ CHECK(send_status == tmb::MessageBus::SendStatus::kOK)
+ << "Message could not be sent from Foreman with TMB client ID "
+ << foreman_client_id_ << " to main thread with TMB client ID"
+ << main_thread_client_id_;
+ }
+ }
+}
+
+bool ForemanSingleNode::canCollectNewMessages(const tmb::message_type_id message_type) {
+ if (QUICKSTEP_EQUALS_ANY_CONSTANT(message_type,
+ kCatalogRelationNewBlockMessage,
+ kWorkOrderFeedbackMessage)) {
+ return false;
+ } else if (worker_directory_->getLeastLoadedWorker().second <=
+ FLAGS_min_load_per_worker) {
+ // If the least loaded worker has only one pending work order, we should
+ // collect new messages and dispatch them.
+ return true;
+ } else {
+ return false;
+ }
+}
+
+void ForemanSingleNode::dispatchWorkerMessages(const vector<unique_ptr<WorkerMessage>> &messages) {
+ for (const auto &message : messages) {
+ DCHECK(message != nullptr);
+ const int recipient_worker_thread_index = message->getRecipientHint();
+ if (recipient_worker_thread_index != WorkerMessage::kInvalidRecipientIndexHint) {
+ sendWorkerMessage(static_cast<size_t>(recipient_worker_thread_index),
+ *message);
+ worker_directory_->incrementNumQueuedWorkOrders(recipient_worker_thread_index);
+ } else {
+ const size_t least_loaded_worker_thread_index = worker_directory_->getLeastLoadedWorker().first;
+ sendWorkerMessage(least_loaded_worker_thread_index, *message);
+ worker_directory_->incrementNumQueuedWorkOrders(least_loaded_worker_thread_index);
+ }
+ }
+}
+
+void ForemanSingleNode::sendWorkerMessage(const size_t worker_thread_index,
+ const WorkerMessage &message) {
+ tmb::message_type_id type;
+ if (message.getType() == WorkerMessage::WorkerMessageType::kRebuildWorkOrder) {
+ type = kRebuildWorkOrderMessage;
+ } else if (message.getType() == WorkerMessage::WorkerMessageType::kWorkOrder) {
+ type = kWorkOrderMessage;
+ } else {
+ FATAL_ERROR("Invalid WorkerMessageType");
+ }
+ TaggedMessage worker_tagged_message(&message, sizeof(message), type);
+
+ const tmb::MessageBus::SendStatus send_status =
+ QueryExecutionUtil::SendTMBMessage(bus_,
+ foreman_client_id_,
+ worker_directory_->getClientID(worker_thread_index),
+ move(worker_tagged_message));
+ CHECK(send_status == tmb::MessageBus::SendStatus::kOK) <<
+ "Message could not be sent from Foreman with TMB client ID "
+ << foreman_client_id_ << " to Foreman with TMB client ID "
+ << worker_directory_->getClientID(worker_thread_index);
+}
+
+void ForemanSingleNode::printWorkOrderProfilingResults(const std::size_t query_id,
+ std::FILE *out) const {
+ const std::vector<
+ std::tuple<std::size_t, std::size_t, std::size_t>>
+ &recorded_times = policy_enforcer_->getProfilingResults(query_id);
+ fputs("Query ID,Worker ID,NUMA Socket,Operator ID,Time (microseconds)\n", out);
+ for (auto workorder_entry : recorded_times) {
+ // Note: Index of the "worker thread index" in the tuple is 0.
+ const std::size_t worker_id = std::get<0>(workorder_entry);
+ fprintf(out,
+ "%lu,%lu,%d,%lu,%lu\n",
+ query_id,
+ worker_id,
+ worker_directory_->getNUMANode(worker_id),
+ std::get<1>(workorder_entry), // Operator ID.
+ std::get<2>(workorder_entry)); // Time.
+ }
+}
+
+} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4fb884c1/query_execution/ForemanSingleNode.hpp
----------------------------------------------------------------------
diff --git a/query_execution/ForemanSingleNode.hpp b/query_execution/ForemanSingleNode.hpp
new file mode 100644
index 0000000..7506d35
--- /dev/null
+++ b/query_execution/ForemanSingleNode.hpp
@@ -0,0 +1,140 @@
+/**
+ * Copyright 2011-2015 Quickstep Technologies LLC.
+ * Copyright 2015-2016 Pivotal Software, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#ifndef QUICKSTEP_QUERY_EXECUTION_FOREMAN_HPP_
+#define QUICKSTEP_QUERY_EXECUTION_FOREMAN_HPP_
+
+#include <cstddef>
+#include <cstdio>
+#include <memory>
+#include <vector>
+
+#include "query_execution/ForemanBase.hpp"
+#include "query_execution/PolicyEnforcer.hpp"
+#include "utility/Macros.hpp"
+
+#include "tmb/id_typedefs.h"
+#include "tmb/message_bus.h"
+
+namespace quickstep {
+
+class CatalogDatabaseLite;
+class StorageManager;
+class WorkerDirectory;
+class WorkerMessage;
+
+/** \addtogroup QueryExecution
+ * @{
+ */
+
+/**
+ * @brief The Foreman receives queries from the main thread, messages from the
+ * policy enforcer and dispatches the work to worker threads. It also
+ * receives work completion messages from workers.
+ **/
+class ForemanSingleNode final : public ForemanBase {
+ public:
+ /**
+ * @brief Constructor.
+ *
+ * @param main_thread_client_id The TMB client ID of the main thread.
+ * @param worker_directory The worker directory.
+ * @param bus A pointer to the TMB.
+ * @param catalog_database The catalog database where this query is executed.
+ * @param storage_manager The StorageManager to use.
+ * @param cpu_id The ID of the CPU to which the Foreman thread can be pinned.
+ * @param num_numa_nodes The number of NUMA nodes in the system.
+ * @param profile_individual_workorders Whether every workorder's execution
+ * be profiled or not.
+ *
+ * @note If cpu_id is not specified, Foreman thread can be possibly moved
+ * around on different CPUs by the OS.
+ **/
+ ForemanSingleNode(const tmb::client_id main_thread_client_id,
+ WorkerDirectory *worker_directory,
+ tmb::MessageBus *bus,
+ CatalogDatabaseLite *catalog_database,
+ StorageManager *storage_manager,
+ const int cpu_id = -1,
+ const std::size_t num_numa_nodes = 1,
+ const bool profile_individual_workorders = false);
+
+ ~ForemanSingleNode() override {}
+
+ /**
+ * @brief Print the results of profiling individual work orders for a given
+ * query.
+ *
+ * TODO(harshad) - Add the name of the operator to the output.
+ * TODO(harshad) - Add the CPU core ID of the operator to the output. This
+ * will require modifying the WorkerDirectory to remember worker affinities.
+ * Until then, the users can refer to the worker_affinities provided to the
+ * cli to infer the CPU core ID where a given worker is pinned.
+ *
+ * @param query_id The ID of the query for which the results are to be printed.
+ * @param out The file stream.
+ **/
+ void printWorkOrderProfilingResults(const std::size_t query_id,
+ std::FILE *out) const;
+
+ protected:
+ void run() override;
+
+ private:
+ /**
+ * @brief Dispatch schedulable WorkOrders, wrapped in WorkerMessages to the
+ * worker threads.
+ *
+ * @param messages The messages to be dispatched.
+ **/
+ void dispatchWorkerMessages(
+ const std::vector<std::unique_ptr<WorkerMessage>> &messages);
+
+ /**
+ * @brief Send the given message to the specified worker.
+ *
+ * @param worker_thread_index The logical index of the recipient worker thread
+ * in WorkerDirectory.
+ * @param message The WorkerMessage to be sent.
+ **/
+ void sendWorkerMessage(const std::size_t worker_thread_index,
+ const WorkerMessage &message);
+
+ /**
+ * @brief Check if we can collect new messages from the PolicyEnforcer.
+ *
+ * @param message_type The type of the last received message.
+ **/
+ bool canCollectNewMessages(const tmb::message_type_id message_type);
+
+ const tmb::client_id main_thread_client_id_;
+
+ WorkerDirectory *worker_directory_;
+
+ CatalogDatabaseLite *catalog_database_;
+ StorageManager *storage_manager_;
+
+ std::unique_ptr<PolicyEnforcer> policy_enforcer_;
+
+ DISALLOW_COPY_AND_ASSIGN(ForemanSingleNode);
+};
+
+/** @} */
+
+} // namespace quickstep
+
+#endif // QUICKSTEP_QUERY_EXECUTION_FOREMAN_HPP_
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4fb884c1/query_optimizer/tests/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_optimizer/tests/CMakeLists.txt b/query_optimizer/tests/CMakeLists.txt
index 5b58f75..9cad47f 100644
--- a/query_optimizer/tests/CMakeLists.txt
+++ b/query_optimizer/tests/CMakeLists.txt
@@ -117,7 +117,7 @@ target_link_libraries(quickstep_queryoptimizer_tests_ExecutionGeneratorTest
quickstep_parser_ParseStatement
quickstep_parser_SqlParserWrapper
quickstep_queryexecution_AdmitRequestMessage
- quickstep_queryexecution_Foreman
+ quickstep_queryexecution_ForemanSingleNode
quickstep_queryexecution_QueryContext
quickstep_queryexecution_QueryExecutionTypedefs
quickstep_queryexecution_QueryExecutionUtil
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4fb884c1/query_optimizer/tests/ExecutionGeneratorTestRunner.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/tests/ExecutionGeneratorTestRunner.cpp b/query_optimizer/tests/ExecutionGeneratorTestRunner.cpp
index 8c1d306..563a777 100644
--- a/query_optimizer/tests/ExecutionGeneratorTestRunner.cpp
+++ b/query_optimizer/tests/ExecutionGeneratorTestRunner.cpp
@@ -25,7 +25,7 @@
#include "cli/PrintToScreen.hpp"
#include "parser/ParseStatement.hpp"
#include "query_execution/AdmitRequestMessage.hpp"
-#include "query_execution/Foreman.hpp"
+#include "query_execution/ForemanSingleNode.hpp"
#include "query_execution/QueryExecutionUtil.hpp"
#include "query_execution/Worker.hpp"
#include "query_optimizer/ExecutionGenerator.hpp"
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4fb884c1/query_optimizer/tests/ExecutionGeneratorTestRunner.hpp
----------------------------------------------------------------------
diff --git a/query_optimizer/tests/ExecutionGeneratorTestRunner.hpp b/query_optimizer/tests/ExecutionGeneratorTestRunner.hpp
index bb2a26f..d1d9380 100644
--- a/query_optimizer/tests/ExecutionGeneratorTestRunner.hpp
+++ b/query_optimizer/tests/ExecutionGeneratorTestRunner.hpp
@@ -25,7 +25,7 @@
#include <vector>
#include "parser/SqlParserWrapper.hpp"
-#include "query_execution/Foreman.hpp"
+#include "query_execution/ForemanSingleNode.hpp"
#include "query_execution/QueryExecutionTypedefs.hpp"
#include "query_execution/Worker.hpp"
#include "query_execution/WorkerDirectory.hpp"
@@ -80,11 +80,12 @@ class ExecutionGeneratorTestRunner : public TextBasedTestRunner {
workers_.reset(new WorkerDirectory(1 /* number of workers */,
worker_client_ids, numa_nodes));
- foreman_.reset(new Foreman(main_thread_client_id_,
- workers_.get(),
- &bus_,
- test_database_loader_.catalog_database(),
- test_database_loader_.storage_manager()));
+ foreman_.reset(
+ new ForemanSingleNode(main_thread_client_id_,
+ workers_.get(),
+ &bus_,
+ test_database_loader_.catalog_database(),
+ test_database_loader_.storage_manager()));
foreman_->start();
worker_->start();
@@ -105,7 +106,7 @@ class ExecutionGeneratorTestRunner : public TextBasedTestRunner {
TestDatabaseLoader test_database_loader_;
MessageBusImpl bus_;
- std::unique_ptr<Foreman> foreman_;
+ std::unique_ptr<ForemanSingleNode> foreman_;
std::unique_ptr<Worker> worker_;
std::unique_ptr<WorkerDirectory> workers_;
[09/25] incubator-quickstep git commit: Add a quick start guide. Move
the old README to a developer README.
Posted by zu...@apache.org.
Add a quick start guide. Move the old README to a developer README.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/ca09ec4e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/ca09ec4e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/ca09ec4e
Branch: refs/heads/dist-exe-test-new
Commit: ca09ec4ea27bd96a233e2fb6fd43547874e7b3f4
Parents: 65af8c7
Author: Jignesh Patel <jm...@hotmail.com>
Authored: Wed Jul 13 16:09:54 2016 -0500
Committer: Zuyu Zhang <zu...@twitter.com>
Committed: Fri Jul 29 16:42:10 2016 -0700
----------------------------------------------------------------------
DEV_README.md | 108 ++++++++++++++++++++++++++++++++
README.md | 176 +++++++++++++++++++++--------------------------------
2 files changed, 176 insertions(+), 108 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/ca09ec4e/DEV_README.md
----------------------------------------------------------------------
diff --git a/DEV_README.md b/DEV_README.md
new file mode 100644
index 0000000..04d5d66
--- /dev/null
+++ b/DEV_README.md
@@ -0,0 +1,108 @@
+# Apache Quickstep (Incubating)
+
+[![Travis Widget]][Travis]
+
+[Travis]: https://travis-ci.org/apache/incubator-quickstep
+[Travis Widget]: https://travis-ci.org/apache/incubator-quickstep.svg?branch=master
+
+Apache Quickstep is an experimental high-performance database engine designed with the
+aim of Data at Bare-Metal Speed. It began life in 2011 as a
+[research project at the University of Wisconsin](https://quickstep.cs.wisc.edu)
+and was acquired by [Pivotal](https://pivotal.io) in 2015.
+Quickstep entered incubation at the
+[Apache Software Foundation](https://www.apache.org) in April, 2016.
+
+## Getting Started (Building)
+
+A [build guide](BUILDING.md) is available which includes instructions for
+building Quickstep for the first time. You may also find it useful to use one
+of the [pre-made Vagrant boxes](build/vagrant) for Quickstep that are already
+set up with all of the development tools needed to build Quickstep.
+
+## Documentation
+
+All publicly-visible classes and functions in the Quickstep code base have
+Doxygen documentation. Simply run `doxygen` in the root of the Quickstep source
+to generate browsable HTML documentation. Of course, the Doxygen comments
+should also be useful when reading header files directly.
+
+In addition to the Doxygen and inline code comments explaining implementation
+details, a high-level overview for each module that comprises Quickstep is
+included in the README files in each subdirectory.
+
+## Architectural Overview
+
+Quickstep is composed of several different modules that handle different
+concerns of a database system. The main modules are:
+
+* [Utility](utility) - Reusable general-purpose code that is used by many
+ other modules.
+* [Threading](threading) - Provides a cross-platform abstraction for threads
+ and synchronization primitives that abstracts the underlying OS threading
+ features.
+* [Types](types) - The core type system used across all of Quickstep. Handles
+ details of how SQL types are stored, parsed, serialized & deserialized, and
+ converted. Also includes basic containers for typed values (tuples and
+ column-vectors) and low-level operations that apply to typed values (e.g.
+ basic arithmetic and comparisons).
+* [Catalog](catalog) - Keeps track of database schema as well as physical
+ storage information for relations (e.g. which physical blocks store a
+ relation's data, and any physical partitioning and placement information).
+* [Storage](storage) - Handles the physical storage of relation data in
+ self-contained, self-describing blocks, both in-memory and on persistent
+ storage (disk or a distributed filesystem). Also includes some heavyweight
+ run-time data structures used in query processing (e.g. hash tables for join
+ and aggregation). Includes a buffer manager component for managing memory
+ use and a file manager component that handles data persistence.
+* [Compression](compression) - A simple implementation of ordered dictionary
+ compression. Several storage formats in the Storage module are capable of
+ storing compressed column data and evaluating some expressions directly on
+ compressed data without decompressing. The common code supporting compression
+ is in this module.
+* [Expressions](expressions) - This module builds on the simple operations
+ provided by the Types module to support arbitrarily complex expressions over
+ data, including scalar expressions, predicates, and aggregate functions with
+ and without grouping.
+* [Relational Operators](relational_operators) - This module provides the
+ building blocks for queries in Quickstep. A query is represented as a
+ directed acyclic graph of relational operators, each of which is responsible
+ for applying some relational-algebraic operation(s) to tranform its input.
+ Operators generate individual self-contained "work orders" that can be
+ executed independently. Most operators are parallelism-friendly and generate
+ one work-order per storage block of input.
+* [Query Execution](query_execution) - Handles the actual scheduling and
+ execution of work from a query at runtime. The central class is the Foreman,
+ an independent thread with a global view of the query plan and progress. The
+ Foreman dispatches work-orders to stateless Worker threads and monitors their
+ progress, and also coordinates streaming of partial results between producers
+ and consumers in a query plan DAG to maximize parallelism. This module also
+ includes the QueryContext class, which holds global shared state for an
+ individual query and is designed to support easy
+ serialization/deserialization for distributed execution.
+* [Parser](parser) - A simple SQL lexer and parser that parses SQL syntax into
+ an abstract syntax tree for consumption by the Query Optimizer.
+* [Query Optimizer](query_optimizer) - Takes the abstract syntax tree generated
+ by the parser and transforms it into a runable query-plan DAG for the Query
+ Execution module. The Query Optimizer is responsible for resolving references
+ to relations and attributes in the query, checking it for semantic
+ correctness, and applying optimizations (e.g. filter pushdown, column
+ pruning, join ordering) as part of the transformation process.
+* [Command-Line Interface](cli) - An interactive SQL shell interface to
+ Quickstep.
+
+## Licensing
+
+Quickstep is licensed under the Apache License, Version 2.0. See [LICENSE](https://github.com/pivotalsoftware/quickstep/blob/master/LICENSE) for the full license text.
+
+## Disclaimer
+Apache Quickstep is an effort undergoing incubation at the Apache Software
+Foundation (ASF), sponsored by the Apache Incubator PMC.
+
+Incubation is required of all newly accepted projects until a further
+review indicates that the infrastructure, communications, and decision
+making process have stabilized in a manner consistent with other
+successful ASF projects.
+
+While incubation status is not necessarily a reflection of the
+completeness or stability of the code, it does indicate that the
+project has yet to be fully endorsed by the ASF.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/ca09ec4e/README.md
----------------------------------------------------------------------
diff --git a/README.md b/README.md
index 04d5d66..bdfb7fc 100644
--- a/README.md
+++ b/README.md
@@ -1,108 +1,68 @@
-# Apache Quickstep (Incubating)
-
-[![Travis Widget]][Travis]
-
-[Travis]: https://travis-ci.org/apache/incubator-quickstep
-[Travis Widget]: https://travis-ci.org/apache/incubator-quickstep.svg?branch=master
-
-Apache Quickstep is an experimental high-performance database engine designed with the
-aim of Data at Bare-Metal Speed. It began life in 2011 as a
-[research project at the University of Wisconsin](https://quickstep.cs.wisc.edu)
-and was acquired by [Pivotal](https://pivotal.io) in 2015.
-Quickstep entered incubation at the
-[Apache Software Foundation](https://www.apache.org) in April, 2016.
-
-## Getting Started (Building)
-
-A [build guide](BUILDING.md) is available which includes instructions for
-building Quickstep for the first time. You may also find it useful to use one
-of the [pre-made Vagrant boxes](build/vagrant) for Quickstep that are already
-set up with all of the development tools needed to build Quickstep.
-
-## Documentation
-
-All publicly-visible classes and functions in the Quickstep code base have
-Doxygen documentation. Simply run `doxygen` in the root of the Quickstep source
-to generate browsable HTML documentation. Of course, the Doxygen comments
-should also be useful when reading header files directly.
-
-In addition to the Doxygen and inline code comments explaining implementation
-details, a high-level overview for each module that comprises Quickstep is
-included in the README files in each subdirectory.
-
-## Architectural Overview
-
-Quickstep is composed of several different modules that handle different
-concerns of a database system. The main modules are:
-
-* [Utility](utility) - Reusable general-purpose code that is used by many
- other modules.
-* [Threading](threading) - Provides a cross-platform abstraction for threads
- and synchronization primitives that abstracts the underlying OS threading
- features.
-* [Types](types) - The core type system used across all of Quickstep. Handles
- details of how SQL types are stored, parsed, serialized & deserialized, and
- converted. Also includes basic containers for typed values (tuples and
- column-vectors) and low-level operations that apply to typed values (e.g.
- basic arithmetic and comparisons).
-* [Catalog](catalog) - Keeps track of database schema as well as physical
- storage information for relations (e.g. which physical blocks store a
- relation's data, and any physical partitioning and placement information).
-* [Storage](storage) - Handles the physical storage of relation data in
- self-contained, self-describing blocks, both in-memory and on persistent
- storage (disk or a distributed filesystem). Also includes some heavyweight
- run-time data structures used in query processing (e.g. hash tables for join
- and aggregation). Includes a buffer manager component for managing memory
- use and a file manager component that handles data persistence.
-* [Compression](compression) - A simple implementation of ordered dictionary
- compression. Several storage formats in the Storage module are capable of
- storing compressed column data and evaluating some expressions directly on
- compressed data without decompressing. The common code supporting compression
- is in this module.
-* [Expressions](expressions) - This module builds on the simple operations
- provided by the Types module to support arbitrarily complex expressions over
- data, including scalar expressions, predicates, and aggregate functions with
- and without grouping.
-* [Relational Operators](relational_operators) - This module provides the
- building blocks for queries in Quickstep. A query is represented as a
- directed acyclic graph of relational operators, each of which is responsible
- for applying some relational-algebraic operation(s) to tranform its input.
- Operators generate individual self-contained "work orders" that can be
- executed independently. Most operators are parallelism-friendly and generate
- one work-order per storage block of input.
-* [Query Execution](query_execution) - Handles the actual scheduling and
- execution of work from a query at runtime. The central class is the Foreman,
- an independent thread with a global view of the query plan and progress. The
- Foreman dispatches work-orders to stateless Worker threads and monitors their
- progress, and also coordinates streaming of partial results between producers
- and consumers in a query plan DAG to maximize parallelism. This module also
- includes the QueryContext class, which holds global shared state for an
- individual query and is designed to support easy
- serialization/deserialization for distributed execution.
-* [Parser](parser) - A simple SQL lexer and parser that parses SQL syntax into
- an abstract syntax tree for consumption by the Query Optimizer.
-* [Query Optimizer](query_optimizer) - Takes the abstract syntax tree generated
- by the parser and transforms it into a runable query-plan DAG for the Query
- Execution module. The Query Optimizer is responsible for resolving references
- to relations and attributes in the query, checking it for semantic
- correctness, and applying optimizations (e.g. filter pushdown, column
- pruning, join ordering) as part of the transformation process.
-* [Command-Line Interface](cli) - An interactive SQL shell interface to
- Quickstep.
-
-## Licensing
-
-Quickstep is licensed under the Apache License, Version 2.0. See [LICENSE](https://github.com/pivotalsoftware/quickstep/blob/master/LICENSE) for the full license text.
-
-## Disclaimer
-Apache Quickstep is an effort undergoing incubation at the Apache Software
-Foundation (ASF), sponsored by the Apache Incubator PMC.
-
-Incubation is required of all newly accepted projects until a further
-review indicates that the infrastructure, communications, and decision
-making process have stabilized in a manner consistent with other
-successful ASF projects.
-
-While incubation status is not necessarily a reflection of the
-completeness or stability of the code, it does indicate that the
-project has yet to be fully endorsed by the ASF.
+# Apache Quickstep (Incubating)
+
+[![Travis Widget]][Travis]
+
+[Travis]: https://travis-ci.org/apache/incubator-quickstep
+[Travis Widget]: https://travis-ci.org/apache/incubator-quickstep.svg?branch=master
+
+## What is Quickstep?
+Apache Quickstep is high-performance database engine designed to exploit the full potential of hardware that is packed in modern computing boxes (servers and laptops). The initial version (available now!) targets single-node in-memory environments. If your data spills overs the memory limit Quickstep will still work, so you don't have to obsessively worry about the in-memory part. Also, if your working set fits in memory then Quickstep will transparently and automatically figure that out, and cache that hot set to deliver in-memory performance.
+
+Distributed execution is the next big feature for Quickstep.
+
+Quickstep began life in 2011 as a
+[research project at the University of Wisconsin](https://www.cs.wisc.edu/~jignesh)
+and entered incubation at the
+[Apache Software Foundation](https://www.apache.org) in April, 2016.
+
+## Why Quickstep?
+Did you know that the hardware that you have in your laptop was spread across a small cluster just a decade ago? (PS: Hopefully you are not using a very old laptop!) If you look at a high-end server box, then that packs compute and storage power that was a full rack about 5 years ago! And, the way hardware technology is going, that box is going to become even more powerful in the future. In fact, it is likely that the computing power in each box is going to grow faster than other hardware components (e.g. networking) in data centers. So, if you care about performance and/or total operating costs, paying attention to single box performance is likely to be super important in the long run.
+
+In other words there is a small data center in an individual compute boxes today! Quickstep aims to allow you to fully exploit the potential of that data center that is hidden in each individual box today. We call this the **scaling-in approach**, and it complements a scaling-out approach. But without scaling-in, you are overpaying (by a lot!) when you run your data service.
+
+## What are the key ingredients?
+
+Modern computing boxes contain a large number of computing cores and large main memory configuration. Quickstep allows you to fully exploit these hardware resources using novel data processing, data storage, and query processing methods that include:
+
+1. A unique **decoupling of data-flow from control-flow** for query execution that allows for unlimited intra and inter-query parallelism. Thus, using all the processing core effectively.
+
+2. A **template meta-programming** framework that provides fast vectorized query execution. Thus, using each processor cycle very efficiently.
+
+3. A **hybrid data storage** architecture that includes columnar and row-store. Yes, this may surprise some of you, but sometimes a row-store beats a column-store!
+
+And, it is **open source!**
+
+## Giving it a spin
+
+1. Checkout the code: ```git clone https://git-wip-us.apache.org/repos/asf/incubator-quickstep.git quickstep```
+2. Then, go to the code directory: ```cd quickstep```
+3. Initialize the dependencies: ```git submodule init```
+4. Checkout the dependencies: ```git submodule update```
+5. Go into the build directory: ```cd build```
+6. Create the Makefile: ```cmake -D CMAKE_BUILD_TYPE=Release ..```
+7. Build: ```make -j4```. Note you may replace the 4 with the number of cores on your machine.
+8. Start quickstep: ```./quickstep_cli_shell --initialize_db=true```. You can now fire SQL queries. To quit, you can type in ```quit;``` Your data is stored in the directory ```qsstor```
+
+
+## Additional pointers
+
+1. For other build options, see the more comprehensive [build guide](BUILDING.md).
+2. To get started as a developer, you should start with the [code organization guide](DEV_README.md).
+
+
+## Licensing
+
+Quickstep is licensed under the Apache License, Version 2.0. See [LICENSE](https://github.com/pivotalsoftware/quickstep/blob/master/LICENSE) for the full license text.
+
+## Disclaimer
+Apache Quickstep is an effort undergoing incubation at the Apache Software
+Foundation (ASF), sponsored by the Apache Incubator PMC.
+
+Incubation is required of all newly accepted projects until a further
+review indicates that the infrastructure, communications, and decision
+making process have stabilized in a manner consistent with other
+successful ASF projects.
+
+While incubation status is not necessarily a reflection of the
+completeness or stability of the code, it does indicate that the
+project has yet to be fully endorsed by the ASF.
[18/25] incubator-quickstep git commit: Introduced PolicyEnforcerBase
and its single-node implementation.
Posted by zu...@apache.org.
Introduced PolicyEnforcerBase and its single-node implementation.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/95c451a5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/95c451a5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/95c451a5
Branch: refs/heads/dist-exe-test-new
Commit: 95c451a5241c5f929a76b7412c38b9ef1f3bb74f
Parents: 3bb5ca8
Author: Zuyu Zhang <zu...@twitter.com>
Authored: Mon Jul 18 12:18:04 2016 -0700
Committer: Zuyu Zhang <zu...@twitter.com>
Committed: Fri Jul 29 16:43:23 2016 -0700
----------------------------------------------------------------------
query_execution/CMakeLists.txt | 23 +-
query_execution/ForemanSingleNode.cpp | 6 +-
query_execution/ForemanSingleNode.hpp | 7 +-
query_execution/PolicyEnforcer.cpp | 254 ----------------------
query_execution/PolicyEnforcer.hpp | 223 -------------------
query_execution/PolicyEnforcerBase.cpp | 177 +++++++++++++++
query_execution/PolicyEnforcerBase.hpp | 196 +++++++++++++++++
query_execution/PolicyEnforcerSingleNode.cpp | 110 ++++++++++
query_execution/PolicyEnforcerSingleNode.hpp | 104 +++++++++
9 files changed, 612 insertions(+), 488 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/95c451a5/query_execution/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_execution/CMakeLists.txt b/query_execution/CMakeLists.txt
index 8c12a5d..f582ba5 100644
--- a/query_execution/CMakeLists.txt
+++ b/query_execution/CMakeLists.txt
@@ -34,7 +34,8 @@ if (ENABLE_DISTRIBUTED)
endif()
add_library(quickstep_queryexecution_ForemanBase ../empty_src.cpp ForemanBase.hpp)
add_library(quickstep_queryexecution_ForemanSingleNode ForemanSingleNode.cpp ForemanSingleNode.hpp)
-add_library(quickstep_queryexecution_PolicyEnforcer PolicyEnforcer.cpp PolicyEnforcer.hpp)
+add_library(quickstep_queryexecution_PolicyEnforcerBase PolicyEnforcerBase.cpp PolicyEnforcerBase.hpp)
+add_library(quickstep_queryexecution_PolicyEnforcerSingleNode PolicyEnforcerSingleNode.cpp PolicyEnforcerSingleNode.hpp)
add_library(quickstep_queryexecution_QueryContext QueryContext.cpp QueryContext.hpp)
add_library(quickstep_queryexecution_QueryContext_proto
${queryexecution_QueryContext_proto_srcs}
@@ -85,7 +86,7 @@ target_link_libraries(quickstep_queryexecution_ForemanSingleNode
glog
quickstep_queryexecution_AdmitRequestMessage
quickstep_queryexecution_ForemanBase
- quickstep_queryexecution_PolicyEnforcer
+ quickstep_queryexecution_PolicyEnforcerSingleNode
quickstep_queryexecution_QueryExecutionTypedefs
quickstep_queryexecution_QueryExecutionUtil
quickstep_queryexecution_WorkerDirectory
@@ -95,22 +96,29 @@ target_link_libraries(quickstep_queryexecution_ForemanSingleNode
quickstep_utility_Macros
tmb
${GFLAGS_LIB_NAME})
-target_link_libraries(quickstep_queryexecution_PolicyEnforcer
+target_link_libraries(quickstep_queryexecution_PolicyEnforcerBase
glog
quickstep_catalog_CatalogDatabase
quickstep_catalog_CatalogRelation
- quickstep_catalog_CatalogTypedefs
quickstep_catalog_PartitionScheme
quickstep_queryexecution_QueryExecutionMessages_proto
quickstep_queryexecution_QueryExecutionState
quickstep_queryexecution_QueryExecutionTypedefs
quickstep_queryexecution_QueryManagerBase
+ quickstep_relationaloperators_WorkOrder
+ quickstep_storage_StorageBlockInfo
+ quickstep_utility_Macros
+ tmb)
+target_link_libraries(quickstep_queryexecution_PolicyEnforcerSingleNode
+ glog
+ quickstep_catalog_CatalogTypedefs
+ quickstep_queryexecution_PolicyEnforcerBase
+ quickstep_queryexecution_QueryExecutionState
+ quickstep_queryexecution_QueryManagerBase
quickstep_queryexecution_QueryManagerSingleNode
quickstep_queryexecution_WorkerDirectory
quickstep_queryexecution_WorkerMessage
quickstep_queryoptimizer_QueryHandle
- quickstep_relationaloperators_WorkOrder
- quickstep_storage_StorageBlockInfo
quickstep_utility_Macros
tmb
${GFLAGS_LIB_NAME})
@@ -244,7 +252,8 @@ target_link_libraries(quickstep_queryexecution
quickstep_queryexecution_AdmitRequestMessage
quickstep_queryexecution_ForemanBase
quickstep_queryexecution_ForemanSingleNode
- quickstep_queryexecution_PolicyEnforcer
+ quickstep_queryexecution_PolicyEnforcerBase
+ quickstep_queryexecution_PolicyEnforcerSingleNode
quickstep_queryexecution_QueryContext
quickstep_queryexecution_QueryContext_proto
quickstep_queryexecution_QueryExecutionMessages_proto
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/95c451a5/query_execution/ForemanSingleNode.cpp
----------------------------------------------------------------------
diff --git a/query_execution/ForemanSingleNode.cpp b/query_execution/ForemanSingleNode.cpp
index 3aa1f0b..cda02a7 100644
--- a/query_execution/ForemanSingleNode.cpp
+++ b/query_execution/ForemanSingleNode.cpp
@@ -25,6 +25,7 @@
#include <vector>
#include "query_execution/AdmitRequestMessage.hpp"
+#include "query_execution/PolicyEnforcerSingleNode.hpp"
#include "query_execution/QueryExecutionTypedefs.hpp"
#include "query_execution/QueryExecutionUtil.hpp"
#include "query_execution/WorkerDirectory.hpp"
@@ -36,6 +37,7 @@
#include "gflags/gflags.h"
#include "glog/logging.h"
+#include "tmb/id_typedefs.h"
#include "tmb/message_bus.h"
#include "tmb/tagged_message.h"
@@ -46,6 +48,8 @@ using std::vector;
namespace quickstep {
+class QueryHandle;
+
DEFINE_uint64(min_load_per_worker, 2, "The minimum load defined as the number "
"of pending work orders for the worker. This information is used "
"by the Foreman to assign work orders to worker threads");
@@ -88,7 +92,7 @@ ForemanSingleNode::ForemanSingleNode(
bus_->RegisterClientAsReceiver(foreman_client_id_, message_type);
}
- policy_enforcer_.reset(new PolicyEnforcer(
+ policy_enforcer_.reset(new PolicyEnforcerSingleNode(
foreman_client_id_,
num_numa_nodes,
catalog_database_,
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/95c451a5/query_execution/ForemanSingleNode.hpp
----------------------------------------------------------------------
diff --git a/query_execution/ForemanSingleNode.hpp b/query_execution/ForemanSingleNode.hpp
index 7506d35..caef5e0 100644
--- a/query_execution/ForemanSingleNode.hpp
+++ b/query_execution/ForemanSingleNode.hpp
@@ -24,11 +24,12 @@
#include <vector>
#include "query_execution/ForemanBase.hpp"
-#include "query_execution/PolicyEnforcer.hpp"
+#include "query_execution/PolicyEnforcerSingleNode.hpp"
#include "utility/Macros.hpp"
#include "tmb/id_typedefs.h"
-#include "tmb/message_bus.h"
+
+namespace tmb { class MessageBus; }
namespace quickstep {
@@ -128,7 +129,7 @@ class ForemanSingleNode final : public ForemanBase {
CatalogDatabaseLite *catalog_database_;
StorageManager *storage_manager_;
- std::unique_ptr<PolicyEnforcer> policy_enforcer_;
+ std::unique_ptr<PolicyEnforcerSingleNode> policy_enforcer_;
DISALLOW_COPY_AND_ASSIGN(ForemanSingleNode);
};
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/95c451a5/query_execution/PolicyEnforcer.cpp
----------------------------------------------------------------------
diff --git a/query_execution/PolicyEnforcer.cpp b/query_execution/PolicyEnforcer.cpp
deleted file mode 100644
index 4cba8c5..0000000
--- a/query_execution/PolicyEnforcer.cpp
+++ /dev/null
@@ -1,254 +0,0 @@
-/**
- * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
- * University of Wisconsin\u2014Madison.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- **/
-
-#include "query_execution/PolicyEnforcer.hpp"
-
-#include <cstddef>
-#include <memory>
-#include <queue>
-#include <utility>
-#include <unordered_map>
-#include <vector>
-
-#include "catalog/CatalogDatabase.hpp"
-#include "catalog/CatalogRelation.hpp"
-#include "catalog/CatalogTypedefs.hpp"
-#include "catalog/PartitionScheme.hpp"
-#include "query_execution/QueryExecutionMessages.pb.h"
-#include "query_execution/QueryExecutionState.hpp"
-#include "query_execution/QueryManagerBase.hpp"
-#include "query_execution/QueryManagerSingleNode.hpp"
-#include "query_execution/WorkerDirectory.hpp"
-#include "query_execution/WorkerMessage.hpp"
-#include "query_optimizer/QueryHandle.hpp"
-#include "relational_operators/WorkOrder.hpp"
-#include "storage/StorageBlockInfo.hpp"
-
-#include "gflags/gflags.h"
-#include "glog/logging.h"
-
-namespace quickstep {
-
-DEFINE_uint64(max_msgs_per_dispatch_round, 20, "Maximum number of messages that"
- " can be allocated in a single round of dispatch of messages to"
- " the workers.");
-
-bool PolicyEnforcer::admitQuery(QueryHandle *query_handle) {
- if (admitted_queries_.size() < kMaxConcurrentQueries) {
- // Ok to admit the query.
- const std::size_t query_id = query_handle->query_id();
- if (admitted_queries_.find(query_id) == admitted_queries_.end()) {
- // Query with the same ID not present, ok to admit.
- admitted_queries_[query_id].reset(
- new QueryManagerSingleNode(foreman_client_id_, num_numa_nodes_, query_handle,
- catalog_database_, storage_manager_, bus_));
- return true;
- } else {
- LOG(ERROR) << "Query with the same ID " << query_id << " exists";
- return false;
- }
- } else {
- // This query will have to wait.
- waiting_queries_.push(query_handle);
- return false;
- }
-}
-
-void PolicyEnforcer::processMessage(const TaggedMessage &tagged_message) {
- std::size_t query_id;
- QueryManagerBase::dag_node_index op_index;
-
- switch (tagged_message.message_type()) {
- case kWorkOrderCompleteMessage: {
- serialization::NormalWorkOrderCompletionMessage proto;
- // Note: This proto message contains the time it took to execute the
- // WorkOrder. It can be accessed in this scope.
- CHECK(proto.ParseFromArray(tagged_message.message(),
- tagged_message.message_bytes()));
- worker_directory_->decrementNumQueuedWorkOrders(
- proto.worker_thread_index());
- if (profile_individual_workorders_) {
- recordTimeForWorkOrder(proto);
- }
-
- query_id = proto.query_id();
- DCHECK(admitted_queries_.find(query_id) != admitted_queries_.end());
-
- op_index = proto.operator_index();
- admitted_queries_[query_id]->processWorkOrderCompleteMessage(op_index);
- break;
- }
- case kRebuildWorkOrderCompleteMessage: {
- serialization::RebuildWorkOrderCompletionMessage proto;
- // Note: This proto message contains the time it took to execute the
- // rebuild WorkOrder. It can be accessed in this scope.
- CHECK(proto.ParseFromArray(tagged_message.message(),
- tagged_message.message_bytes()));
- worker_directory_->decrementNumQueuedWorkOrders(
- proto.worker_thread_index());
-
- query_id = proto.query_id();
- DCHECK(admitted_queries_.find(query_id) != admitted_queries_.end());
-
- op_index = proto.operator_index();
- admitted_queries_[query_id]->processRebuildWorkOrderCompleteMessage(op_index);
- break;
- }
- case kCatalogRelationNewBlockMessage: {
- serialization::CatalogRelationNewBlockMessage proto;
- CHECK(proto.ParseFromArray(tagged_message.message(),
- tagged_message.message_bytes()));
-
- const block_id block = proto.block_id();
-
- CatalogRelation *relation =
- static_cast<CatalogDatabase*>(catalog_database_)->getRelationByIdMutable(proto.relation_id());
- relation->addBlock(block);
-
- if (proto.has_partition_id()) {
- relation->getPartitionSchemeMutable()->addBlockToPartition(
- proto.partition_id(), block);
- }
- return;
- }
- case kDataPipelineMessage: {
- serialization::DataPipelineMessage proto;
- CHECK(proto.ParseFromArray(tagged_message.message(),
- tagged_message.message_bytes()));
- query_id = proto.query_id();
- DCHECK(admitted_queries_.find(query_id) != admitted_queries_.end());
-
- op_index = proto.operator_index();
- admitted_queries_[query_id]->processDataPipelineMessage(
- op_index, proto.block_id(), proto.relation_id());
- break;
- }
- case kWorkOrdersAvailableMessage: {
- serialization::WorkOrdersAvailableMessage proto;
- CHECK(proto.ParseFromArray(tagged_message.message(),
- tagged_message.message_bytes()));
- query_id = proto.query_id();
- DCHECK(admitted_queries_.find(query_id) != admitted_queries_.end());
-
- op_index = proto.operator_index();
-
- // Check if new work orders are available.
- admitted_queries_[query_id]->fetchNormalWorkOrders(op_index);
- break;
- }
- case kWorkOrderFeedbackMessage: {
- WorkOrder::FeedbackMessage msg(
- const_cast<void *>(tagged_message.message()),
- tagged_message.message_bytes());
- query_id = msg.header().query_id;
- DCHECK(admitted_queries_.find(query_id) != admitted_queries_.end());
-
- op_index = msg.header().rel_op_index;
- admitted_queries_[query_id]->processFeedbackMessage(op_index, msg);
- break;
- }
- default:
- LOG(FATAL) << "Unknown message type found in PolicyEnforcer";
- }
- if (admitted_queries_[query_id]->queryStatus(op_index) ==
- QueryManagerBase::QueryStatusCode::kQueryExecuted) {
- removeQuery(query_id);
- if (!waiting_queries_.empty()) {
- // Admit the earliest waiting query.
- QueryHandle *new_query = waiting_queries_.front();
- waiting_queries_.pop();
- admitQuery(new_query);
- }
- }
-}
-
-void PolicyEnforcer::getWorkerMessages(
- std::vector<std::unique_ptr<WorkerMessage>> *worker_messages) {
- // Iterate over admitted queries until either there are no more
- // messages available, or the maximum number of messages have
- // been collected.
- DCHECK(worker_messages->empty());
- // TODO(harshad) - Make this function generic enough so that it
- // works well when multiple queries are getting executed.
- std::size_t per_query_share = 0;
- if (!admitted_queries_.empty()) {
- per_query_share = FLAGS_max_msgs_per_dispatch_round / admitted_queries_.size();
- } else {
- LOG(WARNING) << "Requesting WorkerMessages when no query is running";
- return;
- }
- DCHECK_GT(per_query_share, 0u);
- std::vector<std::size_t> finished_queries_ids;
-
- for (const auto &admitted_query_info : admitted_queries_) {
- QueryManagerBase *curr_query_manager = admitted_query_info.second.get();
- DCHECK(curr_query_manager != nullptr);
- std::size_t messages_collected_curr_query = 0;
- while (messages_collected_curr_query < per_query_share) {
- WorkerMessage *next_worker_message =
- static_cast<QueryManagerSingleNode*>(curr_query_manager)->getNextWorkerMessage(0, kAnyNUMANodeID);
- if (next_worker_message != nullptr) {
- ++messages_collected_curr_query;
- worker_messages->push_back(std::unique_ptr<WorkerMessage>(next_worker_message));
- } else {
- // No more work ordes from the current query at this time.
- // Check if the query's execution is over.
- if (curr_query_manager->getQueryExecutionState().hasQueryExecutionFinished()) {
- // If the query has been executed, remove it.
- finished_queries_ids.push_back(admitted_query_info.first);
- }
- break;
- }
- }
- }
- for (const std::size_t finished_qid : finished_queries_ids) {
- removeQuery(finished_qid);
- }
-}
-
-void PolicyEnforcer::removeQuery(const std::size_t query_id) {
- DCHECK(admitted_queries_.find(query_id) != admitted_queries_.end());
- if (!admitted_queries_[query_id]->getQueryExecutionState().hasQueryExecutionFinished()) {
- LOG(WARNING) << "Removing query with ID " << query_id
- << " that hasn't finished its execution";
- }
- admitted_queries_.erase(query_id);
-}
-
-bool PolicyEnforcer::admitQueries(
- const std::vector<QueryHandle*> &query_handles) {
- for (QueryHandle *curr_query : query_handles) {
- if (!admitQuery(curr_query)) {
- return false;
- }
- }
- return true;
-}
-
-void PolicyEnforcer::recordTimeForWorkOrder(
- const serialization::NormalWorkOrderCompletionMessage &proto) {
- const std::size_t query_id = proto.query_id();
- if (workorder_time_recorder_.find(query_id) == workorder_time_recorder_.end()) {
- workorder_time_recorder_[query_id];
- }
- workorder_time_recorder_[query_id].emplace_back(
- proto.worker_thread_index(),
- proto.operator_index(),
- proto.execution_time_in_microseconds());
-}
-
-} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/95c451a5/query_execution/PolicyEnforcer.hpp
----------------------------------------------------------------------
diff --git a/query_execution/PolicyEnforcer.hpp b/query_execution/PolicyEnforcer.hpp
deleted file mode 100644
index 8bd6d92..0000000
--- a/query_execution/PolicyEnforcer.hpp
+++ /dev/null
@@ -1,223 +0,0 @@
-/**
- * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
- * University of Wisconsin\u2014Madison.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- **/
-
-#ifndef QUICKSTEP_QUERY_EXECUTION_POLICY_ENFORCER_HPP_
-#define QUICKSTEP_QUERY_EXECUTION_POLICY_ENFORCER_HPP_
-
-#include <cstddef>
-#include <memory>
-#include <queue>
-#include <tuple>
-#include <unordered_map>
-#include <vector>
-
-#include "query_execution/QueryExecutionTypedefs.hpp"
-#include "query_execution/QueryManagerBase.hpp"
-#include "query_execution/WorkerMessage.hpp"
-#include "utility/Macros.hpp"
-
-#include "glog/logging.h"
-
-#include "tmb/id_typedefs.h"
-
-namespace tmb { class MessageBus; }
-
-namespace quickstep {
-
-class CatalogDatabaseLite;
-class QueryHandle;
-class StorageManager;
-class WorkerDirectory;
-
-namespace serialization { class NormalWorkOrderCompletionMessage; }
-
-/** \addtogroup QueryExecution
- * @{
- */
-
-/**
- * @brief A class that ensures that a high level policy is maintained
- * in sharing resources among concurrent queries.
- **/
-class PolicyEnforcer {
- public:
- /**
- * @brief Constructor.
- *
- * @param foreman_client_id The TMB client ID of the Foreman.
- * @param num_numa_nodes Number of NUMA nodes used by the system.
- * @param catalog_database The CatalogDatabase used.
- * @param storage_manager The StorageManager used.
- * @param bus The TMB.
- **/
- PolicyEnforcer(const tmb::client_id foreman_client_id,
- const std::size_t num_numa_nodes,
- CatalogDatabaseLite *catalog_database,
- StorageManager *storage_manager,
- WorkerDirectory *worker_directory,
- tmb::MessageBus *bus,
- const bool profile_individual_workorders = false)
- : foreman_client_id_(foreman_client_id),
- num_numa_nodes_(num_numa_nodes),
- catalog_database_(catalog_database),
- storage_manager_(storage_manager),
- worker_directory_(worker_directory),
- bus_(bus),
- profile_individual_workorders_(profile_individual_workorders) {}
-
- /**
- * @brief Destructor.
- **/
- ~PolicyEnforcer() {
- if (hasQueries()) {
- LOG(WARNING) << "Destructing PolicyEnforcer with some unfinished or "
- "waiting queries";
- }
- }
-
- /**
- * @brief Admit a query to the system.
- *
- * @param query_handle The QueryHandle for the new query.
- *
- * @return Whether the query was admitted to the system.
- **/
- bool admitQuery(QueryHandle *query_handle);
-
- /**
- * @brief Admit multiple queries in the system.
- *
- * @note In the current simple implementation, we only allow one active
- * query in the system. Other queries will have to wait.
- *
- * @param query_handles A vector of QueryHandles for the queries to be
- * admitted.
- *
- * @return True if all the queries were admitted, false if at least one query
- * was not admitted.
- **/
- bool admitQueries(const std::vector<QueryHandle*> &query_handles);
-
- /**
- * @brief Remove a given query that is under execution.
- *
- * @note This function is made public so that it is possible for a query
- * to be killed. Otherwise, it should only be used privately by the
- * class.
- *
- * TODO(harshad) - Extend this function to support removal of waiting queries.
- *
- * @param query_id The ID of the query to be removed.
- **/
- void removeQuery(const std::size_t query_id);
-
- /**
- * @brief Get worker messages to be dispatched. These worker messages come
- * from the active queries.
- *
- * @param worker_messages The worker messages to be dispatched.
- **/
- void getWorkerMessages(
- std::vector<std::unique_ptr<WorkerMessage>> *worker_messages);
-
- /**
- * @brief Process a message sent to the Foreman, which gets passed on to the
- * policy enforcer.
- *
- * @param message The message.
- **/
- void processMessage(const TaggedMessage &tagged_message);
-
- /**
- * @brief Check if there are any queries to be executed.
- *
- * @return True if there is at least one active or waiting query, false if
- * the policy enforcer doesn't have any query.
- **/
- inline bool hasQueries() const {
- return !(admitted_queries_.empty() && waiting_queries_.empty());
- }
-
- /**
- * @brief Get the profiling results for individual work order execution for a
- * given query.
- *
- * @note This function should only be called if profiling individual work
- * orders option is enabled.
- *
- * @param query_id The ID of the query for which the profiling results are
- * requested.
- *
- * @return A vector of tuples, each being a single profiling entry.
- **/
- inline const std::vector<std::tuple<std::size_t, std::size_t, std::size_t>>&
- getProfilingResults(const std::size_t query_id) const {
- DCHECK(profile_individual_workorders_);
- DCHECK(workorder_time_recorder_.find(query_id) !=
- workorder_time_recorder_.end());
- return workorder_time_recorder_.at(query_id);
- }
-
- private:
- static constexpr std::size_t kMaxConcurrentQueries = 1;
-
- /**
- * @brief Record the execution time for a finished WorkOrder.
- *
- * TODO(harshad) - Extend the functionality to rebuild work orders.
- *
- * @param proto The completion message proto sent after the WorkOrder
- * execution.
- **/
- void recordTimeForWorkOrder(
- const serialization::NormalWorkOrderCompletionMessage &proto);
-
- const tmb::client_id foreman_client_id_;
- const std::size_t num_numa_nodes_;
-
- CatalogDatabaseLite *catalog_database_;
- StorageManager *storage_manager_;
- WorkerDirectory *worker_directory_;
-
- tmb::MessageBus *bus_;
- const bool profile_individual_workorders_;
-
- // Key = query ID, value = QueryManagerBase* for the key query.
- std::unordered_map<std::size_t, std::unique_ptr<QueryManagerBase>> admitted_queries_;
-
- // The queries which haven't been admitted yet.
- std::queue<QueryHandle*> waiting_queries_;
-
- // Key = Query ID.
- // Value = A tuple indicating a record of executing a work order.
- // Within a tuple ...
- // 1st element: Logical worker ID.
- // 2nd element: Operator ID.
- // 3rd element: Time in microseconds to execute the work order.
- std::unordered_map<
- std::size_t,
- std::vector<std::tuple<std::size_t, std::size_t, std::size_t>>>
- workorder_time_recorder_;
-
- DISALLOW_COPY_AND_ASSIGN(PolicyEnforcer);
-};
-
-/** @} */
-
-} // namespace quickstep
-
-#endif // QUICKSTEP_QUERY_EXECUTION_POLICY_ENFORCER_HPP_
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/95c451a5/query_execution/PolicyEnforcerBase.cpp
----------------------------------------------------------------------
diff --git a/query_execution/PolicyEnforcerBase.cpp b/query_execution/PolicyEnforcerBase.cpp
new file mode 100644
index 0000000..d16a502
--- /dev/null
+++ b/query_execution/PolicyEnforcerBase.cpp
@@ -0,0 +1,177 @@
+/**
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#include "query_execution/PolicyEnforcerBase.hpp"
+
+#include <cstddef>
+#include <memory>
+#include <queue>
+#include <unordered_map>
+#include <vector>
+
+#include "catalog/CatalogDatabase.hpp"
+#include "catalog/CatalogRelation.hpp"
+#include "catalog/PartitionScheme.hpp"
+#include "query_execution/QueryExecutionMessages.pb.h"
+#include "query_execution/QueryExecutionState.hpp"
+#include "query_execution/QueryManagerBase.hpp"
+#include "relational_operators/WorkOrder.hpp"
+#include "storage/StorageBlockInfo.hpp"
+
+#include "glog/logging.h"
+
+namespace quickstep {
+
+void PolicyEnforcerBase::processMessage(const TaggedMessage &tagged_message) {
+ std::size_t query_id;
+ QueryManagerBase::dag_node_index op_index;
+
+ switch (tagged_message.message_type()) {
+ case kWorkOrderCompleteMessage: {
+ serialization::NormalWorkOrderCompletionMessage proto;
+ // Note: This proto message contains the time it took to execute the
+ // WorkOrder. It can be accessed in this scope.
+ CHECK(proto.ParseFromArray(tagged_message.message(),
+ tagged_message.message_bytes()));
+ decrementNumQueuedWorkOrders(proto.worker_thread_index());
+
+ if (profile_individual_workorders_) {
+ recordTimeForWorkOrder(proto);
+ }
+
+ query_id = proto.query_id();
+ DCHECK(admitted_queries_.find(query_id) != admitted_queries_.end());
+
+ op_index = proto.operator_index();
+ admitted_queries_[query_id]->processWorkOrderCompleteMessage(op_index);
+ break;
+ }
+ case kRebuildWorkOrderCompleteMessage: {
+ serialization::RebuildWorkOrderCompletionMessage proto;
+ // Note: This proto message contains the time it took to execute the
+ // rebuild WorkOrder. It can be accessed in this scope.
+ CHECK(proto.ParseFromArray(tagged_message.message(),
+ tagged_message.message_bytes()));
+ decrementNumQueuedWorkOrders(proto.worker_thread_index());
+
+ query_id = proto.query_id();
+ DCHECK(admitted_queries_.find(query_id) != admitted_queries_.end());
+
+ op_index = proto.operator_index();
+ admitted_queries_[query_id]->processRebuildWorkOrderCompleteMessage(op_index);
+ break;
+ }
+ case kCatalogRelationNewBlockMessage: {
+ serialization::CatalogRelationNewBlockMessage proto;
+ CHECK(proto.ParseFromArray(tagged_message.message(),
+ tagged_message.message_bytes()));
+
+ const block_id block = proto.block_id();
+
+ CatalogRelation *relation =
+ static_cast<CatalogDatabase*>(catalog_database_)->getRelationByIdMutable(proto.relation_id());
+ relation->addBlock(block);
+
+ if (proto.has_partition_id()) {
+ relation->getPartitionSchemeMutable()->addBlockToPartition(
+ proto.partition_id(), block);
+ }
+ return;
+ }
+ case kDataPipelineMessage: {
+ serialization::DataPipelineMessage proto;
+ CHECK(proto.ParseFromArray(tagged_message.message(),
+ tagged_message.message_bytes()));
+ query_id = proto.query_id();
+ DCHECK(admitted_queries_.find(query_id) != admitted_queries_.end());
+
+ op_index = proto.operator_index();
+ admitted_queries_[query_id]->processDataPipelineMessage(
+ op_index, proto.block_id(), proto.relation_id());
+ break;
+ }
+ case kWorkOrdersAvailableMessage: {
+ serialization::WorkOrdersAvailableMessage proto;
+ CHECK(proto.ParseFromArray(tagged_message.message(),
+ tagged_message.message_bytes()));
+ query_id = proto.query_id();
+ DCHECK(admitted_queries_.find(query_id) != admitted_queries_.end());
+
+ op_index = proto.operator_index();
+
+ // Check if new work orders are available.
+ admitted_queries_[query_id]->fetchNormalWorkOrders(op_index);
+ break;
+ }
+ case kWorkOrderFeedbackMessage: {
+ WorkOrder::FeedbackMessage msg(
+ const_cast<void *>(tagged_message.message()),
+ tagged_message.message_bytes());
+ query_id = msg.header().query_id;
+ DCHECK(admitted_queries_.find(query_id) != admitted_queries_.end());
+
+ op_index = msg.header().rel_op_index;
+ admitted_queries_[query_id]->processFeedbackMessage(op_index, msg);
+ break;
+ }
+ default:
+ LOG(FATAL) << "Unknown message type found in PolicyEnforcer";
+ }
+ if (admitted_queries_[query_id]->queryStatus(op_index) ==
+ QueryManagerBase::QueryStatusCode::kQueryExecuted) {
+ removeQuery(query_id);
+ if (!waiting_queries_.empty()) {
+ // Admit the earliest waiting query.
+ QueryHandle *new_query = waiting_queries_.front();
+ waiting_queries_.pop();
+ admitQuery(new_query);
+ }
+ }
+}
+
+void PolicyEnforcerBase::removeQuery(const std::size_t query_id) {
+ DCHECK(admitted_queries_.find(query_id) != admitted_queries_.end());
+ if (!admitted_queries_[query_id]->getQueryExecutionState().hasQueryExecutionFinished()) {
+ LOG(WARNING) << "Removing query with ID " << query_id
+ << " that hasn't finished its execution";
+ }
+ admitted_queries_.erase(query_id);
+}
+
+bool PolicyEnforcerBase::admitQueries(
+ const std::vector<QueryHandle*> &query_handles) {
+ for (QueryHandle *curr_query : query_handles) {
+ if (!admitQuery(curr_query)) {
+ return false;
+ }
+ }
+ return true;
+}
+
+void PolicyEnforcerBase::recordTimeForWorkOrder(
+ const serialization::NormalWorkOrderCompletionMessage &proto) {
+ const std::size_t query_id = proto.query_id();
+ if (workorder_time_recorder_.find(query_id) == workorder_time_recorder_.end()) {
+ workorder_time_recorder_[query_id];
+ }
+ workorder_time_recorder_[query_id].emplace_back(
+ proto.worker_thread_index(),
+ proto.operator_index(),
+ proto.execution_time_in_microseconds());
+}
+
+} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/95c451a5/query_execution/PolicyEnforcerBase.hpp
----------------------------------------------------------------------
diff --git a/query_execution/PolicyEnforcerBase.hpp b/query_execution/PolicyEnforcerBase.hpp
new file mode 100644
index 0000000..0482ebc
--- /dev/null
+++ b/query_execution/PolicyEnforcerBase.hpp
@@ -0,0 +1,196 @@
+/**
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#ifndef QUICKSTEP_QUERY_EXECUTION_POLICY_ENFORCER_BASE_HPP_
+#define QUICKSTEP_QUERY_EXECUTION_POLICY_ENFORCER_BASE_HPP_
+
+#include <cstddef>
+#include <memory>
+#include <queue>
+#include <tuple>
+#include <unordered_map>
+#include <vector>
+
+#include "query_execution/QueryExecutionTypedefs.hpp"
+#include "query_execution/QueryManagerBase.hpp"
+#include "utility/Macros.hpp"
+
+#include "glog/logging.h"
+
+namespace quickstep {
+
+class CatalogDatabaseLite;
+class QueryHandle;
+
+namespace serialization { class NormalWorkOrderCompletionMessage; }
+
+/** \addtogroup QueryExecution
+ * @{
+ */
+
+/**
+ * @brief A base class that ensures that a high level policy is maintained
+ * in sharing resources among concurrent queries.
+ **/
+class PolicyEnforcerBase {
+ public:
+ /**
+ * @brief Constructor.
+ *
+ * @param catalog_database The CatalogDatabase used.
+ * @param profile_individual_workorders If true, profile each normal work order.
+ **/
+ PolicyEnforcerBase(CatalogDatabaseLite *catalog_database,
+ const bool profile_individual_workorders)
+ : catalog_database_(catalog_database),
+ profile_individual_workorders_(profile_individual_workorders) {}
+
+ /**
+ * @brief Virtual Destructor.
+ **/
+ virtual ~PolicyEnforcerBase() {
+ if (hasQueries()) {
+ LOG(WARNING) << "Destructing PolicyEnforcer with some unfinished or "
+ "waiting queries";
+ }
+ }
+
+ /**
+ * @brief Admit multiple queries in the system.
+ *
+ * @note In the current simple implementation, we only allow one active
+ * query in the system. Other queries will have to wait.
+ *
+ * @param query_handles A vector of QueryHandles for the queries to be
+ * admitted.
+ *
+ * @return True if all the queries were admitted, false if at least one query
+ * was not admitted.
+ **/
+ bool admitQueries(const std::vector<QueryHandle*> &query_handles);
+
+ /**
+ * @brief Remove a given query that is under execution.
+ *
+ * @note This function is made public so that it is possible for a query
+ * to be killed. Otherwise, it should only be used privately by the
+ * class.
+ *
+ * TODO(harshad) - Extend this function to support removal of waiting queries.
+ *
+ * @param query_id The ID of the query to be removed.
+ **/
+ void removeQuery(const std::size_t query_id);
+
+ /**
+ * @brief Process a message sent to the Foreman, which gets passed on to the
+ * policy enforcer.
+ *
+ * @param message The message.
+ **/
+ void processMessage(const TaggedMessage &tagged_message);
+
+ /**
+ * @brief Check if there are any queries to be executed.
+ *
+ * @return True if there is at least one active or waiting query, false if
+ * the policy enforcer doesn't have any query.
+ **/
+ inline bool hasQueries() const {
+ return !(admitted_queries_.empty() && waiting_queries_.empty());
+ }
+
+ /**
+ * @brief Get the profiling results for individual work order execution for a
+ * given query.
+ *
+ * @note This function should only be called if profiling individual work
+ * orders option is enabled.
+ *
+ * @param query_id The ID of the query for which the profiling results are
+ * requested.
+ *
+ * @return A vector of tuples, each being a single profiling entry.
+ **/
+ inline const std::vector<std::tuple<std::size_t, std::size_t, std::size_t>>&
+ getProfilingResults(const std::size_t query_id) const {
+ DCHECK(profile_individual_workorders_);
+ DCHECK(workorder_time_recorder_.find(query_id) !=
+ workorder_time_recorder_.end());
+ return workorder_time_recorder_.at(query_id);
+ }
+
+ protected:
+ static constexpr std::size_t kMaxConcurrentQueries = 1;
+
+ /**
+ * @brief Record the execution time for a finished WorkOrder.
+ *
+ * TODO(harshad) - Extend the functionality to rebuild work orders.
+ *
+ * @param proto The completion message proto sent after the WorkOrder
+ * execution.
+ **/
+ void recordTimeForWorkOrder(
+ const serialization::NormalWorkOrderCompletionMessage &proto);
+
+ CatalogDatabaseLite *catalog_database_;
+
+ const bool profile_individual_workorders_;
+
+ // Key = query ID, value = QueryManagerBase* for the key query.
+ std::unordered_map<std::size_t, std::unique_ptr<QueryManagerBase>> admitted_queries_;
+
+ // The queries which haven't been admitted yet.
+ std::queue<QueryHandle*> waiting_queries_;
+
+ // Key = Query ID.
+ // Value = A tuple indicating a record of executing a work order.
+ // Within a tuple ...
+ // 1st element: Logical worker ID.
+ // 2nd element: Operator ID.
+ // 3rd element: Time in microseconds to execute the work order.
+ std::unordered_map<
+ std::size_t,
+ std::vector<std::tuple<std::size_t, std::size_t, std::size_t>>>
+ workorder_time_recorder_;
+
+ private:
+ /**
+ * @brief Admit a query to the system.
+ *
+ * @param query_handle The QueryHandle for the new query.
+ *
+ * @return Whether the query was admitted to the system.
+ **/
+ virtual bool admitQuery(QueryHandle *query_handle) = 0;
+
+ /**
+ * @brief Decrement the number of queued workorders for the given worker by 1.
+ *
+ * @param worker_index The logical ID of the given worker.
+ **/
+ virtual void decrementNumQueuedWorkOrders(const std::size_t worker_index) = 0;
+
+ DISALLOW_COPY_AND_ASSIGN(PolicyEnforcerBase);
+};
+
+/** @} */
+
+} // namespace quickstep
+
+#endif // QUICKSTEP_QUERY_EXECUTION_POLICY_ENFORCER_BASE_HPP_
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/95c451a5/query_execution/PolicyEnforcerSingleNode.cpp
----------------------------------------------------------------------
diff --git a/query_execution/PolicyEnforcerSingleNode.cpp b/query_execution/PolicyEnforcerSingleNode.cpp
new file mode 100644
index 0000000..549e39f
--- /dev/null
+++ b/query_execution/PolicyEnforcerSingleNode.cpp
@@ -0,0 +1,110 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#include "query_execution/PolicyEnforcerSingleNode.hpp"
+
+#include <cstddef>
+#include <memory>
+#include <queue>
+#include <utility>
+#include <unordered_map>
+#include <vector>
+
+#include "catalog/CatalogTypedefs.hpp"
+#include "query_execution/QueryExecutionState.hpp"
+#include "query_execution/QueryManagerBase.hpp"
+#include "query_execution/QueryManagerSingleNode.hpp"
+#include "query_execution/WorkerDirectory.hpp"
+#include "query_execution/WorkerMessage.hpp"
+#include "query_optimizer/QueryHandle.hpp"
+
+#include "gflags/gflags.h"
+#include "glog/logging.h"
+
+namespace quickstep {
+
+DEFINE_uint64(max_msgs_per_dispatch_round, 20, "Maximum number of messages that"
+ " can be allocated in a single round of dispatch of messages to"
+ " the workers.");
+
+void PolicyEnforcerSingleNode::getWorkerMessages(
+ std::vector<std::unique_ptr<WorkerMessage>> *worker_messages) {
+ // Iterate over admitted queries until either there are no more
+ // messages available, or the maximum number of messages have
+ // been collected.
+ DCHECK(worker_messages->empty());
+ // TODO(harshad) - Make this function generic enough so that it
+ // works well when multiple queries are getting executed.
+ std::size_t per_query_share = 0;
+ if (!admitted_queries_.empty()) {
+ per_query_share = FLAGS_max_msgs_per_dispatch_round / admitted_queries_.size();
+ } else {
+ LOG(WARNING) << "Requesting WorkerMessages when no query is running";
+ return;
+ }
+ DCHECK_GT(per_query_share, 0u);
+ std::vector<std::size_t> finished_queries_ids;
+
+ for (const auto &admitted_query_info : admitted_queries_) {
+ QueryManagerBase *curr_query_manager = admitted_query_info.second.get();
+ DCHECK(curr_query_manager != nullptr);
+ std::size_t messages_collected_curr_query = 0;
+ while (messages_collected_curr_query < per_query_share) {
+ WorkerMessage *next_worker_message =
+ static_cast<QueryManagerSingleNode*>(curr_query_manager)->getNextWorkerMessage(0, kAnyNUMANodeID);
+ if (next_worker_message != nullptr) {
+ ++messages_collected_curr_query;
+ worker_messages->push_back(std::unique_ptr<WorkerMessage>(next_worker_message));
+ } else {
+ // No more work ordes from the current query at this time.
+ // Check if the query's execution is over.
+ if (curr_query_manager->getQueryExecutionState().hasQueryExecutionFinished()) {
+ // If the query has been executed, remove it.
+ finished_queries_ids.push_back(admitted_query_info.first);
+ }
+ break;
+ }
+ }
+ }
+ for (const std::size_t finished_qid : finished_queries_ids) {
+ removeQuery(finished_qid);
+ }
+}
+
+bool PolicyEnforcerSingleNode::admitQuery(QueryHandle *query_handle) {
+ if (admitted_queries_.size() < PolicyEnforcerBase::kMaxConcurrentQueries) {
+ // Ok to admit the query.
+ const std::size_t query_id = query_handle->query_id();
+ if (admitted_queries_.find(query_id) == admitted_queries_.end()) {
+ // Query with the same ID not present, ok to admit.
+ admitted_queries_[query_id].reset(
+ new QueryManagerSingleNode(foreman_client_id_, num_numa_nodes_, query_handle,
+ catalog_database_, storage_manager_, bus_));
+ return true;
+ } else {
+ LOG(ERROR) << "Query with the same ID " << query_id << " exists";
+ return false;
+ }
+ } else {
+ // This query will have to wait.
+ waiting_queries_.push(query_handle);
+ return false;
+ }
+}
+
+void PolicyEnforcerSingleNode::decrementNumQueuedWorkOrders(const std::size_t worker_index) {
+ worker_directory_->decrementNumQueuedWorkOrders(worker_index);
+}
+
+} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/95c451a5/query_execution/PolicyEnforcerSingleNode.hpp
----------------------------------------------------------------------
diff --git a/query_execution/PolicyEnforcerSingleNode.hpp b/query_execution/PolicyEnforcerSingleNode.hpp
new file mode 100644
index 0000000..671fd83
--- /dev/null
+++ b/query_execution/PolicyEnforcerSingleNode.hpp
@@ -0,0 +1,104 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#ifndef QUICKSTEP_QUERY_EXECUTION_POLICY_ENFORCER_SINGLE_NODE_HPP_
+#define QUICKSTEP_QUERY_EXECUTION_POLICY_ENFORCER_SINGLE_NODE_HPP_
+
+#include <cstddef>
+#include <memory>
+#include <vector>
+
+#include "query_execution/PolicyEnforcerBase.hpp"
+#include "utility/Macros.hpp"
+
+#include "tmb/id_typedefs.h"
+
+namespace tmb { class MessageBus; }
+
+namespace quickstep {
+
+class CatalogDatabaseLite;
+class QueryHandle;
+class StorageManager;
+class WorkerDirectory;
+class WorkerMessage;
+
+/** \addtogroup QueryExecution
+ * @{
+ */
+
+/**
+ * @brief A class that ensures that a high level policy is maintained
+ * in sharing resources among concurrent queries.
+ **/
+class PolicyEnforcerSingleNode final : public PolicyEnforcerBase {
+ public:
+ /**
+ * @brief Constructor.
+ *
+ * @param foreman_client_id The TMB client ID of the Foreman.
+ * @param num_numa_nodes Number of NUMA nodes used by the system.
+ * @param catalog_database The CatalogDatabase used.
+ * @param storage_manager The StorageManager used.
+ * @param bus The TMB.
+ **/
+ PolicyEnforcerSingleNode(const tmb::client_id foreman_client_id,
+ const std::size_t num_numa_nodes,
+ CatalogDatabaseLite *catalog_database,
+ StorageManager *storage_manager,
+ WorkerDirectory *worker_directory,
+ tmb::MessageBus *bus,
+ const bool profile_individual_workorders = false)
+ : PolicyEnforcerBase(catalog_database, profile_individual_workorders),
+ foreman_client_id_(foreman_client_id),
+ num_numa_nodes_(num_numa_nodes),
+ storage_manager_(storage_manager),
+ worker_directory_(worker_directory),
+ bus_(bus) {}
+
+ /**
+ * @brief Destructor.
+ **/
+ ~PolicyEnforcerSingleNode() override {}
+
+ bool admitQuery(QueryHandle *query_handle) override;
+
+ /**
+ * @brief Get worker messages to be dispatched. These worker messages come
+ * from the active queries.
+ *
+ * @param worker_messages The worker messages to be dispatched.
+ **/
+ void getWorkerMessages(
+ std::vector<std::unique_ptr<WorkerMessage>> *worker_messages);
+
+ private:
+ void decrementNumQueuedWorkOrders(const std::size_t worker_index) override;
+
+ const tmb::client_id foreman_client_id_;
+ const std::size_t num_numa_nodes_;
+
+ StorageManager *storage_manager_;
+ WorkerDirectory *worker_directory_;
+
+ tmb::MessageBus *bus_;
+
+ DISALLOW_COPY_AND_ASSIGN(PolicyEnforcerSingleNode);
+};
+
+/** @} */
+
+} // namespace quickstep
+
+#endif // QUICKSTEP_QUERY_EXECUTION_POLICY_ENFORCER_SINGLE_NODE_HPP_
[22/25] incubator-quickstep git commit: Added the distributed
execution engine and tests.
Posted by zu...@apache.org.
Added the distributed execution engine and tests.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/603b6149
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/603b6149
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/603b6149
Branch: refs/heads/dist-exe-test-new
Commit: 603b61498ae61f78f61286fa9c11d5ea68bef833
Parents: 784b1ed
Author: Zuyu Zhang <zu...@twitter.com>
Authored: Fri Jul 22 11:31:33 2016 -0700
Committer: Zuyu Zhang <zu...@twitter.com>
Committed: Fri Jul 29 16:43:24 2016 -0700
----------------------------------------------------------------------
query_execution/AdmitRequestMessage.hpp | 7 +
query_execution/CMakeLists.txt | 50 ++-
query_execution/ForemanDistributed.cpp | 333 +++++++++++++++++++
query_execution/ForemanDistributed.hpp | 130 ++++++++
query_execution/PolicyEnforcerBase.cpp | 3 +
query_execution/PolicyEnforcerBase.hpp | 7 +
query_execution/PolicyEnforcerDistributed.cpp | 253 ++++++++++++++
query_execution/PolicyEnforcerDistributed.hpp | 112 +++++++
query_execution/QueryExecutionMessages.proto | 16 +-
query_execution/QueryExecutionTypedefs.hpp | 4 +
query_execution/QueryManagerBase.cpp | 3 +-
query_execution/QueryManagerBase.hpp | 26 +-
query_execution/QueryManagerDistributed.cpp | 41 ++-
query_execution/QueryManagerDistributed.hpp | 10 +-
query_execution/Shiftboss.cpp | 89 +++--
query_optimizer/CMakeLists.txt | 4 +
query_optimizer/QueryHandle.hpp | 26 ++
query_optimizer/tests/CMakeLists.txt | 41 +++
.../tests/DistributedExecutionGeneratorTest.cpp | 57 ++++
.../DistributedExecutionGeneratorTestRunner.cpp | 122 +++++++
.../DistributedExecutionGeneratorTestRunner.hpp | 146 ++++++++
.../tests/execution_generator/CMakeLists.txt | 68 +++-
third_party/tmb/include/tmb/tagged_message.h | 9 +
23 files changed, 1511 insertions(+), 46 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/603b6149/query_execution/AdmitRequestMessage.hpp
----------------------------------------------------------------------
diff --git a/query_execution/AdmitRequestMessage.hpp b/query_execution/AdmitRequestMessage.hpp
index e33b354..75c5ff6 100644
--- a/query_execution/AdmitRequestMessage.hpp
+++ b/query_execution/AdmitRequestMessage.hpp
@@ -60,6 +60,13 @@ class AdmitRequestMessage {
return query_handles_;
}
+ /**
+ * @brief Get the mutable query handles from this message.
+ **/
+ std::vector<QueryHandle*>* getQueryHandlesMutable() {
+ return &query_handles_;
+ }
+
private:
std::vector<QueryHandle*> query_handles_;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/603b6149/query_execution/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_execution/CMakeLists.txt b/query_execution/CMakeLists.txt
index 8bf1ab1..cfb72d7 100644
--- a/query_execution/CMakeLists.txt
+++ b/query_execution/CMakeLists.txt
@@ -33,8 +33,14 @@ if (ENABLE_DISTRIBUTED)
add_library(quickstep_queryexecution_BlockLocator BlockLocator.cpp BlockLocator.hpp)
endif()
add_library(quickstep_queryexecution_ForemanBase ../empty_src.cpp ForemanBase.hpp)
+if (ENABLE_DISTRIBUTED)
+ add_library(quickstep_queryexecution_ForemanDistributed ForemanDistributed.cpp ForemanDistributed.hpp)
+endif(ENABLE_DISTRIBUTED)
add_library(quickstep_queryexecution_ForemanSingleNode ForemanSingleNode.cpp ForemanSingleNode.hpp)
add_library(quickstep_queryexecution_PolicyEnforcerBase PolicyEnforcerBase.cpp PolicyEnforcerBase.hpp)
+if (ENABLE_DISTRIBUTED)
+ add_library(quickstep_queryexecution_PolicyEnforcerDistributed PolicyEnforcerDistributed.cpp PolicyEnforcerDistributed.hpp)
+endif(ENABLE_DISTRIBUTED)
add_library(quickstep_queryexecution_PolicyEnforcerSingleNode PolicyEnforcerSingleNode.cpp PolicyEnforcerSingleNode.hpp)
add_library(quickstep_queryexecution_QueryContext QueryContext.cpp QueryContext.hpp)
add_library(quickstep_queryexecution_QueryContext_proto
@@ -83,6 +89,26 @@ target_link_libraries(quickstep_queryexecution_ForemanBase
quickstep_threading_Thread
quickstep_utility_Macros
tmb)
+if (ENABLE_DISTRIBUTED)
+ target_link_libraries(quickstep_queryexecution_ForemanDistributed
+ glog
+ quickstep_catalog_CatalogDatabase
+ quickstep_catalog_CatalogRelation
+ quickstep_catalog_CatalogTypedefs
+ quickstep_queryexecution_AdmitRequestMessage
+ quickstep_queryexecution_ForemanBase
+ quickstep_queryexecution_PolicyEnforcerDistributed
+ quickstep_queryexecution_QueryExecutionMessages_proto
+ quickstep_queryexecution_QueryExecutionTypedefs
+ quickstep_queryexecution_QueryExecutionUtil
+ quickstep_queryexecution_ShiftbossDirectory
+ quickstep_queryoptimizer_QueryHandle
+ quickstep_threading_ThreadUtil
+ quickstep_utility_EqualsAnyConstant
+ quickstep_utility_Macros
+ tmb
+ ${GFLAGS_LIB_NAME})
+endif(ENABLE_DISTRIBUTED)
target_link_libraries(quickstep_queryexecution_ForemanSingleNode
glog
quickstep_queryexecution_AdmitRequestMessage
@@ -110,6 +136,26 @@ target_link_libraries(quickstep_queryexecution_PolicyEnforcerBase
quickstep_storage_StorageBlockInfo
quickstep_utility_Macros
tmb)
+if (ENABLE_DISTRIBUTED)
+ target_link_libraries(quickstep_queryexecution_PolicyEnforcerDistributed
+ glog
+ quickstep_catalog_CatalogRelation
+ quickstep_catalog_Catalog_proto
+ quickstep_queryexecution_PolicyEnforcerBase
+ quickstep_queryexecution_QueryContext_proto
+ quickstep_queryexecution_QueryExecutionMessages_proto
+ quickstep_queryexecution_QueryExecutionState
+ quickstep_queryexecution_QueryExecutionTypedefs
+ quickstep_queryexecution_QueryExecutionUtil
+ quickstep_queryexecution_QueryManagerBase
+ quickstep_queryexecution_QueryManagerDistributed
+ quickstep_queryexecution_ShiftbossDirectory
+ quickstep_queryoptimizer_QueryHandle
+ quickstep_storage_StorageBlockInfo
+ quickstep_utility_Macros
+ tmb
+ ${GFLAGS_LIB_NAME})
+endif(ENABLE_DISTRIBUTED)
target_link_libraries(quickstep_queryexecution_PolicyEnforcerSingleNode
glog
quickstep_catalog_CatalogTypedefs
@@ -294,10 +340,12 @@ target_link_libraries(quickstep_queryexecution
if (ENABLE_DISTRIBUTED)
target_link_libraries(quickstep_queryexecution
quickstep_queryexecution_BlockLocator
+ quickstep_queryexecution_ForemanDistributed
+ quickstep_queryexecution_PolicyEnforcerDistributed
quickstep_queryexecution_QueryManagerDistributed
quickstep_queryexecution_Shiftboss
quickstep_queryexecution_ShiftbossDirectory)
-endif()
+endif(ENABLE_DISTRIBUTED)
# Tests:
if (ENABLE_DISTRIBUTED)
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/603b6149/query_execution/ForemanDistributed.cpp
----------------------------------------------------------------------
diff --git a/query_execution/ForemanDistributed.cpp b/query_execution/ForemanDistributed.cpp
new file mode 100644
index 0000000..1c0fba8
--- /dev/null
+++ b/query_execution/ForemanDistributed.cpp
@@ -0,0 +1,333 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#include "query_execution/ForemanDistributed.hpp"
+
+#include <cstddef>
+#include <cstdio>
+#include <cstdlib>
+#include <memory>
+#include <tuple>
+#include <utility>
+#include <vector>
+
+#include "catalog/CatalogDatabase.hpp"
+#include "catalog/CatalogRelation.hpp"
+#include "catalog/CatalogTypedefs.hpp"
+#include "query_execution/AdmitRequestMessage.hpp"
+#include "query_execution/PolicyEnforcerDistributed.hpp"
+#include "query_execution/QueryExecutionMessages.pb.h"
+#include "query_execution/QueryExecutionTypedefs.hpp"
+#include "query_execution/QueryExecutionUtil.hpp"
+#include "query_execution/ShiftbossDirectory.hpp"
+#include "query_optimizer/QueryHandle.hpp"
+#include "threading/ThreadUtil.hpp"
+#include "utility/EqualsAnyConstant.hpp"
+
+#include "glog/logging.h"
+
+#include "tmb/address.h"
+#include "tmb/id_typedefs.h"
+#include "tmb/message_bus.h"
+#include "tmb/message_style.h"
+#include "tmb/tagged_message.h"
+
+using std::move;
+using std::size_t;
+using std::unique_ptr;
+using std::vector;
+
+using tmb::AnnotatedMessage;
+using tmb::TaggedMessage;
+
+namespace quickstep {
+
+namespace S = serialization;
+
+ForemanDistributed::ForemanDistributed(
+ tmb::MessageBus *bus,
+ CatalogDatabaseLite *catalog_database,
+ const int cpu_id,
+ const bool profile_individual_workorders)
+ : ForemanBase(bus, cpu_id),
+ catalog_database_(DCHECK_NOTNULL(catalog_database)) {
+ const std::vector<QueryExecutionMessageType> sender_message_types{
+ kShiftbossRegistrationResponseMessage,
+ kQueryInitiateMessage,
+ kWorkOrderMessage,
+ kInitiateRebuildMessage,
+ kSaveQueryResultMessage,
+ kQueryExecutionSuccessMessage,
+ kPoisonMessage};
+
+ for (const auto message_type : sender_message_types) {
+ bus_->RegisterClientAsSender(foreman_client_id_, message_type);
+ }
+
+ const std::vector<QueryExecutionMessageType> receiver_message_types{
+ kShiftbossRegistrationMessage,
+ kAdmitRequestMessage,
+ kQueryInitiateResponseMessage,
+ kCatalogRelationNewBlockMessage,
+ kDataPipelineMessage,
+ kInitiateRebuildResponseMessage,
+ kWorkOrderCompleteMessage,
+ kRebuildWorkOrderCompleteMessage,
+ kWorkOrderFeedbackMessage,
+ kWorkOrdersAvailableMessage,
+ kSaveQueryResultResponseMessage,
+ kPoisonMessage};
+
+ for (const auto message_type : receiver_message_types) {
+ bus_->RegisterClientAsReceiver(foreman_client_id_, message_type);
+ }
+
+ policy_enforcer_.reset(new PolicyEnforcerDistributed(
+ foreman_client_id_,
+ catalog_database_,
+ &shiftboss_directory_,
+ bus_,
+ profile_individual_workorders));
+}
+
+void ForemanDistributed::run() {
+ if (cpu_id_ >= 0) {
+ // We can pin the foreman thread to a CPU if specified.
+ ThreadUtil::BindToCPU(cpu_id_);
+ }
+
+ // Ensure that at least one Shiftboss to register.
+ if (shiftboss_directory_.empty()) {
+ const AnnotatedMessage annotated_message = bus_->Receive(foreman_client_id_, 0, true);
+ const TaggedMessage &tagged_message = annotated_message.tagged_message;
+ DCHECK_EQ(kShiftbossRegistrationMessage, tagged_message.message_type());
+ LOG(INFO) << "ForemanDistributed received typed '" << tagged_message.message_type()
+ << "' message from client " << annotated_message.sender;
+
+ S::ShiftbossRegistrationMessage proto;
+ CHECK(proto.ParseFromArray(tagged_message.message(), tagged_message.message_bytes()));
+
+ processShiftbossRegisterationMessage(annotated_message.sender, proto.work_order_capacity());
+ DCHECK_EQ(1u, shiftboss_directory_.size());
+ }
+
+ // Event loop
+ for (;;) {
+ // Receive() causes this thread to sleep until next message is received.
+ const AnnotatedMessage annotated_message =
+ bus_->Receive(foreman_client_id_, 0, true);
+ const TaggedMessage &tagged_message = annotated_message.tagged_message;
+ const tmb::message_type_id message_type = tagged_message.message_type();
+ LOG(INFO) << "ForemanDistributed received typed '" << message_type
+ << "' message from client " << annotated_message.sender;
+ switch (message_type) {
+ case kShiftbossRegistrationMessage: {
+ S::ShiftbossRegistrationMessage proto;
+ CHECK(proto.ParseFromArray(tagged_message.message(), tagged_message.message_bytes()));
+
+ processShiftbossRegisterationMessage(annotated_message.sender, proto.work_order_capacity());
+ break;
+ }
+ case kAdmitRequestMessage: {
+ AdmitRequestMessage *request_message =
+ const_cast<AdmitRequestMessage*>(
+ static_cast<const AdmitRequestMessage*>(tagged_message.message()));
+
+ vector<QueryHandle *> *query_handles = request_message->getQueryHandlesMutable();
+ DCHECK(!query_handles->empty());
+
+ for (QueryHandle *handle : *query_handles) {
+ handle->setClientId(annotated_message.sender);
+ }
+
+ bool all_queries_admitted = true;
+ if (query_handles->size() == 1u) {
+ all_queries_admitted =
+ policy_enforcer_->admitQuery(query_handles->front());
+ } else {
+ all_queries_admitted = policy_enforcer_->admitQueries(*query_handles);
+ }
+ if (!all_queries_admitted) {
+ LOG(WARNING) << "The scheduler could not admit all the queries";
+ // TODO(harshad) - Inform the main thread about the failure.
+ }
+ break;
+ }
+ case kQueryInitiateResponseMessage: {
+ // TODO(zuyu): check the query id.
+ break;
+ }
+ case kCatalogRelationNewBlockMessage: // Fall through
+ case kDataPipelineMessage:
+ case kRebuildWorkOrderCompleteMessage:
+ case kWorkOrderCompleteMessage:
+ case kWorkOrderFeedbackMessage:
+ case kWorkOrdersAvailableMessage: {
+ policy_enforcer_->processMessage(tagged_message);
+ break;
+ }
+ case kInitiateRebuildResponseMessage: {
+ // A unique case in the distributed version.
+ policy_enforcer_->processInitiateRebuildResponseMessage(tagged_message);
+ break;
+ }
+ case kSaveQueryResultResponseMessage: {
+ S::SaveQueryResultResponseMessage proto;
+ CHECK(proto.ParseFromArray(tagged_message.message(), tagged_message.message_bytes()));
+
+ processSaveQueryResultResponseMessage(proto.cli_id(), proto.relation_id());
+ break;
+ }
+ case kPoisonMessage: {
+ if (policy_enforcer_->hasQueries()) {
+ LOG(WARNING) << "Foreman thread exiting while some queries are "
+ "under execution or waiting to be admitted";
+ }
+
+ // Shutdown all Shiftbosses.
+ tmb::Address shiftboss_addresses;
+ for (std::size_t i = 0; i < shiftboss_directory_.size(); ++i) {
+ shiftboss_addresses.AddRecipient(shiftboss_directory_.getClientId(i));
+ }
+
+ tmb::MessageStyle broadcast_style;
+ broadcast_style.Broadcast(true);
+
+ TaggedMessage poison_message(kPoisonMessage);
+
+ const tmb::MessageBus::SendStatus send_status =
+ bus_->Send(foreman_client_id_,
+ shiftboss_addresses,
+ broadcast_style,
+ move(poison_message));
+ DCHECK(send_status == tmb::MessageBus::SendStatus::kOK);
+ return;
+ }
+ default:
+ LOG(FATAL) << "Unknown message type to Foreman";
+ }
+
+ if (canCollectNewMessages(message_type)) {
+ vector<unique_ptr<S::WorkOrderMessage>> new_messages;
+ policy_enforcer_->getWorkOrderMessages(&new_messages);
+ dispatchWorkOrderMessages(new_messages);
+ }
+ }
+}
+
+bool ForemanDistributed::canCollectNewMessages(const tmb::message_type_id message_type) {
+ return !QUICKSTEP_EQUALS_ANY_CONSTANT(message_type,
+ kCatalogRelationNewBlockMessage,
+ kWorkOrderFeedbackMessage) &&
+ // TODO(zuyu): Multiple Shiftbosses support.
+ !shiftboss_directory_.hasReachedCapacity(0);
+}
+
+void ForemanDistributed::dispatchWorkOrderMessages(const vector<unique_ptr<S::WorkOrderMessage>> &messages) {
+ for (const auto &message : messages) {
+ DCHECK(message != nullptr);
+ // TODO(zuyu): Multiple Shiftbosses support.
+ sendWorkOrderMessage(0, *message);
+ shiftboss_directory_.incrementNumQueuedWorkOrders(0);
+ }
+}
+
+void ForemanDistributed::sendWorkOrderMessage(const size_t shiftboss_index,
+ const S::WorkOrderMessage &proto) {
+ const size_t proto_length = proto.ByteSize();
+ char *proto_bytes = static_cast<char*>(malloc(proto_length));
+ CHECK(proto.SerializeToArray(proto_bytes, proto_length));
+
+ TaggedMessage message(static_cast<const void*>(proto_bytes),
+ proto_length,
+ kWorkOrderMessage);
+ free(proto_bytes);
+
+ const tmb::MessageBus::SendStatus send_status =
+ QueryExecutionUtil::SendTMBMessage(bus_,
+ foreman_client_id_,
+ shiftboss_directory_.getClientId(shiftboss_index),
+ move(message));
+ CHECK(send_status == tmb::MessageBus::SendStatus::kOK)
+ << "Message could not be sent from Foreman with TMB client ID " << foreman_client_id_
+ << " to Shiftboss with TMB client ID " << shiftboss_directory_.getClientId(shiftboss_index);
+}
+
+void ForemanDistributed::printWorkOrderProfilingResults(const std::size_t query_id,
+ std::FILE *out) const {
+ const std::vector<
+ std::tuple<std::size_t, std::size_t, std::size_t>>
+ &recorded_times = policy_enforcer_->getProfilingResults(query_id);
+ fputs("Query ID,Worker ID,Operator ID,Time (microseconds)\n", out);
+ for (const auto &workorder_entry : recorded_times) {
+ // Note: Index of the "worker thread index" in the tuple is 0.
+ const std::size_t worker_id = std::get<0>(workorder_entry);
+ fprintf(out,
+ "%lu,%lu,%lu,%lu\n",
+ query_id,
+ worker_id,
+ std::get<1>(workorder_entry), // Operator ID.
+ std::get<2>(workorder_entry)); // Time.
+ }
+}
+
+void ForemanDistributed::processShiftbossRegisterationMessage(const client_id shiftboss_client_id,
+ const std::size_t work_order_capacity) {
+ shiftboss_directory_.addShiftboss(shiftboss_client_id, work_order_capacity);
+
+ S::ShiftbossRegistrationResponseMessage proto;
+
+ const size_t proto_length = proto.ByteSize();
+ char *proto_bytes = static_cast<char*>(malloc(proto_length));
+ CHECK(proto.SerializeToArray(proto_bytes, proto_length));
+
+ TaggedMessage message(static_cast<const void*>(proto_bytes),
+ proto_length,
+ kShiftbossRegistrationResponseMessage);
+ free(proto_bytes);
+
+ LOG(INFO) << "ForemanDistributed sent ShiftbossRegistrationResponseMessage (typed '"
+ << kShiftbossRegistrationResponseMessage
+ << "') to Shiftboss with TMB client id " << shiftboss_client_id;
+ QueryExecutionUtil::SendTMBMessage(bus_,
+ foreman_client_id_,
+ shiftboss_client_id,
+ move(message));
+}
+
+void ForemanDistributed::processSaveQueryResultResponseMessage(const tmb::client_id cli_id,
+ const relation_id result_relation_id) {
+ S::QueryExecutionSuccessMessage proto;
+ proto.mutable_result_relation()->MergeFrom(
+ static_cast<CatalogDatabase*>(catalog_database_)->getRelationById(result_relation_id)->getProto());
+
+ const size_t proto_length = proto.ByteSize();
+ char *proto_bytes = static_cast<char*>(malloc(proto_length));
+ CHECK(proto.SerializeToArray(proto_bytes, proto_length));
+
+ TaggedMessage message(static_cast<const void*>(proto_bytes),
+ proto_length,
+ kQueryExecutionSuccessMessage);
+ free(proto_bytes);
+
+ // Notify the CLI regarding the query result.
+ LOG(INFO) << "ForemanDistributed sent QueryExecutionSuccessMessage (typed '" << kQueryExecutionSuccessMessage
+ << "') to CLI with TMB client id " << cli_id;
+ QueryExecutionUtil::SendTMBMessage(bus_,
+ foreman_client_id_,
+ cli_id,
+ move(message));
+}
+
+} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/603b6149/query_execution/ForemanDistributed.hpp
----------------------------------------------------------------------
diff --git a/query_execution/ForemanDistributed.hpp b/query_execution/ForemanDistributed.hpp
new file mode 100644
index 0000000..8a4a97c
--- /dev/null
+++ b/query_execution/ForemanDistributed.hpp
@@ -0,0 +1,130 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#ifndef QUICKSTEP_QUERY_EXECUTION_FOREMAN_DISTRIBUTED_HPP_
+#define QUICKSTEP_QUERY_EXECUTION_FOREMAN_DISTRIBUTED_HPP_
+
+#include <cstddef>
+#include <cstdio>
+#include <memory>
+#include <vector>
+
+#include "catalog/CatalogTypedefs.hpp"
+#include "query_execution/ForemanBase.hpp"
+#include "query_execution/PolicyEnforcerDistributed.hpp"
+#include "query_execution/ShiftbossDirectory.hpp"
+#include "utility/Macros.hpp"
+
+#include "tmb/id_typedefs.h"
+
+namespace tmb { class MessageBus; }
+
+namespace quickstep {
+
+class CatalogDatabaseLite;
+
+namespace serialization { class WorkOrderMessage; }
+
+/** \addtogroup QueryExecution
+ * @{
+ */
+
+/**
+ * @brief The Foreman receives queries from the main thread, messages from the
+ * policy enforcer and dispatches the work to Shiftbosses. It also
+ * receives work completion messages from Shiftbosses.
+ **/
+class ForemanDistributed final : public ForemanBase {
+ public:
+ /**
+ * @brief Constructor.
+ *
+ * @param bus A pointer to the TMB.
+ * @param catalog_database The catalog database where this query is executed.
+ * @param cpu_id The ID of the CPU to which the Foreman thread can be pinned.
+ * @param profile_individual_workorders Whether every workorder's execution
+ * be profiled or not.
+ *
+ * @note If cpu_id is not specified, Foreman thread can be possibly moved
+ * around on different CPUs by the OS.
+ **/
+ ForemanDistributed(tmb::MessageBus *bus,
+ CatalogDatabaseLite *catalog_database,
+ const int cpu_id = -1,
+ const bool profile_individual_workorders = false);
+
+ ~ForemanDistributed() override {}
+
+ /**
+ * @brief Print the results of profiling individual work orders for a given
+ * query.
+ *
+ * TODO(harshad) - Add the name of the operator to the output.
+ *
+ * @param query_id The ID of the query for which the results are to be printed.
+ * @param out The file stream.
+ **/
+ void printWorkOrderProfilingResults(const std::size_t query_id,
+ std::FILE *out) const;
+
+ protected:
+ void run() override;
+
+ private:
+ /**
+ * @brief Dispatch schedulable WorkOrders, wrapped in WorkOrderMessages to the
+ * worker threads.
+ *
+ * @param messages The messages to be dispatched.
+ **/
+ void dispatchWorkOrderMessages(
+ const std::vector<std::unique_ptr<serialization::WorkOrderMessage>> &messages);
+
+ /**
+ * @brief Send the given message to the specified worker.
+ *
+ * @param worker_index The logical index of the recipient worker in
+ * ShiftbossDirectory.
+ * @param proto The WorkOrderMessage to be sent.
+ **/
+ void sendWorkOrderMessage(const std::size_t worker_index,
+ const serialization::WorkOrderMessage &proto);
+
+ void processShiftbossRegisterationMessage(const tmb::client_id shiftboss_client_id,
+ const std::size_t work_order_capacity);
+
+ void processSaveQueryResultResponseMessage(const tmb::client_id cli_id,
+ const relation_id result_relation_id);
+
+ /**
+ * @brief Check if we can collect new messages from the PolicyEnforcer.
+ *
+ * @param message_type The type of the last received message.
+ **/
+ bool canCollectNewMessages(const tmb::message_type_id message_type);
+
+ ShiftbossDirectory shiftboss_directory_;
+
+ CatalogDatabaseLite *catalog_database_;
+
+ std::unique_ptr<PolicyEnforcerDistributed> policy_enforcer_;
+
+ DISALLOW_COPY_AND_ASSIGN(ForemanDistributed);
+};
+
+/** @} */
+
+} // namespace quickstep
+
+#endif // QUICKSTEP_QUERY_EXECUTION_FOREMAN_DISTRIBUTED_HPP_
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/603b6149/query_execution/PolicyEnforcerBase.cpp
----------------------------------------------------------------------
diff --git a/query_execution/PolicyEnforcerBase.cpp b/query_execution/PolicyEnforcerBase.cpp
index d16a502..a28fa3b 100644
--- a/query_execution/PolicyEnforcerBase.cpp
+++ b/query_execution/PolicyEnforcerBase.cpp
@@ -131,8 +131,11 @@ void PolicyEnforcerBase::processMessage(const TaggedMessage &tagged_message) {
default:
LOG(FATAL) << "Unknown message type found in PolicyEnforcer";
}
+
if (admitted_queries_[query_id]->queryStatus(op_index) ==
QueryManagerBase::QueryStatusCode::kQueryExecuted) {
+ onQueryCompletion(admitted_queries_[query_id]->query_handle());
+
removeQuery(query_id);
if (!waiting_queries_.empty()) {
// Admit the earliest waiting query.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/603b6149/query_execution/PolicyEnforcerBase.hpp
----------------------------------------------------------------------
diff --git a/query_execution/PolicyEnforcerBase.hpp b/query_execution/PolicyEnforcerBase.hpp
index 0482ebc..1de0677 100644
--- a/query_execution/PolicyEnforcerBase.hpp
+++ b/query_execution/PolicyEnforcerBase.hpp
@@ -148,6 +148,13 @@ class PolicyEnforcerBase {
void recordTimeForWorkOrder(
const serialization::NormalWorkOrderCompletionMessage &proto);
+ /**
+ * @brief Add custom actions upon the completion of a query.
+ *
+ * @param query_handle The query handle.
+ **/
+ virtual void onQueryCompletion(QueryHandle *query_handle) {}
+
CatalogDatabaseLite *catalog_database_;
const bool profile_individual_workorders_;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/603b6149/query_execution/PolicyEnforcerDistributed.cpp
----------------------------------------------------------------------
diff --git a/query_execution/PolicyEnforcerDistributed.cpp b/query_execution/PolicyEnforcerDistributed.cpp
new file mode 100644
index 0000000..59df3de
--- /dev/null
+++ b/query_execution/PolicyEnforcerDistributed.cpp
@@ -0,0 +1,253 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#include "query_execution/PolicyEnforcerDistributed.hpp"
+
+#include <cstddef>
+#include <cstdlib>
+#include <memory>
+#include <queue>
+#include <utility>
+#include <unordered_map>
+#include <vector>
+
+#include "catalog/Catalog.pb.h"
+#include "catalog/CatalogRelation.hpp"
+#include "query_execution/QueryContext.pb.h"
+#include "query_execution/QueryExecutionMessages.pb.h"
+#include "query_execution/QueryExecutionState.hpp"
+#include "query_execution/QueryExecutionTypedefs.hpp"
+#include "query_execution/QueryExecutionUtil.hpp"
+#include "query_execution/QueryManagerBase.hpp"
+#include "query_execution/QueryManagerDistributed.hpp"
+#include "query_optimizer/QueryHandle.hpp"
+#include "storage/StorageBlockInfo.hpp"
+
+#include "gflags/gflags.h"
+#include "glog/logging.h"
+
+#include "tmb/id_typedefs.h"
+#include "tmb/message_bus.h"
+#include "tmb/tagged_message.h"
+
+using std::free;
+using std::malloc;
+using std::move;
+using std::size_t;
+using std::unique_ptr;
+using std::vector;
+
+using tmb::TaggedMessage;
+
+namespace quickstep {
+
+namespace S = serialization;
+
+DEFINE_uint64(max_msgs_per_dispatch_round, 20, "Maximum number of messages that"
+ " can be allocated in a single round of dispatch of messages to"
+ " the workers.");
+
+void PolicyEnforcerDistributed::getWorkOrderMessages(
+ vector<unique_ptr<S::WorkOrderMessage>> *work_order_messages) {
+ // Iterate over admitted queries until either there are no more
+ // messages available, or the maximum number of messages have
+ // been collected.
+ DCHECK(work_order_messages->empty());
+ // TODO(harshad) - Make this function generic enough so that it
+ // works well when multiple queries are getting executed.
+ if (admitted_queries_.empty()) {
+ LOG(WARNING) << "Requesting WorkerMessages when no query is running";
+ return;
+ }
+
+ const std::size_t per_query_share =
+ FLAGS_max_msgs_per_dispatch_round / admitted_queries_.size();
+ DCHECK_GT(per_query_share, 0u);
+
+ vector<std::size_t> finished_queries_ids;
+
+ for (const auto &admitted_query_info : admitted_queries_) {
+ QueryManagerBase *curr_query_manager = admitted_query_info.second.get();
+ DCHECK(curr_query_manager != nullptr);
+ std::size_t messages_collected_curr_query = 0;
+ while (messages_collected_curr_query < per_query_share) {
+ S::WorkOrderMessage *next_work_order_message =
+ static_cast<QueryManagerDistributed*>(curr_query_manager)->getNextWorkOrderMessage(0);
+ if (next_work_order_message != nullptr) {
+ ++messages_collected_curr_query;
+ work_order_messages->push_back(unique_ptr<S::WorkOrderMessage>(next_work_order_message));
+ } else {
+ // No more work ordes from the current query at this time.
+ // Check if the query's execution is over.
+ if (curr_query_manager->getQueryExecutionState().hasQueryExecutionFinished()) {
+ // If the query has been executed, remove it.
+ finished_queries_ids.push_back(admitted_query_info.first);
+ }
+ break;
+ }
+ }
+ }
+ for (const std::size_t finished_qid : finished_queries_ids) {
+ onQueryCompletion(admitted_queries_[finished_qid]->query_handle());
+ removeQuery(finished_qid);
+ }
+}
+
+bool PolicyEnforcerDistributed::admitQuery(QueryHandle *query_handle) {
+ if (admitted_queries_.size() < PolicyEnforcerBase::kMaxConcurrentQueries) {
+ // Ok to admit the query.
+ const std::size_t query_id = query_handle->query_id();
+ if (admitted_queries_.find(query_id) == admitted_queries_.end()) {
+ // NOTE(zuyu): Should call before constructing a 'QueryManager'.
+ // Otherwise, an InitiateRebuildMessage may be sent before 'QueryContext'
+ // initializes.
+ initiateQueryInShiftboss(query_handle);
+
+ // Query with the same ID not present, ok to admit.
+ admitted_queries_[query_id].reset(
+ new QueryManagerDistributed(query_handle, shiftboss_directory_, foreman_client_id_, bus_));
+ return true;
+ } else {
+ LOG(ERROR) << "Query with the same ID " << query_id << " exists";
+ return false;
+ }
+ } else {
+ // This query will have to wait.
+ waiting_queries_.push(query_handle);
+ return false;
+ }
+}
+
+void PolicyEnforcerDistributed::processInitiateRebuildResponseMessage(const tmb::TaggedMessage &tagged_message) {
+ S::InitiateRebuildResponseMessage proto;
+ CHECK(proto.ParseFromArray(tagged_message.message(), tagged_message.message_bytes()));
+
+ const std::size_t query_id = proto.query_id();
+ DCHECK(admitted_queries_.find(query_id) != admitted_queries_.end());
+
+ QueryManagerBase *query_manager = admitted_queries_[query_id].get();
+
+ const std::size_t num_rebuild_work_orders = proto.num_rebuild_work_orders();
+ query_manager->processInitiateRebuildResponseMessage(proto.operator_index(), num_rebuild_work_orders);
+ shiftboss_directory_->addNumQueuedWorkOrders(proto.shiftboss_index(), num_rebuild_work_orders);
+
+ if (query_manager->getQueryExecutionState().hasQueryExecutionFinished()) {
+ onQueryCompletion(query_manager->query_handle());
+
+ removeQuery(query_id);
+ if (!waiting_queries_.empty()) {
+ // Admit the earliest waiting query.
+ QueryHandle *new_query = waiting_queries_.front();
+ waiting_queries_.pop();
+ admitQuery(new_query);
+ }
+ }
+}
+
+void PolicyEnforcerDistributed::initiateQueryInShiftboss(QueryHandle *query_handle) {
+ S::QueryInitiateMessage proto;
+ proto.set_query_id(query_handle->query_id());
+ proto.mutable_catalog_database_cache()->MergeFrom(query_handle->getCatalogDatabaseCacheProto());
+ proto.mutable_query_context()->MergeFrom(query_handle->getQueryContextProto());
+
+ const size_t proto_length = proto.ByteSize();
+ char *proto_bytes = static_cast<char*>(malloc(proto_length));
+ CHECK(proto.SerializeToArray(proto_bytes, proto_length));
+
+ TaggedMessage message(static_cast<const void*>(proto_bytes),
+ proto_length,
+ kQueryInitiateMessage);
+ free(proto_bytes);
+
+ LOG(INFO) << "PolicyEnforcerDistributed sent QueryInitiateMessage (typed '" << kQueryInitiateMessage
+ << "') to Shiftboss 0";
+
+ // TODO(zuyu): Multiple Shiftbosses support.
+ const tmb::MessageBus::SendStatus send_status =
+ QueryExecutionUtil::SendTMBMessage(bus_,
+ foreman_client_id_,
+ shiftboss_directory_->getClientId(0),
+ move(message));
+ CHECK(send_status == tmb::MessageBus::SendStatus::kOK)
+ << "Message could not be sent from Foreman with TMB client ID " << foreman_client_id_
+ << " to Shiftboss with TMB client ID " << shiftboss_directory_->getClientId(0);
+
+ // Wait Shiftboss for QueryInitiateResponseMessage.
+ const tmb::AnnotatedMessage annotated_message = bus_->Receive(foreman_client_id_, 0, true);
+ const TaggedMessage &tagged_message = annotated_message.tagged_message;
+ DCHECK_EQ(kQueryInitiateResponseMessage, tagged_message.message_type());
+ LOG(INFO) << "PolicyEnforcerDistributed received typed '" << tagged_message.message_type()
+ << "' message from client " << annotated_message.sender;
+
+ S::QueryInitiateResponseMessage proto_response;
+ CHECK(proto_response.ParseFromArray(tagged_message.message(), tagged_message.message_bytes()));
+}
+
+void PolicyEnforcerDistributed::onQueryCompletion(QueryHandle *query_handle) {
+ const CatalogRelation *query_result = query_handle->getQueryResultRelation();
+ if (query_result == nullptr) {
+ // TODO(zuyu): notify Shiftboss to remove QueryContext.
+ TaggedMessage message(kQueryExecutionSuccessMessage);
+
+ const tmb::client_id cli_id = query_handle->getClientId();
+
+ // Notify the CLI regarding the query execution result.
+ LOG(INFO) << "PolicyEnforcerDistributed sent QueryExecutionSuccessMessage (typed '" << kQueryExecutionSuccessMessage
+ << "') to CLI with TMB client id " << cli_id;
+ const tmb::MessageBus::SendStatus send_status =
+ QueryExecutionUtil::SendTMBMessage(bus_,
+ foreman_client_id_,
+ cli_id,
+ move(message));
+ CHECK(send_status == tmb::MessageBus::SendStatus::kOK)
+ << "Message could not be sent from Foreman with TMB client ID " << foreman_client_id_
+ << " to CLI with TMB client ID " << cli_id;
+ return;
+ }
+
+ // SaveQueryResultMessage implies QueryContext clean up in Shiftboss.
+ S::SaveQueryResultMessage proto;
+ proto.set_query_id(query_handle->query_id());
+ proto.set_relation_id(query_result->getID());
+
+ const vector<block_id> blocks(query_result->getBlocksSnapshot());
+ for (const block_id block : blocks) {
+ proto.add_blocks(block);
+ }
+
+ proto.set_cli_id(query_handle->getClientId());
+
+ const size_t proto_length = proto.ByteSize();
+ char *proto_bytes = static_cast<char*>(malloc(proto_length));
+ CHECK(proto.SerializeToArray(proto_bytes, proto_length));
+
+ TaggedMessage message(static_cast<const void*>(proto_bytes),
+ proto_length,
+ kSaveQueryResultMessage);
+ free(proto_bytes);
+
+ LOG(INFO) << "PolicyEnforcerDistributed sent SaveQueryResultMessage (typed '" << kSaveQueryResultMessage
+ << "') to Shiftboss 0";
+ // TODO(zuyu): Support multiple shiftbosses.
+ const tmb::MessageBus::SendStatus send_status =
+ QueryExecutionUtil::SendTMBMessage(bus_,
+ foreman_client_id_,
+ shiftboss_directory_->getClientId(0),
+ move(message));
+ CHECK(send_status == tmb::MessageBus::SendStatus::kOK)
+ << "Message could not be sent from Foreman with TMB client ID " << foreman_client_id_
+ << " to Shiftboss with TMB client ID " << shiftboss_directory_->getClientId(0);
+}
+
+} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/603b6149/query_execution/PolicyEnforcerDistributed.hpp
----------------------------------------------------------------------
diff --git a/query_execution/PolicyEnforcerDistributed.hpp b/query_execution/PolicyEnforcerDistributed.hpp
new file mode 100644
index 0000000..8b07748
--- /dev/null
+++ b/query_execution/PolicyEnforcerDistributed.hpp
@@ -0,0 +1,112 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#ifndef QUICKSTEP_QUERY_EXECUTION_POLICY_ENFORCER_DISTRIBUTED_HPP_
+#define QUICKSTEP_QUERY_EXECUTION_POLICY_ENFORCER_DISTRIBUTED_HPP_
+
+#include <cstddef>
+#include <memory>
+#include <vector>
+
+#include "query_execution/PolicyEnforcerBase.hpp"
+#include "query_execution/ShiftbossDirectory.hpp"
+#include "utility/Macros.hpp"
+
+#include "tmb/id_typedefs.h"
+
+namespace tmb {
+class MessageBus;
+class TaggedMessage;
+}
+
+namespace quickstep {
+
+class CatalogDatabaseLite;
+class QueryHandle;
+
+namespace serialization { class WorkOrderMessage; }
+
+/** \addtogroup QueryExecution
+ * @{
+ */
+
+/**
+ * @brief A class that ensures that a high level policy is maintained
+ * in sharing resources among concurrent queries.
+ **/
+class PolicyEnforcerDistributed final : public PolicyEnforcerBase {
+ public:
+ /**
+ * @brief Constructor.
+ *
+ * @param foreman_client_id The TMB client ID of the Foreman.
+ * @param catalog_database The CatalogDatabase used.
+ * @param bus The TMB.
+ **/
+ PolicyEnforcerDistributed(const tmb::client_id foreman_client_id,
+ CatalogDatabaseLite *catalog_database,
+ ShiftbossDirectory *shiftboss_directory,
+ tmb::MessageBus *bus,
+ const bool profile_individual_workorders = false)
+ : PolicyEnforcerBase(catalog_database, profile_individual_workorders),
+ foreman_client_id_(foreman_client_id),
+ shiftboss_directory_(shiftboss_directory),
+ bus_(bus) {}
+
+ /**
+ * @brief Destructor.
+ **/
+ ~PolicyEnforcerDistributed() override {}
+
+ bool admitQuery(QueryHandle *query_handle) override;
+
+ /**
+ * @brief Get work order messages to be dispatched. These messages come from
+ * the active queries.
+ *
+ * @param work_order_messages The work order messages to be dispatched.
+ **/
+ void getWorkOrderMessages(
+ std::vector<std::unique_ptr<serialization::WorkOrderMessage>> *work_order_messages);
+
+ /**
+ * @brief Process the initiate rebuild work order response message.
+ *
+ * @param tagged_message The message.
+ **/
+ void processInitiateRebuildResponseMessage(const tmb::TaggedMessage &tagged_message);
+
+ private:
+ void decrementNumQueuedWorkOrders(const std::size_t shiftboss_index) override {
+ shiftboss_directory_->decrementNumQueuedWorkOrders(shiftboss_index);
+ }
+
+ void onQueryCompletion(QueryHandle *query_handle) override;
+
+ void initiateQueryInShiftboss(QueryHandle *query_handle);
+
+ const tmb::client_id foreman_client_id_;
+
+ ShiftbossDirectory *shiftboss_directory_;
+
+ tmb::MessageBus *bus_;
+
+ DISALLOW_COPY_AND_ASSIGN(PolicyEnforcerDistributed);
+};
+
+/** @} */
+
+} // namespace quickstep
+
+#endif // QUICKSTEP_QUERY_EXECUTION_POLICY_ENFORCER_DISTRIBUTED_HPP_
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/603b6149/query_execution/QueryExecutionMessages.proto
----------------------------------------------------------------------
diff --git a/query_execution/QueryExecutionMessages.proto b/query_execution/QueryExecutionMessages.proto
index 308d736..99de75c 100644
--- a/query_execution/QueryExecutionMessages.proto
+++ b/query_execution/QueryExecutionMessages.proto
@@ -111,15 +111,27 @@ message InitiateRebuildResponseMessage {
required uint64 query_id = 1;
required uint64 operator_index = 2;
required uint64 num_rebuild_work_orders = 3;
+ required uint64 shiftboss_index = 4;
}
message SaveQueryResultMessage {
- required int32 relation_id = 1;
- repeated fixed64 blocks = 2 [packed=true];
+ required uint64 query_id = 1;
+ required int32 relation_id = 2;
+ repeated fixed64 blocks = 3 [packed=true];
+
+ // Defined in "tmb/id_typedefs.h".
+ required uint32 cli_id = 4;
}
message SaveQueryResultResponseMessage {
required int32 relation_id = 1;
+
+ // Defined in "tmb/id_typedefs.h".
+ required uint32 cli_id = 2;
+}
+
+message QueryExecutionSuccessMessage {
+ optional CatalogRelationSchema result_relation = 1;
}
// BlockLocator related messages.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/603b6149/query_execution/QueryExecutionTypedefs.hpp
----------------------------------------------------------------------
diff --git a/query_execution/QueryExecutionTypedefs.hpp b/query_execution/QueryExecutionTypedefs.hpp
index b67209f..0d43237 100644
--- a/query_execution/QueryExecutionTypedefs.hpp
+++ b/query_execution/QueryExecutionTypedefs.hpp
@@ -84,6 +84,10 @@ enum QueryExecutionMessageType : message_type_id {
kSaveQueryResultMessage, // From Foreman to Shiftboss.
kSaveQueryResultResponseMessage, // From Shiftboss to Foreman.
+ // From Foreman to CLI.
+ kQueryExecutionSuccessMessage,
+ kQueryExecutionErrorMessage,
+
// BlockLocator related messages, sorted in a life cycle of StorageManager
// with a unique block domain.
kBlockDomainRegistrationMessage, // From Worker to BlockLocator.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/603b6149/query_execution/QueryManagerBase.cpp
----------------------------------------------------------------------
diff --git a/query_execution/QueryManagerBase.cpp b/query_execution/QueryManagerBase.cpp
index d2a3341..4ee51c3 100644
--- a/query_execution/QueryManagerBase.cpp
+++ b/query_execution/QueryManagerBase.cpp
@@ -35,7 +35,8 @@ using std::pair;
namespace quickstep {
QueryManagerBase::QueryManagerBase(QueryHandle *query_handle)
- : query_id_(DCHECK_NOTNULL(query_handle)->query_id()),
+ : query_handle_(query_handle),
+ query_id_(DCHECK_NOTNULL(query_handle)->query_id()),
query_dag_(DCHECK_NOTNULL(
DCHECK_NOTNULL(query_handle->getQueryPlanMutable())->getQueryPlanDAGMutable())),
num_operators_in_dag_(query_dag_->size()),
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/603b6149/query_execution/QueryManagerBase.hpp
----------------------------------------------------------------------
diff --git a/query_execution/QueryManagerBase.hpp b/query_execution/QueryManagerBase.hpp
index 6edfd5c..3338478 100644
--- a/query_execution/QueryManagerBase.hpp
+++ b/query_execution/QueryManagerBase.hpp
@@ -24,6 +24,7 @@
#include "catalog/CatalogTypedefs.hpp"
#include "query_execution/QueryExecutionState.hpp"
+#include "query_optimizer/QueryOptimizerConfig.h" // For QUICKSTEP_DISTRIBUTED.
#include "relational_operators/RelationalOperator.hpp"
#include "relational_operators/WorkOrder.hpp"
#include "storage/StorageBlockInfo.hpp"
@@ -79,6 +80,13 @@ class QueryManagerBase {
}
/**
+ * @brief Get the query handle.
+ **/
+ inline QueryHandle* query_handle() const {
+ return query_handle_;
+ }
+
+ /**
* @brief Process the received WorkOrder complete message.
*
* @param op_index The index of the specified operator node in the query DAG
@@ -128,6 +136,20 @@ class QueryManagerBase {
void processFeedbackMessage(const dag_node_index op_index,
const WorkOrder::FeedbackMessage &message);
+#ifdef QUICKSTEP_DISTRIBUTED
+ /**
+ * @brief Process the initiate rebuild work order response message.
+ *
+ * @param shiftboss_index The Shiftboss index for the rebuild work orders.
+ * @param op_index The index of the specified operator node in the query DAG
+ * for initiating the rebuild work order.
+ * @param num_rebuild_work_orders The number of the rebuild work orders
+ * generated for the operator indexed by 'op_index'.
+ **/
+ virtual void processInitiateRebuildResponseMessage(const dag_node_index op_index,
+ const std::size_t num_rebuild_work_orders) {}
+#endif // QUICKSTEP_DISTRIBUTED
+
/**
* @brief Get the query status after processing an incoming message.
*
@@ -250,9 +272,11 @@ class QueryManagerBase {
return query_exec_state_->hasRebuildInitiated(index);
}
+ QueryHandle *query_handle_; // Owned by the optimizer.
+
const std::size_t query_id_;
- DAG<RelationalOperator, bool> *query_dag_;
+ DAG<RelationalOperator, bool> *query_dag_; // Owned by 'query_handle_'.
const dag_node_index num_operators_in_dag_;
// For all nodes, store their receiving dependents.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/603b6149/query_execution/QueryManagerDistributed.cpp
----------------------------------------------------------------------
diff --git a/query_execution/QueryManagerDistributed.cpp b/query_execution/QueryManagerDistributed.cpp
index e906fa5..bed3e45 100644
--- a/query_execution/QueryManagerDistributed.cpp
+++ b/query_execution/QueryManagerDistributed.cpp
@@ -32,6 +32,7 @@
#include "glog/logging.h"
#include "tmb/id_typedefs.h"
+#include "tmb/message_bus.h"
using std::free;
using std::malloc;
@@ -42,11 +43,11 @@ using std::unique_ptr;
namespace quickstep {
QueryManagerDistributed::QueryManagerDistributed(QueryHandle *query_handle,
- ShiftbossDirectory *shiftbosses,
+ ShiftbossDirectory *shiftboss_directory,
const tmb::client_id foreman_client_id,
tmb::MessageBus *bus)
: QueryManagerBase(query_handle),
- shiftbosses_(shiftbosses),
+ shiftboss_directory_(shiftboss_directory),
foreman_client_id_(foreman_client_id),
bus_(bus),
normal_workorder_protos_container_(
@@ -119,6 +120,27 @@ bool QueryManagerDistributed::fetchNormalWorkOrders(const dag_node_index index)
return generated_new_workorder_protos;
}
+void QueryManagerDistributed::processInitiateRebuildResponseMessage(const dag_node_index op_index,
+ const std::size_t num_rebuild_work_orders) {
+ // TODO(zuyu): Multiple workers support.
+ query_exec_state_->setRebuildStatus(op_index, num_rebuild_work_orders, true);
+
+ if (num_rebuild_work_orders != 0u) {
+ // Wait for the rebuild work orders finish.
+ return;
+ }
+
+ markOperatorFinished(op_index);
+
+ for (const std::pair<dag_node_index, bool> &dependent_link :
+ query_dag_->getDependents(op_index)) {
+ const dag_node_index dependent_op_index = dependent_link.first;
+ if (checkAllBlockingDependenciesMet(dependent_op_index)) {
+ processOperator(dependent_op_index, true);
+ }
+ }
+}
+
bool QueryManagerDistributed::initiateRebuild(const dag_node_index index) {
DCHECK(checkRebuildRequired(index));
DCHECK(!checkRebuildInitiated(index));
@@ -127,6 +149,7 @@ bool QueryManagerDistributed::initiateRebuild(const dag_node_index index) {
DCHECK_NE(op.getInsertDestinationID(), QueryContext::kInvalidInsertDestinationId);
serialization::InitiateRebuildMessage proto;
+ proto.set_query_id(query_id_);
proto.set_operator_index(index);
proto.set_insert_destination_index(op.getInsertDestinationID());
proto.set_relation_id(op.getOutputRelationID());
@@ -140,13 +163,17 @@ bool QueryManagerDistributed::initiateRebuild(const dag_node_index index) {
kInitiateRebuildMessage);
free(proto_bytes);
- LOG(INFO) << "ForemanDistributed sent InitiateRebuildMessage (typed '" << kInitiateRebuildMessage
+ LOG(INFO) << "QueryManagerDistributed sent InitiateRebuildMessage (typed '" << kInitiateRebuildMessage
<< "') to Shiftboss";
// TODO(zuyu): Multiple workers support.
- QueryExecutionUtil::SendTMBMessage(bus_,
- foreman_client_id_,
- shiftbosses_->getClientId(0),
- move(tagged_msg));
+ const tmb::MessageBus::SendStatus send_status =
+ QueryExecutionUtil::SendTMBMessage(bus_,
+ foreman_client_id_,
+ shiftboss_directory_->getClientId(0),
+ move(tagged_msg));
+ CHECK(send_status == tmb::MessageBus::SendStatus::kOK)
+ << "Message could not be sent from Foreman with TMB client ID " << foreman_client_id_
+ << " to Shiftboss with TMB client ID " << shiftboss_directory_->getClientId(0);
// The negative value indicates that the number of rebuild work orders is to be
// determined.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/603b6149/query_execution/QueryManagerDistributed.hpp
----------------------------------------------------------------------
diff --git a/query_execution/QueryManagerDistributed.hpp b/query_execution/QueryManagerDistributed.hpp
index 8641c22..9a3f44b 100644
--- a/query_execution/QueryManagerDistributed.hpp
+++ b/query_execution/QueryManagerDistributed.hpp
@@ -15,6 +15,7 @@
#ifndef QUICKSTEP_QUERY_EXECUTION_QUERY_MANAGER_DISTRIBUTED_HPP_
#define QUICKSTEP_QUERY_EXECUTION_QUERY_MANAGER_DISTRIBUTED_HPP_
+#include <cstddef>
#include <memory>
#include "query_execution/QueryExecutionState.hpp"
@@ -47,12 +48,12 @@ class QueryManagerDistributed final : public QueryManagerBase {
* @brief Constructor.
*
* @param query_handle The QueryHandle object for this query.
- * @param shiftbosses The ShiftbossDirectory to use.
+ * @param shiftboss_directory The ShiftbossDirectory to use.
* @param foreman_client_id The TMB client ID of the foreman thread.
* @param bus The TMB used for communication.
**/
QueryManagerDistributed(QueryHandle *query_handle,
- ShiftbossDirectory *shiftbosses,
+ ShiftbossDirectory *shiftboss_directory,
const tmb::client_id foreman_client_id,
tmb::MessageBus *bus);
@@ -60,6 +61,9 @@ class QueryManagerDistributed final : public QueryManagerBase {
bool fetchNormalWorkOrders(const dag_node_index index) override;
+ void processInitiateRebuildResponseMessage(const dag_node_index op_index,
+ const std::size_t num_rebuild_work_orders) override;
+
/**
* @brief Get the next normal workorder to be excuted, wrapped in a
* WorkOrderMessage proto.
@@ -88,7 +92,7 @@ class QueryManagerDistributed final : public QueryManagerBase {
(query_exec_state_->getNumRebuildWorkOrders(index) == 0);
}
- ShiftbossDirectory *shiftbosses_;
+ ShiftbossDirectory *shiftboss_directory_;
const tmb::client_id foreman_client_id_;
tmb::MessageBus *bus_;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/603b6149/query_execution/Shiftboss.cpp
----------------------------------------------------------------------
diff --git a/query_execution/Shiftboss.cpp b/query_execution/Shiftboss.cpp
index 7f655c6..925dc1f 100644
--- a/query_execution/Shiftboss.cpp
+++ b/query_execution/Shiftboss.cpp
@@ -113,10 +113,14 @@ void Shiftboss::run() {
<< "') forwarded WorkOrderMessage (typed '" << kWorkOrderMessage
<< "') from Foreman to worker " << worker_index;
- QueryExecutionUtil::SendTMBMessage(bus_,
- shiftboss_client_id_,
- workers_->getClientID(worker_index),
- move(worker_tagged_message));
+ const tmb::MessageBus::SendStatus send_status =
+ QueryExecutionUtil::SendTMBMessage(bus_,
+ shiftboss_client_id_,
+ workers_->getClientID(worker_index),
+ move(worker_tagged_message));
+ DCHECK(send_status == tmb::MessageBus::SendStatus::kOK)
+ << "Message could not be sent from Shiftboss with TMB client ID " << shiftboss_client_id_
+ << " to Worker with TMB client ID " << workers_->getClientID(worker_index);
break;
}
case kInitiateRebuildMessage: {
@@ -143,10 +147,14 @@ void Shiftboss::run() {
<< "' message from worker (client " << annotated_message.sender << ") to Foreman";
DCHECK_NE(foreman_client_id_, tmb::kClientIdNone);
- QueryExecutionUtil::SendTMBMessage(bus_,
- shiftboss_client_id_,
- foreman_client_id_,
- move(annotated_message.tagged_message));
+ const tmb::MessageBus::SendStatus send_status =
+ QueryExecutionUtil::SendTMBMessage(bus_,
+ shiftboss_client_id_,
+ foreman_client_id_,
+ move(annotated_message.tagged_message));
+ DCHECK(send_status == tmb::MessageBus::SendStatus::kOK)
+ << "Message could not be sent from Shiftboss with TMB client ID " << shiftboss_client_id_
+ << " to Foreman with TMB client ID " << foreman_client_id_;
break;
}
case kSaveQueryResultMessage: {
@@ -167,8 +175,11 @@ void Shiftboss::run() {
}
}
+ query_contexts_.erase(proto.query_id());
+
serialization::SaveQueryResultResponseMessage proto_response;
proto_response.set_relation_id(proto.relation_id());
+ proto_response.set_cli_id(proto.cli_id());
const size_t proto_response_length = proto_response.ByteSize();
char *proto_response_bytes = static_cast<char*>(malloc(proto_response_length));
@@ -182,10 +193,14 @@ void Shiftboss::run() {
LOG(INFO) << "Shiftboss (id '" << shiftboss_client_id_
<< "') sent SaveQueryResultResponseMessage (typed '" << kSaveQueryResultResponseMessage
<< "') to Foreman";
- QueryExecutionUtil::SendTMBMessage(bus_,
- shiftboss_client_id_,
- foreman_client_id_,
- move(message_response));
+ const tmb::MessageBus::SendStatus send_status =
+ QueryExecutionUtil::SendTMBMessage(bus_,
+ shiftboss_client_id_,
+ foreman_client_id_,
+ move(message_response));
+ DCHECK(send_status == tmb::MessageBus::SendStatus::kOK)
+ << "Message could not be sent from Shiftboss with TMB client ID " << shiftboss_client_id_
+ << " to Foreman with TMB client ID " << foreman_client_id_;
break;
}
case kPoisonMessage: {
@@ -196,7 +211,7 @@ void Shiftboss::run() {
tmb::MessageStyle broadcast_style;
broadcast_style.Broadcast(true);
- tmb::MessageBus::SendStatus send_status =
+ const tmb::MessageBus::SendStatus send_status =
bus_->Send(shiftboss_client_id_,
worker_addresses_,
broadcast_style,
@@ -249,7 +264,7 @@ void Shiftboss::registerWithForeman() {
kShiftbossRegistrationMessage);
free(proto_bytes);
- tmb::MessageBus::SendStatus send_status =
+ const tmb::MessageBus::SendStatus send_status =
bus_->Send(shiftboss_client_id_, all_addresses, style, move(message));
DCHECK(send_status == tmb::MessageBus::SendStatus::kOK);
}
@@ -268,10 +283,6 @@ void Shiftboss::processQueryInitiateMessage(
bus_));
query_contexts_.emplace(query_id, move(query_context));
- LOG(INFO) << "Shiftboss (id '" << shiftboss_client_id_
- << "') sent QueryInitiateResponseMessage (typed '" << kQueryInitiateResponseMessage
- << "') to Foreman";
-
serialization::QueryInitiateResponseMessage proto;
proto.set_query_id(query_id);
@@ -284,10 +295,18 @@ void Shiftboss::processQueryInitiateMessage(
kQueryInitiateResponseMessage);
free(proto_bytes);
- QueryExecutionUtil::SendTMBMessage(bus_,
- shiftboss_client_id_,
- foreman_client_id_,
- move(message_response));
+ LOG(INFO) << "Shiftboss (id '" << shiftboss_client_id_
+ << "') sent QueryInitiateResponseMessage (typed '" << kQueryInitiateResponseMessage
+ << "') to Foreman";
+
+ const tmb::MessageBus::SendStatus send_status =
+ QueryExecutionUtil::SendTMBMessage(bus_,
+ shiftboss_client_id_,
+ foreman_client_id_,
+ move(message_response));
+ DCHECK(send_status == tmb::MessageBus::SendStatus::kOK)
+ << "Message could not be sent from Shiftboss with TMB client ID " << shiftboss_client_id_
+ << " to Foreman with TMB client ID " << foreman_client_id_;
}
void Shiftboss::processInitiateRebuildMessage(const std::size_t query_id,
@@ -311,6 +330,8 @@ void Shiftboss::processInitiateRebuildMessage(const std::size_t query_id,
proto.set_query_id(query_id);
proto.set_operator_index(op_index);
proto.set_num_rebuild_work_orders(partially_filled_block_refs.size());
+ // TODO(zuyu): Multiple Shiftboss support.
+ proto.set_shiftboss_index(0);
const size_t proto_length = proto.ByteSize();
char *proto_bytes = static_cast<char*>(malloc(proto_length));
@@ -321,10 +342,14 @@ void Shiftboss::processInitiateRebuildMessage(const std::size_t query_id,
kInitiateRebuildResponseMessage);
free(proto_bytes);
- QueryExecutionUtil::SendTMBMessage(bus_,
- shiftboss_client_id_,
- foreman_client_id_,
- move(message_response));
+ const tmb::MessageBus::SendStatus send_status =
+ QueryExecutionUtil::SendTMBMessage(bus_,
+ shiftboss_client_id_,
+ foreman_client_id_,
+ move(message_response));
+ DCHECK(send_status == tmb::MessageBus::SendStatus::kOK)
+ << "Message could not be sent from Shiftboss with TMB client ID " << shiftboss_client_id_
+ << " to Foreman with TMB client ID " << foreman_client_id_;
for (size_t i = 0; i < partially_filled_block_refs.size(); ++i) {
// NOTE(zuyu): Worker releases the memory after the execution of
@@ -349,10 +374,14 @@ void Shiftboss::processInitiateRebuildMessage(const std::size_t query_id,
<< "') sent RebuildWorkOrderMessage (typed '" << kRebuildWorkOrderMessage
<< "') to worker " << worker_index;
- QueryExecutionUtil::SendTMBMessage(bus_,
- shiftboss_client_id_,
- workers_->getClientID(worker_index),
- move(worker_tagged_message));
+ const tmb::MessageBus::SendStatus send_status =
+ QueryExecutionUtil::SendTMBMessage(bus_,
+ shiftboss_client_id_,
+ workers_->getClientID(worker_index),
+ move(worker_tagged_message));
+ DCHECK(send_status == tmb::MessageBus::SendStatus::kOK)
+ << "Message could not be sent from Shiftboss with TMB client ID " << shiftboss_client_id_
+ << " to Worker with TMB client ID " << workers_->getClientID(worker_index);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/603b6149/query_optimizer/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_optimizer/CMakeLists.txt b/query_optimizer/CMakeLists.txt
index a56b714..b6b97a0 100644
--- a/query_optimizer/CMakeLists.txt
+++ b/query_optimizer/CMakeLists.txt
@@ -212,6 +212,10 @@ target_link_libraries(quickstep_queryoptimizer_QueryHandle
quickstep_queryexecution_QueryContext_proto
quickstep_queryoptimizer_QueryPlan
quickstep_utility_Macros)
+if (ENABLE_DISTRIBUTED)
+ target_link_libraries(quickstep_queryoptimizer_QueryHandle
+ tmb)
+endif(ENABLE_DISTRIBUTED)
target_link_libraries(quickstep_queryoptimizer_QueryPlan
quickstep_relationaloperators_RelationalOperator
quickstep_utility_DAG
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/603b6149/query_optimizer/QueryHandle.hpp
----------------------------------------------------------------------
diff --git a/query_optimizer/QueryHandle.hpp b/query_optimizer/QueryHandle.hpp
index 5f3649a..bbf1918 100644
--- a/query_optimizer/QueryHandle.hpp
+++ b/query_optimizer/QueryHandle.hpp
@@ -24,9 +24,14 @@
#include "catalog/Catalog.pb.h"
#include "query_execution/QueryContext.pb.h"
+#include "query_optimizer/QueryOptimizerConfig.h" // For QUICKSTEP_DISTRIBUTED.
#include "query_optimizer/QueryPlan.hpp"
#include "utility/Macros.hpp"
+#ifdef QUICKSTEP_DISTRIBUTED
+#include "tmb/id_typedefs.h"
+#endif // QUICKSTEP_DISTRIBUTED
+
namespace quickstep {
class CatalogRelation;
@@ -119,6 +124,22 @@ class QueryHandle {
query_result_relation_ = relation;
}
+#ifdef QUICKSTEP_DISTRIBUTED
+ /**
+ * @brief Get the client id.
+ */
+ tmb::client_id getClientId() const {
+ return cli_id_;
+ }
+
+ /**
+ * @brief Set the client id.
+ */
+ void setClientId(const tmb::client_id cli_id) {
+ cli_id_ = cli_id;
+ }
+#endif // QUICKSTEP_DISTRIBUTED
+
private:
const std::size_t query_id_;
const std::uint64_t query_priority_;
@@ -134,6 +155,11 @@ class QueryHandle {
// and deleted by the Cli shell.
const CatalogRelation *query_result_relation_;
+#ifdef QUICKSTEP_DISTRIBUTED
+ // The client id of the CLI which sends the query.
+ tmb::client_id cli_id_;
+#endif // QUICKSTEP_DISTRIBUTED
+
DISALLOW_COPY_AND_ASSIGN(QueryHandle);
};
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/603b6149/query_optimizer/tests/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_optimizer/tests/CMakeLists.txt b/query_optimizer/tests/CMakeLists.txt
index 9cad47f..6522117 100644
--- a/query_optimizer/tests/CMakeLists.txt
+++ b/query_optimizer/tests/CMakeLists.txt
@@ -78,6 +78,14 @@ target_link_libraries(quickstep_queryoptimizer_tests_TestDatabaseLoader
quickstep_utility_Macros
tmb)
+if (ENABLE_DISTRIBUTED)
+ add_executable(quickstep_queryoptimizer_tests_DistributedExecutionGeneratorTest
+ DistributedExecutionGeneratorTest.cpp
+ DistributedExecutionGeneratorTestRunner.cpp
+ DistributedExecutionGeneratorTestRunner.hpp
+ "${PROJECT_SOURCE_DIR}/utility/textbased_test/TextBasedTest.cpp"
+ "${PROJECT_SOURCE_DIR}/utility/textbased_test/TextBasedTest.hpp")
+endif(ENABLE_DISTRIBUTED)
add_executable(quickstep_queryoptimizer_tests_ExecutionGeneratorTest
ExecutionGeneratorTest.cpp
ExecutionGeneratorTestRunner.cpp
@@ -107,6 +115,39 @@ add_executable(quickstep_queryoptimizer_tests_OptimizerTextTest
"${PROJECT_SOURCE_DIR}/utility/textbased_test/TextBasedTest.cpp"
"${PROJECT_SOURCE_DIR}/utility/textbased_test/TextBasedTest.hpp")
+if (ENABLE_DISTRIBUTED)
+ target_link_libraries(quickstep_queryoptimizer_tests_DistributedExecutionGeneratorTest
+ glog
+ gtest
+ gtest_main
+ quickstep_catalog_CatalogDatabase
+ quickstep_catalog_CatalogTypedefs
+ quickstep_cli_DropRelation
+ quickstep_cli_PrintToScreen
+ quickstep_parser_ParseStatement
+ quickstep_parser_SqlParserWrapper
+ quickstep_queryexecution_ForemanDistributed
+ quickstep_queryexecution_QueryContext
+ quickstep_queryexecution_QueryExecutionTypedefs
+ quickstep_queryexecution_QueryExecutionUtil
+ quickstep_queryexecution_Shiftboss
+ quickstep_queryexecution_Worker
+ quickstep_queryexecution_WorkerDirectory
+ quickstep_queryexecution_WorkerMessage
+ quickstep_queryoptimizer_ExecutionGenerator
+ quickstep_queryoptimizer_LogicalGenerator
+ quickstep_queryoptimizer_OptimizerContext
+ quickstep_queryoptimizer_PhysicalGenerator
+ quickstep_queryoptimizer_QueryHandle
+ quickstep_queryoptimizer_physical_Physical
+ quickstep_queryoptimizer_tests_TestDatabaseLoader
+ quickstep_utility_Macros
+ quickstep_utility_MemStream
+ quickstep_utility_SqlError
+ quickstep_utility_TextBasedTestDriver
+ tmb
+ ${LIBS})
+endif(ENABLE_DISTRIBUTED)
target_link_libraries(quickstep_queryoptimizer_tests_ExecutionGeneratorTest
${GFLAGS_LIB_NAME}
glog
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/603b6149/query_optimizer/tests/DistributedExecutionGeneratorTest.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/tests/DistributedExecutionGeneratorTest.cpp b/query_optimizer/tests/DistributedExecutionGeneratorTest.cpp
new file mode 100644
index 0000000..fc0c67d
--- /dev/null
+++ b/query_optimizer/tests/DistributedExecutionGeneratorTest.cpp
@@ -0,0 +1,57 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#include <fstream>
+#include <memory>
+
+#include "query_optimizer/tests/DistributedExecutionGeneratorTestRunner.hpp"
+#include "utility/textbased_test/TextBasedTestDriver.hpp"
+#include "utility/textbased_test/TextBasedTest.hpp"
+
+#include "gflags/gflags.h"
+#include "glog/logging.h"
+#include "gtest/gtest.h"
+
+using quickstep::TextBasedTest;
+
+QUICKSTEP_GENERATE_TEXT_TEST(DISTRIBUTED_EXECUTION_GENERATOR_TEST);
+
+int main(int argc, char** argv) {
+ google::InitGoogleLogging(argv[0]);
+ // Honor FLAGS_buffer_pool_slots in StorageManager.
+ gflags::ParseCommandLineFlags(&argc, &argv, true);
+
+ if (argc < 4) {
+ LOG(ERROR) << "Must have at least 3 arguments, but " << argc - 1
+ << " are provided";
+ }
+
+ std::ifstream input_file(argv[1]);
+ CHECK(input_file.is_open()) << argv[1];
+ std::unique_ptr<quickstep::optimizer::DistributedExecutionGeneratorTestRunner>
+ test_runner(
+ new quickstep::optimizer::DistributedExecutionGeneratorTestRunner(argv[3]));
+ test_driver.reset(
+ new quickstep::TextBasedTestDriver(&input_file, test_runner.get()));
+ test_driver->registerOption(
+ quickstep::optimizer::DistributedExecutionGeneratorTestRunner::kResetOption);
+
+ ::testing::InitGoogleTest(&argc, argv);
+ const int success = RUN_ALL_TESTS();
+ if (success != 0) {
+ test_driver->writeActualOutputToFile(argv[2]);
+ }
+
+ return success;
+}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/603b6149/query_optimizer/tests/DistributedExecutionGeneratorTestRunner.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/tests/DistributedExecutionGeneratorTestRunner.cpp b/query_optimizer/tests/DistributedExecutionGeneratorTestRunner.cpp
new file mode 100644
index 0000000..ffed4f0
--- /dev/null
+++ b/query_optimizer/tests/DistributedExecutionGeneratorTestRunner.cpp
@@ -0,0 +1,122 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#include "query_optimizer/tests/DistributedExecutionGeneratorTestRunner.hpp"
+
+#include <cstdio>
+#include <set>
+#include <string>
+
+#include "cli/DropRelation.hpp"
+#include "cli/PrintToScreen.hpp"
+#include "parser/ParseStatement.hpp"
+#include "query_execution/ForemanDistributed.hpp"
+#include "query_execution/QueryExecutionTypedefs.hpp"
+#include "query_optimizer/ExecutionGenerator.hpp"
+#include "query_optimizer/LogicalGenerator.hpp"
+#include "query_optimizer/OptimizerContext.hpp"
+#include "query_optimizer/PhysicalGenerator.hpp"
+#include "query_optimizer/QueryHandle.hpp"
+#include "query_optimizer/physical/Physical.hpp"
+#include "utility/MemStream.hpp"
+#include "utility/SqlError.hpp"
+
+#include "glog/logging.h"
+
+#include "tmb/message_bus.h"
+#include "tmb/tagged_message.h"
+
+namespace quickstep {
+
+class CatalogRelation;
+
+namespace optimizer {
+
+const char *DistributedExecutionGeneratorTestRunner::kResetOption =
+ "reset_before_execution";
+
+void DistributedExecutionGeneratorTestRunner::runTestCase(
+ const std::string &input, const std::set<std::string> &options,
+ std::string *output) {
+ // TODO(qzeng): Test multi-threaded query execution when we have a Sort operator.
+
+ VLOG(4) << "Test SQL(s): " << input;
+
+ if (options.find(kResetOption) != options.end()) {
+ test_database_loader_.clear();
+ test_database_loader_.createTestRelation(false /* allow_vchar */);
+ test_database_loader_.loadTestRelation();
+ }
+
+ MemStream output_stream;
+ sql_parser_.feedNextBuffer(new std::string(input));
+
+ while (true) {
+ ParseResult result = sql_parser_.getNextStatement();
+
+ OptimizerContext optimizer_context(query_id_++,
+ test_database_loader_.catalog_database(),
+ test_database_loader_.storage_manager());
+
+ if (result.condition != ParseResult::kSuccess) {
+ if (result.condition == ParseResult::kError) {
+ *output = result.error_message;
+ }
+ break;
+ }
+
+ std::printf("%s\n", result.parsed_statement->toString().c_str());
+ try {
+ QueryHandle query_handle(optimizer_context.query_id());
+ LogicalGenerator logical_generator(&optimizer_context);
+ PhysicalGenerator physical_generator;
+ ExecutionGenerator execution_generator(&optimizer_context,
+ &query_handle);
+
+ const physical::PhysicalPtr physical_plan =
+ physical_generator.generatePlan(
+ logical_generator.generatePlan(*result.parsed_statement));
+ execution_generator.generatePlan(physical_plan);
+
+ QueryExecutionUtil::ConstructAndSendAdmitRequestMessage(
+ cli_id_,
+ foreman_->getBusClientID(),
+ &query_handle,
+ &bus_);
+
+ const tmb::AnnotatedMessage annotated_message = bus_.Receive(cli_id_, 0, true);
+ DCHECK_EQ(kQueryExecutionSuccessMessage, annotated_message.tagged_message.message_type());
+
+ const CatalogRelation *query_result_relation = query_handle.getQueryResultRelation();
+ if (query_result_relation) {
+ PrintToScreen::PrintRelation(*query_result_relation,
+ test_database_loader_.storage_manager(),
+ output_stream.file());
+ DropRelation::Drop(*query_result_relation,
+ test_database_loader_.catalog_database(),
+ test_database_loader_.storage_manager());
+ }
+ } catch (const SqlError &error) {
+ *output = error.formatMessage(input);
+ break;
+ }
+ }
+
+ if (output->empty()) {
+ *output = output_stream.str();
+ }
+}
+
+} // namespace optimizer
+} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/603b6149/query_optimizer/tests/DistributedExecutionGeneratorTestRunner.hpp
----------------------------------------------------------------------
diff --git a/query_optimizer/tests/DistributedExecutionGeneratorTestRunner.hpp b/query_optimizer/tests/DistributedExecutionGeneratorTestRunner.hpp
new file mode 100644
index 0000000..cd59596
--- /dev/null
+++ b/query_optimizer/tests/DistributedExecutionGeneratorTestRunner.hpp
@@ -0,0 +1,146 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#ifndef QUICKSTEP_QUERY_OPTIMIZER_TESTS_DISTRIBUTED_EXECUTION_GENERATOR_TEST_RUNNER_HPP_
+#define QUICKSTEP_QUERY_OPTIMIZER_TESTS_DISTRIBUTED_EXECUTION_GENERATOR_TEST_RUNNER_HPP_
+
+#include <memory>
+#include <set>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "catalog/CatalogDatabase.hpp"
+#include "catalog/CatalogTypedefs.hpp"
+#include "parser/SqlParserWrapper.hpp"
+#include "query_execution/ForemanDistributed.hpp"
+#include "query_execution/QueryExecutionTypedefs.hpp"
+#include "query_execution/QueryExecutionUtil.hpp"
+#include "query_execution/Shiftboss.hpp"
+#include "query_execution/Worker.hpp"
+#include "query_execution/WorkerDirectory.hpp"
+#include "query_execution/WorkerMessage.hpp"
+#include "query_optimizer/tests/TestDatabaseLoader.hpp"
+#include "utility/Macros.hpp"
+#include "utility/textbased_test/TextBasedTestDriver.hpp"
+#include "utility/textbased_test/TextBasedTestRunner.hpp"
+
+#include "glog/logging.h"
+
+#include "tmb/id_typedefs.h"
+#include "tmb/message_bus.h"
+#include "tmb/tagged_message.h"
+
+namespace quickstep {
+namespace optimizer {
+
+/**
+ * @brief TextBasedTestRunner for testing the ExecutionGenerator in the
+ * distributed version.
+ */
+class DistributedExecutionGeneratorTestRunner : public TextBasedTestRunner {
+ public:
+ /**
+ * @brief If this option is enabled, recreate the entire database and
+ * repopulate the data before every test.
+ */
+ static const char *kResetOption;
+
+ /**
+ * @brief Constructor.
+ */
+ explicit DistributedExecutionGeneratorTestRunner(const std::string &storage_path)
+ : query_id_(0),
+ test_database_loader_(storage_path) {
+ test_database_loader_.createTestRelation(false /* allow_vchar */);
+ test_database_loader_.loadTestRelation();
+
+ bus_.Initialize();
+
+ // NOTE(zuyu): Foreman should initialize before Shiftboss so that the former
+ // could receive a registration message from the latter.
+ foreman_.reset(new ForemanDistributed(&bus_, test_database_loader_.catalog_database()));
+
+ worker_.reset(new Worker(0 /* worker_thread_index */, &bus_));
+
+ std::vector<tmb::client_id> worker_client_ids;
+ worker_client_ids.push_back(worker_->getBusClientID());
+
+ // We don't use the NUMA aware version of worker code.
+ const std::vector<numa_node_id> numa_nodes(worker_client_ids.size(), kAnyNUMANodeID);
+
+ workers_.reset(
+ new WorkerDirectory(worker_client_ids.size(), worker_client_ids, numa_nodes));
+
+ shiftboss_.reset(new Shiftboss(&bus_, test_database_loader_.storage_manager(), workers_.get()));
+
+ cli_id_ = bus_.Connect();
+ bus_.RegisterClientAsSender(cli_id_, kAdmitRequestMessage);
+ bus_.RegisterClientAsSender(cli_id_, kPoisonMessage);
+ bus_.RegisterClientAsReceiver(cli_id_, kQueryExecutionSuccessMessage);
+
+ foreman_->start();
+
+ shiftboss_->start();
+ worker_->start();
+ }
+
+ ~DistributedExecutionGeneratorTestRunner() {
+ std::unique_ptr<WorkerMessage> poison_message(WorkerMessage::PoisonMessage());
+ tmb::TaggedMessage poison_tagged_message(poison_message.get(),
+ sizeof(*poison_message),
+ quickstep::kPoisonMessage);
+
+ const tmb::MessageBus::SendStatus send_status =
+ QueryExecutionUtil::SendTMBMessage(
+ &bus_,
+ cli_id_,
+ foreman_->getBusClientID(),
+ std::move(poison_tagged_message));
+ CHECK(send_status == tmb::MessageBus::SendStatus::kOK);
+
+ worker_->join();
+ shiftboss_->join();
+
+ foreman_->join();
+ }
+
+ void runTestCase(const std::string &input,
+ const std::set<std::string> &options,
+ std::string *output) override;
+
+ private:
+ std::size_t query_id_;
+
+ SqlParserWrapper sql_parser_;
+ TestDatabaseLoader test_database_loader_;
+
+ MessageBusImpl bus_;
+
+ tmb::client_id cli_id_;
+
+ std::unique_ptr<ForemanDistributed> foreman_;
+
+ std::unique_ptr<Worker> worker_;
+ std::unique_ptr<WorkerDirectory> workers_;
+
+ std::unique_ptr<Shiftboss> shiftboss_;
+
+ DISALLOW_COPY_AND_ASSIGN(DistributedExecutionGeneratorTestRunner);
+};
+
+} // namespace optimizer
+} // namespace quickstep
+
+#endif // QUICKSTEP_QUERY_OPTIMIZER_TESTS_DISTRIBUTED_EXECUTION_GENERATOR_TEST_RUNNER_HPP_
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/603b6149/query_optimizer/tests/execution_generator/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_optimizer/tests/execution_generator/CMakeLists.txt b/query_optimizer/tests/execution_generator/CMakeLists.txt
index 56bae16..cd0e626 100644
--- a/query_optimizer/tests/execution_generator/CMakeLists.txt
+++ b/query_optimizer/tests/execution_generator/CMakeLists.txt
@@ -13,6 +13,61 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+add_test(quickstep_queryoptimizer_tests_distributed_executiongenerator_create
+ "../quickstep_queryoptimizer_tests_DistributedExecutionGeneratorTest"
+ "${CMAKE_CURRENT_SOURCE_DIR}/Create.test"
+ "${CMAKE_CURRENT_BINARY_DIR}/Create.test"
+ "${CMAKE_CURRENT_BINARY_DIR}/DistributedCreate/")
+add_test(quickstep_queryoptimizer_tests_distributed_executiongenerator_delete
+ "../quickstep_queryoptimizer_tests_ExecutionGeneratorTest"
+ "${CMAKE_CURRENT_SOURCE_DIR}/Delete.test"
+ "${CMAKE_CURRENT_BINARY_DIR}/Delete.test"
+ "${CMAKE_CURRENT_BINARY_DIR}/DistributedDelete/")
+add_test(quickstep_queryoptimizer_tests_distributed_executiongenerator_distinct
+ "../quickstep_queryoptimizer_tests_ExecutionGeneratorTest"
+ "${CMAKE_CURRENT_SOURCE_DIR}/Distinct.test"
+ "${CMAKE_CURRENT_BINARY_DIR}/Distinct.test"
+ "${CMAKE_CURRENT_BINARY_DIR}/DistributedDistinct/")
+add_test(quickstep_queryoptimizer_tests_distributed_executiongenerator_drop
+ "../quickstep_queryoptimizer_tests_ExecutionGeneratorTest"
+ "${CMAKE_CURRENT_SOURCE_DIR}/Drop.test"
+ "${CMAKE_CURRENT_BINARY_DIR}/Drop.test"
+ "${CMAKE_CURRENT_BINARY_DIR}/DistributedDrop/")
+add_test(quickstep_queryoptimizer_tests_distributed_executiongenerator_index
+ "../quickstep_queryoptimizer_tests_ExecutionGeneratorTest"
+ "${CMAKE_CURRENT_SOURCE_DIR}/Index.test"
+ "${CMAKE_CURRENT_BINARY_DIR}/Index.test"
+ "${CMAKE_CURRENT_BINARY_DIR}/DistributedIndex/")
+add_test(quickstep_queryoptimizer_tests_distributed_executiongenerator_insert
+ "../quickstep_queryoptimizer_tests_ExecutionGeneratorTest"
+ "${CMAKE_CURRENT_SOURCE_DIR}/Insert.test"
+ "${CMAKE_CURRENT_BINARY_DIR}/Insert.test"
+ "${CMAKE_CURRENT_BINARY_DIR}/DistributedInsert/")
+add_test(quickstep_queryoptimizer_tests_distributed_executiongenerator_join
+ "../quickstep_queryoptimizer_tests_ExecutionGeneratorTest"
+ "${CMAKE_CURRENT_SOURCE_DIR}/Join.test"
+ "${CMAKE_CURRENT_BINARY_DIR}/Join.test"
+ "${CMAKE_CURRENT_BINARY_DIR}/DistributedJoin/")
+add_test(quickstep_queryoptimizer_tests_distributed_executiongenerator_select
+ "../quickstep_queryoptimizer_tests_ExecutionGeneratorTest"
+ "${CMAKE_CURRENT_SOURCE_DIR}/Select.test"
+ "${CMAKE_CURRENT_BINARY_DIR}/Select.test"
+ "${CMAKE_CURRENT_BINARY_DIR}/DistributedSelect/")
+add_test(quickstep_queryoptimizer_tests_distributed_executiongenerator_stringpatternmatching
+ "../quickstep_queryoptimizer_tests_ExecutionGeneratorTest"
+ "${CMAKE_CURRENT_SOURCE_DIR}/StringPatternMatching.test"
+ "${CMAKE_CURRENT_BINARY_DIR}/StringPatternMatching.test"
+ "${CMAKE_CURRENT_BINARY_DIR}/DistributedStringPatternMatching/")
+add_test(quickstep_queryoptimizer_tests_distributed_executiongenerator_tablegenerator
+ "../quickstep_queryoptimizer_tests_ExecutionGeneratorTest"
+ "${CMAKE_CURRENT_SOURCE_DIR}/TableGenerator.test"
+ "${CMAKE_CURRENT_BINARY_DIR}/TableGenerator.test"
+ "${CMAKE_CURRENT_BINARY_DIR}/DistributedTableGenerator/")
+add_test(quickstep_queryoptimizer_tests_distributed_executiongenerator_update
+ "../quickstep_queryoptimizer_tests_ExecutionGeneratorTest"
+ "${CMAKE_CURRENT_SOURCE_DIR}/Update.test"
+ "${CMAKE_CURRENT_BINARY_DIR}/Update.test"
+ "${CMAKE_CURRENT_BINARY_DIR}/DistributedUpdate/")
add_test(quickstep_queryoptimizer_tests_executiongenerator_create
"../quickstep_queryoptimizer_tests_ExecutionGeneratorTest"
"${CMAKE_CURRENT_SOURCE_DIR}/Create.test"
@@ -74,6 +129,17 @@ add_test(quickstep_queryoptimizer_tests_executiongenerator_update
file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/Create)
file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/Delete)
file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/Distinct)
+file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/DistributedCreate)
+file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/DistributedDelete)
+file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/DistributedDistinct)
+file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/DistributedDrop)
+file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/DistributedIndex)
+file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/DistributedInsert)
+file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/DistributedJoin)
+file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/DistributedSelect)
+file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/DistributedStringPatternMatching)
+file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/DistributedTableGenerator)
+file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/DistributedUpdate)
file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/Drop)
file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/Index)
file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/Insert)
@@ -81,4 +147,4 @@ file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/Join)
file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/Select)
file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/StringPatternMatching)
file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/TableGenerator)
-file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/Update)
\ No newline at end of file
+file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/Update)
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/603b6149/third_party/tmb/include/tmb/tagged_message.h
----------------------------------------------------------------------
diff --git a/third_party/tmb/include/tmb/tagged_message.h b/third_party/tmb/include/tmb/tagged_message.h
index 49dcee7..75b980e 100644
--- a/third_party/tmb/include/tmb/tagged_message.h
+++ b/third_party/tmb/include/tmb/tagged_message.h
@@ -63,6 +63,15 @@ class TaggedMessage {
}
/**
+ * @brief Constructor which creates an empty, typed message.
+ **/
+ explicit TaggedMessage(const message_type_id message_type)
+ : payload_inline_(true),
+ message_type_(message_type) {
+ payload_.in_line.size = 0;
+ }
+
+ /**
* @brief Constructor.
*
* @param msg A pointer to the message contents in memory, which will be
[12/25] incubator-quickstep git commit: Disallow negative number of
worker threads.
Posted by zu...@apache.org.
Disallow negative number of worker threads.
- Fixed a bug thereby Quickstep command line now disallows negative
number of worker threads.
- If the user provides zero or fewer worker threads, we switch to the
default number of worker threasd, instead of terminating the process.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/e5224a1d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/e5224a1d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/e5224a1d
Branch: refs/heads/dist-exe-test-new
Commit: e5224a1d64faacdd8516482aa2f0ab581755c00e
Parents: 6bbdb4d
Author: Harshad Deshmukh <hb...@apache.org>
Authored: Thu Jun 30 11:04:29 2016 -0500
Committer: Zuyu Zhang <zu...@twitter.com>
Committed: Fri Jul 29 16:42:10 2016 -0700
----------------------------------------------------------------------
cli/QuickstepCli.cpp | 23 +++++++++++------------
1 file changed, 11 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/e5224a1d/cli/QuickstepCli.cpp
----------------------------------------------------------------------
diff --git a/cli/QuickstepCli.cpp b/cli/QuickstepCli.cpp
index 3f99130..02a55a0 100644
--- a/cli/QuickstepCli.cpp
+++ b/cli/QuickstepCli.cpp
@@ -200,19 +200,18 @@ int main(int argc, char* argv[]) {
// that we computed above, provided it did return a valid value.
// TODO(jmp): May need to change this at some point to keep one thread
// available for the OS if the hardware concurrency level is high.
- const unsigned int real_num_workers = quickstep::FLAGS_num_workers != 0
- ? quickstep::FLAGS_num_workers
- : (num_hw_threads != 0 ?
- num_hw_threads
- : 1);
-
- if (real_num_workers > 0) {
- printf("Starting Quickstep with %d worker thread(s) and a %.2f GB buffer pool\n",
- real_num_workers,
- (static_cast<double>(quickstep::FLAGS_buffer_pool_slots) * quickstep::kSlotSizeBytes)/quickstep::kAGigaByte);
- } else {
- LOG(FATAL) << "Quickstep needs at least one worker thread to run";
+ if (quickstep::FLAGS_num_workers <= 0) {
+ LOG(INFO) << "Quickstep expects at least one worker thread, switching to "
+ "the default number of worker threads";
}
+ const int real_num_workers = quickstep::FLAGS_num_workers > 0
+ ? quickstep::FLAGS_num_workers
+ : (num_hw_threads != 0 ? num_hw_threads : 1);
+
+ DCHECK_GT(real_num_workers, 0);
+ printf("Starting Quickstep with %d worker thread(s) and a %.2f GB buffer pool\n",
+ real_num_workers,
+ (static_cast<double>(quickstep::FLAGS_buffer_pool_slots) * quickstep::kSlotSizeBytes)/quickstep::kAGigaByte);
#ifdef QUICKSTEP_HAVE_FILE_MANAGER_HDFS
if (quickstep::FLAGS_use_hdfs) {
[24/25] incubator-quickstep git commit: Added README for types module.
Posted by zu...@apache.org.
Added README for types module.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/614abab9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/614abab9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/614abab9
Branch: refs/heads/dist-exe-test-new
Commit: 614abab92224c6a12ee53252edba87a1912b4511
Parents: db66fde
Author: Craig Chasseur <sp...@gmail.com>
Authored: Wed Jul 27 20:01:53 2016 -0700
Committer: Zuyu Zhang <zu...@twitter.com>
Committed: Fri Jul 29 16:43:24 2016 -0700
----------------------------------------------------------------------
types/README.md | 102 +++++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 102 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/614abab9/types/README.md
----------------------------------------------------------------------
diff --git a/types/README.md b/types/README.md
new file mode 100644
index 0000000..baf01aa
--- /dev/null
+++ b/types/README.md
@@ -0,0 +1,102 @@
+# The Quickstep Type System
+
+The types module is used across Quickstep and handles details of how date values
+are stored and represented, how they are parsed from and printed to
+human-readable text, and low-level operations on values that form the building
+blocks for more complex [expressions](../expressions).
+
+## The Type Class
+
+Every distinct concrete type in Quickstep is represented by a single object of
+a class derived from the base `quickstep::Type` class. All types have some
+common properties, including the following:
+
+ * A `TypeID` - an enum identifying the type, e.g. `kInt` for 32-bit integers,
+ or `kVarChar` for variable-length strings.
+ * Nullability - whether the type allows NULL values. All types have both a
+ nullable and a non-nullable flavor, except for NullType, a special type that
+ can ONLY store NULLs and has no non-nullable version.
+ * Storage size - minimum and maximum byte length. For fixed-length types like
+ basic numeric types and fixed length `CHAR(X)` strings, these lengths are
+ the same. For variable-length types like `VARCHAR(X)`, they can be
+ different (and the `Type` class has a method `estimateAverageByteLength()`
+ that can be used to make educated guesses when allocating storage). Note
+ that storage requirements really only apply to uncompressed, non-NULL
+ values. The actual bytes needed to store the values in the
+ [storage system](../storage) may be different if
+ [compression](../compression) is used, and some storage formats might store
+ NULLs differently.
+
+Some categories of types have additional properties (e.g. `CharType` and
+`VarCharType` also have a length parameter that indicates the maximum length of
+string that can be stored).
+
+### Getting a Type
+
+Each distinct, concrete Type is represented by a single object in the entire
+Quickstep process. To actually get a reference to usable `Type`, most code will
+go through the `TypeFactory`. `TypeFactory` provides static methods to access
+specific types by `TypeID` and other parameters. It can also deserialize a type
+from its protobuf representation (a `quickstep::serialization::Type` message).
+Finally, it also provides methods that can determine a `Type` that two different
+types can be cast to.
+
+### More on the `Type` Interface
+
+In addition to methods that allow inspection of a type's properties (e.g. those
+listed above), the Type class defines an interface with useful functionality
+common to all types:
+
+ * Serialization (of the type itself) - the `getProto()` method produces a
+ protobuf message that can be serialized and deserialized and later passed to
+ the TypeFactory to get back the same type.
+ * Relationship to other types - `equals()` determines if two types are exactly
+ the same, while `isCoercibleFrom()` determines if it is possible to convert
+ from another type to a given type (e.g. with a `CAST`), and
+ `isSafelyCoercibleFrom()` determines if such a conversion can always be done
+ without loss of precision.
+ * Printing to human-readable format - `printValueToString()` and
+ `printValueToFile()` can print out values of a type (see `TypedValue` below)
+ in human-readable format.
+ * Parsing from human-readable format - Similarly, `parseValueFromString()`
+ produces a `TypedValue` that is parsed from a string in human-readable
+ format.
+ * Making values - `makeValue()` creates a `TypedValue` from a bare pointer to
+ a value's representation in storage. For nullable types, `makeNullValue()`
+ makes a NULL value, and for numeric types, `makeZeroValue()` makes a zero
+ of that type.
+ * Coercing values - `coerceValue()` takes a value of another type and converts
+ it to the given type (e.g. as part of a `CAST`).
+
+## The TypedValue Class
+
+An individual typed value in Quickstep is represented by an instance of the
+`TypedValue` class. TypedValues can be created by methods of the `Type` class,
+by operation and expression classes that operate on values, or simply by calling
+one of several constructors provided in the class itself for convenience.
+TypedValues have C++ value semantics (i.e. they are copyable, assignable, and
+movable). A TypedValue may own its own data, or it may be a lightweight
+reference to data that is stored elsewhere in memory (this can be checked with
+`isReference()`, and any reference can be upgraded to own its own data copy by
+calling `ensureNotReference()`).
+
+Here are some of the things you can do with a TypedValue:
+
+ * NULL checks - calling `isNull()` determines if the TypedValue represents a
+ NULL. Several methods of TypedValue are usable only for non-NULL values, so
+ it is often important to check this first if in doubt.
+ * Access to underlying data - `getDataPtr()` returns an untyped `void*`
+ pointer to the underlying data, and `getDataSize()` returns the size of the
+ underlying data in bytes. Depending on the type of the value, the templated
+ method `getLiteral()` can be used to get the underlying data as a literal
+ scalar, or `getAsciiStringLength()` can be used to get the string length of
+ a `CHAR(X)` or `VARCHAR(X)` without counting null-terminators.
+ * Hashing - `getHash()` returns a hash of the value, which is suitable for
+ use in the HashTables of the [storage system](../storage), or in generic
+ hash-based C++ containers. `fastEqualCheck()` is provided to quickly check
+ whether two TypedValues of the same type (e.g. in the same hash table) are
+ actually equal.
+ * Serialization/Deserialization - `getProto()` serializes a TypedValue to a
+ `serialization::TypedValue` protobuf. The static method `ProtoIsValid()`
+ checks whether a serialized TypedValue is valid, and
+ `ReconstructFromProto()` rebuilds a TypedValue from its serialized form.
[25/25] incubator-quickstep git commit: Introduced Shiftboss for the
distributed version.
Posted by zu...@apache.org.
Introduced Shiftboss for the distributed version.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/db66fde2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/db66fde2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/db66fde2
Branch: refs/heads/dist-exe-test-new
Commit: db66fde2c15cc8c9a717c189397818d988a6f00f
Parents: 18b9a1f
Author: Zuyu Zhang <zu...@twitter.com>
Authored: Fri Jul 22 13:29:03 2016 -0700
Committer: Zuyu Zhang <zu...@twitter.com>
Committed: Fri Jul 29 16:43:24 2016 -0700
----------------------------------------------------------------------
catalog/CatalogDatabaseCache.hpp | 5 +
query_execution/CMakeLists.txt | 24 ++
query_execution/QueryExecutionMessages.proto | 30 ++
query_execution/QueryExecutionTypedefs.hpp | 8 +
query_execution/Shiftboss.cpp | 360 ++++++++++++++++++++++
query_execution/Shiftboss.hpp | 241 +++++++++++++++
storage/StorageManager.hpp | 1 +
7 files changed, 669 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/db66fde2/catalog/CatalogDatabaseCache.hpp
----------------------------------------------------------------------
diff --git a/catalog/CatalogDatabaseCache.hpp b/catalog/CatalogDatabaseCache.hpp
index 77afe2a..b3e73a6 100644
--- a/catalog/CatalogDatabaseCache.hpp
+++ b/catalog/CatalogDatabaseCache.hpp
@@ -54,6 +54,11 @@ namespace serialization { class CatalogDatabase; }
class CatalogDatabaseCache : public CatalogDatabaseLite {
public:
/**
+ * @brief Constructor.
+ **/
+ CatalogDatabaseCache() {}
+
+ /**
* @brief Constructor. Reconstruct a database cache from its serialized
* Protocol Buffer form.
*
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/db66fde2/query_execution/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_execution/CMakeLists.txt b/query_execution/CMakeLists.txt
index f582ba5..8bf1ab1 100644
--- a/query_execution/CMakeLists.txt
+++ b/query_execution/CMakeLists.txt
@@ -52,6 +52,7 @@ if (ENABLE_DISTRIBUTED)
endif()
add_library(quickstep_queryexecution_QueryManagerSingleNode QueryManagerSingleNode.cpp QueryManagerSingleNode.hpp)
if (ENABLE_DISTRIBUTED)
+ add_library(quickstep_queryexecution_Shiftboss Shiftboss.cpp Shiftboss.hpp)
add_library(quickstep_queryexecution_ShiftbossDirectory ../empty_src.cpp ShiftbossDirectory.hpp)
endif()
add_library(quickstep_queryexecution_WorkOrderProtosContainer ../empty_src.cpp WorkOrderProtosContainer.hpp)
@@ -157,6 +158,8 @@ target_link_libraries(quickstep_queryexecution_QueryContext_proto
quickstep_utility_SortConfiguration_proto
${PROTOBUF_LIBRARY})
target_link_libraries(quickstep_queryexecution_QueryExecutionMessages_proto
+ quickstep_catalog_Catalog_proto
+ quickstep_queryexecution_QueryContext_proto
quickstep_relationaloperators_WorkOrder_proto
${PROTOBUF_LIBRARY})
target_link_libraries(quickstep_queryexecution_QueryExecutionState
@@ -214,6 +217,26 @@ target_link_libraries(quickstep_queryexecution_QueryManagerSingleNode
quickstep_utility_Macros
tmb)
if (ENABLE_DISTRIBUTED)
+ target_link_libraries(quickstep_queryexecution_Shiftboss
+ glog
+ quickstep_catalog_CatalogDatabaseCache
+ quickstep_catalog_CatalogTypedefs
+ quickstep_queryexecution_QueryContext
+ quickstep_queryexecution_QueryExecutionMessages_proto
+ quickstep_queryexecution_QueryExecutionTypedefs
+ quickstep_queryexecution_QueryExecutionUtil
+ quickstep_queryexecution_WorkerDirectory
+ quickstep_queryexecution_WorkerMessage
+ quickstep_relationaloperators_RebuildWorkOrder
+ quickstep_relationaloperators_WorkOrderFactory
+ quickstep_storage_InsertDestination
+ quickstep_storage_StorageBlock
+ quickstep_storage_StorageBlockInfo
+ quickstep_storage_StorageManager
+ quickstep_threading_Thread
+ quickstep_threading_ThreadUtil
+ quickstep_utility_Macros
+ tmb)
target_link_libraries(quickstep_queryexecution_ShiftbossDirectory
quickstep_utility_Macros
tmb)
@@ -272,6 +295,7 @@ if (ENABLE_DISTRIBUTED)
target_link_libraries(quickstep_queryexecution
quickstep_queryexecution_BlockLocator
quickstep_queryexecution_QueryManagerDistributed
+ quickstep_queryexecution_Shiftboss
quickstep_queryexecution_ShiftbossDirectory)
endif()
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/db66fde2/query_execution/QueryExecutionMessages.proto
----------------------------------------------------------------------
diff --git a/query_execution/QueryExecutionMessages.proto b/query_execution/QueryExecutionMessages.proto
index fa20993..591ca6c 100644
--- a/query_execution/QueryExecutionMessages.proto
+++ b/query_execution/QueryExecutionMessages.proto
@@ -16,6 +16,8 @@ syntax = "proto2";
package quickstep.serialization;
+import "catalog/Catalog.proto";
+import "query_execution/QueryContext.proto";
import "relational_operators/WorkOrder.proto";
// Used for any messages that do not carry payloads.
@@ -73,6 +75,25 @@ message WorkOrdersAvailableMessage {
}
// Distributed version related messages.
+message ShiftbossRegistrationMessage {
+ // The total Work Order processing capacity in Shiftboss, which equals to the
+ // sum of the capacity of each worker managed by Shiftboss.
+ required uint64 work_order_capacity = 1;
+}
+
+message ShiftbossRegistrationResponseMessage {
+}
+
+message QueryInitiateMessage {
+ required uint64 query_id = 1;
+ required CatalogDatabase catalog_database_cache = 2;
+ required QueryContext query_context = 3;
+}
+
+message QueryInitiateResponseMessage {
+ required uint64 query_id = 1;
+}
+
message WorkOrderMessage {
required uint64 query_id = 1;
required uint64 operator_index = 2;
@@ -92,6 +113,15 @@ message InitiateRebuildResponseMessage {
required uint64 num_rebuild_work_orders = 3;
}
+message QueryResultRelationMessage {
+ required int32 relation_id = 1;
+ repeated fixed64 blocks = 2 [packed=true];
+}
+
+message QueryResultRelationResponseMessage {
+ required int32 relation_id = 1;
+}
+
// BlockLocator related messages.
message BlockDomainRegistrationMessage {
// Format IP:Port, i.e., "0.0.0.0:0".
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/db66fde2/query_execution/QueryExecutionTypedefs.hpp
----------------------------------------------------------------------
diff --git a/query_execution/QueryExecutionTypedefs.hpp b/query_execution/QueryExecutionTypedefs.hpp
index 61e76d7..d73d4ee 100644
--- a/query_execution/QueryExecutionTypedefs.hpp
+++ b/query_execution/QueryExecutionTypedefs.hpp
@@ -73,9 +73,17 @@ enum QueryExecutionMessageType : message_type_id {
kPoisonMessage, // From the main thread to Foreman and Workers.
#ifdef QUICKSTEP_DISTRIBUTED
+ kShiftbossRegistrationMessage, // From Shiftboss to Foreman.
+ kShiftbossRegistrationResponseMessage, // From Foreman to Shiftboss.
+ kQueryInitiateMessage, // From Foreman to Shiftboss.
+ kQueryInitiateResponseMessage, // From Shiftboss to Foreman.
+
kInitiateRebuildMessage, // From Foreman to Shiftboss.
kInitiateRebuildResponseMessage, // From Shiftboss to Foreman.
+ kQueryResultRelationMessage, // From Foreman to Shiftboss.
+ kQueryResultRelationResponseMessage, // From Shiftboss to Foreman.
+
// BlockLocator related messages, sorted in a life cycle of StorageManager
// with a unique block domain.
kBlockDomainRegistrationMessage, // From Worker to BlockLocator.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/db66fde2/query_execution/Shiftboss.cpp
----------------------------------------------------------------------
diff --git a/query_execution/Shiftboss.cpp b/query_execution/Shiftboss.cpp
new file mode 100644
index 0000000..af56306
--- /dev/null
+++ b/query_execution/Shiftboss.cpp
@@ -0,0 +1,360 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#include "query_execution/Shiftboss.hpp"
+
+#include <cstddef>
+#include <cstdlib>
+#include <memory>
+#include <string>
+#include <unordered_map>
+#include <utility>
+#include <vector>
+
+#include "catalog/CatalogTypedefs.hpp"
+#include "query_execution/QueryContext.hpp"
+#include "query_execution/QueryExecutionMessages.pb.h"
+#include "query_execution/QueryExecutionTypedefs.hpp"
+#include "query_execution/QueryExecutionUtil.hpp"
+#include "query_execution/WorkerMessage.hpp"
+#include "relational_operators/RebuildWorkOrder.hpp"
+#include "relational_operators/WorkOrderFactory.hpp"
+#include "storage/InsertDestination.hpp"
+#include "storage/StorageBlock.hpp"
+#include "storage/StorageBlockInfo.hpp"
+#include "storage/StorageManager.hpp"
+#include "threading/ThreadUtil.hpp"
+
+#include "glog/logging.h"
+
+#include "tmb/address.h"
+#include "tmb/id_typedefs.h"
+#include "tmb/message_bus.h"
+#include "tmb/message_style.h"
+#include "tmb/tagged_message.h"
+
+using std::free;
+using std::malloc;
+using std::move;
+using std::size_t;
+using std::string;
+using std::unique_ptr;
+using std::vector;
+
+using tmb::TaggedMessage;
+
+namespace quickstep {
+
+class WorkOrder;
+
+void Shiftboss::run() {
+ if (cpu_id_ >= 0) {
+ // We can pin the shiftboss thread to a CPU if specified.
+ ThreadUtil::BindToCPU(cpu_id_);
+ }
+
+ for (;;) {
+ // Receive() is a blocking call, causing this thread to sleep until next
+ // message is received.
+ AnnotatedMessage annotated_message(bus_->Receive(shiftboss_client_id_, 0, true));
+ LOG(INFO) << "Shiftboss (id '" << shiftboss_client_id_
+ << "') received the typed '" << annotated_message.tagged_message.message_type()
+ << "' message from client " << annotated_message.sender;
+ switch (annotated_message.tagged_message.message_type()) {
+ case kShiftbossRegistrationResponseMessage: {
+ foreman_client_id_ = annotated_message.sender;
+ break;
+ }
+ case kQueryInitiateMessage: {
+ const TaggedMessage &tagged_message = annotated_message.tagged_message;
+
+ serialization::QueryInitiateMessage proto;
+ CHECK(proto.ParseFromArray(tagged_message.message(), tagged_message.message_bytes()));
+
+ processQueryInitiateMessage(proto.query_id(), proto.catalog_database_cache(), proto.query_context());
+ break;
+ }
+ case kWorkOrderMessage: {
+ const TaggedMessage &tagged_message = annotated_message.tagged_message;
+
+ serialization::WorkOrderMessage proto;
+ CHECK(proto.ParseFromArray(tagged_message.message(), tagged_message.message_bytes()));
+
+ const std::size_t query_id = proto.query_id();
+ DCHECK_EQ(1u, query_contexts_.count(query_id));
+
+ WorkOrder *work_order = WorkOrderFactory::ReconstructFromProto(proto.work_order(),
+ &database_cache_,
+ query_contexts_[query_id].get(),
+ storage_manager_,
+ shiftboss_client_id_,
+ bus_);
+
+ unique_ptr<WorkerMessage> worker_message(
+ WorkerMessage::WorkOrderMessage(work_order, proto.operator_index()));
+
+ TaggedMessage worker_tagged_message(worker_message.get(),
+ sizeof(*worker_message),
+ kWorkOrderMessage);
+
+ const size_t worker_index = getSchedulableWorker();
+ LOG(INFO) << "Shiftboss (id '" << shiftboss_client_id_
+ << "') forwarded WorkOrderMessage (typed '" << kWorkOrderMessage
+ << "') from Foreman to worker " << worker_index;
+
+ QueryExecutionUtil::SendTMBMessage(bus_,
+ shiftboss_client_id_,
+ workers_->getClientID(worker_index),
+ move(worker_tagged_message));
+ break;
+ }
+ case kInitiateRebuildMessage: {
+ // Construct rebuild work orders, and send back their number to
+ // 'ForemanDistributed'.
+ const TaggedMessage &tagged_message = annotated_message.tagged_message;
+
+ serialization::InitiateRebuildMessage proto;
+ CHECK(proto.ParseFromArray(tagged_message.message(), tagged_message.message_bytes()));
+
+ processInitiateRebuildMessage(proto.query_id(),
+ proto.operator_index(),
+ proto.insert_destination_index(),
+ proto.relation_id());
+ break;
+ }
+ case kWorkOrderCompleteMessage: // Fall through.
+ case kRebuildWorkOrderCompleteMessage:
+ case kDataPipelineMessage:
+ case kWorkOrdersAvailableMessage:
+ case kWorkOrderFeedbackMessage: {
+ LOG(INFO) << "Shiftboss (id '" << shiftboss_client_id_
+ << "') forwarded typed '" << annotated_message.tagged_message.message_type()
+ << "' message from worker (client " << annotated_message.sender << ") to Foreman";
+
+ DCHECK_NE(foreman_client_id_, tmb::kClientIdNone);
+ QueryExecutionUtil::SendTMBMessage(bus_,
+ shiftboss_client_id_,
+ foreman_client_id_,
+ move(annotated_message.tagged_message));
+ break;
+ }
+ case kQueryResultRelationMessage: {
+ // TODO(zuyu): Rename to kSaveQueryResultMessage.
+ const TaggedMessage &tagged_message = annotated_message.tagged_message;
+
+ serialization::QueryResultRelationMessage proto;
+ CHECK(proto.ParseFromArray(tagged_message.message(), tagged_message.message_bytes()));
+
+ for (int i = 0; i < proto.blocks_size(); ++i) {
+ const block_id block = proto.blocks(i);
+ storage_manager_->saveBlockOrBlob(block);
+ if (storage_manager_->blockOrBlobIsLoaded(block)) {
+ // NOTE(zuyu): eviction is required to avoid accesses to the query
+ // result relation schema in CatalogDatabaseCache, for all query
+ // optimizer execution generator unit tests and the single-process
+ // Quickstep CLI.
+ storage_manager_->evictBlockOrBlob(block);
+ }
+ }
+
+ serialization::QueryResultRelationResponseMessage ack_proto;
+ ack_proto.set_relation_id(proto.relation_id());
+
+ const size_t ack_proto_length = ack_proto.ByteSize();
+ char *ack_proto_bytes = static_cast<char*>(malloc(ack_proto_length));
+ CHECK(ack_proto.SerializeToArray(ack_proto_bytes, ack_proto_length));
+
+ TaggedMessage ack_message(static_cast<const void*>(ack_proto_bytes),
+ ack_proto_length,
+ kQueryResultRelationResponseMessage);
+ free(ack_proto_bytes);
+
+ LOG(INFO) << "Shiftboss (id '" << shiftboss_client_id_
+ << "') sent QueryResultRelationResponseMessage (typed '" << kQueryResultRelationResponseMessage
+ << ") to Foreman";
+ QueryExecutionUtil::SendTMBMessage(bus_,
+ shiftboss_client_id_,
+ foreman_client_id_,
+ move(ack_message));
+ break;
+ }
+ case kPoisonMessage: {
+ LOG(INFO) << "Shiftboss (id '" << shiftboss_client_id_
+ << "') forwarded PoisonMessage (typed '" << kPoisonMessage
+ << "') from Foreman to all workers";
+
+ tmb::MessageStyle broadcast_style;
+ broadcast_style.Broadcast(true);
+
+ tmb::MessageBus::SendStatus send_status =
+ bus_->Send(shiftboss_client_id_,
+ worker_addresses_,
+ broadcast_style,
+ move(annotated_message.tagged_message));
+ DCHECK(send_status == tmb::MessageBus::SendStatus::kOK);
+ return;
+ }
+ default: {
+ LOG(FATAL) << "Unknown TMB message type";
+ }
+ }
+ }
+}
+
+size_t Shiftboss::getSchedulableWorker() {
+ const size_t num_workers = workers_->getNumWorkers();
+
+ size_t curr_worker = start_worker_index_;
+ for (;;) {
+ if (workers_->getNumQueuedWorkOrders(curr_worker) < max_msgs_per_worker_) {
+ start_worker_index_ = (curr_worker + 1) % num_workers;
+ // TODO(zuyu): workers_->incrementNumQueuedWorkOrders(curr_worker);
+ // But we need a WorkOrder queue first.
+ return curr_worker;
+ }
+
+ curr_worker = (curr_worker + 1) % num_workers;
+ }
+}
+
+void Shiftboss::registerWithForeman() {
+ LOG(INFO) << "Shiftboss (id '" << shiftboss_client_id_
+ << "') sent ShiftbossRegistrationMessage (typed '" << kShiftbossRegistrationMessage
+ << "') to all";
+
+ tmb::Address all_addresses;
+ all_addresses.All(true);
+
+ tmb::MessageStyle style;
+
+ serialization::ShiftbossRegistrationMessage proto;
+ proto.set_work_order_capacity(getWorkOrderCapacity());
+
+ const size_t proto_length = proto.ByteSize();
+ char *proto_bytes = static_cast<char*>(malloc(proto_length));
+ CHECK(proto.SerializeToArray(proto_bytes, proto_length));
+
+ TaggedMessage message(static_cast<const void*>(proto_bytes),
+ proto_length,
+ kShiftbossRegistrationMessage);
+ free(proto_bytes);
+
+ tmb::MessageBus::SendStatus send_status =
+ bus_->Send(shiftboss_client_id_, all_addresses, style, move(message));
+ DCHECK(send_status == tmb::MessageBus::SendStatus::kOK);
+}
+
+void Shiftboss::processQueryInitiateMessage(
+ const std::size_t query_id,
+ const serialization::CatalogDatabase &catalog_database_cache_proto,
+ const serialization::QueryContext &query_context_proto) {
+ database_cache_.update(catalog_database_cache_proto);
+
+ unique_ptr<QueryContext> query_context(
+ new QueryContext(query_context_proto,
+ database_cache_,
+ storage_manager_,
+ shiftboss_client_id_,
+ bus_));
+ query_contexts_.emplace(query_id, move(query_context));
+
+ LOG(INFO) << "Shiftboss (id '" << shiftboss_client_id_
+ << "') sent QueryInitiateResponseMessage (typed '" << kQueryInitiateResponseMessage
+ << "') to Foreman";
+
+ serialization::QueryInitiateResponseMessage proto;
+ proto.set_query_id(query_id);
+
+ const size_t proto_length = proto.ByteSize();
+ char *proto_bytes = static_cast<char*>(malloc(proto_length));
+ CHECK(proto.SerializeToArray(proto_bytes, proto_length));
+
+ TaggedMessage ack_message(static_cast<const void*>(proto_bytes),
+ proto_length,
+ kQueryInitiateResponseMessage);
+ free(proto_bytes);
+
+ QueryExecutionUtil::SendTMBMessage(bus_,
+ shiftboss_client_id_,
+ foreman_client_id_,
+ move(ack_message));
+}
+
+void Shiftboss::processInitiateRebuildMessage(const std::size_t query_id,
+ const std::size_t op_index,
+ const QueryContext::insert_destination_id dest_index,
+ const relation_id rel_id) {
+ DCHECK_NE(foreman_client_id_, tmb::kClientIdNone);
+
+ DCHECK_EQ(1u, query_contexts_.count(query_id));
+ InsertDestination *insert_destination = query_contexts_[query_id]->getInsertDestination(dest_index);
+ DCHECK(insert_destination != nullptr);
+
+ vector<MutableBlockReference> partially_filled_block_refs;
+ insert_destination->getPartiallyFilledBlocks(&partially_filled_block_refs);
+
+ LOG(INFO) << "Shiftboss (id '" << shiftboss_client_id_
+ << "') sent InitiateRebuildResponseMessage (typed '" << kInitiateRebuildResponseMessage
+ << "') to Foreman";
+
+ serialization::InitiateRebuildResponseMessage proto;
+ proto.set_query_id(query_id);
+ proto.set_operator_index(op_index);
+ proto.set_num_rebuild_work_orders(partially_filled_block_refs.size());
+
+ const size_t proto_length = proto.ByteSize();
+ char *proto_bytes = static_cast<char*>(malloc(proto_length));
+ CHECK(proto.SerializeToArray(proto_bytes, proto_length));
+
+ TaggedMessage ack_message(static_cast<const void*>(proto_bytes),
+ proto_length,
+ kInitiateRebuildResponseMessage);
+ free(proto_bytes);
+
+ QueryExecutionUtil::SendTMBMessage(bus_,
+ shiftboss_client_id_,
+ foreman_client_id_,
+ move(ack_message));
+
+ for (size_t i = 0; i < partially_filled_block_refs.size(); ++i) {
+ // NOTE(zuyu): Worker releases the memory after the execution of
+ // RebuildWorkOrder on the Worker.
+ WorkOrder *rebuild_work_order =
+ new RebuildWorkOrder(query_id,
+ move(partially_filled_block_refs[i]),
+ op_index,
+ rel_id,
+ shiftboss_client_id_,
+ bus_);
+
+ unique_ptr<WorkerMessage> worker_message(
+ WorkerMessage::RebuildWorkOrderMessage(rebuild_work_order, op_index));
+
+ TaggedMessage worker_tagged_message(worker_message.get(),
+ sizeof(*worker_message),
+ kRebuildWorkOrderMessage);
+
+ const size_t worker_index = getSchedulableWorker();
+ LOG(INFO) << "Shiftboss (id '" << shiftboss_client_id_
+ << "') sent RebuildWorkOrderMessage (typed '" << kRebuildWorkOrderMessage
+ << "') to worker " << worker_index;
+
+ QueryExecutionUtil::SendTMBMessage(bus_,
+ shiftboss_client_id_,
+ workers_->getClientID(worker_index),
+ move(worker_tagged_message));
+ }
+}
+
+} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/db66fde2/query_execution/Shiftboss.hpp
----------------------------------------------------------------------
diff --git a/query_execution/Shiftboss.hpp b/query_execution/Shiftboss.hpp
new file mode 100644
index 0000000..096ab74
--- /dev/null
+++ b/query_execution/Shiftboss.hpp
@@ -0,0 +1,241 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#ifndef QUICKSTEP_QUERY_EXECUTION_SHIFTBOSS_HPP_
+#define QUICKSTEP_QUERY_EXECUTION_SHIFTBOSS_HPP_
+
+#include <cstddef>
+#include <memory>
+#include <unordered_map>
+
+#include "catalog/CatalogDatabaseCache.hpp"
+#include "catalog/CatalogTypedefs.hpp"
+#include "query_execution/QueryContext.hpp"
+#include "query_execution/QueryExecutionTypedefs.hpp"
+#include "query_execution/WorkerDirectory.hpp"
+#include "threading/Thread.hpp"
+#include "utility/Macros.hpp"
+
+#include "glog/logging.h"
+
+#include "tmb/address.h"
+#include "tmb/id_typedefs.h"
+#include "tmb/message_bus.h"
+
+namespace quickstep {
+
+class StorageManager;
+
+namespace serialization {
+class CatalogDatabase;
+class QueryContext;
+} // namespace serialization
+
+/** \addtogroup QueryExecution
+ * @{
+ */
+
+/**
+ * @brief The Shiftboss accepts workorder protos from shiftboss, and assigns
+ * the workorders to workers.
+ **/
+class Shiftboss : public Thread {
+ public:
+ /**
+ * @brief Constructor.
+ *
+ * @param bus A pointer to the TMB.
+ * @param storage_manager The StorageManager to use.
+ * @param workers A pointer to the WorkerDirectory.
+ * @param cpu_id The ID of the CPU to which the Shiftboss thread can be pinned.
+ *
+ * @note If cpu_id is not specified, Shiftboss thread can be possibly moved
+ * around on different CPUs by the OS.
+ **/
+ Shiftboss(tmb::MessageBus *bus,
+ StorageManager *storage_manager,
+ WorkerDirectory *workers,
+ const int cpu_id = -1)
+ : bus_(DCHECK_NOTNULL(bus)),
+ storage_manager_(DCHECK_NOTNULL(storage_manager)),
+ workers_(DCHECK_NOTNULL(workers)),
+ cpu_id_(cpu_id),
+ shiftboss_client_id_(tmb::kClientIdNone),
+ foreman_client_id_(tmb::kClientIdNone),
+ max_msgs_per_worker_(1),
+ start_worker_index_(0u) {
+ // Check to have at least one Worker.
+ DCHECK_GT(workers->getNumWorkers(), 0u);
+
+ shiftboss_client_id_ = bus_->Connect();
+ LOG(INFO) << "Shiftboss TMB client ID: " << shiftboss_client_id_;
+ DCHECK_NE(shiftboss_client_id_, tmb::kClientIdNone);
+
+ // Messages between Foreman and Shiftboss.
+ bus_->RegisterClientAsSender(shiftboss_client_id_, kShiftbossRegistrationMessage);
+ bus_->RegisterClientAsReceiver(shiftboss_client_id_, kShiftbossRegistrationResponseMessage);
+
+ bus_->RegisterClientAsReceiver(shiftboss_client_id_, kQueryInitiateMessage);
+ bus_->RegisterClientAsSender(shiftboss_client_id_, kQueryInitiateResponseMessage);
+
+ bus_->RegisterClientAsReceiver(shiftboss_client_id_, kInitiateRebuildMessage);
+ bus_->RegisterClientAsSender(shiftboss_client_id_, kInitiateRebuildResponseMessage);
+
+ // Message sent to Worker.
+ bus_->RegisterClientAsSender(shiftboss_client_id_, kRebuildWorkOrderMessage);
+
+ // Message sent to Foreman.
+ bus_->RegisterClientAsSender(shiftboss_client_id_, kCatalogRelationNewBlockMessage);
+ bus_->RegisterClientAsSender(shiftboss_client_id_, kDataPipelineMessage);
+ bus_->RegisterClientAsSender(shiftboss_client_id_, kWorkOrdersAvailableMessage);
+ bus_->RegisterClientAsSender(shiftboss_client_id_, kWorkOrderFeedbackMessage);
+
+ // Forward the following message types from Foreman to Workers.
+ bus_->RegisterClientAsReceiver(shiftboss_client_id_, kWorkOrderMessage);
+ bus_->RegisterClientAsSender(shiftboss_client_id_, kWorkOrderMessage);
+
+ // Forward the following message types from Workers to Foreman.
+ bus_->RegisterClientAsReceiver(shiftboss_client_id_, kWorkOrderCompleteMessage);
+ bus_->RegisterClientAsSender(shiftboss_client_id_, kWorkOrderCompleteMessage);
+
+ bus_->RegisterClientAsReceiver(shiftboss_client_id_, kRebuildWorkOrderCompleteMessage);
+ bus_->RegisterClientAsSender(shiftboss_client_id_, kRebuildWorkOrderCompleteMessage);
+
+ bus_->RegisterClientAsReceiver(shiftboss_client_id_, kQueryResultRelationMessage);
+ bus_->RegisterClientAsSender(shiftboss_client_id_, kQueryResultRelationResponseMessage);
+
+ // Stop itself.
+ bus_->RegisterClientAsReceiver(shiftboss_client_id_, kPoisonMessage);
+ // Stop all workers.
+ bus_->RegisterClientAsSender(shiftboss_client_id_, kPoisonMessage);
+
+ for (std::size_t i = 0; i < workers_->getNumWorkers(); ++i) {
+ worker_addresses_.AddRecipient(workers_->getClientID(i));
+ }
+
+ registerWithForeman();
+ }
+
+ ~Shiftboss() override {
+ }
+
+ /**
+ * @brief Get the TMB client ID of Shiftboss thread.
+ *
+ * @return TMB client ID of shiftboss thread.
+ **/
+ inline tmb::client_id getBusClientID() const {
+ return shiftboss_client_id_;
+ }
+
+ /**
+ * @brief Get the Work Order processing capacity of all Workers managed by
+ * Shiftboss during a single round of WorkOrder dispatch.
+ **/
+ inline std::size_t getWorkOrderCapacity() const {
+ DCHECK_NE(max_msgs_per_worker_, 0u);
+ return max_msgs_per_worker_ * workers_->getNumWorkers();
+ }
+
+ /**
+ * @brief Get the Worker to assign WorkOrders for execution. Block to wait if
+ * all Workers have reached their capacity for queued WorkOrders.
+ **/
+ // TODO(zuyu): To achieve non-blocking, we need a queue to cache received
+ // normal Work Order protos from Foreman and the generated rebuild Work Orders.
+ inline std::size_t getSchedulableWorker();
+
+ /**
+ * @brief Set the maximum number of messages that should be allocated to each
+ * worker during a single round of WorkOrder dispatch.
+ *
+ * @param max_msgs_per_worker Maximum number of messages.
+ **/
+ inline void setMaxMessagesPerWorker(const std::size_t max_msgs_per_worker) {
+ max_msgs_per_worker_ = max_msgs_per_worker;
+ }
+
+ protected:
+ /**
+ * @brief The shiftboss receives workorders, and based on the response it
+ * assigns workorders to workers.
+ *
+ * @note The workers who get the messages from the Shiftboss execute and
+ * subsequently delete the WorkOrder contained in the message.
+ **/
+ void run() override;
+
+ private:
+ void registerWithForeman();
+
+ /**
+ * @brief Process the Shiftboss initiate message and ack back.
+ *
+ * @param query_id The given query id.
+ * @param catalog_database_cache_proto The proto used to update
+ * CatalogDatabaseCache.
+ * @param query_context_proto The QueryContext proto.
+ **/
+ void processQueryInitiateMessage(const std::size_t query_id,
+ const serialization::CatalogDatabase &catalog_database_cache_proto,
+ const serialization::QueryContext &query_context_proto);
+
+ /**
+ * @brief Process the RebuildWorkOrder initiate message and ack back.
+ *
+ * @param query_id The ID of the query to which this RebuildWorkOrder initiate
+ * message belongs.
+ * @param op_index The index of the operator for rebuild work orders.
+ * @param dest_index The InsertDestination index in QueryContext to rebuild.
+ * @param rel_id The relation that needs to generate rebuild work orders.
+ **/
+ void processInitiateRebuildMessage(const std::size_t query_id,
+ const std::size_t op_index,
+ const QueryContext::insert_destination_id dest_index,
+ const relation_id rel_id);
+
+ // TODO(zuyu): Use two buses for the message communication between Foreman and Shiftboss,
+ // and Shiftboss and Worker thread pool.
+ tmb::MessageBus *bus_;
+
+ CatalogDatabaseCache database_cache_;
+ StorageManager *storage_manager_;
+ WorkerDirectory *workers_;
+
+ // The ID of the CPU that the Shiftboss thread can optionally be pinned to.
+ const int cpu_id_;
+
+ tmb::client_id shiftboss_client_id_, foreman_client_id_;
+
+ // TMB recipients for all workers managed by this Shiftboss.
+ tmb::Address worker_addresses_;
+
+ // During a single round of WorkOrder dispatch, a Worker should be allocated
+ // at most these many WorkOrders.
+ std::size_t max_msgs_per_worker_;
+
+ // The worker index for scheduling Work Order.
+ std::size_t start_worker_index_;
+
+ // QueryContexts per query.
+ std::unordered_map<std::size_t, std::unique_ptr<QueryContext>> query_contexts_;
+
+ DISALLOW_COPY_AND_ASSIGN(Shiftboss);
+};
+
+/** @} */
+
+} // namespace quickstep
+
+#endif // QUICKSTEP_QUERY_EXECUTION_SHIFTBOSS_HPP_
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/db66fde2/storage/StorageManager.hpp
----------------------------------------------------------------------
diff --git a/storage/StorageManager.hpp b/storage/StorageManager.hpp
index 50ddb0f..348018f 100644
--- a/storage/StorageManager.hpp
+++ b/storage/StorageManager.hpp
@@ -619,6 +619,7 @@ class StorageManager {
FRIEND_TEST(BlockLocatorTest, BlockTest);
FRIEND_TEST(BlockLocatorTest, BlobTest);
+ friend class Shiftboss;
FRIEND_TEST(StorageManagerTest, DifferentNUMANodeBlobTestWithEviction);
FRIEND_TEST(StorageManagerTest, EvictFromSameShardTest);
[05/25] incubator-quickstep git commit: Introduced QueryManagerBase,
and renamed QueryManagerSingleNode.
Posted by zu...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/65af8c7b/query_execution/tests/QueryManagerSingleNode_unittest.cpp
----------------------------------------------------------------------
diff --git a/query_execution/tests/QueryManagerSingleNode_unittest.cpp b/query_execution/tests/QueryManagerSingleNode_unittest.cpp
new file mode 100644
index 0000000..52cee20
--- /dev/null
+++ b/query_execution/tests/QueryManagerSingleNode_unittest.cpp
@@ -0,0 +1,942 @@
+/**
+ * Copyright 2011-2015 Quickstep Technologies LLC.
+ * Copyright 2015-2016 Pivotal Software, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#include <climits>
+#include <cstdlib>
+#include <memory>
+#include <utility>
+#include <vector>
+
+#include "catalog/CatalogDatabase.hpp"
+#include "catalog/CatalogRelation.hpp"
+#include "catalog/CatalogTypedefs.hpp"
+#include "query_execution/QueryContext.hpp"
+#include "query_execution/QueryContext.pb.h"
+#include "query_execution/QueryExecutionMessages.pb.h"
+#include "query_execution/QueryExecutionState.hpp"
+#include "query_execution/QueryExecutionTypedefs.hpp"
+#include "query_execution/QueryManagerSingleNode.hpp"
+#include "query_execution/WorkOrdersContainer.hpp"
+#include "query_execution/WorkerDirectory.hpp"
+#include "query_execution/WorkerMessage.hpp"
+#include "query_optimizer/QueryHandle.hpp"
+#include "query_optimizer/QueryPlan.hpp"
+#include "relational_operators/RelationalOperator.hpp"
+#include "relational_operators/WorkOrder.hpp"
+#include "storage/InsertDestination.hpp"
+#include "storage/InsertDestination.pb.h"
+#include "storage/StorageBlock.hpp"
+#include "storage/StorageBlockInfo.hpp"
+#include "storage/StorageManager.hpp"
+#include "utility/DAG.hpp"
+#include "utility/Macros.hpp"
+
+#include "glog/logging.h"
+#include "gtest/gtest.h"
+
+#include "tmb/id_typedefs.h"
+#include "tmb/tagged_message.h"
+
+namespace tmb { class MessageBus; }
+
+using std::move;
+using std::unique_ptr;
+using std::vector;
+
+using tmb::client_id;
+
+namespace quickstep {
+
+class WorkOrderProtosContainer;
+
+class MockWorkOrder : public WorkOrder {
+ public:
+ explicit MockWorkOrder(const int op_index)
+ : WorkOrder(0), op_index_(op_index) {}
+
+ void execute() override {
+ VLOG(3) << "WorkOrder[" << op_index_ << "] executing.";
+ }
+
+ inline QueryPlan::DAGNodeIndex getOpIndex() const {
+ return op_index_;
+ }
+
+ private:
+ const QueryPlan::DAGNodeIndex op_index_;
+
+ DISALLOW_COPY_AND_ASSIGN(MockWorkOrder);
+};
+
+class MockOperator: public RelationalOperator {
+ public:
+ enum function_name {
+ kFeedInputBlock = 0,
+ kFeedInputBlocks,
+ kDoneFeedingInputBlocks,
+ kGetAllWorkOrders
+ };
+
+ MockOperator(const bool produce_workorders,
+ const bool has_streaming_input,
+ const int max_getworkorder_iters = 1,
+ const int max_workorders = INT_MAX)
+ : RelationalOperator(0 /* Query Id */),
+ produce_workorders_(produce_workorders),
+ has_streaming_input_(has_streaming_input),
+ max_workorders_(max_workorders),
+ max_getworkorder_iters_(max_getworkorder_iters),
+ num_calls_get_workorders_(0),
+ num_workorders_generated_(0),
+ num_calls_feedblock_(0),
+ num_calls_feedblocks_(0),
+ num_calls_donefeedingblocks_(0) {
+ }
+
+#define MOCK_OP_LOG(x) VLOG(x) << "Op[" << op_index_ << "]: " << __func__ << ": "
+
+ // The methods below are used to check whether QueryManager calls the Relational
+ // operator, how many times it calls a particular method etc.
+ inline int getNumWorkOrders() const {
+ return num_workorders_generated_;
+ }
+
+ inline int getNumCalls(const function_name fname) const {
+ switch (fname) {
+ case kFeedInputBlock:
+ return num_calls_feedblock_;
+ case kFeedInputBlocks:
+ return num_calls_feedblocks_;
+ case kDoneFeedingInputBlocks:
+ return num_calls_donefeedingblocks_;
+ case kGetAllWorkOrders:
+ return num_calls_get_workorders_;
+ default:
+ return -1;
+ }
+ }
+
+ inline bool getBlockingDependenciesMet() const {
+ MOCK_OP_LOG(3) << "met.";
+ return blocking_dependencies_met_;
+ }
+
+ void setInsertDestinationID(const QueryContext::insert_destination_id insert_destination_index) {
+ insert_destination_index_ = insert_destination_index;
+ }
+
+ // Mock to trigger doneFeedingInputBlocks for the dependent operators
+ // in QueryManager::markOperatorFinished.
+ void setOutputRelationID(const relation_id rel_id) {
+ output_relation_id_ = rel_id;
+ }
+
+ // Override methods from the base class.
+ bool getAllWorkOrders(
+ WorkOrdersContainer *container,
+ QueryContext *query_context,
+ StorageManager *storage_manager,
+ const tmb::client_id foreman_client_id,
+ tmb::MessageBus *bus) override {
+ ++num_calls_get_workorders_;
+ if (produce_workorders_) {
+ if (has_streaming_input_) {
+ if ((num_calls_feedblock_ > 0 || num_calls_feedblocks_ > 0) && (num_workorders_generated_ < max_workorders_)) {
+ MOCK_OP_LOG(3) << "[stream] generate WorkOrder";
+ container->addNormalWorkOrder(new MockWorkOrder(op_index_), op_index_);
+ ++num_workorders_generated_;
+ }
+ } else {
+ if (blocking_dependencies_met_ && (num_workorders_generated_ < max_workorders_)) {
+ MOCK_OP_LOG(3) << "[static] generate WorkOrder";
+ container->addNormalWorkOrder(new MockWorkOrder(op_index_), op_index_);
+ ++num_workorders_generated_;
+ }
+ }
+ }
+ MOCK_OP_LOG(3) << "count(" << num_calls_get_workorders_ << ") "
+ << "return(" << (num_calls_get_workorders_ == max_getworkorder_iters_) << ")";
+ return num_calls_get_workorders_ == max_getworkorder_iters_;
+ }
+
+ bool getAllWorkOrderProtos(WorkOrderProtosContainer *container) override {
+ return true;
+ }
+
+ void feedInputBlock(const block_id input_block_id,
+ const relation_id input_relation_id) override {
+ ++num_calls_feedblock_;
+ MOCK_OP_LOG(3) << "count(" << num_calls_feedblock_ << ")";
+ }
+
+ void feedInputBlocks(const relation_id rel_id,
+ std::vector<block_id> *partially_filled_blocks) override {
+ ++num_calls_feedblocks_;
+ MOCK_OP_LOG(3) << "count(" << num_calls_feedblocks_ << ")";
+ }
+
+ void doneFeedingInputBlocks(const relation_id rel_id) override {
+ ++num_calls_donefeedingblocks_;
+ MOCK_OP_LOG(3) << "count(" << num_calls_donefeedingblocks_ << ")";
+ }
+
+ QueryContext::insert_destination_id getInsertDestinationID() const override {
+ return insert_destination_index_;
+ }
+
+ const relation_id getOutputRelationID() const override {
+ return output_relation_id_;
+ }
+
+ private:
+ const bool produce_workorders_;
+ const bool has_streaming_input_;
+ const int max_workorders_;
+ const int max_getworkorder_iters_;
+
+ int num_calls_get_workorders_;
+ int num_workorders_generated_;
+ int num_calls_feedblock_;
+ int num_calls_feedblocks_;
+ int num_calls_donefeedingblocks_;
+
+ QueryContext::insert_destination_id insert_destination_index_ = QueryContext::kInvalidInsertDestinationId;
+
+ relation_id output_relation_id_ = -1;
+
+#undef MOCK_OP_LOG
+
+ DISALLOW_COPY_AND_ASSIGN(MockOperator);
+};
+
+
+class QueryManagerTest : public ::testing::Test {
+ protected:
+ virtual void SetUp() {
+ db_.reset(new CatalogDatabase(nullptr /* catalog */, "database"));
+ storage_manager_.reset(new StorageManager("./"));
+ bus_.Initialize();
+ query_handle_.reset(new QueryHandle(0)); // dummy query ID.
+ query_plan_ = query_handle_->getQueryPlanMutable();
+ query_handle_->getQueryContextProtoMutable()->set_query_id(query_handle_->query_id());
+ }
+
+ inline void constructQueryManager() {
+ query_manager_.reset(new QueryManagerSingleNode(
+ 0, 1, query_handle_.get(), db_.get(), storage_manager_.get(), &bus_));
+ }
+
+ inline const int getNumWorkOrdersInExecution(const QueryPlan::DAGNodeIndex index) const {
+ return query_manager_->getQueryExecutionState().getNumQueuedWorkOrders(index);
+ }
+
+ inline const int getNumOperatorsFinished() const {
+ return query_manager_->getQueryExecutionState().getNumOperatorsFinished();
+ }
+
+ inline bool getOperatorFinishedStatus(const QueryPlan::DAGNodeIndex index) const {
+ return query_manager_->getQueryExecutionState().hasExecutionFinished(index);
+ }
+
+ inline bool placeDataPipelineMessage(const QueryPlan::DAGNodeIndex source_operator_index) {
+ VLOG(3) << "Place DataPipeline message for Op[" << source_operator_index << "]";
+ serialization::DataPipelineMessage proto;
+ proto.set_operator_index(source_operator_index);
+
+ proto.set_block_id(0); // dummy block ID
+ proto.set_relation_id(0); // dummy relation ID.
+ proto.set_query_id(0); // dummy query ID.
+
+ // NOTE(zuyu): Using the heap memory to serialize proto as a c-like string.
+ const std::size_t proto_length = proto.ByteSize();
+ char *proto_bytes = static_cast<char*>(std::malloc(proto_length));
+ CHECK(proto.SerializeToArray(proto_bytes, proto_length));
+
+ tmb::TaggedMessage tagged_message(static_cast<const void *>(proto_bytes),
+ proto_length,
+ kDataPipelineMessage);
+ std::free(proto_bytes);
+ query_manager_->processMessage(tagged_message);
+ return query_manager_->getQueryExecutionState().hasQueryExecutionFinished();
+ }
+
+ inline bool placeWorkOrderCompleteMessage(const QueryPlan::DAGNodeIndex index) {
+ VLOG(3) << "Place WorkOrderComplete message for Op[" << index << "]";
+ TaggedMessage tagged_message;
+ serialization::NormalWorkOrderCompletionMessage proto;
+ proto.set_operator_index(index);
+ proto.set_worker_thread_index(1); // dummy worker ID.
+ proto.set_query_id(0); // dummy query ID.
+
+ // NOTE(zuyu): Using the heap memory to serialize proto as a c-like string.
+ const size_t proto_length = proto.ByteSize();
+ char *proto_bytes = static_cast<char*>(std::malloc(proto_length));
+ CHECK(proto.SerializeToArray(proto_bytes, proto_length));
+
+ TaggedMessage message(static_cast<const void*>(proto_bytes),
+ proto_length,
+ kWorkOrderCompleteMessage);
+ std::free(proto_bytes);
+ query_manager_->processMessage(message);
+
+ return query_manager_->getQueryExecutionState().hasQueryExecutionFinished();
+ }
+
+ inline bool placeRebuildWorkOrderCompleteMessage(const QueryPlan::DAGNodeIndex index) {
+ VLOG(3) << "Place RebuildWorkOrderComplete message for Op[" << index << "]";
+ serialization::RebuildWorkOrderCompletionMessage proto;
+ proto.set_operator_index(index);
+ proto.set_worker_thread_index(1); // dummy worker thread ID.
+ proto.set_query_id(0); // dummy query ID.
+
+ // NOTE(zuyu): Using the heap memory to serialize proto as a c-like string.
+ const size_t proto_length = proto.ByteSize();
+ char *proto_bytes = static_cast<char*>(std::malloc(proto_length));
+ CHECK(proto.SerializeToArray(proto_bytes, proto_length));
+
+ TaggedMessage message(static_cast<const void*>(proto_bytes),
+ proto_length,
+ kRebuildWorkOrderCompleteMessage);
+
+ std::free(proto_bytes);
+ query_manager_->processMessage(message);
+
+ return query_manager_->getQueryExecutionState().hasQueryExecutionFinished();
+ }
+
+ inline bool placeOutputBlockMessage(const QueryPlan::DAGNodeIndex index) {
+ VLOG(3) << "Place OutputBlock message for Op[" << index << "]";
+ serialization::DataPipelineMessage proto;
+ proto.set_operator_index(index);
+
+ proto.set_block_id(0); // dummy block ID
+ proto.set_relation_id(0); // dummy relation ID.
+ proto.set_query_id(0); // dummy query ID.
+
+ // NOTE(zuyu): Using the heap memory to serialize proto as a c-like string.
+ const std::size_t proto_length = proto.ByteSize();
+ char *proto_bytes = static_cast<char*>(std::malloc(proto_length));
+ CHECK(proto.SerializeToArray(proto_bytes, proto_length));
+
+ tmb::TaggedMessage tagged_message(static_cast<const void *>(proto_bytes),
+ proto_length,
+ kDataPipelineMessage);
+ std::free(proto_bytes);
+ query_manager_->processMessage(tagged_message);
+ return query_manager_->getQueryExecutionState().hasQueryExecutionFinished();
+ }
+
+ unique_ptr<CatalogDatabase> db_;
+ unique_ptr<StorageManager> storage_manager_;
+
+ QueryPlan *query_plan_;
+ unique_ptr<QueryHandle> query_handle_;
+ unique_ptr<QueryManagerSingleNode> query_manager_;
+
+ MessageBusImpl bus_;
+
+ client_id worker_client_id_;
+
+ unique_ptr<WorkerDirectory> workers_;
+};
+
+TEST_F(QueryManagerTest, SingleNodeDAGNoWorkOrdersTest) {
+ // This test creates a DAG of a single node. No workorders are generated.
+ query_plan_->addRelationalOperator(new MockOperator(false, false));
+
+ const MockOperator &op = static_cast<const MockOperator &>(
+ query_plan_->getQueryPlanDAG().getNodePayload(0));
+
+ constructQueryManager();
+
+ // op doesn't have any dependencies.
+ EXPECT_TRUE(op.getBlockingDependenciesMet());
+
+ // We expect one call for op's getAllWorkOrders().
+ EXPECT_EQ(1, op.getNumCalls(MockOperator::kGetAllWorkOrders));
+ EXPECT_EQ(0, op.getNumCalls(MockOperator::kFeedInputBlock));
+ EXPECT_EQ(0, op.getNumCalls(MockOperator::kFeedInputBlocks));
+}
+
+TEST_F(QueryManagerTest, SingleNodeDAGStaticWorkOrdersTest) {
+ // This test creates a DAG of a single node. Static workorders are generated.
+ const QueryPlan::DAGNodeIndex id =
+ query_plan_->addRelationalOperator(new MockOperator(true, false, 1));
+
+ const MockOperator &op = static_cast<const MockOperator &>(
+ query_plan_->getQueryPlanDAG().getNodePayload(id));
+
+ constructQueryManager();
+
+ // op doesn't have any dependencies.
+ EXPECT_TRUE(op.getBlockingDependenciesMet());
+
+ // We expect one call for op's getAllWorkOrders().
+ EXPECT_EQ(1, op.getNumCalls(MockOperator::kGetAllWorkOrders));
+ EXPECT_EQ(0, op.getNumCalls(MockOperator::kFeedInputBlock));
+ EXPECT_EQ(0, op.getNumCalls(MockOperator::kFeedInputBlocks));
+
+ // One workorder is generated.
+ EXPECT_EQ(1, op.getNumWorkOrders());
+
+ unique_ptr<WorkerMessage> worker_message;
+ worker_message.reset(query_manager_->getNextWorkerMessage(0, -1));
+ EXPECT_TRUE(worker_message != nullptr);
+
+ EXPECT_EQ(WorkerMessage::WorkerMessageType::kWorkOrder,
+ worker_message->getType());
+ EXPECT_EQ(0u, worker_message->getRelationalOpIndex());
+
+ delete worker_message->getWorkOrder();
+
+ EXPECT_EQ(1, getNumWorkOrdersInExecution(id));
+ EXPECT_EQ(0, getNumOperatorsFinished());
+
+ // Send a message to QueryManager upon workorder completion.
+ // Last event processed by QueryManager.
+ EXPECT_TRUE(placeWorkOrderCompleteMessage(id));
+
+ EXPECT_EQ(0, getNumWorkOrdersInExecution(id));
+ EXPECT_EQ(1, getNumOperatorsFinished());
+ EXPECT_TRUE(getOperatorFinishedStatus(id));
+}
+
+TEST_F(QueryManagerTest, SingleNodeDAGDynamicWorkOrdersTest) {
+ // This test creates a DAG of a single node. WorkOrders are generated
+ // dynamically as pending work orders complete execution, i.e.,
+ // getAllWorkOrders() is called multiple times. getAllWorkOrders() will be
+ // called 5 times and 3 work orders will be returned, i.e., 1st 3 calls to
+ // getAllWorkOrders() insert 1 WorkOrder and return false, and the next will
+ // insert no WorkOrder and return true.
+
+ // TODO(shoban): This test can not be more robust than this because of fixed
+ // scaffolding of mocking. If we use gMock, we can do much better.
+ const QueryPlan::DAGNodeIndex id =
+ query_plan_->addRelationalOperator(new MockOperator(true, false, 4, 3));
+
+ const MockOperator &op = static_cast<const MockOperator &>(
+ query_plan_->getQueryPlanDAG().getNodePayload(id));
+
+ constructQueryManager();
+
+ // op doesn't have any dependencies.
+ EXPECT_TRUE(op.getBlockingDependenciesMet());
+
+ for (int i = 0; i < 3; ++i) {
+ // We expect one call for op's getAllWorkOrders().
+ EXPECT_EQ(i + 1, op.getNumCalls(MockOperator::kGetAllWorkOrders));
+
+ // One workorder is generated.
+ // EXPECT_EQ(1, getWorkerInputQueueSize());
+ EXPECT_EQ(i + 1, op.getNumWorkOrders());
+
+ unique_ptr<WorkerMessage> worker_message;
+ worker_message.reset(query_manager_->getNextWorkerMessage(id, -1));
+
+ EXPECT_TRUE(worker_message != nullptr);
+ EXPECT_EQ(WorkerMessage::WorkerMessageType::kWorkOrder,
+ worker_message->getType());
+ EXPECT_EQ(id, worker_message->getRelationalOpIndex());
+
+ delete worker_message->getWorkOrder();
+
+ EXPECT_EQ(1, getNumWorkOrdersInExecution(id));
+ EXPECT_EQ(0, getNumOperatorsFinished());
+
+ if (i < 2) {
+ // Send a message to QueryManager upon workorder completion.
+ EXPECT_FALSE(placeWorkOrderCompleteMessage(id));
+ } else {
+ // Send a message to QueryManager upon workorder completion.
+ // Last event.
+ EXPECT_TRUE(placeWorkOrderCompleteMessage(id));
+ }
+ }
+
+ EXPECT_EQ(0, getNumWorkOrdersInExecution(id));
+
+ EXPECT_EQ(1, getNumOperatorsFinished());
+ EXPECT_TRUE(getOperatorFinishedStatus(id));
+
+ // We place this check in the end, since it's true throughout the test.
+ EXPECT_EQ(0, op.getNumCalls(MockOperator::kFeedInputBlock));
+ EXPECT_EQ(0, op.getNumCalls(MockOperator::kFeedInputBlocks));
+}
+
+TEST_F(QueryManagerTest, TwoNodesDAGBlockingLinkTest) {
+ // We use two nodes in the DAG with a blocking link between them.
+ // There is no streaming of data involved in this test.
+ const QueryPlan::DAGNodeIndex id1 =
+ query_plan_->addRelationalOperator(new MockOperator(true, false));
+ const QueryPlan::DAGNodeIndex id2 =
+ query_plan_->addRelationalOperator(new MockOperator(true, false));
+
+ // Create a blocking link.
+ query_plan_->addDirectDependency(id2, id1, true);
+
+ static_cast<MockOperator *>(
+ query_plan_->getQueryPlanDAGMutable()->getNodePayloadMutable(id1))
+ ->setOutputRelationID(0xdead);
+
+ const MockOperator &op1 = static_cast<const MockOperator &>(
+ query_plan_->getQueryPlanDAG().getNodePayload(id1));
+ const MockOperator &op2 = static_cast<const MockOperator &>(
+ query_plan_->getQueryPlanDAG().getNodePayload(id2));
+
+ constructQueryManager();
+
+ // op1 doesn't have any dependencies
+ EXPECT_TRUE(op1.getBlockingDependenciesMet());
+
+ // Only op1 should receive a call to getAllWorkOrders initially.
+ EXPECT_EQ(1, op1.getNumCalls(MockOperator::kGetAllWorkOrders));
+ EXPECT_EQ(0, op1.getNumCalls(MockOperator::kFeedInputBlock));
+ EXPECT_EQ(0, op1.getNumCalls(MockOperator::kFeedInputBlocks));
+
+ EXPECT_EQ(0, op2.getNumCalls(MockOperator::kGetAllWorkOrders));
+ EXPECT_EQ(0, op2.getNumCalls(MockOperator::kFeedInputBlock));
+ EXPECT_EQ(0, op2.getNumCalls(MockOperator::kFeedInputBlocks));
+
+ // Only op1 should produce a workorder.
+ EXPECT_EQ(1, op1.getNumWorkOrders());
+ EXPECT_EQ(0, op2.getNumWorkOrders());
+
+ // Foreman hasn't yet got workorder completion response for the workorder.
+ unique_ptr<WorkerMessage> worker_message;
+ worker_message.reset(query_manager_->getNextWorkerMessage(id1, -1));
+
+ EXPECT_TRUE(worker_message != nullptr);
+ EXPECT_EQ(WorkerMessage::WorkerMessageType::kWorkOrder,
+ worker_message->getType());
+ EXPECT_EQ(id1, worker_message->getRelationalOpIndex());
+
+ delete worker_message->getWorkOrder();
+
+ EXPECT_EQ(1, getNumWorkOrdersInExecution(id1));
+ EXPECT_EQ(0, getNumWorkOrdersInExecution(id2));
+ EXPECT_EQ(0, getNumOperatorsFinished());
+
+ // Send a message to Foreman upon workorder (generated by op1) completion.
+ EXPECT_FALSE(placeWorkOrderCompleteMessage(id1));
+
+ EXPECT_EQ(0, getNumWorkOrdersInExecution(id1));
+ // op1 is over now, op2 still to go.
+ EXPECT_EQ(1, op2.getNumCalls(MockOperator::kDoneFeedingInputBlocks));
+ EXPECT_EQ(1, getNumOperatorsFinished());
+
+ EXPECT_TRUE(getOperatorFinishedStatus(id1));
+ EXPECT_EQ(1, op2.getNumCalls(MockOperator::kDoneFeedingInputBlocks));
+ EXPECT_FALSE(getOperatorFinishedStatus(id2));
+
+ worker_message.reset(query_manager_->getNextWorkerMessage(id2, -1));
+ EXPECT_TRUE(worker_message != nullptr);
+ EXPECT_EQ(WorkerMessage::WorkerMessageType::kWorkOrder,
+ worker_message->getType());
+ EXPECT_EQ(id2, worker_message->getRelationalOpIndex());
+
+ delete worker_message->getWorkOrder();
+
+ EXPECT_EQ(1, getNumWorkOrdersInExecution(id2));
+
+ // op1 is op2's blocking dependency.
+ EXPECT_TRUE(op2.getBlockingDependenciesMet());
+
+ EXPECT_EQ(1, op1.getNumCalls(MockOperator::kGetAllWorkOrders));
+ // op2 should get first call of getAllWorkOrders() when op1 is over.
+ EXPECT_EQ(1, op2.getNumCalls(MockOperator::kGetAllWorkOrders));
+
+ EXPECT_EQ(1, op2.getNumWorkOrders());
+
+ // Send a message to QueryManager upon workorder (generated by op2) completion.
+ // Note that the worker hasn't yet popped the workorder. Usually this won't
+ // happen as workers pop workorders first, execute and then send the response.
+ EXPECT_TRUE(placeWorkOrderCompleteMessage(id2));
+
+ EXPECT_EQ(1, op2.getNumCalls(MockOperator::kGetAllWorkOrders));
+
+ EXPECT_EQ(0, getNumWorkOrdersInExecution(id1));
+ EXPECT_EQ(0, getNumWorkOrdersInExecution(id2));
+
+ EXPECT_EQ(2, getNumOperatorsFinished());
+ EXPECT_TRUE(getOperatorFinishedStatus(id1));
+ EXPECT_TRUE(getOperatorFinishedStatus(id2));
+
+ // Expect no additional calls to getAllWorkOrders.
+ EXPECT_EQ(1, op1.getNumCalls(MockOperator::kGetAllWorkOrders));
+ EXPECT_EQ(1, op2.getNumCalls(MockOperator::kGetAllWorkOrders));
+}
+
+TEST_F(QueryManagerTest, TwoNodesDAGPipeLinkTest) {
+ // We use two nodes in the DAG with a non-blocking link between them.
+ // We stream output of op1 to op2. Sequeuce of events is as follows:
+ // 1. op1 creates a workorder.
+ // 2. We send a "block full" (from op1) to QueryManager.
+ // 3. op2 creates a workorder because of step 2.
+ const QueryPlan::DAGNodeIndex id1 =
+ query_plan_->addRelationalOperator(new MockOperator(true, false, 1));
+ const QueryPlan::DAGNodeIndex id2 =
+ query_plan_->addRelationalOperator(new MockOperator(true, true, 3));
+
+ // Create a non-blocking link.
+ query_plan_->addDirectDependency(id2, id1, false);
+
+ static_cast<MockOperator *>(
+ query_plan_->getQueryPlanDAGMutable()->getNodePayloadMutable(id1))
+ ->setOutputRelationID(0xdead);
+
+ const MockOperator &op1 = static_cast<const MockOperator &>(
+ query_plan_->getQueryPlanDAG().getNodePayload(id1));
+ const MockOperator &op2 = static_cast<const MockOperator &>(
+ query_plan_->getQueryPlanDAG().getNodePayload(id2));
+
+ constructQueryManager();
+
+ // As none of the operators have a blocking link, blocking dependencies should
+ // be met.
+ EXPECT_TRUE(op1.getBlockingDependenciesMet());
+ EXPECT_TRUE(op2.getBlockingDependenciesMet());
+
+ EXPECT_EQ(1, op1.getNumCalls(MockOperator::kGetAllWorkOrders));
+ EXPECT_EQ(1, op1.getNumWorkOrders());
+ EXPECT_EQ(0, op1.getNumCalls(MockOperator::kFeedInputBlock));
+ EXPECT_EQ(0, op1.getNumCalls(MockOperator::kFeedInputBlocks));
+
+ EXPECT_EQ(1, op2.getNumCalls(MockOperator::kGetAllWorkOrders));
+ // op2 will generate workorder only after receiving a streaming input.
+ EXPECT_EQ(0, op2.getNumWorkOrders());
+ EXPECT_EQ(0, op2.getNumCalls(MockOperator::kFeedInputBlock));
+ EXPECT_EQ(0, op2.getNumCalls(MockOperator::kFeedInputBlocks));
+
+ unique_ptr<WorkerMessage> worker_message;
+ worker_message.reset(query_manager_->getNextWorkerMessage(id1, -1));
+
+ EXPECT_TRUE(worker_message != nullptr);
+ EXPECT_EQ(WorkerMessage::WorkerMessageType::kWorkOrder,
+ worker_message->getType());
+ EXPECT_EQ(id1, worker_message->getRelationalOpIndex());
+
+ delete worker_message->getWorkOrder();
+
+ // Send a message to QueryManager upon block getting full (output of op1).
+ EXPECT_FALSE(placeOutputBlockMessage(id1));
+
+ // op1 is not finished yet because the response of workorder completion hasn't
+ // been received yet by the QueryManager.
+ EXPECT_FALSE(getOperatorFinishedStatus(id1));
+ EXPECT_FALSE(getOperatorFinishedStatus(id2));
+
+ // No additional call to op1's getAllWorkOrders.
+ EXPECT_EQ(1, op1.getNumCalls(MockOperator::kGetAllWorkOrders));
+ EXPECT_EQ(0, op1.getNumCalls(MockOperator::kFeedInputBlock));
+ EXPECT_EQ(0, op1.getNumCalls(MockOperator::kFeedInputBlocks));
+
+ // Output from op1 should be fed to op2.
+ EXPECT_EQ(1, op2.getNumCalls(MockOperator::kFeedInputBlock));
+ EXPECT_EQ(0, op2.getNumCalls(MockOperator::kFeedInputBlocks));
+
+ // A call to op2's getAllWorkOrders because of the streamed input.
+ EXPECT_EQ(2, op2.getNumCalls(MockOperator::kGetAllWorkOrders));
+ EXPECT_EQ(1, op2.getNumWorkOrders());
+
+ // Place a message of a workorder completion of op1 on Foreman's input queue.
+ EXPECT_FALSE(placeWorkOrderCompleteMessage(id1));
+
+ EXPECT_EQ(0, getNumWorkOrdersInExecution(id1));
+ EXPECT_TRUE(getOperatorFinishedStatus(id1));
+ EXPECT_EQ(1, op2.getNumCalls(MockOperator::kDoneFeedingInputBlocks));
+
+ // An additional call to op2's getAllWorkOrders because of completion of op1.
+ EXPECT_EQ(3, op2.getNumCalls(MockOperator::kGetAllWorkOrders));
+ EXPECT_EQ(2, op2.getNumWorkOrders());
+
+ worker_message.reset(query_manager_->getNextWorkerMessage(id2, -1));
+
+ EXPECT_TRUE(worker_message != nullptr);
+ EXPECT_EQ(WorkerMessage::WorkerMessageType::kWorkOrder,
+ worker_message->getType());
+ EXPECT_EQ(id2, worker_message->getRelationalOpIndex());
+
+ delete worker_message->getWorkOrder();
+
+ // Place a message of a workorder completion of op2 on Foreman's input queue.
+ EXPECT_FALSE(placeWorkOrderCompleteMessage(id2));
+
+ EXPECT_TRUE(getOperatorFinishedStatus(id1));
+
+ worker_message.reset(query_manager_->getNextWorkerMessage(id2, -1));
+
+ EXPECT_TRUE(worker_message != nullptr);
+ EXPECT_EQ(WorkerMessage::WorkerMessageType::kWorkOrder,
+ worker_message->getType());
+ EXPECT_EQ(id2, worker_message->getRelationalOpIndex());
+
+ delete worker_message->getWorkOrder();
+
+ EXPECT_EQ(1, getNumWorkOrdersInExecution(id2));
+ EXPECT_FALSE(getOperatorFinishedStatus(id2));
+
+ // Send a message to Foreman upon workorder (generated by op2) completion.
+ EXPECT_TRUE(placeWorkOrderCompleteMessage(id2));
+
+ EXPECT_TRUE(getOperatorFinishedStatus(id1));
+ EXPECT_EQ(1, op2.getNumCalls(MockOperator::kDoneFeedingInputBlocks));
+
+ EXPECT_EQ(0, getNumWorkOrdersInExecution(id2));
+ EXPECT_TRUE(getOperatorFinishedStatus(id2));
+}
+
+TEST_F(QueryManagerTest, TwoNodesDAGPartiallyFilledBlocksTest) {
+ // In this test, we create a 2-node DAG with a non-blocking link between them.
+ // There is no streaming of data from op1 to op2 during the execution of op1.
+ // op1 produces a partially filled block at the end of its execution which is
+ // rebuilt and then fed to op2.
+ const QueryPlan::DAGNodeIndex id1 =
+ query_plan_->addRelationalOperator(new MockOperator(true, false, 1));
+ const QueryPlan::DAGNodeIndex id2 =
+ query_plan_->addRelationalOperator(new MockOperator(true, true, 3, 1));
+
+ // Create a non-blocking link.
+ query_plan_->addDirectDependency(id2, id1, false);
+
+ // Create a relation, owned by db_.*/
+ CatalogRelation *relation =
+ new CatalogRelation(nullptr /* catalog_database */, "test_relation");
+ const relation_id output_relation_id = db_->addRelation(relation);
+
+ // Setup the InsertDestination proto in the query context proto.
+ serialization::QueryContext *query_context_proto =
+ query_handle_->getQueryContextProtoMutable();
+
+ const QueryContext::insert_destination_id insert_destination_index =
+ query_context_proto->insert_destinations_size();
+ serialization::InsertDestination *insert_destination_proto =
+ query_context_proto->add_insert_destinations();
+
+ insert_destination_proto->set_insert_destination_type(
+ serialization::InsertDestinationType::BLOCK_POOL);
+ insert_destination_proto->set_relation_id(output_relation_id);
+ insert_destination_proto->set_relational_op_index(id1);
+
+ MockOperator *op1_mutable = static_cast<MockOperator *>(
+ query_plan_->getQueryPlanDAGMutable()->getNodePayloadMutable(id1));
+ op1_mutable->setInsertDestinationID(insert_destination_index);
+ op1_mutable->setOutputRelationID(output_relation_id);
+
+ const MockOperator &op1 = static_cast<const MockOperator &>(
+ query_plan_->getQueryPlanDAG().getNodePayload(id1));
+ const MockOperator &op2 = static_cast<const MockOperator &>(
+ query_plan_->getQueryPlanDAG().getNodePayload(id2));
+
+ constructQueryManager();
+
+ // NOTE(zuyu): An operator generally has no ideas about partially filled
+ // blocks, but InsertDestination in QueryContext does.
+ // Mock to add partially filled blocks in the InsertDestination.
+ InsertDestination *insert_destination =
+ query_manager_->getQueryContextMutable()->getInsertDestination(
+ insert_destination_index);
+ DCHECK(insert_destination != nullptr);
+ MutableBlockReference block_ref;
+ static_cast<BlockPoolInsertDestination *>(insert_destination)
+ ->available_block_refs_.push_back(move(block_ref));
+
+ // There's no blocking dependency in the DAG.
+ EXPECT_TRUE(op1.getBlockingDependenciesMet());
+ EXPECT_TRUE(op2.getBlockingDependenciesMet());
+
+ EXPECT_EQ(1, op1.getNumCalls(MockOperator::kGetAllWorkOrders));
+ EXPECT_EQ(1, op1.getNumWorkOrders());
+
+ EXPECT_EQ(1, op2.getNumCalls(MockOperator::kGetAllWorkOrders));
+ EXPECT_EQ(0, op2.getNumWorkOrders());
+
+ unique_ptr<WorkerMessage> worker_message;
+ worker_message.reset(query_manager_->getNextWorkerMessage(id1, -1));
+
+ EXPECT_TRUE(worker_message != nullptr);
+ EXPECT_EQ(WorkerMessage::WorkerMessageType::kWorkOrder,
+ worker_message->getType());
+ EXPECT_EQ(id1, worker_message->getRelationalOpIndex());
+
+ delete worker_message->getWorkOrder();
+
+ // Send a message to QueryManager upon workorder (generated by op1) completion.
+ EXPECT_FALSE(placeWorkOrderCompleteMessage(id1));
+
+ EXPECT_EQ(0, getNumWorkOrdersInExecution(id1));
+
+ worker_message.reset(query_manager_->getNextWorkerMessage(id1, -1));
+ EXPECT_TRUE(worker_message != nullptr);
+ EXPECT_EQ(WorkerMessage::WorkerMessageType::kRebuildWorkOrder,
+ worker_message->getType());
+
+ EXPECT_EQ(id1, worker_message->getRelationalOpIndex());
+
+ delete worker_message->getWorkOrder();
+
+ // op1 generates a rebuild workorder. The block is rebuilt and streamed
+ // to Foreman.
+ EXPECT_FALSE(placeDataPipelineMessage(id1));
+
+ EXPECT_FALSE(placeRebuildWorkOrderCompleteMessage(id1));
+ // Based on the streamed input, op2's getAllWorkOrders should produce a
+ // workorder.
+ EXPECT_EQ(3, op2.getNumCalls(MockOperator::kGetAllWorkOrders));
+ EXPECT_EQ(1, op2.getNumWorkOrders());
+
+ worker_message.reset(query_manager_->getNextWorkerMessage(id2, -1));
+
+ EXPECT_TRUE(worker_message != nullptr);
+ EXPECT_EQ(WorkerMessage::WorkerMessageType::kWorkOrder,
+ worker_message->getType());
+
+ EXPECT_EQ(id2, worker_message->getRelationalOpIndex());
+
+ delete worker_message->getWorkOrder();
+
+ EXPECT_TRUE(getOperatorFinishedStatus(id1));
+ EXPECT_EQ(1, op2.getNumCalls(MockOperator::kDoneFeedingInputBlocks));
+ EXPECT_FALSE(getOperatorFinishedStatus(id2));
+ EXPECT_EQ(1, getNumWorkOrdersInExecution(id2));
+
+ // Send a message to QueryManager upon workorder (generated by op2) completion.
+ EXPECT_TRUE(placeWorkOrderCompleteMessage(id2));
+
+ EXPECT_EQ(0, getNumWorkOrdersInExecution(id2));
+
+ EXPECT_TRUE(getOperatorFinishedStatus(id2));
+}
+
+TEST_F(QueryManagerTest, MultipleNodesNoOutputTest) {
+ // When an operator produces workorders but no output, the QueryManager should
+ // check the dependents of this operator to make progress.
+ const QueryPlan::DAGNodeIndex kNumNodes = 5;
+ std::vector<QueryPlan::DAGNodeIndex> ids;
+ ids.reserve(kNumNodes);
+
+ for (QueryPlan::DAGNodeIndex i = 0; i < kNumNodes; ++i) {
+ if (i == 0) {
+ ids[i] = query_plan_->addRelationalOperator(new MockOperator(true, false));
+ } else {
+ ids[i] = query_plan_->addRelationalOperator(new MockOperator(true, true));
+ }
+ VLOG(3) << ids[i];
+ }
+
+ /**
+ * The DAG looks like this:
+ *
+ * op1 -> op2 -> op3 -> op4 -> op5
+ *
+ **/
+ for (QueryPlan::DAGNodeIndex i = 0; i < kNumNodes - 1; ++i) {
+ query_plan_->addDirectDependency(ids[i + 1], ids[i], false);
+ static_cast<MockOperator*>(query_plan_->getQueryPlanDAGMutable()->getNodePayloadMutable(ids[i]))
+ ->setOutputRelationID(0xdead);
+ }
+
+ std::vector<const MockOperator*> operators;
+ for (QueryPlan::DAGNodeIndex i = 0; i < kNumNodes; ++i) {
+ operators.push_back(static_cast<const MockOperator*>(&query_plan_->getQueryPlanDAG().getNodePayload(ids[i])));
+ }
+
+ constructQueryManager();
+
+ // operators[0] should have produced a workorder by now.
+ EXPECT_EQ(1, operators[0]->getNumWorkOrders());
+
+ unique_ptr<WorkerMessage> worker_message;
+ worker_message.reset(query_manager_->getNextWorkerMessage(ids[0], -1));
+
+ EXPECT_TRUE(worker_message != nullptr);
+ EXPECT_EQ(WorkerMessage::WorkerMessageType::kWorkOrder,
+ worker_message->getType());
+
+ EXPECT_EQ(ids[0], worker_message->getRelationalOpIndex());
+
+ delete worker_message->getWorkOrder();
+
+ EXPECT_EQ(1, getNumWorkOrdersInExecution(ids[0]));
+ EXPECT_FALSE(getOperatorFinishedStatus(ids[0]));
+
+ for (QueryPlan::DAGNodeIndex i = 0; i < kNumNodes; ++i) {
+ EXPECT_EQ(1, operators[ids[i]]->getNumCalls(MockOperator::kGetAllWorkOrders));
+ }
+
+ // Send a message to QueryManager upon workorder (generated by operators[0])
+ // completion.
+ EXPECT_TRUE(placeWorkOrderCompleteMessage(ids[0]));
+
+ for (QueryPlan::DAGNodeIndex i = 0; i < kNumNodes; ++i) {
+ EXPECT_EQ(0, getNumWorkOrdersInExecution(ids[i]));
+ EXPECT_TRUE(getOperatorFinishedStatus(ids[i]));
+ if (i < kNumNodes - 1) {
+ EXPECT_EQ(1, operators[i + 1]->getNumCalls(MockOperator::kDoneFeedingInputBlocks));
+ }
+ }
+}
+
+TEST_F(QueryManagerTest, OutOfOrderWorkOrderCompletionTest) {
+ // Consider two operators, both generate one workorder each. The dependent's
+ // workorder finishes before dependency's workorder.
+ const QueryPlan::DAGNodeIndex id1 = query_plan_->addRelationalOperator(new MockOperator(true, false, 1));
+ const QueryPlan::DAGNodeIndex id2 = query_plan_->addRelationalOperator(new MockOperator(true, true, 2, 1));
+
+ // Create a non-blocking link.
+ query_plan_->addDirectDependency(id2, id1, false);
+
+ constructQueryManager();
+
+ unique_ptr<WorkerMessage> worker_message;
+ worker_message.reset(query_manager_->getNextWorkerMessage(id1, -1));
+
+ EXPECT_TRUE(worker_message != nullptr);
+ EXPECT_EQ(WorkerMessage::WorkerMessageType::kWorkOrder,
+ worker_message->getType());
+
+ EXPECT_EQ(id1, worker_message->getRelationalOpIndex());
+
+ delete worker_message->getWorkOrder();
+
+ // Send a message to QueryManager upon a block (output of op1) getting full.
+ EXPECT_FALSE(placeOutputBlockMessage(id1));
+
+ // op1 is not finished yet because the response of workorder completion hasn't
+ // been received yet.
+ EXPECT_FALSE(getOperatorFinishedStatus(id1));
+ EXPECT_FALSE(getOperatorFinishedStatus(id2));
+
+ worker_message.reset(query_manager_->getNextWorkerMessage(id2, -1));
+ EXPECT_TRUE(worker_message != nullptr);
+ EXPECT_EQ(WorkerMessage::WorkerMessageType::kWorkOrder,
+ worker_message->getType());
+
+ EXPECT_EQ(id2, worker_message->getRelationalOpIndex());
+
+ delete worker_message->getWorkOrder();
+
+ // As mentioned earlier, op2 finishes before op1.
+ EXPECT_FALSE(placeWorkOrderCompleteMessage(id2));
+
+ // op1's workorder execution is over.
+ EXPECT_TRUE(placeWorkOrderCompleteMessage(id1));
+
+ EXPECT_TRUE(getOperatorFinishedStatus(id1));
+ EXPECT_TRUE(getOperatorFinishedStatus(id2));
+}
+
+} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/65af8c7b/query_execution/tests/QueryManager_unittest.cpp
----------------------------------------------------------------------
diff --git a/query_execution/tests/QueryManager_unittest.cpp b/query_execution/tests/QueryManager_unittest.cpp
deleted file mode 100644
index 37e2cdd..0000000
--- a/query_execution/tests/QueryManager_unittest.cpp
+++ /dev/null
@@ -1,940 +0,0 @@
-/**
- * Copyright 2011-2015 Quickstep Technologies LLC.
- * Copyright 2015-2016 Pivotal Software, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- **/
-
-#include <climits>
-#include <memory>
-#include <utility>
-#include <vector>
-
-#include "catalog/CatalogDatabase.hpp"
-#include "catalog/CatalogRelation.hpp"
-#include "catalog/CatalogTypedefs.hpp"
-#include "query_execution/QueryContext.hpp"
-#include "query_execution/QueryContext.pb.h"
-#include "query_execution/QueryExecutionMessages.pb.h"
-#include "query_execution/QueryExecutionState.hpp"
-#include "query_execution/QueryExecutionTypedefs.hpp"
-#include "query_execution/QueryManager.hpp"
-#include "query_execution/WorkOrdersContainer.hpp"
-#include "query_execution/WorkerDirectory.hpp"
-#include "query_execution/WorkerMessage.hpp"
-#include "query_optimizer/QueryHandle.hpp"
-#include "query_optimizer/QueryPlan.hpp"
-#include "relational_operators/RelationalOperator.hpp"
-#include "relational_operators/WorkOrder.hpp"
-#include "storage/InsertDestination.hpp"
-#include "storage/InsertDestination.pb.h"
-#include "storage/StorageBlock.hpp"
-#include "storage/StorageBlockInfo.hpp"
-#include "storage/StorageManager.hpp"
-#include "utility/DAG.hpp"
-#include "utility/Macros.hpp"
-
-#include "glog/logging.h"
-#include "gtest/gtest.h"
-
-#include "tmb/id_typedefs.h"
-#include "tmb/message_bus.h"
-#include "tmb/tagged_message.h"
-
-using std::move;
-using std::unique_ptr;
-using std::vector;
-
-using tmb::client_id;
-
-namespace quickstep {
-
-class WorkOrderProtosContainer;
-
-class MockWorkOrder : public WorkOrder {
- public:
- explicit MockWorkOrder(const int op_index)
- : WorkOrder(0), op_index_(op_index) {}
-
- void execute() override {
- VLOG(3) << "WorkOrder[" << op_index_ << "] executing.";
- }
-
- inline QueryPlan::DAGNodeIndex getOpIndex() const {
- return op_index_;
- }
-
- private:
- const QueryPlan::DAGNodeIndex op_index_;
-
- DISALLOW_COPY_AND_ASSIGN(MockWorkOrder);
-};
-
-class MockOperator: public RelationalOperator {
- public:
- enum function_name {
- kFeedInputBlock = 0,
- kFeedInputBlocks,
- kDoneFeedingInputBlocks,
- kGetAllWorkOrders
- };
-
- MockOperator(const bool produce_workorders,
- const bool has_streaming_input,
- const int max_getworkorder_iters = 1,
- const int max_workorders = INT_MAX)
- : RelationalOperator(0 /* Query Id */),
- produce_workorders_(produce_workorders),
- has_streaming_input_(has_streaming_input),
- max_workorders_(max_workorders),
- max_getworkorder_iters_(max_getworkorder_iters),
- num_calls_get_workorders_(0),
- num_workorders_generated_(0),
- num_calls_feedblock_(0),
- num_calls_feedblocks_(0),
- num_calls_donefeedingblocks_(0) {
- }
-
-#define MOCK_OP_LOG(x) VLOG(x) << "Op[" << op_index_ << "]: " << __func__ << ": "
-
- // The methods below are used to check whether QueryManager calls the Relational
- // operator, how many times it calls a particular method etc.
- inline int getNumWorkOrders() const {
- return num_workorders_generated_;
- }
-
- inline int getNumCalls(const function_name fname) const {
- switch (fname) {
- case kFeedInputBlock:
- return num_calls_feedblock_;
- case kFeedInputBlocks:
- return num_calls_feedblocks_;
- case kDoneFeedingInputBlocks:
- return num_calls_donefeedingblocks_;
- case kGetAllWorkOrders:
- return num_calls_get_workorders_;
- default:
- return -1;
- }
- }
-
- inline bool getBlockingDependenciesMet() const {
- MOCK_OP_LOG(3) << "met.";
- return blocking_dependencies_met_;
- }
-
- void setInsertDestinationID(const QueryContext::insert_destination_id insert_destination_index) {
- insert_destination_index_ = insert_destination_index;
- }
-
- // Mock to trigger doneFeedingInputBlocks for the dependent operators
- // in QueryManager::markOperatorFinished.
- void setOutputRelationID(const relation_id rel_id) {
- output_relation_id_ = rel_id;
- }
-
- // Override methods from the base class.
- bool getAllWorkOrders(
- WorkOrdersContainer *container,
- QueryContext *query_context,
- StorageManager *storage_manager,
- const tmb::client_id foreman_client_id,
- tmb::MessageBus *bus) override {
- ++num_calls_get_workorders_;
- if (produce_workorders_) {
- if (has_streaming_input_) {
- if ((num_calls_feedblock_ > 0 || num_calls_feedblocks_ > 0) && (num_workorders_generated_ < max_workorders_)) {
- MOCK_OP_LOG(3) << "[stream] generate WorkOrder";
- container->addNormalWorkOrder(new MockWorkOrder(op_index_), op_index_);
- ++num_workorders_generated_;
- }
- } else {
- if (blocking_dependencies_met_ && (num_workorders_generated_ < max_workorders_)) {
- MOCK_OP_LOG(3) << "[static] generate WorkOrder";
- container->addNormalWorkOrder(new MockWorkOrder(op_index_), op_index_);
- ++num_workorders_generated_;
- }
- }
- }
- MOCK_OP_LOG(3) << "count(" << num_calls_get_workorders_ << ") "
- << "return(" << (num_calls_get_workorders_ == max_getworkorder_iters_) << ")";
- return num_calls_get_workorders_ == max_getworkorder_iters_;
- }
-
- bool getAllWorkOrderProtos(WorkOrderProtosContainer *container) override {
- return true;
- }
-
- void feedInputBlock(const block_id input_block_id,
- const relation_id input_relation_id) override {
- ++num_calls_feedblock_;
- MOCK_OP_LOG(3) << "count(" << num_calls_feedblock_ << ")";
- }
-
- void feedInputBlocks(const relation_id rel_id,
- std::vector<block_id> *partially_filled_blocks) override {
- ++num_calls_feedblocks_;
- MOCK_OP_LOG(3) << "count(" << num_calls_feedblocks_ << ")";
- }
-
- void doneFeedingInputBlocks(const relation_id rel_id) override {
- ++num_calls_donefeedingblocks_;
- MOCK_OP_LOG(3) << "count(" << num_calls_donefeedingblocks_ << ")";
- }
-
- QueryContext::insert_destination_id getInsertDestinationID() const override {
- return insert_destination_index_;
- }
-
- const relation_id getOutputRelationID() const override {
- return output_relation_id_;
- }
-
- private:
- const bool produce_workorders_;
- const bool has_streaming_input_;
- const int max_workorders_;
- const int max_getworkorder_iters_;
-
- int num_calls_get_workorders_;
- int num_workorders_generated_;
- int num_calls_feedblock_;
- int num_calls_feedblocks_;
- int num_calls_donefeedingblocks_;
-
- QueryContext::insert_destination_id insert_destination_index_ = QueryContext::kInvalidInsertDestinationId;
-
- relation_id output_relation_id_ = -1;
-
-#undef MOCK_OP_LOG
-
- DISALLOW_COPY_AND_ASSIGN(MockOperator);
-};
-
-
-class QueryManagerTest : public ::testing::Test {
- protected:
- virtual void SetUp() {
- db_.reset(new CatalogDatabase(nullptr /* catalog */, "database"));
- storage_manager_.reset(new StorageManager("./"));
- bus_.Initialize();
- query_handle_.reset(new QueryHandle(0)); // dummy query ID.
- query_plan_ = query_handle_->getQueryPlanMutable();
- query_handle_->getQueryContextProtoMutable()->set_query_id(query_handle_->query_id());
- }
-
- inline void constructQueryManager() {
- query_manager_.reset(new QueryManager(
- 0, 1, query_handle_.get(), db_.get(), storage_manager_.get(), &bus_));
- }
-
- inline const int getNumWorkOrdersInExecution(const QueryPlan::DAGNodeIndex index) const {
- return query_manager_->getQueryExecutionState().getNumQueuedWorkOrders(index);
- }
-
- inline const int getNumOperatorsFinished() const {
- return query_manager_->getQueryExecutionState().getNumOperatorsFinished();
- }
-
- inline bool getOperatorFinishedStatus(const QueryPlan::DAGNodeIndex index) const {
- return query_manager_->getQueryExecutionState().hasExecutionFinished(index);
- }
-
- inline bool placeDataPipelineMessage(const QueryPlan::DAGNodeIndex source_operator_index) {
- VLOG(3) << "Place DataPipeline message for Op[" << source_operator_index << "]";
- serialization::DataPipelineMessage proto;
- proto.set_operator_index(source_operator_index);
-
- proto.set_block_id(0); // dummy block ID
- proto.set_relation_id(0); // dummy relation ID.
- proto.set_query_id(0); // dummy query ID.
-
- // NOTE(zuyu): Using the heap memory to serialize proto as a c-like string.
- const std::size_t proto_length = proto.ByteSize();
- char *proto_bytes = static_cast<char*>(std::malloc(proto_length));
- CHECK(proto.SerializeToArray(proto_bytes, proto_length));
-
- tmb::TaggedMessage tagged_message(static_cast<const void *>(proto_bytes),
- proto_length,
- kDataPipelineMessage);
- std::free(proto_bytes);
- query_manager_->processMessage(tagged_message);
- return query_manager_->getQueryExecutionState().hasQueryExecutionFinished();
- }
-
- inline bool placeWorkOrderCompleteMessage(const QueryPlan::DAGNodeIndex index) {
- VLOG(3) << "Place WorkOrderComplete message for Op[" << index << "]";
- TaggedMessage tagged_message;
- serialization::NormalWorkOrderCompletionMessage proto;
- proto.set_operator_index(index);
- proto.set_worker_thread_index(1); // dummy worker ID.
- proto.set_query_id(0); // dummy query ID.
-
- // NOTE(zuyu): Using the heap memory to serialize proto as a c-like string.
- const size_t proto_length = proto.ByteSize();
- char *proto_bytes = static_cast<char*>(std::malloc(proto_length));
- CHECK(proto.SerializeToArray(proto_bytes, proto_length));
-
- TaggedMessage message(static_cast<const void*>(proto_bytes),
- proto_length,
- kWorkOrderCompleteMessage);
- std::free(proto_bytes);
- query_manager_->processMessage(message);
-
- return query_manager_->getQueryExecutionState().hasQueryExecutionFinished();
- }
-
- inline bool placeRebuildWorkOrderCompleteMessage(const QueryPlan::DAGNodeIndex index) {
- VLOG(3) << "Place RebuildWorkOrderComplete message for Op[" << index << "]";
- serialization::RebuildWorkOrderCompletionMessage proto;
- proto.set_operator_index(index);
- proto.set_worker_thread_index(1); // dummy worker thread ID.
- proto.set_query_id(0); // dummy query ID.
-
- // NOTE(zuyu): Using the heap memory to serialize proto as a c-like string.
- const size_t proto_length = proto.ByteSize();
- char *proto_bytes = static_cast<char*>(std::malloc(proto_length));
- CHECK(proto.SerializeToArray(proto_bytes, proto_length));
-
- TaggedMessage message(static_cast<const void*>(proto_bytes),
- proto_length,
- kRebuildWorkOrderCompleteMessage);
-
- std::free(proto_bytes);
- query_manager_->processMessage(message);
-
- return query_manager_->getQueryExecutionState().hasQueryExecutionFinished();
- }
-
- inline bool placeOutputBlockMessage(const QueryPlan::DAGNodeIndex index) {
- VLOG(3) << "Place OutputBlock message for Op[" << index << "]";
- serialization::DataPipelineMessage proto;
- proto.set_operator_index(index);
-
- proto.set_block_id(0); // dummy block ID
- proto.set_relation_id(0); // dummy relation ID.
- proto.set_query_id(0); // dummy query ID.
-
- // NOTE(zuyu): Using the heap memory to serialize proto as a c-like string.
- const std::size_t proto_length = proto.ByteSize();
- char *proto_bytes = static_cast<char*>(std::malloc(proto_length));
- CHECK(proto.SerializeToArray(proto_bytes, proto_length));
-
- tmb::TaggedMessage tagged_message(static_cast<const void *>(proto_bytes),
- proto_length,
- kDataPipelineMessage);
- std::free(proto_bytes);
- query_manager_->processMessage(tagged_message);
- return query_manager_->getQueryExecutionState().hasQueryExecutionFinished();
- }
-
- unique_ptr<CatalogDatabase> db_;
- unique_ptr<StorageManager> storage_manager_;
-
- QueryPlan *query_plan_;
- unique_ptr<QueryHandle> query_handle_;
- unique_ptr<QueryManager> query_manager_;
-
- MessageBusImpl bus_;
-
- client_id worker_client_id_;
-
- unique_ptr<WorkerDirectory> workers_;
-};
-
-TEST_F(QueryManagerTest, SingleNodeDAGNoWorkOrdersTest) {
- // This test creates a DAG of a single node. No workorders are generated.
- query_plan_->addRelationalOperator(new MockOperator(false, false));
-
- const MockOperator &op = static_cast<const MockOperator &>(
- query_plan_->getQueryPlanDAG().getNodePayload(0));
-
- constructQueryManager();
-
- // op doesn't have any dependencies.
- EXPECT_TRUE(op.getBlockingDependenciesMet());
-
- // We expect one call for op's getAllWorkOrders().
- EXPECT_EQ(1, op.getNumCalls(MockOperator::kGetAllWorkOrders));
- EXPECT_EQ(0, op.getNumCalls(MockOperator::kFeedInputBlock));
- EXPECT_EQ(0, op.getNumCalls(MockOperator::kFeedInputBlocks));
-}
-
-TEST_F(QueryManagerTest, SingleNodeDAGStaticWorkOrdersTest) {
- // This test creates a DAG of a single node. Static workorders are generated.
- const QueryPlan::DAGNodeIndex id =
- query_plan_->addRelationalOperator(new MockOperator(true, false, 1));
-
- const MockOperator &op = static_cast<const MockOperator &>(
- query_plan_->getQueryPlanDAG().getNodePayload(id));
-
- constructQueryManager();
-
- // op doesn't have any dependencies.
- EXPECT_TRUE(op.getBlockingDependenciesMet());
-
- // We expect one call for op's getAllWorkOrders().
- EXPECT_EQ(1, op.getNumCalls(MockOperator::kGetAllWorkOrders));
- EXPECT_EQ(0, op.getNumCalls(MockOperator::kFeedInputBlock));
- EXPECT_EQ(0, op.getNumCalls(MockOperator::kFeedInputBlocks));
-
- // One workorder is generated.
- EXPECT_EQ(1, op.getNumWorkOrders());
-
- unique_ptr<WorkerMessage> worker_message;
- worker_message.reset(query_manager_->getNextWorkerMessage(0, -1));
- EXPECT_TRUE(worker_message != nullptr);
-
- EXPECT_EQ(WorkerMessage::WorkerMessageType::kWorkOrder,
- worker_message->getType());
- EXPECT_EQ(0u, worker_message->getRelationalOpIndex());
-
- delete worker_message->getWorkOrder();
-
- EXPECT_EQ(1, getNumWorkOrdersInExecution(id));
- EXPECT_EQ(0, getNumOperatorsFinished());
-
- // Send a message to QueryManager upon workorder completion.
- // Last event processed by QueryManager.
- EXPECT_TRUE(placeWorkOrderCompleteMessage(id));
-
- EXPECT_EQ(0, getNumWorkOrdersInExecution(id));
- EXPECT_EQ(1, getNumOperatorsFinished());
- EXPECT_TRUE(getOperatorFinishedStatus(id));
-}
-
-TEST_F(QueryManagerTest, SingleNodeDAGDynamicWorkOrdersTest) {
- // This test creates a DAG of a single node. WorkOrders are generated
- // dynamically as pending work orders complete execution, i.e.,
- // getAllWorkOrders() is called multiple times. getAllWorkOrders() will be
- // called 5 times and 3 work orders will be returned, i.e., 1st 3 calls to
- // getAllWorkOrders() insert 1 WorkOrder and return false, and the next will
- // insert no WorkOrder and return true.
-
- // TODO(shoban): This test can not be more robust than this because of fixed
- // scaffolding of mocking. If we use gMock, we can do much better.
- const QueryPlan::DAGNodeIndex id =
- query_plan_->addRelationalOperator(new MockOperator(true, false, 4, 3));
-
- const MockOperator &op = static_cast<const MockOperator &>(
- query_plan_->getQueryPlanDAG().getNodePayload(id));
-
- constructQueryManager();
-
- // op doesn't have any dependencies.
- EXPECT_TRUE(op.getBlockingDependenciesMet());
-
- for (int i = 0; i < 3; ++i) {
- // We expect one call for op's getAllWorkOrders().
- EXPECT_EQ(i + 1, op.getNumCalls(MockOperator::kGetAllWorkOrders));
-
- // One workorder is generated.
- // EXPECT_EQ(1, getWorkerInputQueueSize());
- EXPECT_EQ(i + 1, op.getNumWorkOrders());
-
- unique_ptr<WorkerMessage> worker_message;
- worker_message.reset(query_manager_->getNextWorkerMessage(id, -1));
-
- EXPECT_TRUE(worker_message != nullptr);
- EXPECT_EQ(WorkerMessage::WorkerMessageType::kWorkOrder,
- worker_message->getType());
- EXPECT_EQ(id, worker_message->getRelationalOpIndex());
-
- delete worker_message->getWorkOrder();
-
- EXPECT_EQ(1, getNumWorkOrdersInExecution(id));
- EXPECT_EQ(0, getNumOperatorsFinished());
-
- if (i < 2) {
- // Send a message to QueryManager upon workorder completion.
- EXPECT_FALSE(placeWorkOrderCompleteMessage(id));
- } else {
- // Send a message to QueryManager upon workorder completion.
- // Last event.
- EXPECT_TRUE(placeWorkOrderCompleteMessage(id));
- }
- }
-
- EXPECT_EQ(0, getNumWorkOrdersInExecution(id));
-
- EXPECT_EQ(1, getNumOperatorsFinished());
- EXPECT_TRUE(getOperatorFinishedStatus(id));
-
- // We place this check in the end, since it's true throughout the test.
- EXPECT_EQ(0, op.getNumCalls(MockOperator::kFeedInputBlock));
- EXPECT_EQ(0, op.getNumCalls(MockOperator::kFeedInputBlocks));
-}
-
-TEST_F(QueryManagerTest, TwoNodesDAGBlockingLinkTest) {
- // We use two nodes in the DAG with a blocking link between them.
- // There is no streaming of data involved in this test.
- const QueryPlan::DAGNodeIndex id1 =
- query_plan_->addRelationalOperator(new MockOperator(true, false));
- const QueryPlan::DAGNodeIndex id2 =
- query_plan_->addRelationalOperator(new MockOperator(true, false));
-
- // Create a blocking link.
- query_plan_->addDirectDependency(id2, id1, true);
-
- static_cast<MockOperator *>(
- query_plan_->getQueryPlanDAGMutable()->getNodePayloadMutable(id1))
- ->setOutputRelationID(0xdead);
-
- const MockOperator &op1 = static_cast<const MockOperator &>(
- query_plan_->getQueryPlanDAG().getNodePayload(id1));
- const MockOperator &op2 = static_cast<const MockOperator &>(
- query_plan_->getQueryPlanDAG().getNodePayload(id2));
-
- constructQueryManager();
-
- // op1 doesn't have any dependencies
- EXPECT_TRUE(op1.getBlockingDependenciesMet());
-
- // Only op1 should receive a call to getAllWorkOrders initially.
- EXPECT_EQ(1, op1.getNumCalls(MockOperator::kGetAllWorkOrders));
- EXPECT_EQ(0, op1.getNumCalls(MockOperator::kFeedInputBlock));
- EXPECT_EQ(0, op1.getNumCalls(MockOperator::kFeedInputBlocks));
-
- EXPECT_EQ(0, op2.getNumCalls(MockOperator::kGetAllWorkOrders));
- EXPECT_EQ(0, op2.getNumCalls(MockOperator::kFeedInputBlock));
- EXPECT_EQ(0, op2.getNumCalls(MockOperator::kFeedInputBlocks));
-
- // Only op1 should produce a workorder.
- EXPECT_EQ(1, op1.getNumWorkOrders());
- EXPECT_EQ(0, op2.getNumWorkOrders());
-
- // Foreman hasn't yet got workorder completion response for the workorder.
- unique_ptr<WorkerMessage> worker_message;
- worker_message.reset(query_manager_->getNextWorkerMessage(id1, -1));
-
- EXPECT_TRUE(worker_message != nullptr);
- EXPECT_EQ(WorkerMessage::WorkerMessageType::kWorkOrder,
- worker_message->getType());
- EXPECT_EQ(id1, worker_message->getRelationalOpIndex());
-
- delete worker_message->getWorkOrder();
-
- EXPECT_EQ(1, getNumWorkOrdersInExecution(id1));
- EXPECT_EQ(0, getNumWorkOrdersInExecution(id2));
- EXPECT_EQ(0, getNumOperatorsFinished());
-
- // Send a message to Foreman upon workorder (generated by op1) completion.
- EXPECT_FALSE(placeWorkOrderCompleteMessage(id1));
-
- EXPECT_EQ(0, getNumWorkOrdersInExecution(id1));
- // op1 is over now, op2 still to go.
- EXPECT_EQ(1, op2.getNumCalls(MockOperator::kDoneFeedingInputBlocks));
- EXPECT_EQ(1, getNumOperatorsFinished());
-
- EXPECT_TRUE(getOperatorFinishedStatus(id1));
- EXPECT_EQ(1, op2.getNumCalls(MockOperator::kDoneFeedingInputBlocks));
- EXPECT_FALSE(getOperatorFinishedStatus(id2));
-
- worker_message.reset(query_manager_->getNextWorkerMessage(id2, -1));
- EXPECT_TRUE(worker_message != nullptr);
- EXPECT_EQ(WorkerMessage::WorkerMessageType::kWorkOrder,
- worker_message->getType());
- EXPECT_EQ(id2, worker_message->getRelationalOpIndex());
-
- delete worker_message->getWorkOrder();
-
- EXPECT_EQ(1, getNumWorkOrdersInExecution(id2));
-
- // op1 is op2's blocking dependency.
- EXPECT_TRUE(op2.getBlockingDependenciesMet());
-
- EXPECT_EQ(1, op1.getNumCalls(MockOperator::kGetAllWorkOrders));
- // op2 should get first call of getAllWorkOrders() when op1 is over.
- EXPECT_EQ(1, op2.getNumCalls(MockOperator::kGetAllWorkOrders));
-
- EXPECT_EQ(1, op2.getNumWorkOrders());
-
- // Send a message to QueryManager upon workorder (generated by op2) completion.
- // Note that the worker hasn't yet popped the workorder. Usually this won't
- // happen as workers pop workorders first, execute and then send the response.
- EXPECT_TRUE(placeWorkOrderCompleteMessage(id2));
-
- EXPECT_EQ(1, op2.getNumCalls(MockOperator::kGetAllWorkOrders));
-
- EXPECT_EQ(0, getNumWorkOrdersInExecution(id1));
- EXPECT_EQ(0, getNumWorkOrdersInExecution(id2));
-
- EXPECT_EQ(2, getNumOperatorsFinished());
- EXPECT_TRUE(getOperatorFinishedStatus(id1));
- EXPECT_TRUE(getOperatorFinishedStatus(id2));
-
- // Expect no additional calls to getAllWorkOrders.
- EXPECT_EQ(1, op1.getNumCalls(MockOperator::kGetAllWorkOrders));
- EXPECT_EQ(1, op2.getNumCalls(MockOperator::kGetAllWorkOrders));
-}
-
-TEST_F(QueryManagerTest, TwoNodesDAGPipeLinkTest) {
- // We use two nodes in the DAG with a non-blocking link between them.
- // We stream output of op1 to op2. Sequeuce of events is as follows:
- // 1. op1 creates a workorder.
- // 2. We send a "block full" (from op1) to QueryManager.
- // 3. op2 creates a workorder because of step 2.
- const QueryPlan::DAGNodeIndex id1 =
- query_plan_->addRelationalOperator(new MockOperator(true, false, 1));
- const QueryPlan::DAGNodeIndex id2 =
- query_plan_->addRelationalOperator(new MockOperator(true, true, 3));
-
- // Create a non-blocking link.
- query_plan_->addDirectDependency(id2, id1, false);
-
- static_cast<MockOperator *>(
- query_plan_->getQueryPlanDAGMutable()->getNodePayloadMutable(id1))
- ->setOutputRelationID(0xdead);
-
- const MockOperator &op1 = static_cast<const MockOperator &>(
- query_plan_->getQueryPlanDAG().getNodePayload(id1));
- const MockOperator &op2 = static_cast<const MockOperator &>(
- query_plan_->getQueryPlanDAG().getNodePayload(id2));
-
- constructQueryManager();
-
- // As none of the operators have a blocking link, blocking dependencies should
- // be met.
- EXPECT_TRUE(op1.getBlockingDependenciesMet());
- EXPECT_TRUE(op2.getBlockingDependenciesMet());
-
- EXPECT_EQ(1, op1.getNumCalls(MockOperator::kGetAllWorkOrders));
- EXPECT_EQ(1, op1.getNumWorkOrders());
- EXPECT_EQ(0, op1.getNumCalls(MockOperator::kFeedInputBlock));
- EXPECT_EQ(0, op1.getNumCalls(MockOperator::kFeedInputBlocks));
-
- EXPECT_EQ(1, op2.getNumCalls(MockOperator::kGetAllWorkOrders));
- // op2 will generate workorder only after receiving a streaming input.
- EXPECT_EQ(0, op2.getNumWorkOrders());
- EXPECT_EQ(0, op2.getNumCalls(MockOperator::kFeedInputBlock));
- EXPECT_EQ(0, op2.getNumCalls(MockOperator::kFeedInputBlocks));
-
- unique_ptr<WorkerMessage> worker_message;
- worker_message.reset(query_manager_->getNextWorkerMessage(id1, -1));
-
- EXPECT_TRUE(worker_message != nullptr);
- EXPECT_EQ(WorkerMessage::WorkerMessageType::kWorkOrder,
- worker_message->getType());
- EXPECT_EQ(id1, worker_message->getRelationalOpIndex());
-
- delete worker_message->getWorkOrder();
-
- // Send a message to QueryManager upon block getting full (output of op1).
- EXPECT_FALSE(placeOutputBlockMessage(id1));
-
- // op1 is not finished yet because the response of workorder completion hasn't
- // been received yet by the QueryManager.
- EXPECT_FALSE(getOperatorFinishedStatus(id1));
- EXPECT_FALSE(getOperatorFinishedStatus(id2));
-
- // No additional call to op1's getAllWorkOrders.
- EXPECT_EQ(1, op1.getNumCalls(MockOperator::kGetAllWorkOrders));
- EXPECT_EQ(0, op1.getNumCalls(MockOperator::kFeedInputBlock));
- EXPECT_EQ(0, op1.getNumCalls(MockOperator::kFeedInputBlocks));
-
- // Output from op1 should be fed to op2.
- EXPECT_EQ(1, op2.getNumCalls(MockOperator::kFeedInputBlock));
- EXPECT_EQ(0, op2.getNumCalls(MockOperator::kFeedInputBlocks));
-
- // A call to op2's getAllWorkOrders because of the streamed input.
- EXPECT_EQ(2, op2.getNumCalls(MockOperator::kGetAllWorkOrders));
- EXPECT_EQ(1, op2.getNumWorkOrders());
-
- // Place a message of a workorder completion of op1 on Foreman's input queue.
- EXPECT_FALSE(placeWorkOrderCompleteMessage(id1));
-
- EXPECT_EQ(0, getNumWorkOrdersInExecution(id1));
- EXPECT_TRUE(getOperatorFinishedStatus(id1));
- EXPECT_EQ(1, op2.getNumCalls(MockOperator::kDoneFeedingInputBlocks));
-
- // An additional call to op2's getAllWorkOrders because of completion of op1.
- EXPECT_EQ(3, op2.getNumCalls(MockOperator::kGetAllWorkOrders));
- EXPECT_EQ(2, op2.getNumWorkOrders());
-
- worker_message.reset(query_manager_->getNextWorkerMessage(id2, -1));
-
- EXPECT_TRUE(worker_message != nullptr);
- EXPECT_EQ(WorkerMessage::WorkerMessageType::kWorkOrder,
- worker_message->getType());
- EXPECT_EQ(id2, worker_message->getRelationalOpIndex());
-
- delete worker_message->getWorkOrder();
-
- // Place a message of a workorder completion of op2 on Foreman's input queue.
- EXPECT_FALSE(placeWorkOrderCompleteMessage(id2));
-
- EXPECT_TRUE(getOperatorFinishedStatus(id1));
-
- worker_message.reset(query_manager_->getNextWorkerMessage(id2, -1));
-
- EXPECT_TRUE(worker_message != nullptr);
- EXPECT_EQ(WorkerMessage::WorkerMessageType::kWorkOrder,
- worker_message->getType());
- EXPECT_EQ(id2, worker_message->getRelationalOpIndex());
-
- delete worker_message->getWorkOrder();
-
- EXPECT_EQ(1, getNumWorkOrdersInExecution(id2));
- EXPECT_FALSE(getOperatorFinishedStatus(id2));
-
- // Send a message to Foreman upon workorder (generated by op2) completion.
- EXPECT_TRUE(placeWorkOrderCompleteMessage(id2));
-
- EXPECT_TRUE(getOperatorFinishedStatus(id1));
- EXPECT_EQ(1, op2.getNumCalls(MockOperator::kDoneFeedingInputBlocks));
-
- EXPECT_EQ(0, getNumWorkOrdersInExecution(id2));
- EXPECT_TRUE(getOperatorFinishedStatus(id2));
-}
-
-TEST_F(QueryManagerTest, TwoNodesDAGPartiallyFilledBlocksTest) {
- // In this test, we create a 2-node DAG with a non-blocking link between them.
- // There is no streaming of data from op1 to op2 during the execution of op1.
- // op1 produces a partially filled block at the end of its execution which is
- // rebuilt and then fed to op2.
- const QueryPlan::DAGNodeIndex id1 =
- query_plan_->addRelationalOperator(new MockOperator(true, false, 1));
- const QueryPlan::DAGNodeIndex id2 =
- query_plan_->addRelationalOperator(new MockOperator(true, true, 3, 1));
-
- // Create a non-blocking link.
- query_plan_->addDirectDependency(id2, id1, false);
-
- // Create a relation, owned by db_.*/
- CatalogRelation *relation =
- new CatalogRelation(nullptr /* catalog_database */, "test_relation");
- const relation_id output_relation_id = db_->addRelation(relation);
-
- // Setup the InsertDestination proto in the query context proto.
- serialization::QueryContext *query_context_proto =
- query_handle_->getQueryContextProtoMutable();
-
- const QueryContext::insert_destination_id insert_destination_index =
- query_context_proto->insert_destinations_size();
- serialization::InsertDestination *insert_destination_proto =
- query_context_proto->add_insert_destinations();
-
- insert_destination_proto->set_insert_destination_type(
- serialization::InsertDestinationType::BLOCK_POOL);
- insert_destination_proto->set_relation_id(output_relation_id);
- insert_destination_proto->set_relational_op_index(id1);
-
- MockOperator *op1_mutable = static_cast<MockOperator *>(
- query_plan_->getQueryPlanDAGMutable()->getNodePayloadMutable(id1));
- op1_mutable->setInsertDestinationID(insert_destination_index);
- op1_mutable->setOutputRelationID(output_relation_id);
-
- const MockOperator &op1 = static_cast<const MockOperator &>(
- query_plan_->getQueryPlanDAG().getNodePayload(id1));
- const MockOperator &op2 = static_cast<const MockOperator &>(
- query_plan_->getQueryPlanDAG().getNodePayload(id2));
-
- constructQueryManager();
-
- // NOTE(zuyu): An operator generally has no ideas about partially filled
- // blocks, but InsertDestination in QueryContext does.
- // Mock to add partially filled blocks in the InsertDestination.
- InsertDestination *insert_destination =
- query_manager_->getQueryContextMutable()->getInsertDestination(
- insert_destination_index);
- DCHECK(insert_destination != nullptr);
- MutableBlockReference block_ref;
- static_cast<BlockPoolInsertDestination *>(insert_destination)
- ->available_block_refs_.push_back(move(block_ref));
-
- // There's no blocking dependency in the DAG.
- EXPECT_TRUE(op1.getBlockingDependenciesMet());
- EXPECT_TRUE(op2.getBlockingDependenciesMet());
-
- EXPECT_EQ(1, op1.getNumCalls(MockOperator::kGetAllWorkOrders));
- EXPECT_EQ(1, op1.getNumWorkOrders());
-
- EXPECT_EQ(1, op2.getNumCalls(MockOperator::kGetAllWorkOrders));
- EXPECT_EQ(0, op2.getNumWorkOrders());
-
- unique_ptr<WorkerMessage> worker_message;
- worker_message.reset(query_manager_->getNextWorkerMessage(id1, -1));
-
- EXPECT_TRUE(worker_message != nullptr);
- EXPECT_EQ(WorkerMessage::WorkerMessageType::kWorkOrder,
- worker_message->getType());
- EXPECT_EQ(id1, worker_message->getRelationalOpIndex());
-
- delete worker_message->getWorkOrder();
-
- // Send a message to QueryManager upon workorder (generated by op1) completion.
- EXPECT_FALSE(placeWorkOrderCompleteMessage(id1));
-
- EXPECT_EQ(0, getNumWorkOrdersInExecution(id1));
-
- worker_message.reset(query_manager_->getNextWorkerMessage(id1, -1));
- EXPECT_TRUE(worker_message != nullptr);
- EXPECT_EQ(WorkerMessage::WorkerMessageType::kRebuildWorkOrder,
- worker_message->getType());
-
- EXPECT_EQ(id1, worker_message->getRelationalOpIndex());
-
- delete worker_message->getWorkOrder();
-
- // op1 generates a rebuild workorder. The block is rebuilt and streamed
- // to Foreman.
- EXPECT_FALSE(placeDataPipelineMessage(id1));
-
- EXPECT_FALSE(placeRebuildWorkOrderCompleteMessage(id1));
- // Based on the streamed input, op2's getAllWorkOrders should produce a
- // workorder.
- EXPECT_EQ(3, op2.getNumCalls(MockOperator::kGetAllWorkOrders));
- EXPECT_EQ(1, op2.getNumWorkOrders());
-
- worker_message.reset(query_manager_->getNextWorkerMessage(id2, -1));
-
- EXPECT_TRUE(worker_message != nullptr);
- EXPECT_EQ(WorkerMessage::WorkerMessageType::kWorkOrder,
- worker_message->getType());
-
- EXPECT_EQ(id2, worker_message->getRelationalOpIndex());
-
- delete worker_message->getWorkOrder();
-
- EXPECT_TRUE(getOperatorFinishedStatus(id1));
- EXPECT_EQ(1, op2.getNumCalls(MockOperator::kDoneFeedingInputBlocks));
- EXPECT_FALSE(getOperatorFinishedStatus(id2));
- EXPECT_EQ(1, getNumWorkOrdersInExecution(id2));
-
- // Send a message to QueryManager upon workorder (generated by op2) completion.
- EXPECT_TRUE(placeWorkOrderCompleteMessage(id2));
-
- EXPECT_EQ(0, getNumWorkOrdersInExecution(id2));
-
- EXPECT_TRUE(getOperatorFinishedStatus(id2));
-}
-
-TEST_F(QueryManagerTest, MultipleNodesNoOutputTest) {
- // When an operator produces workorders but no output, the QueryManager should
- // check the dependents of this operator to make progress.
- const QueryPlan::DAGNodeIndex kNumNodes = 5;
- std::vector<QueryPlan::DAGNodeIndex> ids;
- ids.reserve(kNumNodes);
-
- for (QueryPlan::DAGNodeIndex i = 0; i < kNumNodes; ++i) {
- if (i == 0) {
- ids[i] = query_plan_->addRelationalOperator(new MockOperator(true, false));
- } else {
- ids[i] = query_plan_->addRelationalOperator(new MockOperator(true, true));
- }
- VLOG(3) << ids[i];
- }
-
- /**
- * The DAG looks like this:
- *
- * op1 -> op2 -> op3 -> op4 -> op5
- *
- **/
- for (QueryPlan::DAGNodeIndex i = 0; i < kNumNodes - 1; ++i) {
- query_plan_->addDirectDependency(ids[i + 1], ids[i], false);
- static_cast<MockOperator*>(query_plan_->getQueryPlanDAGMutable()->getNodePayloadMutable(ids[i]))
- ->setOutputRelationID(0xdead);
- }
-
- std::vector<const MockOperator*> operators;
- for (QueryPlan::DAGNodeIndex i = 0; i < kNumNodes; ++i) {
- operators.push_back(static_cast<const MockOperator*>(&query_plan_->getQueryPlanDAG().getNodePayload(ids[i])));
- }
-
- constructQueryManager();
-
- // operators[0] should have produced a workorder by now.
- EXPECT_EQ(1, operators[0]->getNumWorkOrders());
-
- unique_ptr<WorkerMessage> worker_message;
- worker_message.reset(query_manager_->getNextWorkerMessage(ids[0], -1));
-
- EXPECT_TRUE(worker_message != nullptr);
- EXPECT_EQ(WorkerMessage::WorkerMessageType::kWorkOrder,
- worker_message->getType());
-
- EXPECT_EQ(ids[0], worker_message->getRelationalOpIndex());
-
- delete worker_message->getWorkOrder();
-
- EXPECT_EQ(1, getNumWorkOrdersInExecution(ids[0]));
- EXPECT_FALSE(getOperatorFinishedStatus(ids[0]));
-
- for (QueryPlan::DAGNodeIndex i = 0; i < kNumNodes; ++i) {
- EXPECT_EQ(1, operators[ids[i]]->getNumCalls(MockOperator::kGetAllWorkOrders));
- }
-
- // Send a message to QueryManager upon workorder (generated by operators[0])
- // completion.
- EXPECT_TRUE(placeWorkOrderCompleteMessage(ids[0]));
-
- for (QueryPlan::DAGNodeIndex i = 0; i < kNumNodes; ++i) {
- EXPECT_EQ(0, getNumWorkOrdersInExecution(ids[i]));
- EXPECT_TRUE(getOperatorFinishedStatus(ids[i]));
- if (i < kNumNodes - 1) {
- EXPECT_EQ(1, operators[i + 1]->getNumCalls(MockOperator::kDoneFeedingInputBlocks));
- }
- }
-}
-
-TEST_F(QueryManagerTest, OutOfOrderWorkOrderCompletionTest) {
- // Consider two operators, both generate one workorder each. The dependent's
- // workorder finishes before dependency's workorder.
- const QueryPlan::DAGNodeIndex id1 = query_plan_->addRelationalOperator(new MockOperator(true, false, 1));
- const QueryPlan::DAGNodeIndex id2 = query_plan_->addRelationalOperator(new MockOperator(true, true, 2, 1));
-
- // Create a non-blocking link.
- query_plan_->addDirectDependency(id2, id1, false);
-
- constructQueryManager();
-
- unique_ptr<WorkerMessage> worker_message;
- worker_message.reset(query_manager_->getNextWorkerMessage(id1, -1));
-
- EXPECT_TRUE(worker_message != nullptr);
- EXPECT_EQ(WorkerMessage::WorkerMessageType::kWorkOrder,
- worker_message->getType());
-
- EXPECT_EQ(id1, worker_message->getRelationalOpIndex());
-
- delete worker_message->getWorkOrder();
-
- // Send a message to QueryManager upon a block (output of op1) getting full.
- EXPECT_FALSE(placeOutputBlockMessage(id1));
-
- // op1 is not finished yet because the response of workorder completion hasn't
- // been received yet.
- EXPECT_FALSE(getOperatorFinishedStatus(id1));
- EXPECT_FALSE(getOperatorFinishedStatus(id2));
-
- worker_message.reset(query_manager_->getNextWorkerMessage(id2, -1));
- EXPECT_TRUE(worker_message != nullptr);
- EXPECT_EQ(WorkerMessage::WorkerMessageType::kWorkOrder,
- worker_message->getType());
-
- EXPECT_EQ(id2, worker_message->getRelationalOpIndex());
-
- delete worker_message->getWorkOrder();
-
- // As mentioned earlier, op2 finishes before op1.
- EXPECT_FALSE(placeWorkOrderCompleteMessage(id2));
-
- // op1's workorder execution is over.
- EXPECT_TRUE(placeWorkOrderCompleteMessage(id1));
-
- EXPECT_TRUE(getOperatorFinishedStatus(id1));
- EXPECT_TRUE(getOperatorFinishedStatus(id2));
-}
-
-} // namespace quickstep