You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@quickstep.apache.org by sp...@apache.org on 2016/12/11 17:45:39 UTC
[16/51] [abbrv] [partial] incubator-quickstep git commit: remove c++
files
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/156290a4/query_execution/tests/QueryManagerSingleNode_unittest.cpp
----------------------------------------------------------------------
diff --git a/query_execution/tests/QueryManagerSingleNode_unittest.cpp b/query_execution/tests/QueryManagerSingleNode_unittest.cpp
deleted file mode 100644
index f65ec53..0000000
--- a/query_execution/tests/QueryManagerSingleNode_unittest.cpp
+++ /dev/null
@@ -1,890 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- **/
-
-#include <climits>
-#include <memory>
-#include <string>
-#include <utility>
-#include <vector>
-
-#include "catalog/CatalogDatabase.hpp"
-#include "catalog/CatalogRelation.hpp"
-#include "catalog/CatalogTypedefs.hpp"
-#include "query_execution/QueryContext.hpp"
-#include "query_execution/QueryContext.pb.h"
-#include "query_execution/QueryExecutionState.hpp"
-#include "query_execution/QueryExecutionTypedefs.hpp"
-#include "query_execution/QueryManagerSingleNode.hpp"
-#include "query_execution/WorkOrdersContainer.hpp"
-#include "query_execution/WorkerDirectory.hpp"
-#include "query_execution/WorkerMessage.hpp"
-#include "query_optimizer/QueryHandle.hpp"
-#include "query_optimizer/QueryPlan.hpp"
-#include "relational_operators/RelationalOperator.hpp"
-#include "relational_operators/WorkOrder.hpp"
-#include "storage/InsertDestination.hpp"
-#include "storage/InsertDestination.pb.h"
-#include "storage/StorageBlock.hpp"
-#include "storage/StorageBlockInfo.hpp"
-#include "storage/StorageManager.hpp"
-#include "utility/DAG.hpp"
-#include "utility/Macros.hpp"
-
-#include "glog/logging.h"
-#include "gtest/gtest.h"
-
-#include "tmb/id_typedefs.h"
-
-namespace tmb { class MessageBus; }
-
-using std::move;
-using std::unique_ptr;
-using std::vector;
-
-using tmb::client_id;
-
-namespace quickstep {
-
-class WorkOrderProtosContainer;
-
-class MockWorkOrder : public WorkOrder {
- public:
- explicit MockWorkOrder(const int op_index)
- : WorkOrder(0), op_index_(op_index) {}
-
- void execute() override {
- VLOG(3) << "WorkOrder[" << op_index_ << "] executing.";
- }
-
- inline QueryPlan::DAGNodeIndex getOpIndex() const {
- return op_index_;
- }
-
- private:
- const QueryPlan::DAGNodeIndex op_index_;
-
- DISALLOW_COPY_AND_ASSIGN(MockWorkOrder);
-};
-
-class MockOperator: public RelationalOperator {
- public:
- enum function_name {
- kFeedInputBlock = 0,
- kFeedInputBlocks,
- kDoneFeedingInputBlocks,
- kGetAllWorkOrders
- };
-
- MockOperator(const bool produce_workorders,
- const bool has_streaming_input,
- const int max_getworkorder_iters = 1,
- const int max_workorders = INT_MAX)
- : RelationalOperator(0 /* Query Id */),
- produce_workorders_(produce_workorders),
- has_streaming_input_(has_streaming_input),
- max_workorders_(max_workorders),
- max_getworkorder_iters_(max_getworkorder_iters),
- num_calls_get_workorders_(0),
- num_workorders_generated_(0),
- num_calls_feedblock_(0),
- num_calls_feedblocks_(0),
- num_calls_donefeedingblocks_(0) {
- }
-
- std::string getName() const override {
- return "MockOperator";
- }
-
-#define MOCK_OP_LOG(x) VLOG(x) << "Op[" << op_index_ << "]: " << __func__ << ": "
-
- // The methods below are used to check whether QueryManager calls the Relational
- // operator, how many times it calls a particular method etc.
- inline int getNumWorkOrders() const {
- return num_workorders_generated_;
- }
-
- inline int getNumCalls(const function_name fname) const {
- switch (fname) {
- case kFeedInputBlock:
- return num_calls_feedblock_;
- case kFeedInputBlocks:
- return num_calls_feedblocks_;
- case kDoneFeedingInputBlocks:
- return num_calls_donefeedingblocks_;
- case kGetAllWorkOrders:
- return num_calls_get_workorders_;
- default:
- return -1;
- }
- }
-
- inline bool getBlockingDependenciesMet() const {
- MOCK_OP_LOG(3) << "met.";
- return blocking_dependencies_met_;
- }
-
- void setInsertDestinationID(const QueryContext::insert_destination_id insert_destination_index) {
- insert_destination_index_ = insert_destination_index;
- }
-
- // Mock to trigger doneFeedingInputBlocks for the dependent operators
- // in QueryManager::markOperatorFinished.
- void setOutputRelationID(const relation_id rel_id) {
- output_relation_id_ = rel_id;
- }
-
- // Override methods from the base class.
- bool getAllWorkOrders(
- WorkOrdersContainer *container,
- QueryContext *query_context,
- StorageManager *storage_manager,
- const tmb::client_id foreman_client_id,
- tmb::MessageBus *bus) override {
- ++num_calls_get_workorders_;
- if (produce_workorders_) {
- if (has_streaming_input_) {
- if ((num_calls_feedblock_ > 0 || num_calls_feedblocks_ > 0) && (num_workorders_generated_ < max_workorders_)) {
- MOCK_OP_LOG(3) << "[stream] generate WorkOrder";
- container->addNormalWorkOrder(new MockWorkOrder(op_index_), op_index_);
- ++num_workorders_generated_;
- }
- } else {
- if (blocking_dependencies_met_ && (num_workorders_generated_ < max_workorders_)) {
- MOCK_OP_LOG(3) << "[static] generate WorkOrder";
- container->addNormalWorkOrder(new MockWorkOrder(op_index_), op_index_);
- ++num_workorders_generated_;
- }
- }
- }
- MOCK_OP_LOG(3) << "count(" << num_calls_get_workorders_ << ") "
- << "return(" << (num_calls_get_workorders_ == max_getworkorder_iters_) << ")";
- return num_calls_get_workorders_ == max_getworkorder_iters_;
- }
-
- bool getAllWorkOrderProtos(WorkOrderProtosContainer *container) override {
- return true;
- }
-
- void feedInputBlock(const block_id input_block_id,
- const relation_id input_relation_id) override {
- ++num_calls_feedblock_;
- MOCK_OP_LOG(3) << "count(" << num_calls_feedblock_ << ")";
- }
-
- void feedInputBlocks(const relation_id rel_id,
- std::vector<block_id> *partially_filled_blocks) override {
- ++num_calls_feedblocks_;
- MOCK_OP_LOG(3) << "count(" << num_calls_feedblocks_ << ")";
- }
-
- void doneFeedingInputBlocks(const relation_id rel_id) override {
- ++num_calls_donefeedingblocks_;
- MOCK_OP_LOG(3) << "count(" << num_calls_donefeedingblocks_ << ")";
- }
-
- QueryContext::insert_destination_id getInsertDestinationID() const override {
- return insert_destination_index_;
- }
-
- const relation_id getOutputRelationID() const override {
- return output_relation_id_;
- }
-
- private:
- const bool produce_workorders_;
- const bool has_streaming_input_;
- const int max_workorders_;
- const int max_getworkorder_iters_;
-
- int num_calls_get_workorders_;
- int num_workorders_generated_;
- int num_calls_feedblock_;
- int num_calls_feedblocks_;
- int num_calls_donefeedingblocks_;
-
- QueryContext::insert_destination_id insert_destination_index_ = QueryContext::kInvalidInsertDestinationId;
-
- relation_id output_relation_id_ = -1;
-
-#undef MOCK_OP_LOG
-
- DISALLOW_COPY_AND_ASSIGN(MockOperator);
-};
-
-
-class QueryManagerTest : public ::testing::Test {
- protected:
- virtual void SetUp() {
- db_.reset(new CatalogDatabase(nullptr /* catalog */, "database"));
- storage_manager_.reset(new StorageManager("./"));
- bus_.Initialize();
- query_handle_.reset(new QueryHandle(0 /* dummy query ID */, tmb::kClientIdNone /* cli_id */));
- query_plan_ = query_handle_->getQueryPlanMutable();
- query_handle_->getQueryContextProtoMutable()->set_query_id(query_handle_->query_id());
- }
-
- inline void constructQueryManager() {
- query_manager_.reset(new QueryManagerSingleNode(
- 0, 1, query_handle_.get(), db_.get(), storage_manager_.get(), &bus_));
- }
-
- inline const int getNumWorkOrdersInExecution(const QueryPlan::DAGNodeIndex index) const {
- return query_manager_->getQueryExecutionState().getNumQueuedWorkOrders(index);
- }
-
- inline const int getNumOperatorsFinished() const {
- return query_manager_->getQueryExecutionState().getNumOperatorsFinished();
- }
-
- inline bool getOperatorFinishedStatus(const QueryPlan::DAGNodeIndex index) const {
- return query_manager_->getQueryExecutionState().hasExecutionFinished(index);
- }
-
- inline bool placeDataPipelineMessage(const QueryPlan::DAGNodeIndex source_operator_index) {
- VLOG(3) << "Place DataPipeline message for Op[" << source_operator_index << "]";
-
- query_manager_->processDataPipelineMessage(source_operator_index,
- 0 /* dummy block ID */,
- 0 /* dummy relation ID */);
- return query_manager_->getQueryExecutionState().hasQueryExecutionFinished();
- }
-
- inline bool placeWorkOrderCompleteMessage(const QueryPlan::DAGNodeIndex index) {
- VLOG(3) << "Place WorkOrderComplete message for Op[" << index << "]";
-
- query_manager_->processWorkOrderCompleteMessage(index);
- return query_manager_->getQueryExecutionState().hasQueryExecutionFinished();
- }
-
- inline bool placeRebuildWorkOrderCompleteMessage(const QueryPlan::DAGNodeIndex index) {
- VLOG(3) << "Place RebuildWorkOrderComplete message for Op[" << index << "]";
-
- query_manager_->processRebuildWorkOrderCompleteMessage(index);
- return query_manager_->getQueryExecutionState().hasQueryExecutionFinished();
- }
-
- inline bool placeOutputBlockMessage(const QueryPlan::DAGNodeIndex index) {
- VLOG(3) << "Place OutputBlock message for Op[" << index << "]";
-
- query_manager_->processDataPipelineMessage(index,
- 0 /* dummy block ID */,
- 0 /* dummy relation ID */);
- return query_manager_->getQueryExecutionState().hasQueryExecutionFinished();
- }
-
- unique_ptr<CatalogDatabase> db_;
- unique_ptr<StorageManager> storage_manager_;
-
- QueryPlan *query_plan_;
- unique_ptr<QueryHandle> query_handle_;
- unique_ptr<QueryManagerSingleNode> query_manager_;
-
- MessageBusImpl bus_;
-
- client_id worker_client_id_;
-
- unique_ptr<WorkerDirectory> workers_;
-};
-
-TEST_F(QueryManagerTest, SingleNodeDAGNoWorkOrdersTest) {
- // This test creates a DAG of a single node. No workorders are generated.
- query_plan_->addRelationalOperator(new MockOperator(false, false));
-
- const MockOperator &op = static_cast<const MockOperator &>(
- query_plan_->getQueryPlanDAG().getNodePayload(0));
-
- constructQueryManager();
-
- // op doesn't have any dependencies.
- EXPECT_TRUE(op.getBlockingDependenciesMet());
-
- // We expect one call for op's getAllWorkOrders().
- EXPECT_EQ(1, op.getNumCalls(MockOperator::kGetAllWorkOrders));
- EXPECT_EQ(0, op.getNumCalls(MockOperator::kFeedInputBlock));
- EXPECT_EQ(0, op.getNumCalls(MockOperator::kFeedInputBlocks));
-}
-
-TEST_F(QueryManagerTest, SingleNodeDAGStaticWorkOrdersTest) {
- // This test creates a DAG of a single node. Static workorders are generated.
- const QueryPlan::DAGNodeIndex id =
- query_plan_->addRelationalOperator(new MockOperator(true, false, 1));
-
- const MockOperator &op = static_cast<const MockOperator &>(
- query_plan_->getQueryPlanDAG().getNodePayload(id));
-
- constructQueryManager();
-
- // op doesn't have any dependencies.
- EXPECT_TRUE(op.getBlockingDependenciesMet());
-
- // We expect one call for op's getAllWorkOrders().
- EXPECT_EQ(1, op.getNumCalls(MockOperator::kGetAllWorkOrders));
- EXPECT_EQ(0, op.getNumCalls(MockOperator::kFeedInputBlock));
- EXPECT_EQ(0, op.getNumCalls(MockOperator::kFeedInputBlocks));
-
- // One workorder is generated.
- EXPECT_EQ(1, op.getNumWorkOrders());
-
- unique_ptr<WorkerMessage> worker_message;
- worker_message.reset(query_manager_->getNextWorkerMessage(0, -1));
- EXPECT_TRUE(worker_message != nullptr);
-
- EXPECT_EQ(WorkerMessage::WorkerMessageType::kWorkOrder,
- worker_message->getType());
- EXPECT_EQ(0u, worker_message->getRelationalOpIndex());
-
- delete worker_message->getWorkOrder();
-
- EXPECT_EQ(1, getNumWorkOrdersInExecution(id));
- EXPECT_EQ(0, getNumOperatorsFinished());
-
- // Send a message to QueryManager upon workorder completion.
- // Last event processed by QueryManager.
- EXPECT_TRUE(placeWorkOrderCompleteMessage(id));
-
- EXPECT_EQ(0, getNumWorkOrdersInExecution(id));
- EXPECT_EQ(1, getNumOperatorsFinished());
- EXPECT_TRUE(getOperatorFinishedStatus(id));
-}
-
-TEST_F(QueryManagerTest, SingleNodeDAGDynamicWorkOrdersTest) {
- // This test creates a DAG of a single node. WorkOrders are generated
- // dynamically as pending work orders complete execution, i.e.,
- // getAllWorkOrders() is called multiple times. getAllWorkOrders() will be
- // called 5 times and 3 work orders will be returned, i.e., 1st 3 calls to
- // getAllWorkOrders() insert 1 WorkOrder and return false, and the next will
- // insert no WorkOrder and return true.
-
- // TODO(shoban): This test can not be more robust than this because of fixed
- // scaffolding of mocking. If we use gMock, we can do much better.
- const QueryPlan::DAGNodeIndex id =
- query_plan_->addRelationalOperator(new MockOperator(true, false, 4, 3));
-
- const MockOperator &op = static_cast<const MockOperator &>(
- query_plan_->getQueryPlanDAG().getNodePayload(id));
-
- constructQueryManager();
-
- // op doesn't have any dependencies.
- EXPECT_TRUE(op.getBlockingDependenciesMet());
-
- for (int i = 0; i < 3; ++i) {
- // We expect one call for op's getAllWorkOrders().
- EXPECT_EQ(i + 1, op.getNumCalls(MockOperator::kGetAllWorkOrders));
-
- // One workorder is generated.
- // EXPECT_EQ(1, getWorkerInputQueueSize());
- EXPECT_EQ(i + 1, op.getNumWorkOrders());
-
- unique_ptr<WorkerMessage> worker_message;
- worker_message.reset(query_manager_->getNextWorkerMessage(id, -1));
-
- EXPECT_TRUE(worker_message != nullptr);
- EXPECT_EQ(WorkerMessage::WorkerMessageType::kWorkOrder,
- worker_message->getType());
- EXPECT_EQ(id, worker_message->getRelationalOpIndex());
-
- delete worker_message->getWorkOrder();
-
- EXPECT_EQ(1, getNumWorkOrdersInExecution(id));
- EXPECT_EQ(0, getNumOperatorsFinished());
-
- if (i < 2) {
- // Send a message to QueryManager upon workorder completion.
- EXPECT_FALSE(placeWorkOrderCompleteMessage(id));
- } else {
- // Send a message to QueryManager upon workorder completion.
- // Last event.
- EXPECT_TRUE(placeWorkOrderCompleteMessage(id));
- }
- }
-
- EXPECT_EQ(0, getNumWorkOrdersInExecution(id));
-
- EXPECT_EQ(1, getNumOperatorsFinished());
- EXPECT_TRUE(getOperatorFinishedStatus(id));
-
- // We place this check in the end, since it's true throughout the test.
- EXPECT_EQ(0, op.getNumCalls(MockOperator::kFeedInputBlock));
- EXPECT_EQ(0, op.getNumCalls(MockOperator::kFeedInputBlocks));
-}
-
-TEST_F(QueryManagerTest, TwoNodesDAGBlockingLinkTest) {
- // We use two nodes in the DAG with a blocking link between them.
- // There is no streaming of data involved in this test.
- const QueryPlan::DAGNodeIndex id1 =
- query_plan_->addRelationalOperator(new MockOperator(true, false));
- const QueryPlan::DAGNodeIndex id2 =
- query_plan_->addRelationalOperator(new MockOperator(true, false));
-
- // Create a blocking link.
- query_plan_->addDirectDependency(id2, id1, true);
-
- static_cast<MockOperator *>(
- query_plan_->getQueryPlanDAGMutable()->getNodePayloadMutable(id1))
- ->setOutputRelationID(0xdead);
-
- const MockOperator &op1 = static_cast<const MockOperator &>(
- query_plan_->getQueryPlanDAG().getNodePayload(id1));
- const MockOperator &op2 = static_cast<const MockOperator &>(
- query_plan_->getQueryPlanDAG().getNodePayload(id2));
-
- constructQueryManager();
-
- // op1 doesn't have any dependencies
- EXPECT_TRUE(op1.getBlockingDependenciesMet());
-
- // Only op1 should receive a call to getAllWorkOrders initially.
- EXPECT_EQ(1, op1.getNumCalls(MockOperator::kGetAllWorkOrders));
- EXPECT_EQ(0, op1.getNumCalls(MockOperator::kFeedInputBlock));
- EXPECT_EQ(0, op1.getNumCalls(MockOperator::kFeedInputBlocks));
-
- EXPECT_EQ(0, op2.getNumCalls(MockOperator::kGetAllWorkOrders));
- EXPECT_EQ(0, op2.getNumCalls(MockOperator::kFeedInputBlock));
- EXPECT_EQ(0, op2.getNumCalls(MockOperator::kFeedInputBlocks));
-
- // Only op1 should produce a workorder.
- EXPECT_EQ(1, op1.getNumWorkOrders());
- EXPECT_EQ(0, op2.getNumWorkOrders());
-
- // Foreman hasn't yet got workorder completion response for the workorder.
- unique_ptr<WorkerMessage> worker_message;
- worker_message.reset(query_manager_->getNextWorkerMessage(id1, -1));
-
- EXPECT_TRUE(worker_message != nullptr);
- EXPECT_EQ(WorkerMessage::WorkerMessageType::kWorkOrder,
- worker_message->getType());
- EXPECT_EQ(id1, worker_message->getRelationalOpIndex());
-
- delete worker_message->getWorkOrder();
-
- EXPECT_EQ(1, getNumWorkOrdersInExecution(id1));
- EXPECT_EQ(0, getNumWorkOrdersInExecution(id2));
- EXPECT_EQ(0, getNumOperatorsFinished());
-
- // Send a message to Foreman upon workorder (generated by op1) completion.
- EXPECT_FALSE(placeWorkOrderCompleteMessage(id1));
-
- EXPECT_EQ(0, getNumWorkOrdersInExecution(id1));
- // op1 is over now, op2 still to go.
- EXPECT_EQ(1, op2.getNumCalls(MockOperator::kDoneFeedingInputBlocks));
- EXPECT_EQ(1, getNumOperatorsFinished());
-
- EXPECT_TRUE(getOperatorFinishedStatus(id1));
- EXPECT_EQ(1, op2.getNumCalls(MockOperator::kDoneFeedingInputBlocks));
- EXPECT_FALSE(getOperatorFinishedStatus(id2));
-
- worker_message.reset(query_manager_->getNextWorkerMessage(id2, -1));
- EXPECT_TRUE(worker_message != nullptr);
- EXPECT_EQ(WorkerMessage::WorkerMessageType::kWorkOrder,
- worker_message->getType());
- EXPECT_EQ(id2, worker_message->getRelationalOpIndex());
-
- delete worker_message->getWorkOrder();
-
- EXPECT_EQ(1, getNumWorkOrdersInExecution(id2));
-
- // op1 is op2's blocking dependency.
- EXPECT_TRUE(op2.getBlockingDependenciesMet());
-
- EXPECT_EQ(1, op1.getNumCalls(MockOperator::kGetAllWorkOrders));
- // op2 should get first call of getAllWorkOrders() when op1 is over.
- EXPECT_EQ(1, op2.getNumCalls(MockOperator::kGetAllWorkOrders));
-
- EXPECT_EQ(1, op2.getNumWorkOrders());
-
- // Send a message to QueryManager upon workorder (generated by op2) completion.
- // Note that the worker hasn't yet popped the workorder. Usually this won't
- // happen as workers pop workorders first, execute and then send the response.
- EXPECT_TRUE(placeWorkOrderCompleteMessage(id2));
-
- EXPECT_EQ(1, op2.getNumCalls(MockOperator::kGetAllWorkOrders));
-
- EXPECT_EQ(0, getNumWorkOrdersInExecution(id1));
- EXPECT_EQ(0, getNumWorkOrdersInExecution(id2));
-
- EXPECT_EQ(2, getNumOperatorsFinished());
- EXPECT_TRUE(getOperatorFinishedStatus(id1));
- EXPECT_TRUE(getOperatorFinishedStatus(id2));
-
- // Expect no additional calls to getAllWorkOrders.
- EXPECT_EQ(1, op1.getNumCalls(MockOperator::kGetAllWorkOrders));
- EXPECT_EQ(1, op2.getNumCalls(MockOperator::kGetAllWorkOrders));
-}
-
-TEST_F(QueryManagerTest, TwoNodesDAGPipeLinkTest) {
- // We use two nodes in the DAG with a non-blocking link between them.
- // We stream output of op1 to op2. Sequeuce of events is as follows:
- // 1. op1 creates a workorder.
- // 2. We send a "block full" (from op1) to QueryManager.
- // 3. op2 creates a workorder because of step 2.
- const QueryPlan::DAGNodeIndex id1 =
- query_plan_->addRelationalOperator(new MockOperator(true, false, 1));
- const QueryPlan::DAGNodeIndex id2 =
- query_plan_->addRelationalOperator(new MockOperator(true, true, 3));
-
- // Create a non-blocking link.
- query_plan_->addDirectDependency(id2, id1, false);
-
- static_cast<MockOperator *>(
- query_plan_->getQueryPlanDAGMutable()->getNodePayloadMutable(id1))
- ->setOutputRelationID(0xdead);
-
- const MockOperator &op1 = static_cast<const MockOperator &>(
- query_plan_->getQueryPlanDAG().getNodePayload(id1));
- const MockOperator &op2 = static_cast<const MockOperator &>(
- query_plan_->getQueryPlanDAG().getNodePayload(id2));
-
- constructQueryManager();
-
- // As none of the operators have a blocking link, blocking dependencies should
- // be met.
- EXPECT_TRUE(op1.getBlockingDependenciesMet());
- EXPECT_TRUE(op2.getBlockingDependenciesMet());
-
- EXPECT_EQ(1, op1.getNumCalls(MockOperator::kGetAllWorkOrders));
- EXPECT_EQ(1, op1.getNumWorkOrders());
- EXPECT_EQ(0, op1.getNumCalls(MockOperator::kFeedInputBlock));
- EXPECT_EQ(0, op1.getNumCalls(MockOperator::kFeedInputBlocks));
-
- EXPECT_EQ(1, op2.getNumCalls(MockOperator::kGetAllWorkOrders));
- // op2 will generate workorder only after receiving a streaming input.
- EXPECT_EQ(0, op2.getNumWorkOrders());
- EXPECT_EQ(0, op2.getNumCalls(MockOperator::kFeedInputBlock));
- EXPECT_EQ(0, op2.getNumCalls(MockOperator::kFeedInputBlocks));
-
- unique_ptr<WorkerMessage> worker_message;
- worker_message.reset(query_manager_->getNextWorkerMessage(id1, -1));
-
- EXPECT_TRUE(worker_message != nullptr);
- EXPECT_EQ(WorkerMessage::WorkerMessageType::kWorkOrder,
- worker_message->getType());
- EXPECT_EQ(id1, worker_message->getRelationalOpIndex());
-
- delete worker_message->getWorkOrder();
-
- // Send a message to QueryManager upon block getting full (output of op1).
- EXPECT_FALSE(placeOutputBlockMessage(id1));
-
- // op1 is not finished yet because the response of workorder completion hasn't
- // been received yet by the QueryManager.
- EXPECT_FALSE(getOperatorFinishedStatus(id1));
- EXPECT_FALSE(getOperatorFinishedStatus(id2));
-
- // No additional call to op1's getAllWorkOrders.
- EXPECT_EQ(1, op1.getNumCalls(MockOperator::kGetAllWorkOrders));
- EXPECT_EQ(0, op1.getNumCalls(MockOperator::kFeedInputBlock));
- EXPECT_EQ(0, op1.getNumCalls(MockOperator::kFeedInputBlocks));
-
- // Output from op1 should be fed to op2.
- EXPECT_EQ(1, op2.getNumCalls(MockOperator::kFeedInputBlock));
- EXPECT_EQ(0, op2.getNumCalls(MockOperator::kFeedInputBlocks));
-
- // A call to op2's getAllWorkOrders because of the streamed input.
- EXPECT_EQ(2, op2.getNumCalls(MockOperator::kGetAllWorkOrders));
- EXPECT_EQ(1, op2.getNumWorkOrders());
-
- // Place a message of a workorder completion of op1 on Foreman's input queue.
- EXPECT_FALSE(placeWorkOrderCompleteMessage(id1));
-
- EXPECT_EQ(0, getNumWorkOrdersInExecution(id1));
- EXPECT_TRUE(getOperatorFinishedStatus(id1));
- EXPECT_EQ(1, op2.getNumCalls(MockOperator::kDoneFeedingInputBlocks));
-
- // An additional call to op2's getAllWorkOrders because of completion of op1.
- EXPECT_EQ(3, op2.getNumCalls(MockOperator::kGetAllWorkOrders));
- EXPECT_EQ(2, op2.getNumWorkOrders());
-
- worker_message.reset(query_manager_->getNextWorkerMessage(id2, -1));
-
- EXPECT_TRUE(worker_message != nullptr);
- EXPECT_EQ(WorkerMessage::WorkerMessageType::kWorkOrder,
- worker_message->getType());
- EXPECT_EQ(id2, worker_message->getRelationalOpIndex());
-
- delete worker_message->getWorkOrder();
-
- // Place a message of a workorder completion of op2 on Foreman's input queue.
- EXPECT_FALSE(placeWorkOrderCompleteMessage(id2));
-
- EXPECT_TRUE(getOperatorFinishedStatus(id1));
-
- worker_message.reset(query_manager_->getNextWorkerMessage(id2, -1));
-
- EXPECT_TRUE(worker_message != nullptr);
- EXPECT_EQ(WorkerMessage::WorkerMessageType::kWorkOrder,
- worker_message->getType());
- EXPECT_EQ(id2, worker_message->getRelationalOpIndex());
-
- delete worker_message->getWorkOrder();
-
- EXPECT_EQ(1, getNumWorkOrdersInExecution(id2));
- EXPECT_FALSE(getOperatorFinishedStatus(id2));
-
- // Send a message to Foreman upon workorder (generated by op2) completion.
- EXPECT_TRUE(placeWorkOrderCompleteMessage(id2));
-
- EXPECT_TRUE(getOperatorFinishedStatus(id1));
- EXPECT_EQ(1, op2.getNumCalls(MockOperator::kDoneFeedingInputBlocks));
-
- EXPECT_EQ(0, getNumWorkOrdersInExecution(id2));
- EXPECT_TRUE(getOperatorFinishedStatus(id2));
-}
-
-TEST_F(QueryManagerTest, TwoNodesDAGPartiallyFilledBlocksTest) {
- // In this test, we create a 2-node DAG with a non-blocking link between them.
- // There is no streaming of data from op1 to op2 during the execution of op1.
- // op1 produces a partially filled block at the end of its execution which is
- // rebuilt and then fed to op2.
- const QueryPlan::DAGNodeIndex id1 =
- query_plan_->addRelationalOperator(new MockOperator(true, false, 1));
- const QueryPlan::DAGNodeIndex id2 =
- query_plan_->addRelationalOperator(new MockOperator(true, true, 3, 1));
-
- // Create a non-blocking link.
- query_plan_->addDirectDependency(id2, id1, false);
-
- // Create a relation, owned by db_.*/
- CatalogRelation *relation =
- new CatalogRelation(nullptr /* catalog_database */, "test_relation");
- const relation_id output_relation_id = db_->addRelation(relation);
-
- // Setup the InsertDestination proto in the query context proto.
- serialization::QueryContext *query_context_proto =
- query_handle_->getQueryContextProtoMutable();
-
- const QueryContext::insert_destination_id insert_destination_index =
- query_context_proto->insert_destinations_size();
- serialization::InsertDestination *insert_destination_proto =
- query_context_proto->add_insert_destinations();
-
- insert_destination_proto->set_insert_destination_type(
- serialization::InsertDestinationType::BLOCK_POOL);
- insert_destination_proto->set_relation_id(output_relation_id);
- insert_destination_proto->set_relational_op_index(id1);
-
- MockOperator *op1_mutable = static_cast<MockOperator *>(
- query_plan_->getQueryPlanDAGMutable()->getNodePayloadMutable(id1));
- op1_mutable->setInsertDestinationID(insert_destination_index);
- op1_mutable->setOutputRelationID(output_relation_id);
-
- const MockOperator &op1 = static_cast<const MockOperator &>(
- query_plan_->getQueryPlanDAG().getNodePayload(id1));
- const MockOperator &op2 = static_cast<const MockOperator &>(
- query_plan_->getQueryPlanDAG().getNodePayload(id2));
-
- constructQueryManager();
-
- // NOTE(zuyu): An operator generally has no ideas about partially filled
- // blocks, but InsertDestination in QueryContext does.
- // Mock to add partially filled blocks in the InsertDestination.
- InsertDestination *insert_destination =
- query_manager_->getQueryContextMutable()->getInsertDestination(
- insert_destination_index);
- DCHECK(insert_destination != nullptr);
- MutableBlockReference block_ref;
- static_cast<BlockPoolInsertDestination *>(insert_destination)
- ->available_block_refs_.push_back(move(block_ref));
-
- // There's no blocking dependency in the DAG.
- EXPECT_TRUE(op1.getBlockingDependenciesMet());
- EXPECT_TRUE(op2.getBlockingDependenciesMet());
-
- EXPECT_EQ(1, op1.getNumCalls(MockOperator::kGetAllWorkOrders));
- EXPECT_EQ(1, op1.getNumWorkOrders());
-
- EXPECT_EQ(1, op2.getNumCalls(MockOperator::kGetAllWorkOrders));
- EXPECT_EQ(0, op2.getNumWorkOrders());
-
- unique_ptr<WorkerMessage> worker_message;
- worker_message.reset(query_manager_->getNextWorkerMessage(id1, -1));
-
- EXPECT_TRUE(worker_message != nullptr);
- EXPECT_EQ(WorkerMessage::WorkerMessageType::kWorkOrder,
- worker_message->getType());
- EXPECT_EQ(id1, worker_message->getRelationalOpIndex());
-
- delete worker_message->getWorkOrder();
-
- // Send a message to QueryManager upon workorder (generated by op1) completion.
- EXPECT_FALSE(placeWorkOrderCompleteMessage(id1));
-
- EXPECT_EQ(0, getNumWorkOrdersInExecution(id1));
-
- worker_message.reset(query_manager_->getNextWorkerMessage(id1, -1));
- EXPECT_TRUE(worker_message != nullptr);
- EXPECT_EQ(WorkerMessage::WorkerMessageType::kRebuildWorkOrder,
- worker_message->getType());
-
- EXPECT_EQ(id1, worker_message->getRelationalOpIndex());
-
- delete worker_message->getWorkOrder();
-
- // op1 generates a rebuild workorder. The block is rebuilt and streamed
- // to Foreman.
- EXPECT_FALSE(placeDataPipelineMessage(id1));
-
- EXPECT_FALSE(placeRebuildWorkOrderCompleteMessage(id1));
- // Based on the streamed input, op2's getAllWorkOrders should produce a
- // workorder.
- EXPECT_EQ(3, op2.getNumCalls(MockOperator::kGetAllWorkOrders));
- EXPECT_EQ(1, op2.getNumWorkOrders());
-
- worker_message.reset(query_manager_->getNextWorkerMessage(id2, -1));
-
- EXPECT_TRUE(worker_message != nullptr);
- EXPECT_EQ(WorkerMessage::WorkerMessageType::kWorkOrder,
- worker_message->getType());
-
- EXPECT_EQ(id2, worker_message->getRelationalOpIndex());
-
- delete worker_message->getWorkOrder();
-
- EXPECT_TRUE(getOperatorFinishedStatus(id1));
- EXPECT_EQ(1, op2.getNumCalls(MockOperator::kDoneFeedingInputBlocks));
- EXPECT_FALSE(getOperatorFinishedStatus(id2));
- EXPECT_EQ(1, getNumWorkOrdersInExecution(id2));
-
- // Send a message to QueryManager upon workorder (generated by op2) completion.
- EXPECT_TRUE(placeWorkOrderCompleteMessage(id2));
-
- EXPECT_EQ(0, getNumWorkOrdersInExecution(id2));
-
- EXPECT_TRUE(getOperatorFinishedStatus(id2));
-}
-
-TEST_F(QueryManagerTest, MultipleNodesNoOutputTest) {
- // When an operator produces workorders but no output, the QueryManager should
- // check the dependents of this operator to make progress.
- const QueryPlan::DAGNodeIndex kNumNodes = 5;
- std::vector<QueryPlan::DAGNodeIndex> ids;
- ids.reserve(kNumNodes);
-
- for (QueryPlan::DAGNodeIndex i = 0; i < kNumNodes; ++i) {
- if (i == 0) {
- ids[i] = query_plan_->addRelationalOperator(new MockOperator(true, false));
- } else {
- ids[i] = query_plan_->addRelationalOperator(new MockOperator(true, true));
- }
- VLOG(3) << ids[i];
- }
-
- /**
- * The DAG looks like this:
- *
- * op1 -> op2 -> op3 -> op4 -> op5
- *
- **/
- for (QueryPlan::DAGNodeIndex i = 0; i < kNumNodes - 1; ++i) {
- query_plan_->addDirectDependency(ids[i + 1], ids[i], false);
- static_cast<MockOperator*>(query_plan_->getQueryPlanDAGMutable()->getNodePayloadMutable(ids[i]))
- ->setOutputRelationID(0xdead);
- }
-
- std::vector<const MockOperator*> operators;
- for (QueryPlan::DAGNodeIndex i = 0; i < kNumNodes; ++i) {
- operators.push_back(static_cast<const MockOperator*>(&query_plan_->getQueryPlanDAG().getNodePayload(ids[i])));
- }
-
- constructQueryManager();
-
- // operators[0] should have produced a workorder by now.
- EXPECT_EQ(1, operators[0]->getNumWorkOrders());
-
- unique_ptr<WorkerMessage> worker_message;
- worker_message.reset(query_manager_->getNextWorkerMessage(ids[0], -1));
-
- EXPECT_TRUE(worker_message != nullptr);
- EXPECT_EQ(WorkerMessage::WorkerMessageType::kWorkOrder,
- worker_message->getType());
-
- EXPECT_EQ(ids[0], worker_message->getRelationalOpIndex());
-
- delete worker_message->getWorkOrder();
-
- EXPECT_EQ(1, getNumWorkOrdersInExecution(ids[0]));
- EXPECT_FALSE(getOperatorFinishedStatus(ids[0]));
-
- for (QueryPlan::DAGNodeIndex i = 0; i < kNumNodes; ++i) {
- EXPECT_EQ(1, operators[ids[i]]->getNumCalls(MockOperator::kGetAllWorkOrders));
- }
-
- // Send a message to QueryManager upon workorder (generated by operators[0])
- // completion.
- EXPECT_TRUE(placeWorkOrderCompleteMessage(ids[0]));
-
- for (QueryPlan::DAGNodeIndex i = 0; i < kNumNodes; ++i) {
- EXPECT_EQ(0, getNumWorkOrdersInExecution(ids[i]));
- EXPECT_TRUE(getOperatorFinishedStatus(ids[i]));
- if (i < kNumNodes - 1) {
- EXPECT_EQ(1, operators[i + 1]->getNumCalls(MockOperator::kDoneFeedingInputBlocks));
- }
- }
-}
-
-TEST_F(QueryManagerTest, OutOfOrderWorkOrderCompletionTest) {
- // Consider two operators, both generate one workorder each. The dependent's
- // workorder finishes before dependency's workorder.
- const QueryPlan::DAGNodeIndex id1 = query_plan_->addRelationalOperator(new MockOperator(true, false, 1));
- const QueryPlan::DAGNodeIndex id2 = query_plan_->addRelationalOperator(new MockOperator(true, true, 2, 1));
-
- // Create a non-blocking link.
- query_plan_->addDirectDependency(id2, id1, false);
-
- constructQueryManager();
-
- unique_ptr<WorkerMessage> worker_message;
- worker_message.reset(query_manager_->getNextWorkerMessage(id1, -1));
-
- EXPECT_TRUE(worker_message != nullptr);
- EXPECT_EQ(WorkerMessage::WorkerMessageType::kWorkOrder,
- worker_message->getType());
-
- EXPECT_EQ(id1, worker_message->getRelationalOpIndex());
-
- delete worker_message->getWorkOrder();
-
- // Send a message to QueryManager upon a block (output of op1) getting full.
- EXPECT_FALSE(placeOutputBlockMessage(id1));
-
- // op1 is not finished yet because the response of workorder completion hasn't
- // been received yet.
- EXPECT_FALSE(getOperatorFinishedStatus(id1));
- EXPECT_FALSE(getOperatorFinishedStatus(id2));
-
- worker_message.reset(query_manager_->getNextWorkerMessage(id2, -1));
- EXPECT_TRUE(worker_message != nullptr);
- EXPECT_EQ(WorkerMessage::WorkerMessageType::kWorkOrder,
- worker_message->getType());
-
- EXPECT_EQ(id2, worker_message->getRelationalOpIndex());
-
- delete worker_message->getWorkOrder();
-
- // As mentioned earlier, op2 finishes before op1.
- EXPECT_FALSE(placeWorkOrderCompleteMessage(id2));
-
- // op1's workorder execution is over.
- EXPECT_TRUE(placeWorkOrderCompleteMessage(id1));
-
- EXPECT_TRUE(getOperatorFinishedStatus(id1));
- EXPECT_TRUE(getOperatorFinishedStatus(id2));
-}
-
-} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/156290a4/query_execution/tests/WorkOrdersContainer_unittest.cpp
----------------------------------------------------------------------
diff --git a/query_execution/tests/WorkOrdersContainer_unittest.cpp b/query_execution/tests/WorkOrdersContainer_unittest.cpp
deleted file mode 100644
index a3b9ba1..0000000
--- a/query_execution/tests/WorkOrdersContainer_unittest.cpp
+++ /dev/null
@@ -1,870 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- **/
-
-#include <algorithm>
-#include <cstddef>
-#include <vector>
-
-#include "gtest/gtest.h"
-
-#include "query_execution/WorkOrdersContainer.hpp"
-#include "relational_operators/WorkOrder.hpp"
-#include "utility/PtrVector.hpp"
-#include "utility/Macros.hpp"
-
-namespace quickstep {
-
-class MockNUMAWorkOrder : public WorkOrder {
- public:
- MockNUMAWorkOrder(const int id, const std::vector<int> &numa_nodes)
- : WorkOrder(0), id_(id) {
- for (int numa_node : numa_nodes) {
- preferred_numa_nodes_.push_back(numa_node);
- }
- }
-
- const int getID() const {
- return id_;
- }
-
- void execute() override {}
-
- private:
- const int id_;
-
- DISALLOW_COPY_AND_ASSIGN(MockNUMAWorkOrder);
-};
-
-// Note: In the tests, we create a local WorkOrder object (on stack) and use its
-// address as a pointer while inserting to the container.
-// The real usage of the API works as follows - we create a WorkOrder using "new"
-// (on heap) and pass the pointer to the container. When a WorkOrder is
-// retrieved from the container, its pointer is removed from the container and
-// the caller is responsible to delete the actual WorkOrder object.
-
-TEST(WorkOrdersContainerTest, ZeroNUMANodesTest) {
- // A container for one operator and no NUMA nodes.
- WorkOrdersContainer w(1, 0);
-
- EXPECT_EQ(0u, w.getNumNormalWorkOrders(0));
- EXPECT_EQ(0u, w.getNumRebuildWorkOrders(0));
- EXPECT_FALSE(w.hasNormalWorkOrder(0));
- EXPECT_FALSE(w.hasRebuildWorkOrder(0));
- EXPECT_EQ(nullptr, w.getNormalWorkOrder(0));
- EXPECT_EQ(nullptr, w.getRebuildWorkOrder(0));
-}
-
-TEST(WorkOrdersContainerTest, ZeroNUMANodesAddWorkOrderTest) {
- // Add one normal and rebuild workorder each for one operator DAG. Test if
- // they get inserted and retrieved correctly.
- std::vector<int> numa_node_ids;
- // A container for one operator and no NUMA nodes.
- const std::size_t query_id = 0;
- WorkOrdersContainer w(1, 0);
-
- EXPECT_EQ(0u, w.getNumNormalWorkOrders(0));
- EXPECT_EQ(0u, w.getNumRebuildWorkOrders(0));
- EXPECT_FALSE(w.hasNormalWorkOrder(0));
- EXPECT_FALSE(w.hasRebuildWorkOrder(0));
-
- // Create a NUMA agnostic normal WorkOrder.
- MockNUMAWorkOrder work_order(0, numa_node_ids);
- w.addNormalWorkOrder(&work_order, 0);
-
- // Expect the normal WorkOrder count to be 1.
- EXPECT_EQ(1u, w.getNumNormalWorkOrders(0));
- EXPECT_EQ(0u, w.getNumRebuildWorkOrders(0));
- EXPECT_TRUE(w.hasNormalWorkOrder(0));
- EXPECT_FALSE(w.hasRebuildWorkOrder(0));
-
- // Create a NUMA agnostic rebuild WorkOrder.
- MockNUMAWorkOrder work_order1(1, numa_node_ids);
- w.addRebuildWorkOrder(&work_order1, 0);
-
- // Expect the normal WorkOrder count to be 1.
- EXPECT_EQ(1u, w.getNumNormalWorkOrders(0));
- EXPECT_EQ(1u, w.getNumRebuildWorkOrders(0));
- EXPECT_TRUE(w.hasNormalWorkOrder(0));
- EXPECT_TRUE(w.hasRebuildWorkOrder(0));
-
- // Check if we retrieve the same WorkOrders.
- WorkOrder *returned_work_order = w.getNormalWorkOrder(0);
- ASSERT_TRUE(returned_work_order != nullptr);
- EXPECT_EQ(work_order.getID(), static_cast<MockNUMAWorkOrder*>(returned_work_order)->getID());
-
- EXPECT_EQ(query_id, returned_work_order->getQueryID());
-
- WorkOrder *returned_rebuild_work_order = w.getRebuildWorkOrder(0);
- ASSERT_TRUE(returned_rebuild_work_order != nullptr);
- EXPECT_EQ(work_order1.getID(),
- static_cast<MockNUMAWorkOrder *>(returned_rebuild_work_order)->getID());
-
- EXPECT_EQ(query_id, returned_rebuild_work_order->getQueryID());
-
- // Container should be empty now.
- EXPECT_EQ(0u, w.getNumNormalWorkOrders(0));
- EXPECT_EQ(0u, w.getNumRebuildWorkOrders(0));
- EXPECT_FALSE(w.hasNormalWorkOrder(0));
- EXPECT_FALSE(w.hasRebuildWorkOrder(0));
- EXPECT_EQ(nullptr, w.getNormalWorkOrder(0));
- EXPECT_EQ(nullptr, w.getRebuildWorkOrder(0));
-}
-
-TEST(WorkOrdersContainerTest, ZeroNUMANodesMultipleWorkOrdersTest) {
- // Add multiple normal and rebuild workorders each for one operator DAG. Test
- // if they get inserted and retrieved correctly and the order of retrieval.
- // A container for one operator and no NUMA nodes.
- std::vector<int> numa_node_ids;
- const std::size_t query_id = 0;
- WorkOrdersContainer w(1, 0);
-
- EXPECT_EQ(0u, w.getNumNormalWorkOrders(0));
- EXPECT_EQ(0u, w.getNumRebuildWorkOrders(0));
- EXPECT_FALSE(w.hasNormalWorkOrder(0));
- EXPECT_FALSE(w.hasRebuildWorkOrder(0));
-
- PtrVector<MockNUMAWorkOrder> normal_workorders;
- PtrVector<MockNUMAWorkOrder> rebuild_workorders;
- const std::size_t kNumWorkOrders = 100;
-
- // Push the mock WorkOrders in the vectors and the container.
- for (std::size_t i = 0; i < kNumWorkOrders; ++i) {
- EXPECT_EQ(i, w.getNumNormalWorkOrders(0));
- EXPECT_EQ(i, w.getNumRebuildWorkOrders(0));
-
- normal_workorders.push_back(new MockNUMAWorkOrder(i, numa_node_ids));
- rebuild_workorders.push_back(
- new MockNUMAWorkOrder(kNumWorkOrders + i, numa_node_ids));
-
- w.addNormalWorkOrder(&(normal_workorders.back()), 0);
- w.addRebuildWorkOrder(&(rebuild_workorders.back()), 0);
- }
-
- // Expect the normal WorkOrder count to be kNumWorkOrders.
- EXPECT_EQ(kNumWorkOrders, w.getNumNormalWorkOrders(0));
- EXPECT_EQ(kNumWorkOrders, w.getNumRebuildWorkOrders(0));
- EXPECT_TRUE(w.hasNormalWorkOrder(0));
- EXPECT_TRUE(w.hasRebuildWorkOrder(0));
-
- // Retrieve the WorkOrders and check the order of retrieval.
- for (std::size_t i = 0; i < kNumWorkOrders; ++i) {
- EXPECT_EQ(kNumWorkOrders - i, w.getNumNormalWorkOrders(0));
- EXPECT_EQ(kNumWorkOrders - i, w.getNumRebuildWorkOrders(0));
- WorkOrder *returned_work_order = w.getNormalWorkOrder(0);
- ASSERT_TRUE(returned_work_order != nullptr);
- EXPECT_EQ(static_cast<int>(i), static_cast<MockNUMAWorkOrder *>(returned_work_order)->getID());
- WorkOrder *returned_rebuild_work_order = w.getRebuildWorkOrder(0);
- ASSERT_TRUE(returned_work_order != nullptr);
- EXPECT_EQ(static_cast<int>(kNumWorkOrders + i),
- static_cast<MockNUMAWorkOrder *>(returned_rebuild_work_order)->getID());
- EXPECT_EQ(query_id, returned_work_order->getQueryID());
- EXPECT_EQ(query_id, returned_rebuild_work_order->getQueryID());
- }
-
- // Container should be empty now.
- EXPECT_EQ(0u, w.getNumNormalWorkOrders(0));
- EXPECT_EQ(0u, w.getNumRebuildWorkOrders(0));
- EXPECT_FALSE(w.hasNormalWorkOrder(0));
- EXPECT_FALSE(w.hasRebuildWorkOrder(0));
- EXPECT_EQ(nullptr, w.getNormalWorkOrder(0));
- EXPECT_EQ(nullptr, w.getRebuildWorkOrder(0));
-}
-
-TEST(WorkOrdersContainerTest, MultipleNUMANodesTest) {
- // Create WorkOrders belonging to different NUMA nodes and test if they get
- // inserted and retrieved correctly.
- std::vector<int> numa_node_ids;
- // The sequence of NUMA nodes isn't contiguous.
- numa_node_ids.push_back(4);
- numa_node_ids.push_back(7);
- numa_node_ids.push_back(0);
- const std::size_t kNUMANodes =
- 1 +
- *std::max_element(numa_node_ids.begin(), numa_node_ids.end());
-
- const std::size_t kNUMANodesUsed = numa_node_ids.size();
-
- // A container for one operator and kNUMANodes.
- const std::size_t query_id = 0;
- WorkOrdersContainer w(1, kNUMANodes);
-
- for (std::size_t i = 0; i < kNUMANodesUsed; ++i) {
- std::vector<int> curr_numa_node;
- curr_numa_node.push_back(numa_node_ids[i]);
- }
-
- PtrVector<MockNUMAWorkOrder> normal_workorders;
- PtrVector<MockNUMAWorkOrder> rebuild_workorders;
-
- for (std::size_t i = 0; i < kNUMANodesUsed; ++i) {
- // Create a vector consisting of the current NUMA node as its element.
- std::vector<int> curr_numa_node;
- curr_numa_node.push_back(numa_node_ids[i]);
-
- // Create normal and rebuild WorkOrders belonging to exactly one NUMA node.
- normal_workorders.push_back(new MockNUMAWorkOrder(i, curr_numa_node));
- rebuild_workorders.push_back(
- new MockNUMAWorkOrder(kNUMANodes + i, curr_numa_node));
-
- w.addNormalWorkOrder(&(normal_workorders.back()), 0);
- w.addRebuildWorkOrder(&(rebuild_workorders.back()), 0);
-
- // For each NUMA node, check the count of WorkOrders.
- for (std::size_t j = 0; j < i; ++j) {
- EXPECT_EQ(1u, w.getNumNormalWorkOrdersForNUMANode(0, numa_node_ids[j]));
- EXPECT_EQ(1u, w.getNumRebuildWorkOrdersForNUMANode(0, numa_node_ids[j]));
- }
- }
-
- // Retrieve the WorkOrders.
- for (std::size_t i = 0; i < kNUMANodesUsed; ++i) {
- // For each NUMA node, check the count of WorkOrders.
- for (std::size_t j = 0; j < kNUMANodesUsed; ++j) {
- if (j >= i) {
- // We haven't retrieved the workorders for this NUMA node yet.
- EXPECT_EQ(1u, w.getNumNormalWorkOrdersForNUMANode(0, numa_node_ids[j]));
- EXPECT_EQ(1u, w.getNumRebuildWorkOrdersForNUMANode(0, numa_node_ids[j]));
- } else {
- // We already retrieved the workorders for this NUMA node.
- EXPECT_EQ(0u, w.getNumNormalWorkOrdersForNUMANode(0, numa_node_ids[j]));
- EXPECT_EQ(0u, w.getNumRebuildWorkOrdersForNUMANode(0, numa_node_ids[j]));
- }
- }
-
- // Retrieve the workorders for this NUMA node.
- WorkOrder *returned_work_order = w.getNormalWorkOrderForNUMANode(0, numa_node_ids[i]);
- ASSERT_TRUE(returned_work_order != nullptr);
- EXPECT_EQ(normal_workorders[i].getID(),
- static_cast<MockNUMAWorkOrder *>(returned_work_order)->getID());
-
- WorkOrder *returned_rebuild_work_order =
- w.getRebuildWorkOrderForNUMANode(0, numa_node_ids[i]);
- ASSERT_TRUE(returned_rebuild_work_order != nullptr);
- EXPECT_EQ(rebuild_workorders[i].getID(),
- static_cast<MockNUMAWorkOrder *>(returned_rebuild_work_order)->getID());
-
- EXPECT_EQ(query_id, returned_work_order->getQueryID());
- EXPECT_EQ(query_id, returned_rebuild_work_order->getQueryID());
- }
-
- // No workorder should be left for this operator on any NUMA node.
- for (const int numa_node : numa_node_ids) {
- EXPECT_FALSE(w.hasNormalWorkOrderForNUMANode(0, numa_node));
- EXPECT_FALSE(w.hasRebuildWorkOrderForNUMANode(0, numa_node));
- }
-
- // No workorder should be left for this operator.
- EXPECT_FALSE(w.hasNormalWorkOrder(0));
- EXPECT_FALSE(w.hasRebuildWorkOrder(0));
-}
-
-TEST(WorkOrdersContainerTest, AllTypesWorkOrdersTest) {
- // For a given operator create three types of WorkOrders
- // 1. NUMA agnostic.
- // 2. WorkOrder with single NUMA node.
- // 3. WorkOrder with multiple NUMA nodes.
- std::vector<int> numa_nodes;
-
- // Create a WorkOrder with no NUMA node.
- MockNUMAWorkOrder no_numa_work_order(0, numa_nodes);
- EXPECT_TRUE(no_numa_work_order.getPreferredNUMANodes().empty());
-
- // Add one NUMA node.
- numa_nodes.push_back(0);
-
- // Create a WorkOrder with exactly one NUMA node.
- MockNUMAWorkOrder one_numa_work_order(1, numa_nodes);
- EXPECT_EQ(0, one_numa_work_order.getPreferredNUMANodes().front());
-
- // Add another NUMA node.
- numa_nodes.push_back(3);
-
- // Create a WorkOrder with more than one NUMA node.
- MockNUMAWorkOrder multiple_numa_work_order(2, numa_nodes);
- for (std::size_t i = 0; i < numa_nodes.size(); ++i) {
- EXPECT_EQ(numa_nodes[i], multiple_numa_work_order.getPreferredNUMANodes().at(i));
- }
-
- const std::size_t kNUMANodes =
- 1 + *std::max_element(numa_nodes.begin(), numa_nodes.end());
- const std::size_t kNUMANodesUsed = numa_nodes.size();
-
- // Create the container.
- const std::size_t query_id = 0;
- WorkOrdersContainer w(1, kNUMANodes);
-
- w.addNormalWorkOrder(&multiple_numa_work_order, 0);
-
- for (std::size_t i = 0; i < kNUMANodesUsed; ++i) {
- // Check the count of per NUMA node workorders.
- EXPECT_EQ(1u, w.getNumNormalWorkOrdersForNUMANode(0, numa_nodes[i]));
- }
-
- // Expect 1 normal workorder in total for this operator.
- EXPECT_EQ(1u, w.getNumNormalWorkOrders(0));
- EXPECT_EQ(0u, w.getNumRebuildWorkOrders(0));
-
- // Add the WorkOrder with no NUMA node.
- w.addNormalWorkOrder(&no_numa_work_order, 0);
-
- for (std::size_t i = 0; i < kNUMANodesUsed; ++i) {
- // Check the count of per NUMA node workorders.
- EXPECT_EQ(1u, w.getNumNormalWorkOrdersForNUMANode(0, numa_nodes[i]));
- }
-
- // Expect 2 normal workorders in total for this operator.
- EXPECT_EQ(2u, w.getNumNormalWorkOrders(0));
- EXPECT_EQ(0u, w.getNumRebuildWorkOrders(0));
-
- // Add the WorkOrder with exactly one NUMA node.
- w.addNormalWorkOrder(&one_numa_work_order, 0);
-
- EXPECT_EQ(2u, w.getNumNormalWorkOrdersForNUMANode(0, numa_nodes[0]));
- EXPECT_EQ(1u, w.getNumNormalWorkOrdersForNUMANode(0, numa_nodes[1]));
-
- // Expect 3 normal workorders in total for this operator.
- EXPECT_EQ(3u, w.getNumNormalWorkOrders(0));
- EXPECT_EQ(0u, w.getNumRebuildWorkOrders(0));
-
- // Retrieve the workorders for NUMA node = numa_nodes[0]
- MockNUMAWorkOrder *observed_work_order = static_cast<MockNUMAWorkOrder *>(
- w.getNormalWorkOrderForNUMANode(0, numa_nodes[0]));
- ASSERT_TRUE(observed_work_order != nullptr);
-
- EXPECT_EQ(query_id, observed_work_order->getQueryID());
- EXPECT_EQ(one_numa_work_order.getPreferredNUMANodes().front(),
- observed_work_order->getPreferredNUMANodes().front());
- EXPECT_EQ(one_numa_work_order.getID(), observed_work_order->getID());
-
- EXPECT_EQ(1u, w.getNumNormalWorkOrdersForNUMANode(0, numa_nodes[0]));
- EXPECT_EQ(1u, w.getNumNormalWorkOrdersForNUMANode(0, numa_nodes[1]));
-
- // Expect 2 normal workorders in total for this operator.
- EXPECT_EQ(2u, w.getNumNormalWorkOrders(0));
- EXPECT_EQ(0u, w.getNumRebuildWorkOrders(0));
-
- // Retrieve the non NUMA workorder.
- WorkOrder *observed_non_numa_work_order = w.getNormalWorkOrder(0);
- ASSERT_TRUE(observed_non_numa_work_order != nullptr);
- EXPECT_EQ(no_numa_work_order.getID(),
- static_cast<MockNUMAWorkOrder *>(observed_non_numa_work_order)->getID());
-
- EXPECT_EQ(query_id, observed_non_numa_work_order->getQueryID());
- EXPECT_EQ(1u, w.getNumNormalWorkOrdersForNUMANode(0, numa_nodes[0]));
- EXPECT_EQ(1u, w.getNumNormalWorkOrdersForNUMANode(0, numa_nodes[1]));
-
- // Expect 1 normal workorders in total for this operator.
- EXPECT_EQ(1u, w.getNumNormalWorkOrders(0));
- EXPECT_EQ(0u, w.getNumRebuildWorkOrders(0));
-
- // Retrieve the workorder with multiple NUMA nodes.
- MockNUMAWorkOrder *observed_work_order_multiple_numa_nodes =
- static_cast<MockNUMAWorkOrder *>(w.getNormalWorkOrder(0));
- ASSERT_TRUE(observed_work_order_multiple_numa_nodes != nullptr);
- EXPECT_EQ(multiple_numa_work_order.getID(), observed_work_order_multiple_numa_nodes->getID());
-
- EXPECT_EQ(query_id, observed_work_order_multiple_numa_nodes->getQueryID());
- std::vector<int> observed_numa_nodes(
- observed_work_order_multiple_numa_nodes->getPreferredNUMANodes());
- // Look up the expected numa nodes in the observed_numa_nodes vector.
- EXPECT_TRUE(std::find(observed_numa_nodes.begin(), observed_numa_nodes.end(),
- numa_nodes[0]) != observed_numa_nodes.end());
- EXPECT_TRUE(std::find(observed_numa_nodes.begin(), observed_numa_nodes.end(),
- numa_nodes[1]) != observed_numa_nodes.end());
-
- EXPECT_EQ(0u, w.getNumNormalWorkOrdersForNUMANode(0, numa_nodes[0]));
- EXPECT_EQ(0u, w.getNumNormalWorkOrdersForNUMANode(0, numa_nodes[1]));
-
- // Expect no normal workorders in total for this operator.
- EXPECT_EQ(0u, w.getNumNormalWorkOrders(0));
- EXPECT_EQ(0u, w.getNumRebuildWorkOrders(0));
-}
-
-TEST(WorkOrdersContainerTest, MultipleOperatorsNormalWorkOrderTest) {
- // Generate workorders for multiple operators and test their insertion and
- // retrieval. Use the IDs of the WorkOrders to verify the correctness.
-
- // Create the NUMA node vectors.
- std::vector<int> numa_node_ids;
-
- const std::size_t kNumOperators = 100;
-
- // For each operator create normal workorders with no NUMA node.
- PtrVector<MockNUMAWorkOrder> normal_workorders_no_numa;
- std::vector<int> normal_workorders_no_numa_ids;
- // Set of workorder IDs = {0, ... kNumOperators - 1}
- for (std::size_t i = 0; i < kNumOperators; ++i) {
- normal_workorders_no_numa_ids.push_back(i);
- normal_workorders_no_numa.push_back(new MockNUMAWorkOrder(
- normal_workorders_no_numa_ids.back(), numa_node_ids));
- }
-
- // Insert one NUMA node.
- numa_node_ids.push_back(0);
-
- // For each operator create normal workorders with one NUMA node.
- PtrVector<MockNUMAWorkOrder> normal_workorders_one_numa;
- std::vector<int> normal_workorders_one_numa_ids;
- // Set of workorder IDs = {kNumOperators, .. , 2*kNumOperators - 1}
- for (std::size_t i = 0; i < kNumOperators; ++i) {
- normal_workorders_one_numa_ids.push_back(kNumOperators + i);
- normal_workorders_one_numa.push_back(new MockNUMAWorkOrder(
- normal_workorders_one_numa_ids.back(), numa_node_ids));
- }
-
- // Insert another NUMA node.
- numa_node_ids.push_back(1);
-
- // For each operator create normal workorders with more than one NUMA node.
- PtrVector<MockNUMAWorkOrder> normal_workorders_multiple_numa;
- std::vector<int> normal_workorders_multiple_numa_ids;
- // Set of workorder IDs = {2*kNumOperators, .. , 3*kNumOperators - 1}
- for (std::size_t i = 0; i < kNumOperators; ++i) {
- normal_workorders_multiple_numa_ids.push_back(2*kNumOperators + i);
- normal_workorders_multiple_numa.push_back(new MockNUMAWorkOrder(
- normal_workorders_multiple_numa_ids.back(), numa_node_ids));
- }
-
- // TODO(harshad) : Design a test in which the number of NUMA nodes is
- // configurable.
- const std::size_t kNUMANodes = numa_node_ids.size();
-
- // Create the container.
- const std::size_t query_id = 0;
- WorkOrdersContainer w(kNumOperators, kNUMANodes);
-
- std::vector<std::size_t> operator_ids;
- for (std::size_t i = 0; i < kNumOperators; ++i) {
- operator_ids.push_back(i);
- }
-
- // Randomize the operator IDs.
- std::random_shuffle(operator_ids.begin(), operator_ids.end());
-
- // Insert the workorders.
- for (std::size_t outer_op = 0; outer_op < kNumOperators; ++outer_op) {
- const std::size_t curr_operator_id = operator_ids[outer_op];
- // Check the count of WorkOrders per operator.
- for (std::size_t inner_op = 0; inner_op < kNumOperators; ++inner_op) {
- if (inner_op < outer_op) {
- // WorkOrders inserted already.
- EXPECT_EQ(3u, w.getNumNormalWorkOrders(operator_ids[inner_op]));
- EXPECT_TRUE(w.hasNormalWorkOrder(operator_ids[inner_op]));
- EXPECT_EQ(0u, w.getNumRebuildWorkOrders(operator_ids[inner_op]));
- EXPECT_FALSE(w.hasRebuildWorkOrder(operator_ids[inner_op]));
- // Number of WorkOrders : NUMA Node
- // 2 : 0
- // 1 : 1
- for (int numa_node_used = kNUMANodes - 1; numa_node_used >= 0;
- --numa_node_used) {
- EXPECT_EQ(kNUMANodes - numa_node_used,
- w.getNumNormalWorkOrdersForNUMANode(
- operator_ids[inner_op], numa_node_ids[numa_node_used]));
- EXPECT_TRUE(w.hasNormalWorkOrderForNUMANode(
- operator_ids[inner_op], numa_node_ids[numa_node_used]));
- }
- } else {
- // WorkOrders yet to be inserted.
- EXPECT_EQ(0u, w.getNumNormalWorkOrders(operator_ids[inner_op]));
- EXPECT_FALSE(w.hasNormalWorkOrder(operator_ids[inner_op]));
- EXPECT_EQ(0u, w.getNumRebuildWorkOrders(operator_ids[inner_op]));
- EXPECT_FALSE(w.hasRebuildWorkOrder(operator_ids[inner_op]));
- for (int numa_node_used = kNUMANodes - 1; numa_node_used >= 0;
- --numa_node_used) {
- EXPECT_EQ(
- 0u, w.getNumNormalWorkOrdersForNUMANode(
- operator_ids[inner_op], numa_node_ids[numa_node_used]));
- EXPECT_FALSE(w.hasNormalWorkOrderForNUMANode(
- operator_ids[inner_op], numa_node_ids[numa_node_used]));
- }
- }
- }
- // Insert the workorder with multiple NUMA nodes.
- w.addNormalWorkOrder(&(normal_workorders_multiple_numa[curr_operator_id]),
- curr_operator_id);
- // Insert the workorder with no NUMA node.
- w.addNormalWorkOrder(&(normal_workorders_no_numa[curr_operator_id]),
- curr_operator_id);
- // Insert the workorder with one NUMA node.
- w.addNormalWorkOrder(&(normal_workorders_one_numa[curr_operator_id]),
- curr_operator_id);
- }
-
- // Randomize the operator IDs again and retrieve the WorkOrders.
- std::random_shuffle(operator_ids.begin(), operator_ids.end());
- for (std::size_t outer_op = 0; outer_op < kNumOperators; ++outer_op) {
- const std::size_t curr_operator_id = operator_ids[outer_op];
- // Check if the existing WorkOrder counts are correct.
- for (std::size_t inner_op = 0; inner_op < kNumOperators; ++inner_op) {
- if (inner_op >= outer_op) {
- // WorkOrders not yet retrieved.
- EXPECT_EQ(3u, w.getNumNormalWorkOrders(operator_ids[inner_op]));
- EXPECT_TRUE(w.hasNormalWorkOrder(operator_ids[inner_op]));
- EXPECT_EQ(0u, w.getNumRebuildWorkOrders(operator_ids[inner_op]));
- EXPECT_FALSE(w.hasRebuildWorkOrder(operator_ids[inner_op]));
- // Number of WorkOrders : NUMA Node
- // 2 : 0
- // 1 : 1
- for (int numa_node_used = kNUMANodes - 1; numa_node_used >= 0;
- --numa_node_used) {
- EXPECT_EQ(
- kNUMANodes - numa_node_used,
- w.getNumNormalWorkOrdersForNUMANode(
- operator_ids[inner_op], numa_node_ids[numa_node_used]));
- EXPECT_TRUE(w.hasNormalWorkOrderForNUMANode(
- operator_ids[inner_op], numa_node_ids[numa_node_used]));
- }
- } else {
- // WorkOrders already retrieved.
- EXPECT_EQ(0u, w.getNumNormalWorkOrders(operator_ids[inner_op]));
- EXPECT_FALSE(w.hasNormalWorkOrder(operator_ids[inner_op]));
- EXPECT_EQ(0u, w.getNumRebuildWorkOrders(operator_ids[inner_op]));
- EXPECT_FALSE(w.hasRebuildWorkOrder(operator_ids[inner_op]));
- for (int numa_node_used = kNUMANodes - 1; numa_node_used >= 0;
- --numa_node_used) {
- EXPECT_EQ(
- 0u, w.getNumNormalWorkOrdersForNUMANode(
- operator_ids[inner_op], numa_node_ids[numa_node_used]));
- EXPECT_FALSE(w.hasNormalWorkOrderForNUMANode(
- operator_ids[inner_op], numa_node_ids[numa_node_used]));
- }
- }
- }
- // There is a workorder with exactly one NUMA node, get that node id.
- const std::size_t single_numa_node_id = numa_node_ids.front();
- // There is a workorder with more than one NUMA node, get that node id
- // (which is different than the node id above).
- const std::size_t multiple_numa_node_id = numa_node_ids.back();
-
- // Retrieve a single NUMA node workorder.
- MockNUMAWorkOrder *observed_work_order_single_numa =
- static_cast<MockNUMAWorkOrder *>(w.getNormalWorkOrderForNUMANode(
- curr_operator_id, single_numa_node_id));
- ASSERT_TRUE(observed_work_order_single_numa != nullptr);
-
- EXPECT_EQ(query_id, observed_work_order_single_numa->getQueryID());
- // Verify if the workorder ID is correct.
- const int expected_workorder_id_single_numa =
- normal_workorders_one_numa_ids[curr_operator_id];
- EXPECT_EQ(expected_workorder_id_single_numa,
- observed_work_order_single_numa->getID());
-
- // Retrieve a multiple NUMA node workorder.
- MockNUMAWorkOrder *observed_work_order_multiple_numa =
- static_cast<MockNUMAWorkOrder *>(w.getNormalWorkOrderForNUMANode(
- curr_operator_id, multiple_numa_node_id));
- ASSERT_TRUE(observed_work_order_multiple_numa != nullptr);
-
- EXPECT_EQ(query_id, observed_work_order_multiple_numa->getQueryID());
- // Verify if the workorder ID is correct.
- const int expected_workorder_id_multiple_numa =
- normal_workorders_multiple_numa_ids[curr_operator_id];
-
- EXPECT_EQ(expected_workorder_id_multiple_numa,
- observed_work_order_multiple_numa->getID());
-
- // Retrieve a no NUMA node workorder.
- MockNUMAWorkOrder *observed_work_order_no_numa =
- static_cast<MockNUMAWorkOrder *>(w.getNormalWorkOrder(curr_operator_id));
- ASSERT_TRUE(observed_work_order_no_numa != nullptr);
-
- EXPECT_EQ(query_id, observed_work_order_no_numa->getQueryID());
- // Verify if the workorder ID is correct.
- const int expected_workorder_id_no_numa =
- normal_workorders_no_numa_ids[curr_operator_id];
-
- EXPECT_EQ(expected_workorder_id_no_numa, observed_work_order_no_numa->getID());
- }
-}
-
-TEST(WorkOrdersContainerTest, MultipleOperatorsRebuildWorkOrderTest) {
- // This test is exactly similar to previous one, except that this test checks
- // rebuild workorders related interface.
-
- // Create the NUMA node vectors.
- std::vector<int> numa_node_ids;
-
- const std::size_t kNumOperators = 100;
-
- // For each operator create rebuild workorders with no NUMA node.
- PtrVector<MockNUMAWorkOrder> rebuild_workorders_no_numa;
- std::vector<int> rebuild_workorders_no_numa_ids;
- // Set of workorder IDs = {0, ... kNumOperators - 1}
- for (std::size_t i = 0; i < kNumOperators; ++i) {
- rebuild_workorders_no_numa_ids.push_back(i);
- rebuild_workorders_no_numa.push_back(new MockNUMAWorkOrder(
- rebuild_workorders_no_numa_ids.back(), numa_node_ids));
- }
-
- // Insert one NUMA node.
- numa_node_ids.push_back(0);
-
- // For each operator create rebuild workorders with one NUMA node.
- PtrVector<MockNUMAWorkOrder> rebuild_workorders_one_numa;
- std::vector<int> rebuild_workorders_one_numa_ids;
- // Set of workorder IDs = {kNumOperators, .. , 2*kNumOperators - 1}
- for (std::size_t i = 0; i < kNumOperators; ++i) {
- rebuild_workorders_one_numa_ids.push_back(kNumOperators + i);
- rebuild_workorders_one_numa.push_back(new MockNUMAWorkOrder(
- rebuild_workorders_one_numa_ids.back(), numa_node_ids));
- }
-
- // Insert another NUMA node.
- numa_node_ids.push_back(1);
-
- // For each operator create rebuild workorders with more than one NUMA node.
- PtrVector<MockNUMAWorkOrder> rebuild_workorders_multiple_numa;
- std::vector<int> rebuild_workorders_multiple_numa_ids;
- // Set of workorder IDs = {2*kNumOperators, .. , 3*kNumOperators - 1}
- for (std::size_t i = 0; i < kNumOperators; ++i) {
- rebuild_workorders_multiple_numa_ids.push_back(2*kNumOperators + i);
- rebuild_workorders_multiple_numa.push_back(new MockNUMAWorkOrder(
- rebuild_workorders_multiple_numa_ids.back(), numa_node_ids));
- }
-
- // TODO(harshad) : Design a test in which the number of NUMA nodes is
- // configurable.
- const std::size_t kNUMANodes = numa_node_ids.size();
-
- // Create the container.
- const std::size_t query_id = 0;
- WorkOrdersContainer w(kNumOperators, kNUMANodes);
-
- std::vector<std::size_t> operator_ids;
- for (std::size_t i = 0; i < kNumOperators; ++i) {
- operator_ids.push_back(i);
- }
-
- // Randomize the operator IDs.
- std::random_shuffle(operator_ids.begin(), operator_ids.end());
-
- // Insert the workorders.
- for (std::size_t outer_op = 0; outer_op < kNumOperators; ++outer_op) {
- const std::size_t curr_operator_id = operator_ids[outer_op];
- // Check the count of WorkOrders per operator.
- for (std::size_t inner_op = 0; inner_op < kNumOperators; ++inner_op) {
- if (inner_op < outer_op) {
- // WorkOrders inserted already.
- EXPECT_EQ(3u, w.getNumRebuildWorkOrders(operator_ids[inner_op]));
- EXPECT_TRUE(w.hasRebuildWorkOrder(operator_ids[inner_op]));
- EXPECT_EQ(0u, w.getNumNormalWorkOrders(operator_ids[inner_op]));
- EXPECT_FALSE(w.hasNormalWorkOrder(operator_ids[inner_op]));
- // Number of WorkOrders : NUMA Node
- // 2 : 0
- // 1 : 1
- for (int numa_node_used = kNUMANodes - 1; numa_node_used >= 0;
- --numa_node_used) {
- EXPECT_EQ(
- kNUMANodes - numa_node_used,
- w.getNumRebuildWorkOrdersForNUMANode(
- operator_ids[inner_op], numa_node_ids[numa_node_used]));
- EXPECT_TRUE(w.hasRebuildWorkOrderForNUMANode(
- operator_ids[inner_op], numa_node_ids[numa_node_used]));
- }
- } else {
- // WorkOrders yet to be inserted.
- EXPECT_EQ(0u, w.getNumRebuildWorkOrders(operator_ids[inner_op]));
- EXPECT_FALSE(w.hasRebuildWorkOrder(operator_ids[inner_op]));
- EXPECT_EQ(0u, w.getNumNormalWorkOrders(operator_ids[inner_op]));
- EXPECT_FALSE(w.hasNormalWorkOrder(operator_ids[inner_op]));
- for (int numa_node_used = kNUMANodes - 1; numa_node_used >= 0;
- --numa_node_used) {
- EXPECT_EQ(
- 0u, w.getNumRebuildWorkOrdersForNUMANode(
- operator_ids[inner_op], numa_node_ids[numa_node_used]));
- EXPECT_FALSE(w.hasRebuildWorkOrderForNUMANode(
- operator_ids[inner_op], numa_node_ids[numa_node_used]));
- }
- }
- }
- // Insert the workorder with multiple NUMA nodes.
- w.addRebuildWorkOrder(&(rebuild_workorders_multiple_numa[curr_operator_id]),
- curr_operator_id);
- // Insert the workorder with no NUMA node.
- w.addRebuildWorkOrder(&(rebuild_workorders_no_numa[curr_operator_id]),
- curr_operator_id);
- // Insert the workorder with one NUMA node.
- w.addRebuildWorkOrder(&(rebuild_workorders_one_numa[curr_operator_id]),
- curr_operator_id);
- }
-
- // Randomize the operator IDs again and retrieve the WorkOrders.
- std::random_shuffle(operator_ids.begin(), operator_ids.end());
- for (std::size_t outer_op = 0; outer_op < kNumOperators; ++outer_op) {
- const std::size_t curr_operator_id = operator_ids[outer_op];
- // Check if the existing WorkOrder counts are correct.
- for (std::size_t inner_op = 0; inner_op < kNumOperators; ++inner_op) {
- if (inner_op >= outer_op) {
- // WorkOrders not yet retrieved.
- EXPECT_EQ(3u, w.getNumRebuildWorkOrders(operator_ids[inner_op]));
- EXPECT_TRUE(w.hasRebuildWorkOrder(operator_ids[inner_op]));
- EXPECT_EQ(0u, w.getNumNormalWorkOrders(operator_ids[inner_op]));
- EXPECT_FALSE(w.hasNormalWorkOrder(operator_ids[inner_op]));
- // Number of WorkOrders : NUMA Node
- // 2 : 0
- // 1 : 1
- for (int numa_node_used = kNUMANodes - 1; numa_node_used >= 0;
- --numa_node_used) {
- EXPECT_EQ(
- kNUMANodes - numa_node_used,
- w.getNumRebuildWorkOrdersForNUMANode(
- operator_ids[inner_op], numa_node_ids[numa_node_used]));
- EXPECT_TRUE(w.hasRebuildWorkOrderForNUMANode(
- operator_ids[inner_op], numa_node_ids[numa_node_used]));
- }
- } else {
- // WorkOrders already retrieved.
- EXPECT_EQ(0u, w.getNumRebuildWorkOrders(operator_ids[inner_op]));
- EXPECT_FALSE(w.hasRebuildWorkOrder(operator_ids[inner_op]));
- EXPECT_EQ(0u, w.getNumNormalWorkOrders(operator_ids[inner_op]));
- EXPECT_FALSE(w.hasNormalWorkOrder(operator_ids[inner_op]));
- for (int numa_node_used = kNUMANodes - 1; numa_node_used >= 0;
- --numa_node_used) {
- EXPECT_EQ(
- 0u, w.getNumRebuildWorkOrdersForNUMANode(
- operator_ids[inner_op], numa_node_ids[numa_node_used]));
- EXPECT_FALSE(w.hasRebuildWorkOrderForNUMANode(
- operator_ids[inner_op], numa_node_ids[numa_node_used]));
- }
- }
- }
- // There is a workorder with exactly one NUMA node, get that node id.
- const std::size_t single_numa_node_id = numa_node_ids.front();
- // There is a workorder with more than one NUMA node, get that node id
- // (which is different than the node id above).
- const std::size_t multiple_numa_node_id = numa_node_ids.back();
-
- // Retrieve a single NUMA node workorder.
- MockNUMAWorkOrder *observed_work_order_single_numa =
- static_cast<MockNUMAWorkOrder *>(w.getRebuildWorkOrderForNUMANode(
- curr_operator_id, single_numa_node_id));
- ASSERT_TRUE(observed_work_order_single_numa != nullptr);
-
- EXPECT_EQ(query_id, observed_work_order_single_numa->getQueryID());
- // Verify if the workorder ID is correct.
- const int expected_workorder_id_single_numa =
- rebuild_workorders_one_numa_ids[curr_operator_id];
- EXPECT_EQ(expected_workorder_id_single_numa,
- observed_work_order_single_numa->getID());
-
- // Retrieve a multiple NUMA node workorder.
- MockNUMAWorkOrder *observed_work_order_multiple_numa =
- static_cast<MockNUMAWorkOrder *>(w.getRebuildWorkOrderForNUMANode(
- curr_operator_id, multiple_numa_node_id));
- ASSERT_TRUE(observed_work_order_multiple_numa != nullptr);
-
- EXPECT_EQ(query_id, observed_work_order_multiple_numa->getQueryID());
- // Verify if the workorder ID is correct.
- const int expected_workorder_id_multiple_numa =
- rebuild_workorders_multiple_numa_ids[curr_operator_id];
-
- EXPECT_EQ(expected_workorder_id_multiple_numa,
- observed_work_order_multiple_numa->getID());
-
- // Retrieve a no NUMA node workorder.
- MockNUMAWorkOrder *observed_work_order_no_numa =
- static_cast<MockNUMAWorkOrder *>(w.getRebuildWorkOrder(curr_operator_id));
-
- EXPECT_EQ(query_id, observed_work_order_no_numa->getQueryID());
- // Verify if the workorder ID is correct.
- const int expected_workorder_id_no_numa =
- rebuild_workorders_no_numa_ids[curr_operator_id];
-
- EXPECT_EQ(expected_workorder_id_no_numa, observed_work_order_no_numa->getID());
- }
-}
-
-TEST(WorkOrdersContainerTest, RetrievalOrderTest) {
- // Create only two kinds of workorders -
- // 1. WorkOrder with exactly one NUMA node as input.
- // 2. WorkOrder with more than one NUMA node as input.
- // Vary the parameter prefer_single_NUMA_node and verify the behavior.
- std::vector<int> numa_node_ids;
- numa_node_ids.push_back(0);
- const std::size_t kNumWorkOrdersPerType = 100;
-
- const std::size_t query_id = 0;
- WorkOrdersContainer w(1, 2);
-
- std::vector<int> single_numa_node_workorder_ids;
- std::vector<int> multiple_numa_node_workorder_ids;
-
- PtrVector<MockNUMAWorkOrder> single_numa_node_workorders;
- PtrVector<MockNUMAWorkOrder> multiple_numa_node_workorders;
-
- // Insert WorkOrders with exactly one NUMA node and store the WorkOrder IDs.
- for (std::size_t work_order_num = 0;
- work_order_num < kNumWorkOrdersPerType;
- ++work_order_num) {
- w.addNormalWorkOrder(new MockNUMAWorkOrder(work_order_num, numa_node_ids), 0);
- single_numa_node_workorder_ids.push_back(work_order_num);
- }
-
- numa_node_ids.push_back(1);
-
- // Insert WorkOrders with more than one NUMA node and store the WorkOrder IDs.
- for (std::size_t work_order_num = 0;
- work_order_num < kNumWorkOrdersPerType;
- ++work_order_num) {
- w.addNormalWorkOrder(
- new MockNUMAWorkOrder(work_order_num + kNumWorkOrdersPerType, numa_node_ids),
- 0);
- multiple_numa_node_workorder_ids.push_back(kNumWorkOrdersPerType + work_order_num);
- }
-
- // Create a vector of booleans of size kNumWorkOrdersPerType * 2
- std::vector<bool> retrieval_order;
- for (std::size_t i = 0; i < kNumWorkOrdersPerType; ++i) {
- retrieval_order.push_back(true);
- retrieval_order.push_back(false);
- }
-
- // Randomize the order of retrieval.
- std::random_shuffle(retrieval_order.begin(), retrieval_order.end());
-
- std::vector<int>::iterator single_numa_it =
- single_numa_node_workorder_ids.begin();
- std::vector<int>::iterator multiple_numa_it =
- multiple_numa_node_workorder_ids.begin();
-
- for (bool prefer_single_NUMA_node : retrieval_order) {
- // Retrieve the WorkOrder.
- MockNUMAWorkOrder *observed_work_order = static_cast<MockNUMAWorkOrder *>(
- w.getNormalWorkOrder(0, prefer_single_NUMA_node));
- ASSERT_TRUE(observed_work_order != nullptr);
- EXPECT_EQ(query_id, observed_work_order->getQueryID());
- if (prefer_single_NUMA_node) {
- EXPECT_EQ(*single_numa_it, observed_work_order->getID());
- EXPECT_EQ(1u, observed_work_order->getPreferredNUMANodes().size());
- ++single_numa_it;
- } else {
- EXPECT_EQ(*multiple_numa_it, observed_work_order->getID());
- EXPECT_EQ(2u, observed_work_order->getPreferredNUMANodes().size());
- ++multiple_numa_it;
- }
- }
- DEBUG_ASSERT(single_numa_it == single_numa_node_workorder_ids.end());
- DEBUG_ASSERT(multiple_numa_it == multiple_numa_node_workorder_ids.end());
-
- // Expect no more workorders.
- EXPECT_EQ(nullptr, w.getNormalWorkOrder(0, true));
- EXPECT_EQ(nullptr, w.getNormalWorkOrder(0, false));
-}
-
-} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/156290a4/query_execution/tests/WorkerDirectory_unittest.cpp
----------------------------------------------------------------------
diff --git a/query_execution/tests/WorkerDirectory_unittest.cpp b/query_execution/tests/WorkerDirectory_unittest.cpp
deleted file mode 100644
index 2eab75d..0000000
--- a/query_execution/tests/WorkerDirectory_unittest.cpp
+++ /dev/null
@@ -1,296 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- **/
-
-#include <algorithm>
-#include <cstddef>
-#include <iterator>
-#include <memory>
-#include <random>
-#include <stack>
-#include <unordered_map>
-#include <utility>
-#include <vector>
-
-#include "gtest/gtest.h"
-
-#include "query_execution/QueryExecutionTypedefs.hpp"
-#include "query_execution/WorkerDirectory.hpp"
-
-namespace quickstep {
-
-class WorkerDirectoryTest : public ::testing::Test {
- protected:
- // First element is the index of the min element, second element is the
- // value of the min element.
- static std::pair<std::size_t, std::size_t> getMinElement(
- const std::vector<std::size_t> &input_vector) {
- std::vector<std::size_t>::const_iterator min_element_iter =
- std::min_element(std::begin(input_vector), std::end(input_vector));
- const std::size_t min_element_id =
- std::distance(input_vector.begin(), min_element_iter);
- return std::make_pair(min_element_id, *min_element_iter);
- }
-
- // First element is the index of the max element, second element is the
- // value of the max element.
- static std::pair<std::size_t, std::size_t> getMaxElement(
- const std::vector<std::size_t> &input_vector) {
- std::vector<std::size_t>::const_iterator max_element_iter =
- std::max_element(std::begin(input_vector), std::end(input_vector));
- const std::size_t max_element_id =
- std::distance(input_vector.begin(), max_element_iter);
- return std::make_pair(max_element_id, *max_element_iter);
- }
-
- void SetUp() {
- std::vector<int> numa_nodes;
- std::vector<client_id> client_ids;
-
- numa_nodes.reserve(kNumWorkers);
- client_ids.reserve(kNumWorkers);
-
- for (std::size_t worker_thread_index = 0; worker_thread_index < kNumWorkers; ++worker_thread_index) {
- // NUMA node id = worker_thread_index % 4
- // Client ID = worker_thread_index * 2 + 1
- const client_id cid = worker_thread_index * 2 + 1;
- const int numa_node_id = worker_thread_index % 4;
-
- numa_nodes.push_back(numa_node_id);
- client_ids.push_back(cid);
-
- worker_thread_indexs_.push_back(worker_thread_index);
- actual_workers_[worker_thread_index] = std::make_pair(numa_node_id, cid);
- }
- wd_.reset(new WorkerDirectory(kNumWorkers, client_ids, numa_nodes));
-
- // Randomize the order of worker IDs.
- std::random_shuffle(worker_thread_indexs_.begin(), worker_thread_indexs_.end());
- }
-
- // First element is NUMA Node. Second element is client ID.
- // Input is the logical worker ID.
- std::pair<int, client_id> getNUMANodeAndClientIDForWorker(
- const std::size_t worker_thread_index) {
- return actual_workers_[worker_thread_index];
- }
-
- WorkerDirectory* getWorkerDirectory() {
- return wd_.get();
- }
-
- std::size_t getActualNumWorkers() const {
- return kNumWorkers;
- }
-
- std::vector<size_t>& getRandomizedWorkerIDs() {
- std::random_shuffle(worker_thread_indexs_.begin(), worker_thread_indexs_.end());
- return worker_thread_indexs_;
- }
-
- private:
- std::unique_ptr<WorkerDirectory> wd_;
-
- // Key = Worker's logical ID
- // Value = pair <NUMA node ID, client ID> for the worker.
- std::unordered_map<std::size_t, std::pair<int, client_id>> actual_workers_;
-
- std::vector<std::size_t> worker_thread_indexs_;
-
- const std::size_t kNumWorkers = 100;
-};
-
-TEST_F(WorkerDirectoryTest, NUMANodeAndClientIDTest) {
- // Check if the NUMA node ID and client IDs are set correctly and the number
- // of queued workorders is initialized correctly.
- WorkerDirectory *wd = getWorkerDirectory();
-
- EXPECT_EQ(getActualNumWorkers(), wd->getNumWorkers());
-
- for (std::size_t worker_thread_index : getRandomizedWorkerIDs()) {
- const std::pair<int, client_id> observed_ids =
- getNUMANodeAndClientIDForWorker(worker_thread_index);
- EXPECT_EQ(observed_ids.first, wd->getNUMANode(worker_thread_index));
- EXPECT_EQ(observed_ids.second, wd->getClientID(worker_thread_index));
- EXPECT_EQ(0u, wd->getNumQueuedWorkOrders(worker_thread_index));
- }
-}
-
-TEST_F(WorkerDirectoryTest, IncrementAndDecrementWorkOrdersTest) {
- // Increment and decrement the workorders for workers.
- // Check if the NUMA node ID and client IDs are set correctly and the number
- WorkerDirectory *wd = getWorkerDirectory();
- EXPECT_EQ(getActualNumWorkers(), wd->getNumWorkers());
-
- std::vector<std::size_t> actual_num_workorders;
- actual_num_workorders.resize(getActualNumWorkers(), 0);
-
- // We perform the increment and decrement in workorders as determined by a
- // pre-initialized sequence.
- // true = increment and false = decrement in number of workorders.
- std::vector<bool> increment_decrement_mini_sequence(
- {true, true, false, true, false, false, true, false});
-
- std::vector<bool> increment_decrement_sequence;
-
- // Insert the mini sequence into sequence kNumWorkers number of times.
- for (std::size_t i = 0; i < getActualNumWorkers(); ++i) {
- increment_decrement_sequence.insert(
- increment_decrement_sequence.end(),
- increment_decrement_mini_sequence.cbegin(),
- increment_decrement_mini_sequence.cend());
- }
-
- std::random_device rd;
- std::mt19937 mt(rd());
- std::uniform_int_distribution<std::size_t> dist(0, getActualNumWorkers() - 1);
-
- // Perform the increment or decrement operation as determined by the sequence
- // and check the correctness.
- std::stack<std::size_t> worker_thread_indexs_used;
- for (bool to_increment : increment_decrement_sequence) {
- if (to_increment) {
- // Pick a random worker ID and increment its number of workorders.
- const std::size_t chosen_worker_thread_index = dist(mt);
- worker_thread_indexs_used.push(chosen_worker_thread_index);
- EXPECT_EQ(actual_num_workorders[chosen_worker_thread_index],
- wd->getNumQueuedWorkOrders(chosen_worker_thread_index));
- wd->incrementNumQueuedWorkOrders(chosen_worker_thread_index);
- ++actual_num_workorders[chosen_worker_thread_index];
- EXPECT_EQ(actual_num_workorders[chosen_worker_thread_index],
- wd->getNumQueuedWorkOrders(chosen_worker_thread_index));
- } else {
- // For the worker with ID = top of stack, decrement a workorder.
- const std::size_t chosen_worker_thread_index = worker_thread_indexs_used.top();
- worker_thread_indexs_used.pop();
- EXPECT_EQ(actual_num_workorders[chosen_worker_thread_index],
- wd->getNumQueuedWorkOrders(chosen_worker_thread_index));
- wd->decrementNumQueuedWorkOrders(chosen_worker_thread_index);
- --actual_num_workorders[chosen_worker_thread_index];
- EXPECT_EQ(actual_num_workorders[chosen_worker_thread_index],
- wd->getNumQueuedWorkOrders(chosen_worker_thread_index));
- }
- }
-
- // Stack should be empty.
- EXPECT_TRUE(worker_thread_indexs_used.empty());
- // Expect no queued up workorders for any worker.
- for (const std::size_t random_worker_thread_index : getRandomizedWorkerIDs()) {
- EXPECT_EQ(0u, wd->getNumQueuedWorkOrders(random_worker_thread_index));
- EXPECT_EQ(actual_num_workorders[random_worker_thread_index],
- wd->getNumQueuedWorkOrders(random_worker_thread_index));
- }
-}
-
-TEST_F(WorkerDirectoryTest, AddWorkerTest) {
- // Add a worker to the worker directory after constructor call.
- WorkerDirectory *wd = getWorkerDirectory();
- EXPECT_EQ(getActualNumWorkers(), wd->getNumWorkers());
-
- const client_id new_worker_client_id = getActualNumWorkers() * 2 + 1;
- const int new_worker_numa_node = 4;
- wd->addWorker(new_worker_client_id, new_worker_numa_node);
-
- // The logical ID of the new worker.
- const std::size_t new_worker_thread_index = getActualNumWorkers();
-
- EXPECT_EQ(getActualNumWorkers() + 1, wd->getNumWorkers());
- // Check if the client ID is set correctly.
- EXPECT_EQ(new_worker_client_id, wd->getClientID(new_worker_thread_index));
- // Check if the NUMA node ID is set correctly.
- EXPECT_EQ(new_worker_numa_node, wd->getNUMANode(new_worker_thread_index));
- // Check if the new worker has no queued up workorders.
- EXPECT_EQ(0u, wd->getNumQueuedWorkOrders(new_worker_thread_index));
- // Increment a workorder for the new worker, check if the increment is
- // successful, then perform a decrement and check the correctness.
- wd->incrementNumQueuedWorkOrders(new_worker_thread_index);
- EXPECT_EQ(1u, wd->getNumQueuedWorkOrders(new_worker_thread_index));
- wd->decrementNumQueuedWorkOrders(new_worker_thread_index);
- EXPECT_EQ(0u, wd->getNumQueuedWorkOrders(new_worker_thread_index));
-}
-
-TEST_F(WorkerDirectoryTest, WorkerLoadTest) {
- // Assign load (in terms of number of workorders) to the workers and check if
- // the least and most loaded worker is retrieved correctly.
- WorkerDirectory *wd = getWorkerDirectory();
- EXPECT_EQ(getActualNumWorkers(), wd->getNumWorkers());
-
- std::vector<std::size_t> actual_num_workorders;
- actual_num_workorders.resize(getActualNumWorkers(), 0);
-
- // Loop over workers sequentially and increment workorder of all the workers.
- for (std::size_t worker_thread_index = 0; worker_thread_index < getActualNumWorkers();
- ++worker_thread_index) {
- const std::pair<std::size_t, std::size_t> &actual_min_loaded_worker =
- getMinElement(actual_num_workorders);
- const std::pair<std::size_t, std::size_t> &actual_max_loaded_worker =
- getMaxElement(actual_num_workorders);
-
- EXPECT_EQ(actual_min_loaded_worker, wd->getLeastLoadedWorker());
- EXPECT_EQ(actual_max_loaded_worker, wd->getMostLoadedWorker());
-
- wd->incrementNumQueuedWorkOrders(worker_thread_index);
- ++actual_num_workorders[worker_thread_index];
- EXPECT_EQ(actual_num_workorders[worker_thread_index],
- wd->getNumQueuedWorkOrders(worker_thread_index));
- }
-
- // At this time, every worker has exactly one workorder assigned to it.
- // Now increment workorders in a random order.
- for (const std::size_t random_worker_thread_index : getRandomizedWorkerIDs()) {
- const std::pair<std::size_t, std::size_t> actual_min_loaded_worker =
- getMinElement(actual_num_workorders);
- const std::pair<std::size_t, std::size_t> actual_max_loaded_worker =
- getMaxElement(actual_num_workorders);
-
- EXPECT_EQ(actual_min_loaded_worker, wd->getLeastLoadedWorker());
- EXPECT_EQ(actual_max_loaded_worker, wd->getMostLoadedWorker());
-
- wd->incrementNumQueuedWorkOrders(random_worker_thread_index);
- ++actual_num_workorders[random_worker_thread_index];
- EXPECT_EQ(actual_num_workorders[random_worker_thread_index],
- wd->getNumQueuedWorkOrders(random_worker_thread_index));
- }
-
- // At this time, every worker has two workorders assigned to it.
- // Now decrement workorders in a random order twice.
- for (std::size_t iteration = 0; iteration < 2; ++iteration) {
- for (const std::size_t random_worker_thread_index : getRandomizedWorkerIDs()) {
- const std::pair<std::size_t, std::size_t> actual_min_loaded_worker =
- getMinElement(actual_num_workorders);
- const std::pair<std::size_t, std::size_t> actual_max_loaded_worker =
- getMaxElement(actual_num_workorders);
-
- EXPECT_EQ(actual_min_loaded_worker, wd->getLeastLoadedWorker());
- EXPECT_EQ(actual_max_loaded_worker, wd->getMostLoadedWorker());
-
- wd->decrementNumQueuedWorkOrders(random_worker_thread_index);
- --actual_num_workorders[random_worker_thread_index];
- EXPECT_EQ(actual_num_workorders[random_worker_thread_index],
- wd->getNumQueuedWorkOrders(random_worker_thread_index));
- }
- }
- const std::pair<std::size_t, std::size_t> actual_min_loaded_worker =
- getMinElement(actual_num_workorders);
- const std::pair<std::size_t, std::size_t> actual_max_loaded_worker =
- getMaxElement(actual_num_workorders);
-
- EXPECT_EQ(actual_min_loaded_worker, wd->getLeastLoadedWorker());
- EXPECT_EQ(actual_max_loaded_worker, wd->getMostLoadedWorker());
-}
-} // namespace quickstep