You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@quickstep.apache.org by zu...@apache.org on 2016/05/16 20:55:47 UTC
[10/46] incubator-quickstep git commit: Operator Unit Tests Fixes
(#186)
Operator Unit Tests Fixes (#186)
* Set the unique storage path for each operator tests.
* Delete blocks after use.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/1af93027
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/1af93027
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/1af93027
Branch: refs/heads/master
Commit: 1af930275176d85f624d0bb85ece550288b00a60
Parents: 145990e
Author: Zuyu ZHANG <zu...@users.noreply.github.com>
Authored: Sat Apr 23 13:39:09 2016 -0500
Committer: Jignesh Patel <pa...@users.noreply.github.com>
Committed: Sat Apr 23 13:39:09 2016 -0500
----------------------------------------------------------------------
relational_operators/TextScanOperator.cpp | 2 +-
.../tests/AggregationOperator_unittest.cpp | 21 +++++++++-
.../tests/HashJoinOperator_unittest.cpp | 37 +++++++++++++++-
.../tests/SortMergeRunOperator_unittest.cpp | 44 ++++++++++++++++++--
.../SortRunGenerationOperator_unittest.cpp | 13 +++++-
.../tests/TextScanOperator_unittest.cpp | 2 +-
6 files changed, 111 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1af93027/relational_operators/TextScanOperator.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/TextScanOperator.cpp b/relational_operators/TextScanOperator.cpp
index e6d5200..5ede6f7 100644
--- a/relational_operators/TextScanOperator.cpp
+++ b/relational_operators/TextScanOperator.cpp
@@ -288,7 +288,7 @@ void TextScanWorkOrder::execute() {
std::fclose(file);
} else {
- MutableBlobReference blob = storage_manager_->getBlobMutable(text_blob_);
+ BlobReference blob = storage_manager_->getBlob(text_blob_);
const char *blob_pos = static_cast<const char*>(blob->getMemory());
const char *blob_end = blob_pos + text_size_;
bool have_row = false;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1af93027/relational_operators/tests/AggregationOperator_unittest.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/tests/AggregationOperator_unittest.cpp b/relational_operators/tests/AggregationOperator_unittest.cpp
index da9f267..f2207c2 100644
--- a/relational_operators/tests/AggregationOperator_unittest.cpp
+++ b/relational_operators/tests/AggregationOperator_unittest.cpp
@@ -170,6 +170,17 @@ class AggregationOperatorTest : public ::testing::Test {
virtual void TearDown() {
thread_id_map_->removeValue();
+
+ // Drop blocks from relations.
+ const std::vector<block_id> table_blocks = table_->getBlocksSnapshot();
+ for (const block_id block : table_blocks) {
+ storage_manager_->deleteBlockOrBlobFile(block);
+ }
+
+ const std::vector<block_id> result_table_blocks = result_table_->getBlocksSnapshot();
+ for (const block_id block : result_table_blocks) {
+ storage_manager_->deleteBlockOrBlobFile(block);
+ }
}
Tuple* createTuple(const CatalogRelation &relation, const std::int64_t val) {
@@ -420,6 +431,10 @@ class AggregationOperatorTest : public ::testing::Test {
sub_block.getAttributeValueTyped(0, result_table_->getAttributeByName("result-1")->getID());
test(expected0, actual0);
test(expected1, actual1);
+
+ // Drop the block.
+ block.release();
+ storage_manager_->deleteBlockOrBlobFile(result[0]);
}
template <class FinalDataType>
@@ -463,6 +478,10 @@ class AggregationOperatorTest : public ::testing::Test {
check_fn(group_by_id, actual0, actual1);
}
total_tuples += sub_block.numTuples();
+
+ // Drop the block.
+ block.release();
+ storage_manager_->deleteBlockOrBlobFile(result[bid]);
}
EXPECT_EQ(num_tuples, total_tuples);
}
@@ -500,7 +519,7 @@ class AggregationOperatorTest : public ::testing::Test {
const char AggregationOperatorTest::kDatabaseName[] = "database";
const char AggregationOperatorTest::kTableName[] = "table";
-const char AggregationOperatorTest::kStoragePath[] = "./test_data";
+const char AggregationOperatorTest::kStoragePath[] = "./aggregation_operator_test_data";
namespace {
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1af93027/relational_operators/tests/HashJoinOperator_unittest.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/tests/HashJoinOperator_unittest.cpp b/relational_operators/tests/HashJoinOperator_unittest.cpp
index 0e6d96b..333c3f0 100644
--- a/relational_operators/tests/HashJoinOperator_unittest.cpp
+++ b/relational_operators/tests/HashJoinOperator_unittest.cpp
@@ -114,7 +114,7 @@ class HashJoinOperatorTest : public ::testing::TestWithParam<HashTableImplType>
bus_.RegisterClientAsSender(foreman_client_id_, kCatalogRelationNewBlockMessage);
bus_.RegisterClientAsReceiver(foreman_client_id_, kCatalogRelationNewBlockMessage);
- storage_manager_.reset(new StorageManager("./test_data/"));
+ storage_manager_.reset(new StorageManager("./hash_join_operator_test_data/"));
// Create a database.
db_.reset(new CatalogDatabase(nullptr, "database"));
@@ -193,6 +193,17 @@ class HashJoinOperatorTest : public ::testing::TestWithParam<HashTableImplType>
virtual void TearDown() {
thread_id_map_->removeValue();
+
+ // Drop blocks from relations.
+ const std::vector<block_id> dim_blocks = dim_table_->getBlocksSnapshot();
+ for (const block_id block : dim_blocks) {
+ storage_manager_->deleteBlockOrBlobFile(block);
+ }
+
+ const std::vector<block_id> fact_blocks = fact_table_->getBlocksSnapshot();
+ for (const block_id block : fact_blocks) {
+ storage_manager_->deleteBlockOrBlobFile(block);
+ }
}
StorageBlockLayout* createStorageLayout(const CatalogRelation &relation) {
@@ -398,6 +409,10 @@ TEST_P(HashJoinOperatorTest, LongKeyHashJoinTest) {
++counts[value];
}
}
+
+ // Drop the block.
+ result_block.release();
+ storage_manager_->deleteBlockOrBlobFile(result_blocks[bid]);
}
EXPECT_EQ(static_cast<std::size_t>(kNumDimTuples), num_result_tuples);
@@ -550,6 +565,10 @@ TEST_P(HashJoinOperatorTest, IntDuplicateKeyHashJoinTest) {
++fact_counts[value];
}
}
+
+ // Drop the block.
+ result_block.release();
+ storage_manager_->deleteBlockOrBlobFile(result_blocks[bid]);
}
EXPECT_EQ(static_cast<std::size_t>(kNumDimTuples), num_result_tuples);
@@ -689,6 +708,10 @@ TEST_P(HashJoinOperatorTest, CharKeyCartesianProductHashJoinTest) {
++counts[value];
}
}
+
+ // Drop the block.
+ result_block.release();
+ storage_manager_->deleteBlockOrBlobFile(result_blocks[bid]);
}
EXPECT_EQ(static_cast<std::size_t>(kNumDimTuples * kNumFactTuples),
num_result_tuples);
@@ -835,6 +858,10 @@ TEST_P(HashJoinOperatorTest, VarCharDuplicateKeyHashJoinTest) {
++fact_counts[value];
}
}
+
+ // Drop the block.
+ result_block.release();
+ storage_manager_->deleteBlockOrBlobFile(result_blocks[bid]);
}
EXPECT_EQ(static_cast<std::size_t>(kNumDimTuples), num_result_tuples);
@@ -1004,6 +1031,10 @@ TEST_P(HashJoinOperatorTest, CompositeKeyHashJoinTest) {
++fact_counts[value];
}
}
+
+ // Drop the block.
+ result_block.release();
+ storage_manager_->deleteBlockOrBlobFile(result_blocks[bid]);
}
EXPECT_EQ(static_cast<std::size_t>(kNumDimTuples) / 2, num_result_tuples);
@@ -1184,6 +1215,10 @@ TEST_P(HashJoinOperatorTest, CompositeKeyHashJoinWithResidualPredicateTest) {
++fact_counts[value];
}
}
+
+ // Drop the block.
+ result_block.release();
+ storage_manager_->deleteBlockOrBlobFile(result_blocks[bid]);
}
EXPECT_EQ(8u, num_result_tuples);
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1af93027/relational_operators/tests/SortMergeRunOperator_unittest.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/tests/SortMergeRunOperator_unittest.cpp b/relational_operators/tests/SortMergeRunOperator_unittest.cpp
index 5068391..50c508d 100644
--- a/relational_operators/tests/SortMergeRunOperator_unittest.cpp
+++ b/relational_operators/tests/SortMergeRunOperator_unittest.cpp
@@ -196,6 +196,17 @@ class RunTest : public ::testing::Test {
// Usually the worker thread makes the following call. In this test setup,
// we don't have a worker thread hence we have to explicitly make the call.
thread_id_map_->removeValue();
+
+ // Drop blocks from relations and InsertDestination.
+ const vector<block_id> tmp_blocks = insert_destination_->getTouchedBlocks();
+ for (const block_id block : tmp_blocks) {
+ storage_manager_->deleteBlockOrBlobFile(block);
+ }
+
+ const vector<block_id> blocks = table_->getBlocksSnapshot();
+ for (const block_id block : blocks) {
+ storage_manager_->deleteBlockOrBlobFile(block);
+ }
}
// Helper method to insert test tuples.
@@ -225,7 +236,7 @@ class RunTest : public ::testing::Test {
};
const char RunTest::kTableName[] = "table";
-const char RunTest::kStoragePath[] = "./test_data";
+const char RunTest::kStoragePath[] = "./sort_merge_run_operator_test_data";
const tuple_id RunTest::kNumTuples = 100;
const tuple_id RunTest::kNumTuplesPerBlock = 10;
@@ -429,6 +440,17 @@ class RunMergerTest : public ::testing::Test {
// Usually the worker thread makes the following call. In this test setup,
// we don't have a worker thread hence we have to explicitly make the call.
thread_id_map_->removeValue();
+
+ // Drop blocks from relations and InsertDestination.
+ const vector<block_id> tmp_blocks = insert_destination_->getTouchedBlocks();
+ for (const block_id block : tmp_blocks) {
+ storage_manager_->deleteBlockOrBlobFile(block);
+ }
+
+ const vector<block_id> blocks = table_->getBlocksSnapshot();
+ for (const block_id block : blocks) {
+ storage_manager_->deleteBlockOrBlobFile(block);
+ }
}
// Helper method to create test tuples.
@@ -642,7 +664,7 @@ class RunMergerTest : public ::testing::Test {
};
const char RunMergerTest::kTableName[] = "table";
-const char RunMergerTest::kStoragePath[] = "./test_data";
+const char RunMergerTest::kStoragePath[] = "./sort_merge_run_operator_test_data";
const std::size_t RunMergerTest::kNumTuplesPerBlock = 10;
const tuple_id RunMergerTest::kNumBlocksPerRun = 10;
const std::size_t RunMergerTest::kNumRuns = 10;
@@ -1285,6 +1307,22 @@ class SortMergeRunOperatorTest : public ::testing::Test {
// Usually the worker thread makes the following call. In this test setup,
// we don't have a worker thread hence we have to explicitly make the call.
thread_id_map_->removeValue();
+
+ // Drop blocks from relations.
+ const vector<block_id> input_blocks = input_table_->getBlocksSnapshot();
+ for (const block_id block : input_blocks) {
+ storage_manager_->deleteBlockOrBlobFile(block);
+ }
+
+ const vector<block_id> result_blocks = result_table_->getBlocksSnapshot();
+ for (const block_id block : result_blocks) {
+ storage_manager_->deleteBlockOrBlobFile(block);
+ }
+
+ const vector<block_id> run_blocks = run_table_->getBlocksSnapshot();
+ for (const block_id block : run_blocks) {
+ storage_manager_->deleteBlockOrBlobFile(block);
+ }
}
CatalogRelation *createTable(const char *name, const relation_id rel_id) {
@@ -1681,7 +1719,7 @@ class SortMergeRunOperatorTest : public ::testing::Test {
const char SortMergeRunOperatorTest::kTableName[] = "table";
const char SortMergeRunOperatorTest::kResultTableName[] = "result-table";
const char SortMergeRunOperatorTest::kRunTableName[] = "run-table";
-const char SortMergeRunOperatorTest::kStoragePath[] = "./test_data";
+const char SortMergeRunOperatorTest::kStoragePath[] = "./sort_merge_run_operator_test_data";
const char SortMergeRunOperatorTest::kDatabaseName[] = "database";
namespace {
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1af93027/relational_operators/tests/SortRunGenerationOperator_unittest.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/tests/SortRunGenerationOperator_unittest.cpp b/relational_operators/tests/SortRunGenerationOperator_unittest.cpp
index e11676c..7491778 100644
--- a/relational_operators/tests/SortRunGenerationOperator_unittest.cpp
+++ b/relational_operators/tests/SortRunGenerationOperator_unittest.cpp
@@ -193,6 +193,17 @@ class SortRunGenerationOperatorTest : public ::testing::Test {
// Usually the worker thread makes the following call. In this test setup,
// we don't have a worker thread hence we have to explicitly make the call.
thread_id_map_->removeValue();
+
+ // Drop blocks from relations.
+ const vector<block_id> input_blocks = input_table_->getBlocksSnapshot();
+ for (const block_id block : input_blocks) {
+ storage_manager_->deleteBlockOrBlobFile(block);
+ }
+
+ const vector<block_id> result_blocks = result_table_->getBlocksSnapshot();
+ for (const block_id block : result_blocks) {
+ storage_manager_->deleteBlockOrBlobFile(block);
+ }
}
// Helper method to create catalog relation.
@@ -414,7 +425,7 @@ class SortRunGenerationOperatorTest : public ::testing::Test {
const char SortRunGenerationOperatorTest::kTableName[] = "table";
const char SortRunGenerationOperatorTest::kResultTableName[] = "result-table";
-const char SortRunGenerationOperatorTest::kStoragePath[] = "./test_data";
+const char SortRunGenerationOperatorTest::kStoragePath[] = "./sort_run_generation_operator_test_data";
namespace {
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1af93027/relational_operators/tests/TextScanOperator_unittest.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/tests/TextScanOperator_unittest.cpp b/relational_operators/tests/TextScanOperator_unittest.cpp
index 355c4e2..1dfad7b 100644
--- a/relational_operators/tests/TextScanOperator_unittest.cpp
+++ b/relational_operators/tests/TextScanOperator_unittest.cpp
@@ -94,7 +94,7 @@ class TextScanOperatorTest : public ::testing::Test {
relation_->addAttribute(
new CatalogAttribute(relation_, "varchar_attr", TypeFactory::GetType(kVarChar, 20, true)));
- storage_manager_.reset(new StorageManager("./test_data/"));
+ storage_manager_.reset(new StorageManager("./text_scan_operator_test_data/"));
}
virtual void TearDown() {