You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@quickstep.apache.org by zu...@apache.org on 2017/03/10 22:58:46 UTC
incubator-quickstep git commit: Fixed the catalog relation schema
reference bug in the preloading in the distributed version.
Repository: incubator-quickstep
Updated Branches:
refs/heads/dist-preload-fix [created] 4a95738de
Fixed the catalog relation schema reference bug in the preloading in the distributed version.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/4a95738d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/4a95738d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/4a95738d
Branch: refs/heads/dist-preload-fix
Commit: 4a95738de730b03ccfb2026843f5a4e6838cfe2f
Parents: a5c68dd
Author: Zuyu Zhang <zu...@apache.org>
Authored: Fri Mar 10 14:58:39 2017 -0800
Committer: Zuyu Zhang <zu...@apache.org>
Committed: Fri Mar 10 14:58:39 2017 -0800
----------------------------------------------------------------------
catalog/tests/Catalog_unittest.cpp | 5 ++++-
query_execution/Shiftboss.cpp | 4 +++-
storage/CMakeLists.txt | 1 +
storage/PreloaderThread.cpp | 17 +++++++++++++----
storage/PreloaderThread.hpp | 12 ++++++++++--
5 files changed, 31 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4a95738d/catalog/tests/Catalog_unittest.cpp
----------------------------------------------------------------------
diff --git a/catalog/tests/Catalog_unittest.cpp b/catalog/tests/Catalog_unittest.cpp
index 276bfa9..f761026 100644
--- a/catalog/tests/Catalog_unittest.cpp
+++ b/catalog/tests/Catalog_unittest.cpp
@@ -571,7 +571,10 @@ TEST_F(CatalogTest, CatalogIndexTest) {
}
TEST_F(CatalogTest, CatalogDatabaseCacheTest) {
- CatalogRelationSchema* const rel = createCatalogRelation("rel");
+ CatalogRelation * const rel = createCatalogRelation("rel");
+
+ rel->addBlock(BlockIdUtil::GetBlockId(0 /* domain */, 7));
+ rel->addBlock(BlockIdUtil::GetBlockId(0 /* domain */, 14));
rel->addAttribute(new CatalogAttribute(nullptr, "attr_int", TypeFactory::GetType(kInt), -1 /* id */, "int"));
rel->addAttribute(new CatalogAttribute(nullptr, "attr_long", TypeFactory::GetType(kLong)));
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4a95738d/query_execution/Shiftboss.cpp
----------------------------------------------------------------------
diff --git a/query_execution/Shiftboss.cpp b/query_execution/Shiftboss.cpp
index 905464f..72d8913 100644
--- a/query_execution/Shiftboss.cpp
+++ b/query_execution/Shiftboss.cpp
@@ -340,9 +340,11 @@ void Shiftboss::processShiftbossRegistrationResponseMessage() {
storage_manager_->sendBlockDomainToShiftbossIndexMessage(shiftboss_index_);
if (FLAGS_preload_buffer_pool) {
+ database_cache_.update(proto.catalog_database());
+
const CatalogDatabase catalog_database(proto.catalog_database());
- PreloaderThread preloader(catalog_database, storage_manager_, cpu_id_);
+ PreloaderThread preloader(catalog_database, storage_manager_, cpu_id_, &database_cache_);
printf("Preloading the buffer pool ... \n");
const std::chrono::time_point<std::chrono::steady_clock> preload_start = std::chrono::steady_clock::now();
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4a95738d/storage/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/storage/CMakeLists.txt b/storage/CMakeLists.txt
index 8b68150..6fe6436 100644
--- a/storage/CMakeLists.txt
+++ b/storage/CMakeLists.txt
@@ -827,6 +827,7 @@ target_link_libraries(quickstep_storage_PartitionedHashTablePool
target_link_libraries(quickstep_storage_PreloaderThread
glog
quickstep_catalog_CatalogDatabase
+ quickstep_catalog_CatalogDatabaseLite
quickstep_catalog_CatalogRelation
quickstep_catalog_CatalogTypedefs
quickstep_storage_StorageBlock
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4a95738d/storage/PreloaderThread.cpp
----------------------------------------------------------------------
diff --git a/storage/PreloaderThread.cpp b/storage/PreloaderThread.cpp
index edaa143..354e468 100644
--- a/storage/PreloaderThread.cpp
+++ b/storage/PreloaderThread.cpp
@@ -27,6 +27,7 @@
#endif
#include "catalog/CatalogDatabase.hpp"
+#include "catalog/CatalogDatabaseLite.hpp"
#include "catalog/CatalogRelation.hpp"
#include "catalog/CatalogTypedefs.hpp"
#include "storage/StorageBlock.hpp"
@@ -60,8 +61,12 @@ void PreloaderThread::run() {
std::vector<block_id> blocks = relation.getBlocksSnapshot();
for (block_id current_block_id : blocks) {
try {
- BlockReference current_block =
- storage_manager_->getBlock(current_block_id, relation);
+ const CatalogRelationSchema &relation_schema =
+ database_for_schemas_ ? database_for_schemas_->getRelationSchemaById(relation.getID())
+ : relation;
+
+ const BlockReference current_block =
+ storage_manager_->getBlock(current_block_id, relation_schema);
} catch (...) {
LOG(ERROR) << "Error after loading " << blocks_loaded << "blocks";
throw;
@@ -104,8 +109,12 @@ std::size_t PreloaderThread::preloadNUMAAware(
placement_scheme->getNUMANodeForPartition(part_id);
for (block_id curr_block_id : part_scheme.getBlocksInPartition(part_id)) {
try {
- BlockReference current_block = storage_manager_->getBlock(
- curr_block_id, relation, partition_numa_node_id);
+ const CatalogRelationSchema &relation_schema =
+ database_for_schemas_ ? database_for_schemas_->getRelationSchemaById(relation.getID())
+ : relation;
+
+ const BlockReference current_block = storage_manager_->getBlock(
+ curr_block_id, relation_schema, partition_numa_node_id);
} catch (...) {
LOG(ERROR) << "Error while preloading: After loading total "
<< blocks_loaded + num_previously_loaded_blocks
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4a95738d/storage/PreloaderThread.hpp
----------------------------------------------------------------------
diff --git a/storage/PreloaderThread.hpp b/storage/PreloaderThread.hpp
index d9853b8..af3f6ba 100644
--- a/storage/PreloaderThread.hpp
+++ b/storage/PreloaderThread.hpp
@@ -29,6 +29,7 @@
namespace quickstep {
class CatalogDatabase;
+class CatalogDatabaseLite;
class CatalogRelation;
class StorageManager;
@@ -55,13 +56,18 @@ class PreloaderThread : public Thread {
* evicted and the database will not be fully preloaded.
* @param cpu_id The ID of the CPU to affinitize this thread to, or -1 to
* indicate no affinity.
+ * @param database_for_schemas All non-temporary relations in this database
+ * will be used for referencing the schema in a block for the
+ * distributed version.
**/
PreloaderThread(const CatalogDatabase &database,
StorageManager *storage_manager,
- const int cpu_id = -1)
+ const int cpu_id = -1,
+ const CatalogDatabaseLite *database_for_schemas = nullptr)
: database_(database),
storage_manager_(storage_manager),
- cpu_id_(cpu_id) {
+ cpu_id_(cpu_id),
+ database_for_schemas_(database_for_schemas) {
}
~PreloaderThread() override {
@@ -102,6 +108,8 @@ class PreloaderThread : public Thread {
const int cpu_id_;
+ const CatalogDatabaseLite *database_for_schemas_;
+
DISALLOW_COPY_AND_ASSIGN(PreloaderThread);
};