You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@quickstep.apache.org by zu...@apache.org on 2016/05/30 23:18:36 UTC
[01/50] [abbrv] incubator-quickstep git commit: Fixed IWYU include
paths. (#194) [Forced Update!]
Repository: incubator-quickstep
Updated Branches:
refs/heads/work-order-serialization 0cc647151 -> 3e35844f8 (forced update)
Fixed IWYU include paths. (#194)
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/d353a642
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/d353a642
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/d353a642
Branch: refs/heads/work-order-serialization
Commit: d353a642d02d5150f3949fd4575a76a1d06bfc6e
Parents: b9a21b4
Author: Zuyu ZHANG <zu...@users.noreply.github.com>
Authored: Wed Apr 27 03:23:43 2016 -0700
Committer: Jignesh Patel <pa...@users.noreply.github.com>
Committed: Wed Apr 27 05:23:43 2016 -0500
----------------------------------------------------------------------
third_party/iwyu/iwyu_helper.py | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/d353a642/third_party/iwyu/iwyu_helper.py
----------------------------------------------------------------------
diff --git a/third_party/iwyu/iwyu_helper.py b/third_party/iwyu/iwyu_helper.py
index dff1837..a204c50 100755
--- a/third_party/iwyu/iwyu_helper.py
+++ b/third_party/iwyu/iwyu_helper.py
@@ -21,9 +21,10 @@ QUICKSTEP_INCLUDES = [ '.',
'./build/third_party',
'./build/third_party/protobuf/include',
'./build/third_party/gflags/include',
- './third_party/gtest/include',
- './third_party/glog/src',
'./third_party/benchmark/include',
+ './third_party/glog/src',
+ './third_party/googletest/googletest/include',
+ './third_party/re2',
'./third_party/tmb/include']
QUICKSTEP_DEFINES = [ '-DQUICKSTEP_DEBUG',
'-DQUICKSTEP_ENABLE_VECTOR_COPY_ELISION_SELECTION', ]
[36/50] [abbrv] incubator-quickstep git commit: In ScopedBuffer,
initialize the allocated memory bytes to 0 (#230)
Posted by zu...@apache.org.
In ScopedBuffer, initialize the allocated memory bytes to 0 (#230)
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/be35bb5b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/be35bb5b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/be35bb5b
Branch: refs/heads/work-order-serialization
Commit: be35bb5b008d974faf6638cf40f446128f38030a
Parents: 0fffe50
Author: Jianqiao Zhu <ji...@cs.wisc.edu>
Authored: Thu May 19 18:20:53 2016 -0500
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Mon May 30 15:47:52 2016 -0700
----------------------------------------------------------------------
utility/ScopedBuffer.hpp | 19 ++++++++++++++++---
1 file changed, 16 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/be35bb5b/utility/ScopedBuffer.hpp
----------------------------------------------------------------------
diff --git a/utility/ScopedBuffer.hpp b/utility/ScopedBuffer.hpp
index b6fddc5..03290ff 100644
--- a/utility/ScopedBuffer.hpp
+++ b/utility/ScopedBuffer.hpp
@@ -43,9 +43,13 @@ class ScopedBuffer {
* size.
*
* @param alloc_size The number of bytes of memory to allocate.
+ * @param initialize If true, initialize all the bytes of the allocated memory to 0.
**/
- explicit ScopedBuffer(const std::size_t alloc_size) {
+ explicit ScopedBuffer(const std::size_t alloc_size, bool initialize = true) {
internal_ptr_ = std::malloc(alloc_size);
+ if (initialize) {
+ memset(internal_ptr_, 0x0, alloc_size);
+ }
}
/**
@@ -54,10 +58,15 @@ class ScopedBuffer {
*
* @param alloc_size The number of bytes of memory to allocate.
* @param alloc_alignment The alignment of the memory to allocate.
+ * @param initialize If true, initialize all the bytes of the allocated memory to 0.
**/
ScopedBuffer(const std::size_t alloc_size,
- const std::size_t alloc_alignment) {
+ const std::size_t alloc_alignment,
+ const bool initialize = true) {
internal_ptr_ = malloc_with_alignment(alloc_size, alloc_alignment);
+ if (initialize) {
+ memset(internal_ptr_, 0x0, alloc_size);
+ }
}
/**
@@ -110,12 +119,16 @@ class ScopedBuffer {
* size.
*
* @param alloc_size The number of bytes of memory to allocate.
+ * @param initialize If true, initialize all the bytes of the allocated memory to 0.
**/
- void reset(const std::size_t alloc_size) {
+ void reset(const std::size_t alloc_size, const bool initialize = true) {
if (internal_ptr_ != nullptr) {
std::free(internal_ptr_);
}
internal_ptr_ = std::malloc(alloc_size);
+ if (initialize) {
+ memset(internal_ptr_, 0x0, alloc_size);
+ }
}
/**
[28/50] [abbrv] incubator-quickstep git commit: Added a quick fix to
the count(*) problem (#220)
Posted by zu...@apache.org.
Added a quick fix to the count(*) problem (#220)
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/ee54e1d8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/ee54e1d8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/ee54e1d8
Branch: refs/heads/work-order-serialization
Commit: ee54e1d8bacc6a951ee34f07a9901a8c49d4aa34
Parents: 6b8cc26
Author: Jianqiao Zhu <ji...@cs.wisc.edu>
Authored: Tue May 10 17:12:31 2016 -0500
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Mon May 30 15:47:27 2016 -0700
----------------------------------------------------------------------
query_optimizer/physical/Aggregate.cpp | 14 ++++++++++++++
query_optimizer/tests/execution_generator/Select.test | 9 +++------
2 files changed, 17 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/ee54e1d8/query_optimizer/physical/Aggregate.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/physical/Aggregate.cpp b/query_optimizer/physical/Aggregate.cpp
index 9bf606c..c582bba 100644
--- a/query_optimizer/physical/Aggregate.cpp
+++ b/query_optimizer/physical/Aggregate.cpp
@@ -1,6 +1,8 @@
/**
* Copyright 2011-2015 Quickstep Technologies LLC.
* Copyright 2015 Pivotal Software, Inc.
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -70,6 +72,18 @@ std::vector<E::AttributeReferencePtr> Aggregate::getReferencedAttributes()
referenced_attributes_in_predicate.begin(),
referenced_attributes_in_predicate.end());
}
+
+ // TODO(jianqiao): This is a quick fix to the COUNT(*) problem that we retain
+ // at least one column from the child plan to survive the PruneColumns physical
+ // optimization. The comprehensive approach should also try to optimize the
+ // child plan with regard to the knowledge that the aggregation actually don't
+ // care about the values of the child plan's output but needs only the number
+ // of rows of the output.
+ if (referenced_attributes.size() == 0) {
+ DCHECK_GT(input_->getOutputAttributes().size(), static_cast<std::size_t>(0));
+ referenced_attributes.emplace_back(input_->getOutputAttributes()[0]);
+ }
+
return referenced_attributes;
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/ee54e1d8/query_optimizer/tests/execution_generator/Select.test
----------------------------------------------------------------------
diff --git a/query_optimizer/tests/execution_generator/Select.test b/query_optimizer/tests/execution_generator/Select.test
index 023ad2e..05f7108 100644
--- a/query_optimizer/tests/execution_generator/Select.test
+++ b/query_optimizer/tests/execution_generator/Select.test
@@ -464,8 +464,6 @@ WHERE a.int_col = b.int_col
+-----------+-----------+---------------+-----------------------+---------------+------------------------+-----------+
==
-# FIXME(qzeng): Inner nested-loops join has no output column, causing query
-# result to be INCORRECT.
SELECT COUNT(*)
FROM
(SELECT a.float_col
@@ -476,7 +474,7 @@ FROM
+--------------------+
|COUNT(*) |
+--------------------+
-| 0|
+| 575|
+--------------------+
==
@@ -941,15 +939,14 @@ ORDER BY x;
+-----------+-----------+
==
-# TODO(team): Fix Issue #9 to enable COUNT(*).
-SELECT COUNT(long_col)
+SELECT COUNT(*)
FROM test,
(SELECT AVG(long_col) a FROM test) subquery
WHERE double_col < 0
AND long_col > subquery.a;
--
+--------------------+
-|COUNT(long_col) |
+|COUNT(*) |
+--------------------+
| 5|
+--------------------+
[32/50] [abbrv] incubator-quickstep git commit: Minor improvements in
FileManagers. (#224)
Posted by zu...@apache.org.
Minor improvements in FileManagers. (#224)
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/2c0722e1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/2c0722e1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/2c0722e1
Branch: refs/heads/work-order-serialization
Commit: 2c0722e13afa6caabe2d281228b0e707b0ef28d9
Parents: 1fa81a8
Author: Zuyu ZHANG <zu...@users.noreply.github.com>
Authored: Tue May 17 14:26:19 2016 -0700
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Mon May 30 15:47:45 2016 -0700
----------------------------------------------------------------------
storage/CMakeLists.txt | 21 ++++++++------
storage/FileManager.hpp | 9 ++----
storage/FileManagerHdfs.cpp | 57 ++++++++++++++++++-------------------
storage/FileManagerPosix.cpp | 55 +++++++++++++++++------------------
storage/FileManagerPosix.hpp | 6 ++--
storage/FileManagerWindows.cpp | 53 +++++++++++++++++-----------------
storage/FileManagerWindows.hpp | 4 +--
7 files changed, 103 insertions(+), 102 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2c0722e1/storage/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/storage/CMakeLists.txt b/storage/CMakeLists.txt
index 115248c..26a3e32 100644
--- a/storage/CMakeLists.txt
+++ b/storage/CMakeLists.txt
@@ -585,20 +585,22 @@ target_link_libraries(quickstep_storage_FileManager
quickstep_utility_Macros
quickstep_utility_StringUtil)
if (QUICKSTEP_HAVE_FILE_MANAGER_HDFS)
-target_link_libraries(quickstep_storage_FileManagerHdfs
- gflags_nothreads-static
- quickstep_storage_FileManager
- quickstep_storage_StorageBlockInfo
- quickstep_storage_StorageConstants
- quickstep_storage_StorageErrors
- quickstep_utility_Macros
- quickstep_utility_StringUtil
- ${LIBHDFS3_LIBRARIES})
+ target_link_libraries(quickstep_storage_FileManagerHdfs
+ glog
+ gflags_nothreads-static
+ quickstep_storage_FileManager
+ quickstep_storage_StorageBlockInfo
+ quickstep_storage_StorageConstants
+ quickstep_storage_StorageErrors
+ quickstep_utility_Macros
+ quickstep_utility_StringUtil
+ ${LIBHDFS3_LIBRARIES})
endif()
if (QUICKSTEP_HAVE_FILE_MANAGER_POSIX)
target_link_libraries(quickstep_storage_FileManagerLocal
quickstep_storage_FileManagerPosix)
target_link_libraries(quickstep_storage_FileManagerPosix
+ glog
quickstep_storage_FileManager
quickstep_storage_StorageBlockInfo
quickstep_storage_StorageConstants
@@ -609,6 +611,7 @@ elseif (QUICKSTEP_HAVE_FILE_MANAGER_WINDOWS)
target_link_libraries(quickstep_storage_FileManagerLocal
quickstep_storage_FileManagerWindows)
target_link_libraries(quickstep_storage_FileManagerWindows
+ glog
quickstep_storage_FileManager
quickstep_storage_StorageBlockInfo
quickstep_storage_StorageConstants
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2c0722e1/storage/FileManager.hpp
----------------------------------------------------------------------
diff --git a/storage/FileManager.hpp b/storage/FileManager.hpp
index d56c960..b179071 100644
--- a/storage/FileManager.hpp
+++ b/storage/FileManager.hpp
@@ -1,6 +1,6 @@
/**
* Copyright 2011-2015 Quickstep Technologies LLC.
- * Copyright 2015 Pivotal Software, Inc.
+ * Copyright 2015-2016 Pivotal Software, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -19,10 +19,7 @@
#define QUICKSTEP_STORAGE_FILE_MANAGER_HPP_
#include <cstddef>
-#include <cstdint>
-#include <cstdio>
#include <string>
-#include <utility>
#include "storage/StorageBlockInfo.hpp"
#include "utility/Macros.hpp"
@@ -49,12 +46,12 @@ class FileManager {
* @param block_domain The domain of a block id.
**/
explicit FileManager(const std::string &storage_path)
- : storage_path_(storage_path) { }
+ : storage_path_(storage_path) {}
/**
* @brief Virtual destructor.
**/
- virtual ~FileManager() { }
+ virtual ~FileManager() {}
/**
* @brief Get a block or blob's relative filename, which uses storage_path_
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2c0722e1/storage/FileManagerHdfs.cpp
----------------------------------------------------------------------
diff --git a/storage/FileManagerHdfs.cpp b/storage/FileManagerHdfs.cpp
index 5f9706e..e8f048b 100644
--- a/storage/FileManagerHdfs.cpp
+++ b/storage/FileManagerHdfs.cpp
@@ -1,6 +1,6 @@
/**
* Copyright 2011-2015 Quickstep Technologies LLC.
- * Copyright 2015 Pivotal Software, Inc.
+ * Copyright 2015-2016 Pivotal Software, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -32,10 +32,10 @@
#include "storage/StorageBlockInfo.hpp"
#include "storage/StorageConstants.hpp"
#include "storage/StorageErrors.hpp"
-#include "utility/Macros.hpp"
#include "utility/StringUtil.hpp"
#include "gflags/gflags.h"
+#include "glog/logging.h"
using std::size_t;
using std::sscanf;
@@ -76,20 +76,19 @@ static const bool hdfs_num_replications_dummy
FileManagerHdfs::FileManagerHdfs(const string &storage_path)
: FileManager(storage_path) {
- DEBUG_ASSERT(hdfs_namenode_port_dummy);
- DEBUG_ASSERT(hdfs_num_replications_dummy);
+ DCHECK(hdfs_namenode_port_dummy);
+ DCHECK(hdfs_num_replications_dummy);
struct hdfsBuilder *builder = hdfsNewBuilder();
hdfsBuilderSetNameNode(builder, FLAGS_hdfs_namenode_host.c_str());
hdfsBuilderSetNameNodePort(builder, FLAGS_hdfs_namenode_port);
// hdfsBuilderConnect releases builder.
hdfs_ = hdfsBuilderConnect(builder);
- DEBUG_ASSERT(hdfs_ != nullptr);
+ DCHECK(hdfs_ != nullptr);
}
FileManagerHdfs::~FileManagerHdfs() {
- int status = hdfsDisconnect(hdfs_);
- DEBUG_ASSERT(status == 0);
+ CHECK_EQ(0, hdfsDisconnect(hdfs_));
}
block_id_counter FileManagerHdfs::getMaxUsedBlockCounter(const block_id_domain block_domain) const {
@@ -97,7 +96,7 @@ block_id_counter FileManagerHdfs::getMaxUsedBlockCounter(const block_id_domain b
hdfsFileInfo *file_infos = hdfsListDirectory(hdfs_, storage_path_.c_str(), &num_files);
if (file_infos == nullptr) {
if (errno != ENOENT) {
- LOG_WARNING("Failed to list file info with error: " << strerror(errno));
+ LOG(ERROR) << "Failed to list file info with error: " << strerror(errno);
}
return 0;
}
@@ -113,9 +112,9 @@ block_id_counter FileManagerHdfs::getMaxUsedBlockCounter(const block_id_domain b
// NOTE(zuyu): mName looks like
// "/user/<username>/<storage_path_>/qsblk_<block_domain>_[0-9]*.qsb".
const char *filename = std::strrchr(file_infos[i].mName, '/');
- if (filename != nullptr
- && sscanf(filename, filename_pattern.c_str(), &counter) == 1
- && counter > counter_max) {
+ if (filename != nullptr &&
+ sscanf(filename, filename_pattern.c_str(), &counter) == 1 &&
+ counter > counter_max) {
counter_max = counter;
}
}
@@ -126,12 +125,12 @@ block_id_counter FileManagerHdfs::getMaxUsedBlockCounter(const block_id_domain b
}
size_t FileManagerHdfs::numSlots(const block_id block) const {
- string filename(blockFilename(block));
+ const string filename(blockFilename(block));
hdfsFileInfo *file_info = hdfsGetPathInfo(hdfs_, filename.c_str());
if (file_info == nullptr) {
if (errno != ENOENT) {
- LOG_WARNING("Failed to get size of file " << filename << " with error: " << strerror(errno));
+ LOG(ERROR) << "Failed to get size of file " << filename << " with error: " << strerror(errno);
}
return 0;
}
@@ -147,12 +146,12 @@ size_t FileManagerHdfs::numSlots(const block_id block) const {
}
bool FileManagerHdfs::deleteBlockOrBlob(const block_id block) {
- string filename(blockFilename(block));
+ const string filename(blockFilename(block));
if ((hdfsDelete(hdfs_, filename.c_str(), 0) == 0) || (errno == ENOENT)) {
return true;
} else {
- LOG_WARNING("Failed to delete file " << filename << " with error: " << strerror(errno));
+ LOG(ERROR) << "Failed to delete file " << filename << " with error: " << strerror(errno);
return false;
}
}
@@ -160,10 +159,10 @@ bool FileManagerHdfs::deleteBlockOrBlob(const block_id block) {
bool FileManagerHdfs::readBlockOrBlob(const block_id block,
void *buffer,
const size_t length) {
- DEBUG_ASSERT(buffer);
- DEBUG_ASSERT(length % kSlotSizeBytes == 0);
+ DCHECK(buffer != nullptr);
+ DCHECK_EQ(0u, length % kSlotSizeBytes);
- string filename(blockFilename(block));
+ const string filename(blockFilename(block));
hdfsFile file_handle = hdfsOpenFile(hdfs_,
filename.c_str(),
@@ -172,7 +171,7 @@ bool FileManagerHdfs::readBlockOrBlob(const block_id block,
FLAGS_hdfs_num_replications,
kSlotSizeBytes);
if (file_handle == nullptr) {
- LOG_WARNING("Failed to open file " << filename << " with error: " << strerror(errno));
+ LOG(ERROR) << "Failed to open file " << filename << " with error: " << strerror(errno);
return false;
}
@@ -183,17 +182,17 @@ bool FileManagerHdfs::readBlockOrBlob(const block_id block,
bytes_total += bytes;
} else if (bytes == -1) {
if (errno != EINTR) {
- LOG_WARNING("Failed to read file " << filename << " with error: " << strerror(errno));
+ LOG(ERROR) << "Failed to read file " << filename << " with error: " << strerror(errno);
break;
}
} else {
- LOG_WARNING("Failed to read file " << filename << " since EOF was reached unexpectedly");
+ LOG(ERROR) << "Failed to read file " << filename << " since EOF was reached unexpectedly";
break;
}
}
if (hdfsCloseFile(hdfs_, file_handle) != 0) {
- LOG_WARNING("Failed to close file " << filename << " with error: " << strerror(errno));
+ LOG(ERROR) << "Failed to close file " << filename << " with error: " << strerror(errno);
}
return (bytes_total == length);
@@ -202,10 +201,10 @@ bool FileManagerHdfs::readBlockOrBlob(const block_id block,
bool FileManagerHdfs::writeBlockOrBlob(const block_id block,
const void *buffer,
const size_t length) {
- DEBUG_ASSERT(buffer);
- DEBUG_ASSERT(length % kSlotSizeBytes == 0);
+ DCHECK(buffer != nullptr);
+ DCHECK_EQ(0u, length % kSlotSizeBytes);
- string filename(blockFilename(block));
+ const string filename(blockFilename(block));
hdfsFile file_handle = hdfsOpenFile(hdfs_,
filename.c_str(),
@@ -214,7 +213,7 @@ bool FileManagerHdfs::writeBlockOrBlob(const block_id block,
FLAGS_hdfs_num_replications,
kSlotSizeBytes);
if (file_handle == nullptr) {
- LOG_WARNING("Failed to open file " << filename << " with error: " << strerror(errno));
+ LOG(ERROR) << "Failed to open file " << filename << " with error: " << strerror(errno);
return false;
}
@@ -224,17 +223,17 @@ bool FileManagerHdfs::writeBlockOrBlob(const block_id block,
if (bytes > 0) {
bytes_total += bytes;
} else if (bytes == -1) {
- LOG_WARNING("Failed to write file " << filename << " with error: " << strerror(errno));
+ LOG(ERROR) << "Failed to write file " << filename << " with error: " << strerror(errno);
break;
}
}
if (hdfsSync(hdfs_, file_handle) != 0) {
- LOG_WARNING("Failed to sync file " << filename << " with error: " << strerror(errno));
+ LOG(ERROR) << "Failed to sync file " << filename << " with error: " << strerror(errno);
}
if (hdfsCloseFile(hdfs_, file_handle) != 0) {
- LOG_WARNING("Failed to close file " << filename << " with error: " << strerror(errno));
+ LOG(ERROR) << "Failed to close file " << filename << " with error: " << strerror(errno);
}
return (bytes_total == length);
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2c0722e1/storage/FileManagerPosix.cpp
----------------------------------------------------------------------
diff --git a/storage/FileManagerPosix.cpp b/storage/FileManagerPosix.cpp
index 3bfb69d..0346f0d 100644
--- a/storage/FileManagerPosix.cpp
+++ b/storage/FileManagerPosix.cpp
@@ -1,6 +1,6 @@
/**
* Copyright 2011-2015 Quickstep Technologies LLC.
- * Copyright 2015 Pivotal Software, Inc.
+ * Copyright 2015-2016 Pivotal Software, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -35,9 +35,10 @@
#include "storage/StorageBlockInfo.hpp"
#include "storage/StorageConstants.hpp"
#include "storage/StorageErrors.hpp"
-#include "utility/Macros.hpp"
#include "utility/StringUtil.hpp"
+#include "glog/logging.h"
+
using std::size_t;
using std::sscanf;
using std::strerror;
@@ -64,9 +65,9 @@ block_id_counter FileManagerPosix::getMaxUsedBlockCounter(const block_id_domain
filename_pattern.append(".qsb");
block_id_counter counter_max = 0, counter;
- if (glob_result.gl_pathc > 0
- && sscanf(glob_result.gl_pathv[glob_result.gl_pathc - 1], filename_pattern.c_str(), &counter) == 1
- && counter > counter_max) {
+ if (glob_result.gl_pathc > 0 &&
+ sscanf(glob_result.gl_pathv[glob_result.gl_pathc - 1], filename_pattern.c_str(), &counter) == 1 &&
+ counter > counter_max) {
counter_max = counter;
}
@@ -75,12 +76,12 @@ block_id_counter FileManagerPosix::getMaxUsedBlockCounter(const block_id_domain
}
size_t FileManagerPosix::numSlots(const block_id block) const {
- string filename(blockFilename(block));
+ const string filename(blockFilename(block));
struct stat file_stat;
if (stat(filename.c_str(), &file_stat) == -1) {
if (errno != ENOENT) {
- LOG_WARNING("Failed to retrieve info about file " << filename << " with error: " << strerror(errno));
+ LOG(ERROR) << "Failed to retrieve info about file " << filename << " with error: " << strerror(errno);
}
return 0;
}
@@ -93,12 +94,12 @@ size_t FileManagerPosix::numSlots(const block_id block) const {
}
bool FileManagerPosix::deleteBlockOrBlob(const block_id block) {
- string filename(blockFilename(block));
+ const string filename(blockFilename(block));
if ((unlink(filename.c_str()) == 0) || (errno == ENOENT)) {
return true;
} else {
- LOG_WARNING("Failed to delete file " << filename << " with error: " << strerror(errno));
+ LOG(ERROR) << "Failed to delete file " << filename << " with error: " << strerror(errno);
return false;
}
}
@@ -106,35 +107,35 @@ bool FileManagerPosix::deleteBlockOrBlob(const block_id block) {
bool FileManagerPosix::readBlockOrBlob(const block_id block,
void *buffer,
const std::size_t length) {
- DEBUG_ASSERT(buffer);
- DEBUG_ASSERT(length % kSlotSizeBytes == 0);
+ DCHECK(buffer != nullptr);
+ DCHECK_EQ(0u, length % kSlotSizeBytes);
- string filename(blockFilename(block));
+ const string filename(blockFilename(block));
- int fd = open(filename.c_str(), O_RDONLY);
+ const int fd = open(filename.c_str(), O_RDONLY);
if (fd == -1) {
- LOG_WARNING("Failed to open file " << filename << " with error: " << strerror(errno));
+ LOG(ERROR) << "Failed to open file " << filename << " with error: " << strerror(errno);
return false;
}
size_t bytes_total = 0;
while (bytes_total < length) {
- ssize_t bytes = read(fd, static_cast<char*>(buffer) + bytes_total, length - bytes_total);
+ const ssize_t bytes = read(fd, static_cast<char*>(buffer) + bytes_total, length - bytes_total);
if (bytes > 0) {
bytes_total += bytes;
} else if (bytes == -1) {
if (errno != EINTR) {
- LOG_WARNING("Failed to read file " << filename << " with error: " << strerror(errno));
+ LOG(ERROR) << "Failed to read file " << filename << " with error: " << strerror(errno);
break;
}
} else {
- LOG_WARNING("Failed to read file " << filename << " since EOF was reached unexpectedly");
+ LOG(ERROR) << "Failed to read file " << filename << " since EOF was reached unexpectedly";
break;
}
}
if (close(fd) != 0) {
- LOG_WARNING("Failed to close file " << filename << " with error: " << strerror(errno));
+ LOG(ERROR) << "Failed to close file " << filename << " with error: " << strerror(errno);
}
return (bytes_total == length);
@@ -143,34 +144,34 @@ bool FileManagerPosix::readBlockOrBlob(const block_id block,
bool FileManagerPosix::writeBlockOrBlob(const block_id block,
const void *buffer,
const std::size_t length) {
- DEBUG_ASSERT(buffer);
- DEBUG_ASSERT(length % kSlotSizeBytes == 0);
+ DCHECK(buffer != nullptr);
+ DCHECK_EQ(0u, length % kSlotSizeBytes);
- string filename(blockFilename(block));
+ const string filename(blockFilename(block));
- int fd = open(filename.c_str(), O_WRONLY | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR);
+ const int fd = open(filename.c_str(), O_WRONLY | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR);
if (fd == -1) {
- LOG_WARNING("Failed to open file " << filename << " with error: " << strerror(errno));
+ LOG(ERROR) << "Failed to open file " << filename << " with error: " << strerror(errno);
return false;
}
size_t bytes_total = 0;
while (bytes_total < length) {
- ssize_t bytes = write(fd, static_cast<const char*>(buffer) + bytes_total, length - bytes_total);
+ const ssize_t bytes = write(fd, static_cast<const char*>(buffer) + bytes_total, length - bytes_total);
if (bytes > 0) {
bytes_total += bytes;
} else if (bytes == -1 && errno != EINTR) {
- LOG_WARNING("Failed to write file " << filename << " with error: " << strerror(errno));
+ LOG(ERROR) << "Failed to write file " << filename << " with error: " << strerror(errno);
break;
}
}
if (fsync(fd) != 0) {
- LOG_WARNING("Failed to sync file " << filename << " with error: " << strerror(errno));
+ LOG(ERROR) << "Failed to sync file " << filename << " with error: " << strerror(errno);
}
if (close(fd) != 0) {
- LOG_WARNING("Failed to close file " << filename << " with error: " << strerror(errno));
+ LOG(ERROR) << "Failed to close file " << filename << " with error: " << strerror(errno);
}
return (bytes_total == length);
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2c0722e1/storage/FileManagerPosix.hpp
----------------------------------------------------------------------
diff --git a/storage/FileManagerPosix.hpp b/storage/FileManagerPosix.hpp
index d8c1649..b2aea27 100644
--- a/storage/FileManagerPosix.hpp
+++ b/storage/FileManagerPosix.hpp
@@ -1,6 +1,6 @@
/**
* Copyright 2011-2015 Quickstep Technologies LLC.
- * Copyright 2015 Pivotal Software, Inc.
+ * Copyright 2015-2016 Pivotal Software, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -37,9 +37,9 @@ namespace quickstep {
class FileManagerPosix : public FileManager {
public:
explicit FileManagerPosix(const std::string &storage_path)
- : FileManager(storage_path) { }
+ : FileManager(storage_path) {}
- ~FileManagerPosix() override { }
+ ~FileManagerPosix() override {}
bool deleteBlockOrBlob(const block_id block) override;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2c0722e1/storage/FileManagerWindows.cpp
----------------------------------------------------------------------
diff --git a/storage/FileManagerWindows.cpp b/storage/FileManagerWindows.cpp
index 9e3d4c8..b1763ec 100644
--- a/storage/FileManagerWindows.cpp
+++ b/storage/FileManagerWindows.cpp
@@ -1,6 +1,6 @@
/**
* Copyright 2011-2015 Quickstep Technologies LLC.
- * Copyright 2015 Pivotal Software, Inc.
+ * Copyright 2015-2016 Pivotal Software, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -34,9 +34,10 @@
#include "storage/StorageBlockInfo.hpp"
#include "storage/StorageConstants.hpp"
#include "storage/StorageErrors.hpp"
-#include "utility/Macros.hpp"
#include "utility/StringUtil.hpp"
+#include "glog/logging.h"
+
using std::size_t;
using std::strerror;
using std::string;
@@ -60,7 +61,7 @@ block_id_counter FileManagerWindows::getMaxUsedBlockCounter(const block_id_domai
if (find_handle == INVALID_HANDLE_VALUE) {
error_code = GetLastError();
if (error_code != ERROR_FILE_NOT_FOUND) {
- LOG_WARNING("Failed to retrieve blockfiles with error_code: " << error_code);
+ LOG(ERROR) << "Failed to retrieve blockfiles with error_code: " << error_code;
}
return 0;
}
@@ -73,32 +74,32 @@ block_id_counter FileManagerWindows::getMaxUsedBlockCounter(const block_id_domai
block_id_counter counter_max = 0, counter;
do {
- if (sscanf(find_data.cFileName, filename_pattern.c_str(), &counter) == 1
- && counter > counter_max) {
+ if (sscanf(find_data.cFileName, filename_pattern.c_str(), &counter) == 1 &&
+ counter > counter_max) {
counter_max = counter;
}
} while (FindNextFile(find_handle, &find_data) != 0);
error_code = GetLastError();
if (error_code != ERROR_NO_MORE_FILES) {
- LOG_WARNING("Failed to FindNextFile with error_code: " << error_code);
+ LOG(ERROR) << "Failed to FindNextFile with error_code: " << error_code;
}
if (FindClose(find_handle) == 0) {
- LOG_WARNING("Failed to close the file with error_code: " << GetLastError());
+ LOG(ERROR) << "Failed to close the file with error_code: " << GetLastError();
}
return counter_max;
}
size_t FileManagerWindows::numSlots(const block_id block) const {
- string filename(blockFilename(block));
+ const string filename(blockFilename(block));
WIN32_FILE_ATTRIBUTE_DATA file_stat;
if (!GetFileAttributesEx(filename.c_str(), GetFileExInfoStandard, &file_stat)) {
DWORD error_code = GetLastError();
if (error_code != ERROR_FILE_NOT_FOUND) {
- LOG_WARNING("Failed to retrieve info about file " << filename << " with error_code: " << error_code);
+ LOG(ERROR) << "Failed to retrieve info about file " << filename << " with error_code: " << error_code;
}
return 0;
}
@@ -113,28 +114,28 @@ size_t FileManagerWindows::numSlots(const block_id block) const {
}
bool FileManagerWindows::deleteBlockOrBlob(const block_id block) {
- string filename(blockFilename(block));
+ const string filename(blockFilename(block));
if ((DeleteFile(filename.c_str()) != 0) || (GetLastError() == ERROR_FILE_NOT_FOUND)) {
return true;
} else {
- LOG_WARNING("Failed to delete file " << filename << " with error_code: " << GetLastError());
+ LOG(ERROR) << "Failed to delete file " << filename << " with error_code: " << GetLastError();
}
}
bool FileManagerWindows::readBlockOrBlob(const block_id block,
void *buffer,
const size_t length) {
- DEBUG_ASSERT(buffer);
- DEBUG_ASSERT(length % kSlotSizeBytes == 0);
+ DCHECK(buffer != nullptr);
+ DCHECK_EQ(0u, length % kSlotSizeBytes);
- string filename(blockFilename(block));
+ const string filename(blockFilename(block));
FILE *file = fopen(filename.c_str(), "rb");
if (file == nullptr) {
// Note: On most, but not all, library implementations, the errno variable
// is set to a system-specific error code on failure.
- LOG_WARNING("Failed to open file " << filename << " with error: " << strerror(errno));
+ LOG(ERROR) << "Failed to open file " << filename << " with error: " << strerror(errno);
return false;
}
@@ -142,16 +143,16 @@ bool FileManagerWindows::readBlockOrBlob(const block_id block,
const bool result_is_ok = (bytes == length);
if (!result_is_ok) {
if (std::feof(file)) {
- LOG_WARNING("Failed to read file " << filename << " since EOF was reached unexpectedly");
+ LOG(ERROR) << "Failed to read file " << filename << " since EOF was reached unexpectedly";
} else {
- LOG_WARNING("Failed to read file " << filename << " with error: " << strerror(ferror(file)));
+ LOG(ERROR) << "Failed to read file " << filename << " with error: " << strerror(ferror(file));
clearerr(file);
}
}
if (fclose(file)) {
// Note: fclose does not set errno on failure.
- LOG_WARNING("Failed to close file " << filename);
+ LOG(ERROR) << "Failed to close file " << filename;
}
return result_is_ok;
@@ -160,37 +161,37 @@ bool FileManagerWindows::readBlockOrBlob(const block_id block,
bool FileManagerWindows::writeBlockOrBlob(const block_id block,
const void *buffer,
const size_t length) {
- DEBUG_ASSERT(buffer);
- DEBUG_ASSERT(length % kSlotSizeBytes == 0);
+ DCHECK(buffer != nullptr);
+ DCHECK_EQ(0u, length % kSlotSizeBytes);
- string filename(blockFilename(block));
+ const string filename(blockFilename(block));
FILE *file = fopen(filename.c_str(), "wb");
if (file == nullptr) {
// Note: On most, but not all, library implementations, the errno variable
// is set to a system-specific error code on failure.
- LOG_WARNING("Failed to open file " << filename << " with error: " << strerror(errno));
+ LOG(ERROR) << "Failed to open file " << filename << " with error: " << strerror(errno);
return false;
}
const size_t bytes = std::fwrite(buffer, sizeof(char), length, file);
const bool result_is_ok = (bytes == length);
if (!result_is_ok) {
- LOG_WARNING("Failed to write file " << filename << " with error: " << strerror(ferror(file)));
+ LOG(ERROR) << "Failed to write file " << filename << " with error: " << strerror(ferror(file));
clearerr(file);
}
if (fflush(file)) {
- LOG_WARNING("Failed to flush file " << filename << " with error: " << strerror(ferror(file)));
+ LOG(ERROR) << "Failed to flush file " << filename << " with error: " << strerror(ferror(file));
}
if (!FlushFileBuffers(reinterpret_cast<HANDLE>(_get_osfhandle(_fileno(file))))) {
- LOG_WARNING("Failed to re-flush file " << filename << " with error: " << strerror(ferror(file)));
+ LOG(ERROR) << "Failed to re-flush file " << filename << " with error: " << strerror(ferror(file));
}
if (fclose(file)) {
// Note: fclose does not set errno on failure.
- LOG_WARNING("Failed to close file " << filename);
+ LOG(ERROR) << "Failed to close file " << filename;
}
return result_is_ok;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2c0722e1/storage/FileManagerWindows.hpp
----------------------------------------------------------------------
diff --git a/storage/FileManagerWindows.hpp b/storage/FileManagerWindows.hpp
index 085b418..e7b3aa8 100644
--- a/storage/FileManagerWindows.hpp
+++ b/storage/FileManagerWindows.hpp
@@ -37,9 +37,9 @@ namespace quickstep {
class FileManagerWindows : public FileManager {
public:
explicit FileManagerWindows(const std::string &storage_path)
- : FileManager(storage_path) { }
+ : FileManager(storage_path) {}
- ~FileManagerWindows() override { }
+ ~FileManagerWindows() override {}
bool deleteBlockOrBlob(const block_id block) override;
[27/50] [abbrv] incubator-quickstep git commit: Print number of rows
in the output (#219)
Posted by zu...@apache.org.
Print number of rows in the output (#219)
- Provide an API to calculate the number of rows in a table
- Print output time in milliseconds, upto 3 decimal places.
- Simplify output messages.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/6b8cc265
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/6b8cc265
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/6b8cc265
Branch: refs/heads/work-order-serialization
Commit: 6b8cc2650e4d99c580dcd2618ababcc6b403b166
Parents: 21b8508
Author: Harshad Deshmukh <d....@gmail.com>
Authored: Tue May 10 16:27:06 2016 -0500
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Mon May 30 15:46:47 2016 -0700
----------------------------------------------------------------------
CMakeLists.txt | 3 ++-
cli/CMakeLists.txt | 1 +
cli/PrintToScreen.cpp | 22 ++++++++++++++++++++++
cli/PrintToScreen.hpp | 23 +++++++++++++++++++++++
cli/QuickstepCli.cpp | 12 +++++++++---
storage/StorageBlock.cpp | 5 +++++
storage/StorageBlock.hpp | 11 ++++++++---
7 files changed, 70 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/6b8cc265/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/CMakeLists.txt b/CMakeLists.txt
index c9578b0..87a8f7c 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -712,7 +712,8 @@ target_link_libraries(quickstep_cli_shell
quickstep_threading_ThreadIDBasedMap
quickstep_utility_Macros
quickstep_utility_PtrVector
- quickstep_utility_SqlError)
+ quickstep_utility_SqlError
+ quickstep_utility_StringUtil)
if (ENABLE_HDFS)
target_link_libraries(quickstep_cli_shell
quickstep_storage_FileManagerHdfs)
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/6b8cc265/cli/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/cli/CMakeLists.txt b/cli/CMakeLists.txt
index 9c4073b..a1989d5 100644
--- a/cli/CMakeLists.txt
+++ b/cli/CMakeLists.txt
@@ -102,6 +102,7 @@ target_link_libraries(quickstep_cli_PrintToScreen
quickstep_storage_StorageManager
quickstep_storage_TupleIdSequence
quickstep_storage_TupleStorageSubBlock
+ quickstep_types_IntType
quickstep_types_Type
quickstep_types_TypedValue
quickstep_utility_Macros)
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/6b8cc265/cli/PrintToScreen.cpp
----------------------------------------------------------------------
diff --git a/cli/PrintToScreen.cpp b/cli/PrintToScreen.cpp
index c543b70..227ff39 100644
--- a/cli/PrintToScreen.cpp
+++ b/cli/PrintToScreen.cpp
@@ -29,6 +29,7 @@
#include "storage/StorageManager.hpp"
#include "storage/TupleIdSequence.hpp"
#include "storage/TupleStorageSubBlock.hpp"
+#include "types/IntType.hpp"
#include "types/Type.hpp"
#include "types/TypedValue.hpp"
#include "utility/Macros.hpp"
@@ -147,4 +148,25 @@ void PrintToScreen::printTuple(const TupleStorageSubBlock &tuple_store,
fputc('\n', out);
}
+std::size_t PrintToScreen::GetNumTuplesInRelation(
+ const CatalogRelation &relation, StorageManager *storage_manager) {
+ const std::vector<block_id> &blocks = relation.getBlocksSnapshot();
+ std::size_t total_num_tuples = 0;
+ for (block_id block : blocks) {
+ total_num_tuples +=
+ storage_manager->getBlock(block, relation)->getNumTuples();
+ }
+ return total_num_tuples;
+}
+
+void PrintToScreen::PrintOutputSize(const CatalogRelation &relation,
+ StorageManager *storage_manager,
+ FILE *out) {
+ const std::size_t num_rows = GetNumTuplesInRelation(relation, storage_manager);
+ fprintf(out,
+ "(%lu %s)\n",
+ num_rows,
+ (num_rows == 1) ? "row" : "rows");
+}
+
} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/6b8cc265/cli/PrintToScreen.hpp
----------------------------------------------------------------------
diff --git a/cli/PrintToScreen.hpp b/cli/PrintToScreen.hpp
index 3c9afae..0b57b4b 100644
--- a/cli/PrintToScreen.hpp
+++ b/cli/PrintToScreen.hpp
@@ -46,6 +46,29 @@ class PrintToScreen {
static void printHBar(const std::vector<int> &column_widths,
FILE *out);
+
+ /**
+ * @brief Get the total number of tuples in the given relation.
+ *
+ * @param relation The given relation.
+ * @param storage_manager The storage manager.
+ *
+ * @return The total number of tuples in the relation.
+ **/
+ static std::size_t GetNumTuplesInRelation(const CatalogRelation &relation,
+ StorageManager *storage_manager);
+
+ /**
+ * @brief Print the size of the output (i.e. number of rows in the relation).
+ *
+ * @param relation The given relation.
+ * @param storage_manager The storage manager.
+ * @param out The output stream.
+ **/
+ static void PrintOutputSize(const CatalogRelation &relation,
+ StorageManager *storage_manager,
+ FILE *out);
+
private:
// Undefined default constructor. Class is all-static and should not be
// instantiated.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/6b8cc265/cli/QuickstepCli.cpp
----------------------------------------------------------------------
diff --git a/cli/QuickstepCli.cpp b/cli/QuickstepCli.cpp
index 4c0a166..5881b3e 100644
--- a/cli/QuickstepCli.cpp
+++ b/cli/QuickstepCli.cpp
@@ -72,6 +72,7 @@ typedef quickstep::LineReaderDumb LineReaderImpl;
#include "utility/Macros.hpp"
#include "utility/PtrVector.hpp"
#include "utility/SqlError.hpp"
+#include "utility/StringUtil.hpp"
#include "gflags/gflags.h"
@@ -399,6 +400,10 @@ int main(int argc, char* argv[]) {
PrintToScreen::PrintRelation(*query_result_relation,
query_processor->getStorageManager(),
stdout);
+ PrintToScreen::PrintOutputSize(
+ *query_result_relation,
+ query_processor->getStorageManager(),
+ stdout);
DropRelation::Drop(*query_result_relation,
query_processor->getDefaultDatabase(),
@@ -406,13 +411,14 @@ int main(int argc, char* argv[]) {
}
query_processor->saveCatalog();
- printf("Execution time: %g seconds\n",
- std::chrono::duration<double>(end - start).count());
+ std::chrono::duration<double, std::milli> time_ms = end - start;
+ printf("Time: %s ms\n",
+ quickstep::DoubleToStringWithSignificantDigits(
+ time_ms.count(), 3).c_str());
} catch (const std::exception &e) {
fprintf(stderr, "QUERY EXECUTION ERROR: %s\n", e.what());
break;
}
- printf("Query Complete\n");
} else {
if (result.condition == ParseResult::kError) {
fprintf(stderr, "%s", result.error_message.c_str());
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/6b8cc265/storage/StorageBlock.cpp
----------------------------------------------------------------------
diff --git a/storage/StorageBlock.cpp b/storage/StorageBlock.cpp
index 476a130..fdd438d 100644
--- a/storage/StorageBlock.cpp
+++ b/storage/StorageBlock.cpp
@@ -1361,4 +1361,9 @@ void StorageBlock::invalidateAllIndexes() {
}
}
+const std::size_t StorageBlock::getNumTuples() const {
+ DCHECK(tuple_store_ != nullptr);
+ return tuple_store_->numTuples();
+}
+
} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/6b8cc265/storage/StorageBlock.hpp
----------------------------------------------------------------------
diff --git a/storage/StorageBlock.hpp b/storage/StorageBlock.hpp
index 97813e2..3ae3812 100644
--- a/storage/StorageBlock.hpp
+++ b/storage/StorageBlock.hpp
@@ -311,17 +311,17 @@ class StorageBlock : public StorageBlockBase {
ValueAccessor *accessor);
/**
- * @brief Perform a random sampling of data on the StorageBlock. The number
+ * @brief Perform a random sampling of data on the StorageBlock. The number
* of records sampled is determined by the sample percentage in case of
* tuple sample. For block sample all the records in a block are taken.
*
* @param is_block_sample Flag to indicate if the Sampling method is a tuple
* sample or block sample.
* @param percentage The percentage of tuples to be sampled.Used only when the
- * sampling method is tuple sample.
+ * sampling method is tuple sample.
* @param destination Where to insert the tuples resulting from the SAMPLE.
*
- * @return Randomly sampled tuples whose count is determined by the
+ * @return Randomly sampled tuples whose count is determined by the
* sampling percentage. In the case of block sample the entire
* block is returned.
**/
@@ -583,6 +583,11 @@ class StorageBlock : public StorageBlockBase {
return rebuildIndexes(false);
}
+ /**
+ * @brief Get the number of tuples in this storage block.
+ **/
+ const std::size_t getNumTuples() const;
+
private:
static TupleStorageSubBlock* CreateTupleStorageSubBlock(
const CatalogRelationSchema &relation,
[46/50] [abbrv] incubator-quickstep git commit: Resolves SMA bugs
revealed by TPCH. (#234)
Posted by zu...@apache.org.
Resolves SMA bugs revealed by TPCH. (#234)
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/5a81b803
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/5a81b803
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/5a81b803
Branch: refs/heads/work-order-serialization
Commit: 5a81b803ac205785987ebd7eb5d73ee76d1d5c1e
Parents: 1d8c682
Author: Marc S <cr...@users.noreply.github.com>
Authored: Mon May 23 11:11:44 2016 -0700
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Mon May 30 15:47:53 2016 -0700
----------------------------------------------------------------------
storage/CMakeLists.txt | 1 +
storage/SMAIndexSubBlock.cpp | 55 +++++++++++++++++++-----
storage/SMAIndexSubBlock.hpp | 3 +-
storage/tests/SMAIndexSubBlock_unittest.cpp | 52 ++++++++++++++++++++++
4 files changed, 99 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5a81b803/storage/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/storage/CMakeLists.txt b/storage/CMakeLists.txt
index 26a3e32..a3093df 100644
--- a/storage/CMakeLists.txt
+++ b/storage/CMakeLists.txt
@@ -1451,6 +1451,7 @@ target_link_libraries(SMAIndexSubBlock_unittest
quickstep_types_CharType
quickstep_types_DoubleType
quickstep_types_FloatType
+ quickstep_types_IntType
quickstep_types_LongType
quickstep_types_TypeFactory
quickstep_types_TypeID
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5a81b803/storage/SMAIndexSubBlock.cpp
----------------------------------------------------------------------
diff --git a/storage/SMAIndexSubBlock.cpp b/storage/SMAIndexSubBlock.cpp
index aa9bc54..3a1ccc2 100644
--- a/storage/SMAIndexSubBlock.cpp
+++ b/storage/SMAIndexSubBlock.cpp
@@ -181,19 +181,22 @@ Selectivity getSelectivity(const TypedValue &literal,
}
SMAPredicate* SMAPredicate::ExtractSMAPredicate(const ComparisonPredicate &predicate) {
- if (predicate.getLeftOperand().hasStaticValue()) {
- DLOG_IF(FATAL, predicate.getRightOperand().getDataSource() != Scalar::kAttribute);
+ // Check that the predicate is contains an attribute and a literal value.
+ if (predicate.getLeftOperand().hasStaticValue() &&
+ predicate.getRightOperand().getDataSource() == Scalar::kAttribute) {
return new SMAPredicate(
static_cast<const ScalarAttribute&>(predicate.getRightOperand()).getAttribute().getID(),
flipComparisonID(predicate.getComparison().getComparisonID()),
predicate.getLeftOperand().getStaticValue().makeReferenceToThis());
- } else {
- DLOG_IF(FATAL, predicate.getLeftOperand().getDataSource() != Scalar::kAttribute);
+ } else if (predicate.getRightOperand().hasStaticValue() &&
+ predicate.getLeftOperand().getDataSource() == Scalar::kAttribute) {
return new SMAPredicate(
static_cast<const ScalarAttribute&>(predicate.getLeftOperand()).getAttribute().getID(),
predicate.getComparison().getComparisonID(),
predicate.getRightOperand().getStaticValue().makeReferenceToThis());
}
+ // Predicate is improper form, so return nullptr.
+ return nullptr;
}
/**
@@ -603,12 +606,47 @@ Selectivity SMAIndexSubBlock::getSelectivityForPredicate(const ComparisonPredica
}
std::unique_ptr<SMAPredicate> sma_predicate(SMAPredicate::ExtractSMAPredicate(predicate));
+ // The predicate did not contain a static value to compare against.
+ if (!sma_predicate) {
+ return Selectivity::kUnknown;
+ }
+
const SMAEntry *entry = getEntryChecked(sma_predicate->attribute);
// The attribute wasn't indexed.
if (entry == nullptr || !entry->min_entry_ref.valid || !entry->max_entry_ref.valid) {
return Selectivity::kUnknown;
}
+
+ // Check that the type of the comparison is the same or can be coerced the
+ // type of the underlying attribute.
+ if (sma_predicate->literal.getTypeID() != entry->type_id) {
+ // Try to coerce the literal type to the entry type.
+ // It may have to account for variable length attributes.
+ int literal_length = -1;
+ if (TypeFactory::TypeRequiresLengthParameter(sma_predicate->literal.getTypeID())) {
+ literal_length = sma_predicate->literal.getAsciiStringLength();
+ }
+
+ const Type &literal_type = literal_length == -1 ?
+ TypeFactory::GetType(sma_predicate->literal.getTypeID(), false) :
+ TypeFactory::GetType(sma_predicate->literal.getTypeID(), static_cast<std::size_t>(literal_length), false);
+ const Type &attribute_type = literal_length == -1 ?
+ TypeFactory::GetType(entry->type_id, false) :
+ TypeFactory::GetType(entry->type_id, static_cast<std::size_t>(literal_length), false);
+ if (attribute_type.isSafelyCoercibleFrom(literal_type)) {
+ // Convert the literal's type inside the predicate.
+ SMAPredicate *replacement = new SMAPredicate(
+ sma_predicate->attribute,
+ sma_predicate->comparison,
+ attribute_type.coerceValue(sma_predicate->literal, literal_type));
+ sma_predicate.reset(replacement);
+ } else {
+ // The literal type cannot be converted, so do not evaluate with the SMA.
+ return Selectivity::kUnknown;
+ }
+ }
+
return sma_internal::getSelectivity(
sma_predicate->literal,
sma_predicate->comparison,
@@ -623,12 +661,9 @@ predicate_cost_t SMAIndexSubBlock::estimatePredicateEvaluationCost(
DCHECK(initialized_);
// Check that at least one of the operands has a static value.
- if (predicate.getLeftOperand().hasStaticValue() ||
- predicate.getRightOperand().hasStaticValue()) {
- Selectivity selectivity = getSelectivityForPredicate(predicate);
- if (selectivity == Selectivity::kAll || selectivity == Selectivity::kNone) {
- return predicate_cost::kConstantTime;
- }
+ Selectivity selectivity = getSelectivityForPredicate(predicate);
+ if (selectivity == Selectivity::kAll || selectivity == Selectivity::kNone) {
+ return predicate_cost::kConstantTime;
}
return predicate_cost::kInfinite;
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5a81b803/storage/SMAIndexSubBlock.hpp
----------------------------------------------------------------------
diff --git a/storage/SMAIndexSubBlock.hpp b/storage/SMAIndexSubBlock.hpp
index 71ca75b..895add9 100644
--- a/storage/SMAIndexSubBlock.hpp
+++ b/storage/SMAIndexSubBlock.hpp
@@ -120,7 +120,6 @@ struct SMAPredicate {
// that the predicate had not been used previously.
Selectivity selectivity;
- private:
SMAPredicate(const attribute_id attr,
const ComparisonID comp,
const TypedValue lit)
@@ -376,7 +375,7 @@ class SMAIndexSubBlock : public IndexSubBlock {
private:
bool requiresRebuild() const;
- inline sma_internal::Selectivity getSelectivityForPredicate(const ComparisonPredicate &predicate) const;
+ sma_internal::Selectivity getSelectivityForPredicate(const ComparisonPredicate &predicate) const;
// Retrieves an entry, first checking if the given attribute is indexed.
inline const sma_internal::SMAEntry* getEntryChecked(attribute_id attribute) const {
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5a81b803/storage/tests/SMAIndexSubBlock_unittest.cpp
----------------------------------------------------------------------
diff --git a/storage/tests/SMAIndexSubBlock_unittest.cpp b/storage/tests/SMAIndexSubBlock_unittest.cpp
index b709712..ad842c2 100644
--- a/storage/tests/SMAIndexSubBlock_unittest.cpp
+++ b/storage/tests/SMAIndexSubBlock_unittest.cpp
@@ -41,6 +41,7 @@
#include "types/CharType.hpp"
#include "types/DoubleType.hpp"
#include "types/FloatType.hpp"
+#include "types/IntType.hpp"
#include "types/LongType.hpp"
#include "types/TypeFactory.hpp"
#include "types/TypeID.hpp"
@@ -301,6 +302,10 @@ class SMAIndexSubBlockTest : public ::testing::Test {
return !index_->header_->index_consistent;
}
+ Selectivity getSelectivityForPredicate(const ComparisonPredicate &predicate) const {
+ return index_->getSelectivityForPredicate(predicate);
+ }
+
std::unique_ptr<CatalogRelation> relation_;
std::unique_ptr<MockTupleStorageSubBlock> tuple_store_;
ScopedBuffer index_memory_;
@@ -359,6 +364,53 @@ TEST_F(SMAIndexSubBlockTest, TestConstructor) {
EXPECT_FALSE(requiresRebuild());
}
+TEST_F(SMAIndexSubBlockTest, TestGetSelectivityForPredicate) {
+ // Index long, float, bigchar, and varchar type.
+ createIndex({0, 2, 5, 6}, kIndexSubBlockSize);
+ int min = 0, max = 9010, step = 10;
+ for (std::size_t i = min; i <= static_cast<std::size_t>(max); i += step) {
+ generateAndInsertTuple(i, false, "suffix");
+ }
+ index_->rebuild();
+
+ // Tuple storage block now contains long values [0, 9010], and float values
+ // [0, 2252.5].
+ // Test several value types.
+ // Test with inline values, returning both possible value types.
+ std::unique_ptr<ComparisonPredicate> predicate(
+ generateNumericComparisonPredicate<LongType>(ComparisonID::kLess, 0, min));
+ Selectivity cost = getSelectivityForPredicate(*predicate);
+ EXPECT_EQ(Selectivity::kNone, cost);
+
+ // Test on the float attribute with a type that will need coercion. In this
+ // case, a long type should not coerce to an integer type.
+ predicate.reset(generateNumericComparisonPredicate<LongType>(ComparisonID::kLess, 2, min));
+ cost = getSelectivityForPredicate(*predicate);
+ EXPECT_EQ(Selectivity::kUnknown, cost);
+
+ predicate.reset(generateNumericComparisonPredicate<IntType>(ComparisonID::kLess, 2, min));
+ cost = getSelectivityForPredicate(*predicate);
+ EXPECT_EQ(Selectivity::kNone, cost);
+
+ // This should be unknown because of the loss of precision on coercion.
+ predicate.reset(generateNumericComparisonPredicate<FloatType>(ComparisonID::kLess, 0, min + 0.5));
+ cost = getSelectivityForPredicate(*predicate);
+ EXPECT_EQ(Selectivity::kUnknown, cost);
+
+ // Test coercion with String types as well.
+ predicate.reset(generateStringComparisonPredicate(ComparisonID::kLess, 6, ""));
+ cost = getSelectivityForPredicate(*predicate);
+ EXPECT_EQ(Selectivity::kNone, cost);
+
+ predicate.reset(generateStringComparisonPredicate(ComparisonID::kLess, 5, ""));
+ cost = getSelectivityForPredicate(*predicate);
+ EXPECT_EQ(Selectivity::kNone, cost);
+
+ predicate.reset(generateStringComparisonPredicate(ComparisonID::kLess, 5, "999999999999999"));
+ cost = getSelectivityForPredicate(*predicate);
+ EXPECT_EQ(Selectivity::kAll, cost);
+}
+
TEST_F(SMAIndexSubBlockTest, TestRebuild) {
// This test is checking to see if the min/max values are set correctly on
// a rebuild of the index.
[42/50] [abbrv] incubator-quickstep git commit: Groupby hashtable
pool (#236)
Posted by zu...@apache.org.
Groupby hashtable pool (#236)
- Created a HashTablePool class for group by clause.
- Each thread can checkout it's own hash table while doing group by
aggregation.
- AggregationOperationState uses one hash table pool per group by
clause.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/2ddb67bf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/2ddb67bf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/2ddb67bf
Branch: refs/heads/work-order-serialization
Commit: 2ddb67bf438878b572e997caec2397e4c7ac9b8f
Parents: 5bda90e
Author: Harshad Deshmukh <d....@gmail.com>
Authored: Tue May 24 19:04:39 2016 -0500
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Mon May 30 15:47:53 2016 -0700
----------------------------------------------------------------------
.../aggregation/AggregationConcreteHandle.hpp | 105 ++++++++++++
expressions/aggregation/AggregationHandle.hpp | 15 +-
.../aggregation/AggregationHandleAvg.cpp | 9 +
.../aggregation/AggregationHandleAvg.hpp | 4 +
.../aggregation/AggregationHandleCount.cpp | 11 ++
.../aggregation/AggregationHandleCount.hpp | 4 +
.../aggregation/AggregationHandleDistinct.hpp | 7 +
.../aggregation/AggregationHandleMax.cpp | 9 +
.../aggregation/AggregationHandleMax.hpp | 4 +
.../aggregation/AggregationHandleMin.cpp | 9 +
.../aggregation/AggregationHandleMin.hpp | 4 +
.../aggregation/AggregationHandleSum.cpp | 9 +
.../aggregation/AggregationHandleSum.hpp | 4 +
expressions/aggregation/CMakeLists.txt | 2 +
.../tests/AggregationHandleAvg_unittest.cpp | 109 ++++++++++++
.../tests/AggregationHandleCount_unittest.cpp | 126 +++++++++++++-
.../tests/AggregationHandleMax_unittest.cpp | 122 ++++++++++++++
.../tests/AggregationHandleMin_unittest.cpp | 121 ++++++++++++++
.../tests/AggregationHandleSum_unittest.cpp | 124 ++++++++++++++
query_execution/QueryContext.hpp | 2 +-
storage/AggregationOperationState.cpp | 84 ++++++++--
storage/AggregationOperationState.hpp | 4 +
storage/CMakeLists.txt | 10 ++
storage/HashTablePool.hpp | 166 +++++++++++++++++++
storage/StorageManager.cpp | 3 +-
25 files changed, 1045 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2ddb67bf/expressions/aggregation/AggregationConcreteHandle.hpp
----------------------------------------------------------------------
diff --git a/expressions/aggregation/AggregationConcreteHandle.hpp b/expressions/aggregation/AggregationConcreteHandle.hpp
index 52249f7..0267e17 100644
--- a/expressions/aggregation/AggregationConcreteHandle.hpp
+++ b/expressions/aggregation/AggregationConcreteHandle.hpp
@@ -44,6 +44,90 @@ class ValueAccessor;
* @{
*/
+/**
+ * @brief An upserter class for modifying the destination hash table while
+ * merging two group by hash tables.
+ **/
+template <typename HandleT, typename StateT>
+class HashTableStateUpserter {
+ public:
+ /**
+ * @brief Constructor.
+ *
+ * @param handle The aggregation handle being used.
+ * @param source_state The aggregation state in the source aggregation hash
+ * table. The corresponding state (for the same key) in the destination
+ * hash table will be upserted.
+ **/
+ HashTableStateUpserter(const HandleT &handle, const StateT &source_state)
+ : handle_(handle), source_state_(source_state) {}
+
+ /**
+ * @brief The operator for the functor required for the upsert.
+ *
+ * @param destination_state The aggregation state in the aggregation hash
+ * table that is being upserted.
+ **/
+ void operator()(StateT *destination_state) {
+ handle_.mergeStates(source_state_, destination_state);
+ }
+
+ private:
+ const HandleT &handle_;
+ const StateT &source_state_;
+
+ DISALLOW_COPY_AND_ASSIGN(HashTableStateUpserter);
+};
+
+/**
+ * @brief A class to support the functor for merging group by hash tables.
+ **/
+template <typename HandleT, typename StateT, typename HashTableT>
+class HashTableMerger {
+ public:
+ /**
+ * @brief Constructor
+ *
+ * @param handle The Aggregation handle being used.
+ * @param destination_hash_table The destination hash table to which other
+ * hash tables will be merged.
+ **/
+ HashTableMerger(const HandleT &handle,
+ AggregationStateHashTableBase *destination_hash_table)
+ : handle_(handle),
+ destination_hash_table_(
+ static_cast<HashTableT *>(destination_hash_table)) {}
+
+ /**
+ * @brief The operator for the functor.
+ *
+ * @param group_by_key The group by key being merged.
+ * @param source_state The aggregation state for the given key in the source
+ * aggregation hash table.
+ **/
+ inline void operator()(const std::vector<TypedValue> &group_by_key,
+ const StateT &source_state) {
+ const StateT *original_state =
+ destination_hash_table_->getSingleCompositeKey(group_by_key);
+ if (original_state != nullptr) {
+ HashTableStateUpserter<HandleT, StateT> upserter(
+ handle_, source_state);
+ // The CHECK is required as upsertCompositeKey can return false if the
+ // hash table runs out of space during the upsert process. The ideal
+ // solution will be to retry again if the upsert fails.
+ CHECK(destination_hash_table_->upsertCompositeKey(
+ group_by_key, *original_state, &upserter));
+ } else {
+ destination_hash_table_->putCompositeKey(group_by_key, source_state);
+ }
+ }
+
+ private:
+ const HandleT &handle_;
+ HashTableT *destination_hash_table_;
+
+ DISALLOW_COPY_AND_ASSIGN(HashTableMerger);
+};
/**
* @brief The helper intermediate subclass of AggregationHandle that provides
@@ -140,6 +224,11 @@ class AggregationConcreteHandle : public AggregationHandle {
return static_cast<const HandleT*>(this)->finalizeHashTableEntry(*group_state);
}
+ template <typename HandleT, typename StateT, typename HashTableT>
+ void mergeGroupByHashTablesHelper(
+ const AggregationStateHashTableBase &source_hash_table,
+ AggregationStateHashTableBase *destination_hash_table) const;
+
private:
DISALLOW_COPY_AND_ASSIGN(AggregationConcreteHandle);
};
@@ -373,6 +462,22 @@ ColumnVector* AggregationConcreteHandle::finalizeHashTableHelper(
}
}
+template <typename HandleT,
+ typename StateT,
+ typename HashTableT>
+void AggregationConcreteHandle::mergeGroupByHashTablesHelper(
+ const AggregationStateHashTableBase &source_hash_table,
+ AggregationStateHashTableBase *destination_hash_table) const {
+ const HandleT &handle = static_cast<const HandleT &>(*this);
+ const HashTableT &source_hash_table_concrete =
+ static_cast<const HashTableT &>(source_hash_table);
+
+ HashTableMerger<HandleT, StateT, HashTableT> merger(handle,
+ destination_hash_table);
+
+ source_hash_table_concrete.forEachCompositeKey(&merger);
+}
+
} // namespace quickstep
#endif // QUICKSTEP_EXPRESSIONS_AGGREGATION_AGGREGATION_CONCRETE_HANDLE_HPP_
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2ddb67bf/expressions/aggregation/AggregationHandle.hpp
----------------------------------------------------------------------
diff --git a/expressions/aggregation/AggregationHandle.hpp b/expressions/aggregation/AggregationHandle.hpp
index 625f334..cdebb03 100644
--- a/expressions/aggregation/AggregationHandle.hpp
+++ b/expressions/aggregation/AggregationHandle.hpp
@@ -276,7 +276,7 @@ class AggregationHandle {
* each GROUP BY group. Later, a second-round aggregation on the distinctify
* hash table will be performed to actually compute the aggregated result for
* each GROUP BY group.
- *
+ *
* In the case of single aggregation where there is no GROUP BY expressions,
* we simply treat it as a special GROUP BY case that the GROUP BY expression
* vector is empty.
@@ -349,6 +349,19 @@ class AggregationHandle {
const AggregationStateHashTableBase &distinctify_hash_table,
AggregationStateHashTableBase *aggregation_hash_table) const = 0;
+ /**
+ * @brief Merge two GROUP BY hash tables in one.
+ *
+ * @note Both the hash tables should have the same structure.
+ *
+ * @param source_hash_table The hash table which will get merged.
+ * @param destination_hash_table The hash table to which we will merge the
+ * other hash table.
+ **/
+ virtual void mergeGroupByHashTables(
+ const AggregationStateHashTableBase &source_hash_table,
+ AggregationStateHashTableBase *destination_hash_table) const = 0;
+
protected:
AggregationHandle() {
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2ddb67bf/expressions/aggregation/AggregationHandleAvg.cpp
----------------------------------------------------------------------
diff --git a/expressions/aggregation/AggregationHandleAvg.cpp b/expressions/aggregation/AggregationHandleAvg.cpp
index cb0d63d..42a2fb9 100644
--- a/expressions/aggregation/AggregationHandleAvg.cpp
+++ b/expressions/aggregation/AggregationHandleAvg.cpp
@@ -203,4 +203,13 @@ void AggregationHandleAvg::aggregateOnDistinctifyHashTableForGroupBy(
aggregation_hash_table);
}
+void AggregationHandleAvg::mergeGroupByHashTables(
+ const AggregationStateHashTableBase &source_hash_table,
+ AggregationStateHashTableBase *destination_hash_table) const {
+ mergeGroupByHashTablesHelper<AggregationHandleAvg,
+ AggregationStateAvg,
+ AggregationStateHashTable<AggregationStateAvg>>(
+ source_hash_table, destination_hash_table);
+}
+
} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2ddb67bf/expressions/aggregation/AggregationHandleAvg.hpp
----------------------------------------------------------------------
diff --git a/expressions/aggregation/AggregationHandleAvg.hpp b/expressions/aggregation/AggregationHandleAvg.hpp
index 6a94ee6..4ad4b21 100644
--- a/expressions/aggregation/AggregationHandleAvg.hpp
+++ b/expressions/aggregation/AggregationHandleAvg.hpp
@@ -158,6 +158,10 @@ class AggregationHandleAvg : public AggregationConcreteHandle {
const AggregationStateHashTableBase &distinctify_hash_table,
AggregationStateHashTableBase *aggregation_hash_table) const override;
+ void mergeGroupByHashTables(
+ const AggregationStateHashTableBase &source_hash_table,
+ AggregationStateHashTableBase *destination_hash_table) const override;
+
private:
friend class AggregateFunctionAvg;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2ddb67bf/expressions/aggregation/AggregationHandleCount.cpp
----------------------------------------------------------------------
diff --git a/expressions/aggregation/AggregationHandleCount.cpp b/expressions/aggregation/AggregationHandleCount.cpp
index 5ece8ba..964b7c2 100644
--- a/expressions/aggregation/AggregationHandleCount.cpp
+++ b/expressions/aggregation/AggregationHandleCount.cpp
@@ -206,6 +206,17 @@ void AggregationHandleCount<count_star, nullable_type>
aggregation_hash_table);
}
+template <bool count_star, bool nullable_type>
+void AggregationHandleCount<count_star, nullable_type>::mergeGroupByHashTables(
+ const AggregationStateHashTableBase &source_hash_table,
+ AggregationStateHashTableBase *destination_hash_table) const {
+ mergeGroupByHashTablesHelper<
+ AggregationHandleCount,
+ AggregationStateCount,
+ AggregationStateHashTable<AggregationStateCount>>(source_hash_table,
+ destination_hash_table);
+}
+
// Explicitly instantiate and compile in the different versions of
// AggregationHandleCount we need. Note that we do not compile a version with
// 'count_star == true' and 'nullable_type == true', as that combination is
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2ddb67bf/expressions/aggregation/AggregationHandleCount.hpp
----------------------------------------------------------------------
diff --git a/expressions/aggregation/AggregationHandleCount.hpp b/expressions/aggregation/AggregationHandleCount.hpp
index 6bb4e65..50138b9 100644
--- a/expressions/aggregation/AggregationHandleCount.hpp
+++ b/expressions/aggregation/AggregationHandleCount.hpp
@@ -166,6 +166,10 @@ class AggregationHandleCount : public AggregationConcreteHandle {
const AggregationStateHashTableBase &distinctify_hash_table,
AggregationStateHashTableBase *aggregation_hash_table) const override;
+ void mergeGroupByHashTables(
+ const AggregationStateHashTableBase &source_hash_table,
+ AggregationStateHashTableBase *destination_hash_table) const override;
+
private:
friend class AggregateFunctionCount;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2ddb67bf/expressions/aggregation/AggregationHandleDistinct.hpp
----------------------------------------------------------------------
diff --git a/expressions/aggregation/AggregationHandleDistinct.hpp b/expressions/aggregation/AggregationHandleDistinct.hpp
index 918fdf8..6342c2b 100644
--- a/expressions/aggregation/AggregationHandleDistinct.hpp
+++ b/expressions/aggregation/AggregationHandleDistinct.hpp
@@ -109,6 +109,13 @@ class AggregationHandleDistinct : public AggregationConcreteHandle {
const AggregationStateHashTableBase &hash_table,
std::vector<std::vector<TypedValue>> *group_by_keys) const override;
+ void mergeGroupByHashTables(
+ const AggregationStateHashTableBase &source_hash_table,
+ AggregationStateHashTableBase *destination_hash_table) const override {
+ LOG(FATAL)
+ << "AggregationHandleDistinct does not support mergeGroupByHashTables";
+ }
+
private:
DISALLOW_COPY_AND_ASSIGN(AggregationHandleDistinct);
};
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2ddb67bf/expressions/aggregation/AggregationHandleMax.cpp
----------------------------------------------------------------------
diff --git a/expressions/aggregation/AggregationHandleMax.cpp b/expressions/aggregation/AggregationHandleMax.cpp
index 4703657..a7a4a52 100644
--- a/expressions/aggregation/AggregationHandleMax.cpp
+++ b/expressions/aggregation/AggregationHandleMax.cpp
@@ -139,4 +139,13 @@ void AggregationHandleMax::aggregateOnDistinctifyHashTableForGroupBy(
aggregation_hash_table);
}
+void AggregationHandleMax::mergeGroupByHashTables(
+ const AggregationStateHashTableBase &source_hash_table,
+ AggregationStateHashTableBase *destination_hash_table) const {
+ mergeGroupByHashTablesHelper<AggregationHandleMax,
+ AggregationStateMax,
+ AggregationStateHashTable<AggregationStateMax>>(
+ source_hash_table, destination_hash_table);
+}
+
} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2ddb67bf/expressions/aggregation/AggregationHandleMax.hpp
----------------------------------------------------------------------
diff --git a/expressions/aggregation/AggregationHandleMax.hpp b/expressions/aggregation/AggregationHandleMax.hpp
index 8932ef8..5af5a12 100644
--- a/expressions/aggregation/AggregationHandleMax.hpp
+++ b/expressions/aggregation/AggregationHandleMax.hpp
@@ -151,6 +151,10 @@ class AggregationHandleMax : public AggregationConcreteHandle {
const AggregationStateHashTableBase &distinctify_hash_table,
AggregationStateHashTableBase *aggregation_hash_table) const override;
+ void mergeGroupByHashTables(
+ const AggregationStateHashTableBase &source_hash_table,
+ AggregationStateHashTableBase *destination_hash_table) const override;
+
private:
friend class AggregateFunctionMax;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2ddb67bf/expressions/aggregation/AggregationHandleMin.cpp
----------------------------------------------------------------------
diff --git a/expressions/aggregation/AggregationHandleMin.cpp b/expressions/aggregation/AggregationHandleMin.cpp
index de2709a..ca9b163 100644
--- a/expressions/aggregation/AggregationHandleMin.cpp
+++ b/expressions/aggregation/AggregationHandleMin.cpp
@@ -141,4 +141,13 @@ void AggregationHandleMin::aggregateOnDistinctifyHashTableForGroupBy(
aggregation_hash_table);
}
+void AggregationHandleMin::mergeGroupByHashTables(
+ const AggregationStateHashTableBase &source_hash_table,
+ AggregationStateHashTableBase *destination_hash_table) const {
+ mergeGroupByHashTablesHelper<AggregationHandleMin,
+ AggregationStateMin,
+ AggregationStateHashTable<AggregationStateMin>>(
+ source_hash_table, destination_hash_table);
+}
+
} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2ddb67bf/expressions/aggregation/AggregationHandleMin.hpp
----------------------------------------------------------------------
diff --git a/expressions/aggregation/AggregationHandleMin.hpp b/expressions/aggregation/AggregationHandleMin.hpp
index 4e4c05d..f68bb9d 100644
--- a/expressions/aggregation/AggregationHandleMin.hpp
+++ b/expressions/aggregation/AggregationHandleMin.hpp
@@ -149,6 +149,10 @@ class AggregationHandleMin : public AggregationConcreteHandle {
const AggregationStateHashTableBase &distinctify_hash_table,
AggregationStateHashTableBase *aggregation_hash_table) const override;
+ void mergeGroupByHashTables(
+ const AggregationStateHashTableBase &source_hash_table,
+ AggregationStateHashTableBase *destination_hash_table) const override;
+
private:
friend class AggregateFunctionMin;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2ddb67bf/expressions/aggregation/AggregationHandleSum.cpp
----------------------------------------------------------------------
diff --git a/expressions/aggregation/AggregationHandleSum.cpp b/expressions/aggregation/AggregationHandleSum.cpp
index 14421d2..691ff39 100644
--- a/expressions/aggregation/AggregationHandleSum.cpp
+++ b/expressions/aggregation/AggregationHandleSum.cpp
@@ -190,4 +190,13 @@ void AggregationHandleSum::aggregateOnDistinctifyHashTableForGroupBy(
aggregation_hash_table);
}
+void AggregationHandleSum::mergeGroupByHashTables(
+ const AggregationStateHashTableBase &source_hash_table,
+ AggregationStateHashTableBase *destination_hash_table) const {
+ mergeGroupByHashTablesHelper<AggregationHandleSum,
+ AggregationStateSum,
+ AggregationStateHashTable<AggregationStateSum>>(
+ source_hash_table, destination_hash_table);
+}
+
} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2ddb67bf/expressions/aggregation/AggregationHandleSum.hpp
----------------------------------------------------------------------
diff --git a/expressions/aggregation/AggregationHandleSum.hpp b/expressions/aggregation/AggregationHandleSum.hpp
index b765243..fdc0884 100644
--- a/expressions/aggregation/AggregationHandleSum.hpp
+++ b/expressions/aggregation/AggregationHandleSum.hpp
@@ -148,6 +148,10 @@ class AggregationHandleSum : public AggregationConcreteHandle {
const AggregationStateHashTableBase &distinctify_hash_table,
AggregationStateHashTableBase *aggregation_hash_table) const override;
+ void mergeGroupByHashTables(
+ const AggregationStateHashTableBase &source_hash_table,
+ AggregationStateHashTableBase *destination_hash_table) const override;
+
private:
friend class AggregateFunctionSum;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2ddb67bf/expressions/aggregation/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/expressions/aggregation/CMakeLists.txt b/expressions/aggregation/CMakeLists.txt
index 26cec7f..416c4c6 100644
--- a/expressions/aggregation/CMakeLists.txt
+++ b/expressions/aggregation/CMakeLists.txt
@@ -291,6 +291,8 @@ target_link_libraries(AggregationHandle_tests
quickstep_expressions_aggregation_AggregationHandleMin
quickstep_expressions_aggregation_AggregationHandleSum
quickstep_expressions_aggregation_AggregationID
+ quickstep_storage_HashTableBase
+ quickstep_storage_StorageManager
quickstep_types_CharType
quickstep_types_DateOperatorOverloads
quickstep_types_DatetimeIntervalType
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2ddb67bf/expressions/aggregation/tests/AggregationHandleAvg_unittest.cpp
----------------------------------------------------------------------
diff --git a/expressions/aggregation/tests/AggregationHandleAvg_unittest.cpp b/expressions/aggregation/tests/AggregationHandleAvg_unittest.cpp
index d27b54e..fd82cba 100644
--- a/expressions/aggregation/tests/AggregationHandleAvg_unittest.cpp
+++ b/expressions/aggregation/tests/AggregationHandleAvg_unittest.cpp
@@ -26,6 +26,7 @@
#include "expressions/aggregation/AggregationHandle.hpp"
#include "expressions/aggregation/AggregationHandleAvg.hpp"
#include "expressions/aggregation/AggregationID.hpp"
+#include "storage/StorageManager.hpp"
#include "types/CharType.hpp"
#include "types/DateOperatorOverloads.hpp"
#include "types/DatetimeIntervalType.hpp"
@@ -238,6 +239,7 @@ class AggregationHandleAvgTest : public::testing::Test {
std::unique_ptr<AggregationHandle> aggregation_handle_avg_;
std::unique_ptr<AggregationState> aggregation_handle_avg_state_;
+ std::unique_ptr<StorageManager> storage_manager_;
};
const int AggregationHandleAvgTest::kNumSamples;
@@ -417,4 +419,111 @@ TEST_F(AggregationHandleAvgTest, ResultTypeForArgumentTypeTest) {
EXPECT_TRUE(ResultTypeForArgumentTypeTest(kYearMonthInterval, kYearMonthInterval));
}
+TEST_F(AggregationHandleAvgTest, GroupByTableMergeTestAvg) {
+ const Type &long_non_null_type = LongType::Instance(false);
+ initializeHandle(long_non_null_type);
+ storage_manager_.reset(new StorageManager("./test_avg_data"));
+ std::unique_ptr<AggregationStateHashTableBase> source_hash_table(
+ aggregation_handle_avg_->createGroupByHashTable(
+ HashTableImplType::kSimpleScalarSeparateChaining,
+ std::vector<const Type *>(1, &long_non_null_type),
+ 10,
+ storage_manager_.get()));
+ std::unique_ptr<AggregationStateHashTableBase> destination_hash_table(
+ aggregation_handle_avg_->createGroupByHashTable(
+ HashTableImplType::kSimpleScalarSeparateChaining,
+ std::vector<const Type *>(1, &long_non_null_type),
+ 10,
+ storage_manager_.get()));
+
+ AggregationStateHashTable<AggregationStateAvg> *destination_hash_table_derived =
+ static_cast<AggregationStateHashTable<AggregationStateAvg> *>(
+ destination_hash_table.get());
+
+ AggregationStateHashTable<AggregationStateAvg> *source_hash_table_derived =
+ static_cast<AggregationStateHashTable<AggregationStateAvg> *>(
+ source_hash_table.get());
+
+ AggregationHandleAvg *aggregation_handle_avg_derived =
+ static_cast<AggregationHandleAvg *>(aggregation_handle_avg_.get());
+ // We create three keys: first is present in both the hash tables, second key
+ // is present only in the source hash table while the third key is present
+ // the destination hash table only.
+ std::vector<TypedValue> common_key;
+ common_key.emplace_back(static_cast<std::int64_t>(0));
+ std::vector<TypedValue> exclusive_source_key, exclusive_destination_key;
+ exclusive_source_key.emplace_back(static_cast<std::int64_t>(1));
+ exclusive_destination_key.emplace_back(static_cast<std::int64_t>(2));
+
+ const std::int64_t common_key_source_avg = 355;
+ TypedValue common_key_source_avg_val(common_key_source_avg);
+
+ const std::int64_t common_key_destination_avg = 295;
+ TypedValue common_key_destination_avg_val(common_key_destination_avg);
+
+ const std::int64_t exclusive_key_source_avg = 1;
+ TypedValue exclusive_key_source_avg_val(exclusive_key_source_avg);
+
+ const std::int64_t exclusive_key_destination_avg = 1;
+ TypedValue exclusive_key_destination_avg_val(exclusive_key_destination_avg);
+
+ std::unique_ptr<AggregationStateAvg> common_key_source_state(
+ static_cast<AggregationStateAvg *>(
+ aggregation_handle_avg_->createInitialState()));
+ std::unique_ptr<AggregationStateAvg> common_key_destination_state(
+ static_cast<AggregationStateAvg *>(
+ aggregation_handle_avg_->createInitialState()));
+ std::unique_ptr<AggregationStateAvg> exclusive_key_source_state(
+ static_cast<AggregationStateAvg *>(
+ aggregation_handle_avg_->createInitialState()));
+ std::unique_ptr<AggregationStateAvg> exclusive_key_destination_state(
+ static_cast<AggregationStateAvg *>(
+ aggregation_handle_avg_->createInitialState()));
+
+ // Create avg value states for keys.
+ aggregation_handle_avg_derived->iterateUnaryInl(common_key_source_state.get(),
+ common_key_source_avg_val);
+
+ aggregation_handle_avg_derived->iterateUnaryInl(
+ common_key_destination_state.get(), common_key_destination_avg_val);
+
+ aggregation_handle_avg_derived->iterateUnaryInl(
+ exclusive_key_destination_state.get(), exclusive_key_destination_avg_val);
+
+ aggregation_handle_avg_derived->iterateUnaryInl(
+ exclusive_key_source_state.get(), exclusive_key_source_avg_val);
+
+ // Add the key-state pairs to the hash tables.
+ source_hash_table_derived->putCompositeKey(common_key,
+ *common_key_source_state);
+ destination_hash_table_derived->putCompositeKey(
+ common_key, *common_key_destination_state);
+ source_hash_table_derived->putCompositeKey(exclusive_source_key,
+ *exclusive_key_source_state);
+ destination_hash_table_derived->putCompositeKey(
+ exclusive_destination_key, *exclusive_key_destination_state);
+
+ EXPECT_EQ(2u, destination_hash_table_derived->numEntries());
+ EXPECT_EQ(2u, source_hash_table_derived->numEntries());
+
+ aggregation_handle_avg_->mergeGroupByHashTables(*source_hash_table,
+ destination_hash_table.get());
+
+ EXPECT_EQ(3u, destination_hash_table_derived->numEntries());
+
+ CheckAvgValue<double>(
+ (common_key_destination_avg_val.getLiteral<std::int64_t>() +
+ common_key_source_avg_val.getLiteral<std::int64_t>()) / static_cast<double>(2),
+ *aggregation_handle_avg_derived,
+ *(destination_hash_table_derived->getSingleCompositeKey(common_key)));
+ CheckAvgValue<double>(exclusive_key_destination_avg_val.getLiteral<std::int64_t>(),
+ *aggregation_handle_avg_derived,
+ *(destination_hash_table_derived->getSingleCompositeKey(
+ exclusive_destination_key)));
+ CheckAvgValue<double>(exclusive_key_source_avg_val.getLiteral<std::int64_t>(),
+ *aggregation_handle_avg_derived,
+ *(source_hash_table_derived->getSingleCompositeKey(
+ exclusive_source_key)));
+}
+
} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2ddb67bf/expressions/aggregation/tests/AggregationHandleCount_unittest.cpp
----------------------------------------------------------------------
diff --git a/expressions/aggregation/tests/AggregationHandleCount_unittest.cpp b/expressions/aggregation/tests/AggregationHandleCount_unittest.cpp
index 7bebf6a..bf02523 100644
--- a/expressions/aggregation/tests/AggregationHandleCount_unittest.cpp
+++ b/expressions/aggregation/tests/AggregationHandleCount_unittest.cpp
@@ -27,6 +27,7 @@
#include "expressions/aggregation/AggregationHandle.hpp"
#include "expressions/aggregation/AggregationHandleCount.hpp"
#include "expressions/aggregation/AggregationID.hpp"
+#include "storage/StorageManager.hpp"
#include "types/CharType.hpp"
#include "types/DoubleType.hpp"
#include "types/FloatType.hpp"
@@ -355,6 +356,7 @@ class AggregationHandleCountTest : public::testing::Test {
std::unique_ptr<AggregationHandle> aggregation_handle_count_;
std::unique_ptr<AggregationState> aggregation_handle_count_state_;
+ std::unique_ptr<StorageManager> storage_manager_;
};
typedef AggregationHandleCountTest AggregationHandleCountDeathTest;
@@ -477,5 +479,127 @@ TEST_F(AggregationHandleCountTest, ResultTypeForArgumentTypeTest) {
EXPECT_TRUE(ResultTypeForArgumentTypeTest(kDouble, kLong));
}
-} // namespace quickstep
+TEST_F(AggregationHandleCountTest, GroupByTableMergeTestCount) {
+ const Type &long_non_null_type = LongType::Instance(false);
+ initializeHandle(&long_non_null_type);
+ storage_manager_.reset(new StorageManager("./test_count_data"));
+ std::unique_ptr<AggregationStateHashTableBase> source_hash_table(
+ aggregation_handle_count_->createGroupByHashTable(
+ HashTableImplType::kSimpleScalarSeparateChaining,
+ std::vector<const Type *>(1, &long_non_null_type),
+ 10,
+ storage_manager_.get()));
+ std::unique_ptr<AggregationStateHashTableBase> destination_hash_table(
+ aggregation_handle_count_->createGroupByHashTable(
+ HashTableImplType::kSimpleScalarSeparateChaining,
+ std::vector<const Type *>(1, &long_non_null_type),
+ 10,
+ storage_manager_.get()));
+
+ AggregationStateHashTable<AggregationStateCount> *destination_hash_table_derived =
+ static_cast<AggregationStateHashTable<AggregationStateCount> *>(
+ destination_hash_table.get());
+
+ AggregationStateHashTable<AggregationStateCount> *source_hash_table_derived =
+ static_cast<AggregationStateHashTable<AggregationStateCount> *>(
+ source_hash_table.get());
+
+ // TODO(harshad) - Use TemplateUtil::CreateBoolInstantiatedInstance to
+ // generate all the combinations of the bool template arguments and test them.
+ AggregationHandleCount<true, false> *aggregation_handle_count_derived =
+ static_cast<AggregationHandleCount<true, false> *>(
+ aggregation_handle_count_.get());
+ // We create three keys: first is present in both the hash tables, second key
+ // is present only in the source hash table while the third key is present
+ // the destination hash table only.
+ std::vector<TypedValue> common_key;
+ common_key.emplace_back(static_cast<std::int64_t>(0));
+ std::vector<TypedValue> exclusive_source_key, exclusive_destination_key;
+ exclusive_source_key.emplace_back(static_cast<std::int64_t>(1));
+ exclusive_destination_key.emplace_back(static_cast<std::int64_t>(2));
+
+ const std::int64_t common_key_source_count = 1;
+ TypedValue common_key_source_count_val(common_key_source_count);
+
+ const std::int64_t common_key_destination_count = 1;
+ TypedValue common_key_destination_count_val(common_key_destination_count);
+
+ const std::int64_t exclusive_key_source_count = 1;
+ TypedValue exclusive_key_source_count_val(exclusive_key_source_count);
+
+ const std::int64_t exclusive_key_destination_count = 1;
+ TypedValue exclusive_key_destination_count_val(exclusive_key_destination_count);
+
+ std::unique_ptr<AggregationStateCount> common_key_source_state(
+ static_cast<AggregationStateCount *>(
+ aggregation_handle_count_->createInitialState()));
+ std::unique_ptr<AggregationStateCount> common_key_destination_state(
+ static_cast<AggregationStateCount *>(
+ aggregation_handle_count_->createInitialState()));
+ std::unique_ptr<AggregationStateCount> exclusive_key_source_state(
+ static_cast<AggregationStateCount *>(
+ aggregation_handle_count_->createInitialState()));
+ std::unique_ptr<AggregationStateCount> exclusive_key_destination_state(
+ static_cast<AggregationStateCount *>(
+ aggregation_handle_count_->createInitialState()));
+
+ // Create count value states for keys.
+ aggregation_handle_count_derived->iterateUnaryInl(common_key_source_state.get(),
+ common_key_source_count_val);
+ std::int64_t actual_val = aggregation_handle_count_->finalize(*common_key_source_state)
+ .getLiteral<std::int64_t>();
+ EXPECT_EQ(common_key_source_count_val.getLiteral<std::int64_t>(), actual_val);
+
+ aggregation_handle_count_derived->iterateUnaryInl(
+ common_key_destination_state.get(), common_key_destination_count_val);
+ actual_val = aggregation_handle_count_->finalize(*common_key_destination_state)
+ .getLiteral<std::int64_t>();
+ EXPECT_EQ(common_key_destination_count_val.getLiteral<std::int64_t>(), actual_val);
+
+ aggregation_handle_count_derived->iterateUnaryInl(
+ exclusive_key_destination_state.get(), exclusive_key_destination_count_val);
+ actual_val =
+ aggregation_handle_count_->finalize(*exclusive_key_destination_state)
+ .getLiteral<std::int64_t>();
+ EXPECT_EQ(exclusive_key_destination_count_val.getLiteral<std::int64_t>(), actual_val);
+
+ aggregation_handle_count_derived->iterateUnaryInl(
+ exclusive_key_source_state.get(), exclusive_key_source_count_val);
+ actual_val = aggregation_handle_count_->finalize(*exclusive_key_source_state)
+ .getLiteral<std::int64_t>();
+ EXPECT_EQ(exclusive_key_source_count_val.getLiteral<std::int64_t>(), actual_val);
+
+ // Add the key-state pairs to the hash tables.
+ source_hash_table_derived->putCompositeKey(common_key,
+ *common_key_source_state);
+ destination_hash_table_derived->putCompositeKey(
+ common_key, *common_key_destination_state);
+ source_hash_table_derived->putCompositeKey(exclusive_source_key,
+ *exclusive_key_source_state);
+ destination_hash_table_derived->putCompositeKey(
+ exclusive_destination_key, *exclusive_key_destination_state);
+
+ EXPECT_EQ(2u, destination_hash_table_derived->numEntries());
+ EXPECT_EQ(2u, source_hash_table_derived->numEntries());
+
+ aggregation_handle_count_->mergeGroupByHashTables(*source_hash_table,
+ destination_hash_table.get());
+
+ EXPECT_EQ(3u, destination_hash_table_derived->numEntries());
+
+ CheckCountValue(
+ common_key_destination_count_val.getLiteral<std::int64_t>() +
+ common_key_source_count_val.getLiteral<std::int64_t>(),
+ *aggregation_handle_count_derived,
+ *(destination_hash_table_derived->getSingleCompositeKey(common_key)));
+ CheckCountValue(exclusive_key_destination_count_val.getLiteral<std::int64_t>(),
+ *aggregation_handle_count_derived,
+ *(destination_hash_table_derived->getSingleCompositeKey(
+ exclusive_destination_key)));
+ CheckCountValue(exclusive_key_source_count_val.getLiteral<std::int64_t>(),
+ *aggregation_handle_count_derived,
+ *(source_hash_table_derived->getSingleCompositeKey(
+ exclusive_source_key)));
+}
+} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2ddb67bf/expressions/aggregation/tests/AggregationHandleMax_unittest.cpp
----------------------------------------------------------------------
diff --git a/expressions/aggregation/tests/AggregationHandleMax_unittest.cpp b/expressions/aggregation/tests/AggregationHandleMax_unittest.cpp
index 027f24b..fc25e91 100644
--- a/expressions/aggregation/tests/AggregationHandleMax_unittest.cpp
+++ b/expressions/aggregation/tests/AggregationHandleMax_unittest.cpp
@@ -29,6 +29,8 @@
#include "expressions/aggregation/AggregationHandle.hpp"
#include "expressions/aggregation/AggregationHandleMax.hpp"
#include "expressions/aggregation/AggregationID.hpp"
+#include "storage/HashTableBase.hpp"
+#include "storage/StorageManager.hpp"
#include "types/CharType.hpp"
#include "types/DatetimeIntervalType.hpp"
#include "types/DatetimeLit.hpp"
@@ -413,6 +415,7 @@ class AggregationHandleMaxTest : public ::testing::Test {
std::unique_ptr<AggregationHandle> aggregation_handle_max_;
std::unique_ptr<AggregationState> aggregation_handle_max_state_;
+ std::unique_ptr<StorageManager> storage_manager_;
};
template <>
@@ -637,4 +640,123 @@ TEST_F(AggregationHandleMaxTest, ResultTypeForArgumentTypeTest) {
EXPECT_TRUE(ResultTypeForArgumentTypeTest(kDouble, kDouble));
}
+TEST_F(AggregationHandleMaxTest, GroupByTableMergeTest) {
+ const Type &int_non_null_type = IntType::Instance(false);
+ initializeHandle(int_non_null_type);
+ storage_manager_.reset(new StorageManager("./test_max_data"));
+ std::unique_ptr<AggregationStateHashTableBase> source_hash_table(
+ aggregation_handle_max_->createGroupByHashTable(
+ HashTableImplType::kSimpleScalarSeparateChaining,
+ std::vector<const Type *>(1, &int_non_null_type),
+ 10,
+ storage_manager_.get()));
+ std::unique_ptr<AggregationStateHashTableBase> destination_hash_table(
+ aggregation_handle_max_->createGroupByHashTable(
+ HashTableImplType::kSimpleScalarSeparateChaining,
+ std::vector<const Type *>(1, &int_non_null_type),
+ 10,
+ storage_manager_.get()));
+
+ AggregationStateHashTable<AggregationStateMax> *destination_hash_table_derived =
+ static_cast<AggregationStateHashTable<AggregationStateMax> *>(
+ destination_hash_table.get());
+
+ AggregationStateHashTable<AggregationStateMax> *source_hash_table_derived =
+ static_cast<AggregationStateHashTable<AggregationStateMax> *>(
+ source_hash_table.get());
+
+ AggregationHandleMax *aggregation_handle_max_derived =
+ static_cast<AggregationHandleMax *>(aggregation_handle_max_.get());
+ // We create three keys: first is present in both the hash tables, second key
+ // is present only in the source hash table while the third key is present
+ // the destination hash table only.
+ std::vector<TypedValue> common_key;
+ common_key.emplace_back(0);
+ std::vector<TypedValue> exclusive_source_key, exclusive_destination_key;
+ exclusive_source_key.emplace_back(1);
+ exclusive_destination_key.emplace_back(2);
+
+ const int common_key_source_max = 3000;
+ TypedValue common_key_source_max_val(common_key_source_max);
+
+ const int common_key_destination_max = 4000;
+ TypedValue common_key_destination_max_val(common_key_destination_max);
+
+ const int exclusive_key_source_max = 100;
+ TypedValue exclusive_key_source_max_val(exclusive_key_source_max);
+
+ const int exclusive_key_destination_max = 200;
+ TypedValue exclusive_key_destination_max_val(exclusive_key_destination_max);
+
+ std::unique_ptr<AggregationStateMax> common_key_source_state(
+ static_cast<AggregationStateMax *>(
+ aggregation_handle_max_->createInitialState()));
+ std::unique_ptr<AggregationStateMax> common_key_destination_state(
+ static_cast<AggregationStateMax *>(
+ aggregation_handle_max_->createInitialState()));
+ std::unique_ptr<AggregationStateMax> exclusive_key_source_state(
+ static_cast<AggregationStateMax *>(
+ aggregation_handle_max_->createInitialState()));
+ std::unique_ptr<AggregationStateMax> exclusive_key_destination_state(
+ static_cast<AggregationStateMax *>(
+ aggregation_handle_max_->createInitialState()));
+
+ // Create max value states for keys.
+ aggregation_handle_max_derived->iterateUnaryInl(common_key_source_state.get(),
+ common_key_source_max_val);
+ int actual_val = aggregation_handle_max_->finalize(*common_key_source_state)
+ .getLiteral<int>();
+ EXPECT_EQ(common_key_source_max_val.getLiteral<int>(), actual_val);
+
+ aggregation_handle_max_derived->iterateUnaryInl(
+ common_key_destination_state.get(), common_key_destination_max_val);
+ actual_val = aggregation_handle_max_->finalize(*common_key_destination_state)
+ .getLiteral<int>();
+ EXPECT_EQ(common_key_destination_max_val.getLiteral<int>(), actual_val);
+
+ aggregation_handle_max_derived->iterateUnaryInl(
+ exclusive_key_destination_state.get(), exclusive_key_destination_max_val);
+ actual_val =
+ aggregation_handle_max_->finalize(*exclusive_key_destination_state)
+ .getLiteral<int>();
+ EXPECT_EQ(exclusive_key_destination_max_val.getLiteral<int>(), actual_val);
+
+ aggregation_handle_max_derived->iterateUnaryInl(
+ exclusive_key_source_state.get(), exclusive_key_source_max_val);
+ actual_val = aggregation_handle_max_->finalize(*exclusive_key_source_state)
+ .getLiteral<int>();
+ EXPECT_EQ(exclusive_key_source_max_val.getLiteral<int>(), actual_val);
+
+ // Add the key-state pairs to the hash tables.
+ source_hash_table_derived->putCompositeKey(common_key,
+ *common_key_source_state);
+ destination_hash_table_derived->putCompositeKey(
+ common_key, *common_key_destination_state);
+ source_hash_table_derived->putCompositeKey(exclusive_source_key,
+ *exclusive_key_source_state);
+ destination_hash_table_derived->putCompositeKey(
+ exclusive_destination_key, *exclusive_key_destination_state);
+
+ EXPECT_EQ(2u, destination_hash_table_derived->numEntries());
+ EXPECT_EQ(2u, source_hash_table_derived->numEntries());
+
+ aggregation_handle_max_->mergeGroupByHashTables(*source_hash_table,
+ destination_hash_table.get());
+
+ EXPECT_EQ(3u, destination_hash_table_derived->numEntries());
+
+ CheckMaxValue<int>(
+ common_key_destination_max_val.getLiteral<int>(),
+ *aggregation_handle_max_derived,
+ *(destination_hash_table_derived->getSingleCompositeKey(common_key)));
+ CheckMaxValue<int>(exclusive_key_destination_max_val.getLiteral<int>(),
+ *aggregation_handle_max_derived,
+ *(destination_hash_table_derived->getSingleCompositeKey(
+ exclusive_destination_key)));
+ CheckMaxValue<int>(exclusive_key_source_max_val.getLiteral<int>(),
+ *aggregation_handle_max_derived,
+ *(source_hash_table_derived->getSingleCompositeKey(
+ exclusive_source_key)));
+}
+
} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2ddb67bf/expressions/aggregation/tests/AggregationHandleMin_unittest.cpp
----------------------------------------------------------------------
diff --git a/expressions/aggregation/tests/AggregationHandleMin_unittest.cpp b/expressions/aggregation/tests/AggregationHandleMin_unittest.cpp
index eb64472..a87ace9 100644
--- a/expressions/aggregation/tests/AggregationHandleMin_unittest.cpp
+++ b/expressions/aggregation/tests/AggregationHandleMin_unittest.cpp
@@ -29,6 +29,7 @@
#include "expressions/aggregation/AggregationHandle.hpp"
#include "expressions/aggregation/AggregationHandleMin.hpp"
#include "expressions/aggregation/AggregationID.hpp"
+#include "storage/StorageManager.hpp"
#include "types/CharType.hpp"
#include "types/DatetimeIntervalType.hpp"
#include "types/DatetimeLit.hpp"
@@ -411,6 +412,7 @@ class AggregationHandleMinTest : public ::testing::Test {
std::unique_ptr<AggregationHandle> aggregation_handle_min_;
std::unique_ptr<AggregationState> aggregation_handle_min_state_;
+ std::unique_ptr<StorageManager> storage_manager_;
};
template <>
@@ -634,4 +636,123 @@ TEST_F(AggregationHandleMinTest, ResultTypeForArgumentTypeTest) {
EXPECT_TRUE(ResultTypeForArgumentTypeTest(kDouble, kDouble));
}
+TEST_F(AggregationHandleMinTest, GroupByTableMergeTest) {
+ const Type &int_non_null_type = IntType::Instance(false);
+ initializeHandle(int_non_null_type);
+ storage_manager_.reset(new StorageManager("./test_min_data"));
+ std::unique_ptr<AggregationStateHashTableBase> source_hash_table(
+ aggregation_handle_min_->createGroupByHashTable(
+ HashTableImplType::kSimpleScalarSeparateChaining,
+ std::vector<const Type *>(1, &int_non_null_type),
+ 10,
+ storage_manager_.get()));
+ std::unique_ptr<AggregationStateHashTableBase> destination_hash_table(
+ aggregation_handle_min_->createGroupByHashTable(
+ HashTableImplType::kSimpleScalarSeparateChaining,
+ std::vector<const Type *>(1, &int_non_null_type),
+ 10,
+ storage_manager_.get()));
+
+ AggregationStateHashTable<AggregationStateMin> *destination_hash_table_derived =
+ static_cast<AggregationStateHashTable<AggregationStateMin> *>(
+ destination_hash_table.get());
+
+ AggregationStateHashTable<AggregationStateMin> *source_hash_table_derived =
+ static_cast<AggregationStateHashTable<AggregationStateMin> *>(
+ source_hash_table.get());
+
+ AggregationHandleMin *aggregation_handle_min_derived =
+ static_cast<AggregationHandleMin *>(aggregation_handle_min_.get());
+ // We create three keys: first is present in both the hash tables, second key
+ // is present only in the source hash table while the third key is present
+ // the destination hash table only.
+ std::vector<TypedValue> common_key;
+ common_key.emplace_back(0);
+ std::vector<TypedValue> exclusive_source_key, exclusive_destination_key;
+ exclusive_source_key.emplace_back(1);
+ exclusive_destination_key.emplace_back(2);
+
+ const int common_key_source_min = 3000;
+ TypedValue common_key_source_min_val(common_key_source_min);
+
+ const int common_key_destination_min = 4000;
+ TypedValue common_key_destination_min_val(common_key_destination_min);
+
+ const int exclusive_key_source_min = 100;
+ TypedValue exclusive_key_source_min_val(exclusive_key_source_min);
+
+ const int exclusive_key_destination_min = 200;
+ TypedValue exclusive_key_destination_min_val(exclusive_key_destination_min);
+
+ std::unique_ptr<AggregationStateMin> common_key_source_state(
+ static_cast<AggregationStateMin *>(
+ aggregation_handle_min_->createInitialState()));
+ std::unique_ptr<AggregationStateMin> common_key_destination_state(
+ static_cast<AggregationStateMin *>(
+ aggregation_handle_min_->createInitialState()));
+ std::unique_ptr<AggregationStateMin> exclusive_key_source_state(
+ static_cast<AggregationStateMin *>(
+ aggregation_handle_min_->createInitialState()));
+ std::unique_ptr<AggregationStateMin> exclusive_key_destination_state(
+ static_cast<AggregationStateMin *>(
+ aggregation_handle_min_->createInitialState()));
+
+ // Create min value states for keys.
+ aggregation_handle_min_derived->iterateUnaryInl(common_key_source_state.get(),
+ common_key_source_min_val);
+ int actual_val = aggregation_handle_min_->finalize(*common_key_source_state)
+ .getLiteral<int>();
+ EXPECT_EQ(common_key_source_min_val.getLiteral<int>(), actual_val);
+
+ aggregation_handle_min_derived->iterateUnaryInl(
+ common_key_destination_state.get(), common_key_destination_min_val);
+ actual_val = aggregation_handle_min_->finalize(*common_key_destination_state)
+ .getLiteral<int>();
+ EXPECT_EQ(common_key_destination_min_val.getLiteral<int>(), actual_val);
+
+ aggregation_handle_min_derived->iterateUnaryInl(
+ exclusive_key_destination_state.get(), exclusive_key_destination_min_val);
+ actual_val =
+ aggregation_handle_min_->finalize(*exclusive_key_destination_state)
+ .getLiteral<int>();
+ EXPECT_EQ(exclusive_key_destination_min_val.getLiteral<int>(), actual_val);
+
+ aggregation_handle_min_derived->iterateUnaryInl(
+ exclusive_key_source_state.get(), exclusive_key_source_min_val);
+ actual_val = aggregation_handle_min_->finalize(*exclusive_key_source_state)
+ .getLiteral<int>();
+ EXPECT_EQ(exclusive_key_source_min_val.getLiteral<int>(), actual_val);
+
+ // Add the key-state pairs to the hash tables.
+ source_hash_table_derived->putCompositeKey(common_key,
+ *common_key_source_state);
+ destination_hash_table_derived->putCompositeKey(
+ common_key, *common_key_destination_state);
+ source_hash_table_derived->putCompositeKey(exclusive_source_key,
+ *exclusive_key_source_state);
+ destination_hash_table_derived->putCompositeKey(
+ exclusive_destination_key, *exclusive_key_destination_state);
+
+ EXPECT_EQ(2u, destination_hash_table_derived->numEntries());
+ EXPECT_EQ(2u, source_hash_table_derived->numEntries());
+
+ aggregation_handle_min_->mergeGroupByHashTables(*source_hash_table,
+ destination_hash_table.get());
+
+ EXPECT_EQ(3u, destination_hash_table_derived->numEntries());
+
+ CheckMinValue<int>(
+ common_key_source_min_val.getLiteral<int>(),
+ *aggregation_handle_min_derived,
+ *(destination_hash_table_derived->getSingleCompositeKey(common_key)));
+ CheckMinValue<int>(exclusive_key_destination_min_val.getLiteral<int>(),
+ *aggregation_handle_min_derived,
+ *(destination_hash_table_derived->getSingleCompositeKey(
+ exclusive_destination_key)));
+ CheckMinValue<int>(exclusive_key_source_min_val.getLiteral<int>(),
+ *aggregation_handle_min_derived,
+ *(source_hash_table_derived->getSingleCompositeKey(
+ exclusive_source_key)));
+}
+
} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2ddb67bf/expressions/aggregation/tests/AggregationHandleSum_unittest.cpp
----------------------------------------------------------------------
diff --git a/expressions/aggregation/tests/AggregationHandleSum_unittest.cpp b/expressions/aggregation/tests/AggregationHandleSum_unittest.cpp
index 7dbbeb3..abf8a89 100644
--- a/expressions/aggregation/tests/AggregationHandleSum_unittest.cpp
+++ b/expressions/aggregation/tests/AggregationHandleSum_unittest.cpp
@@ -26,6 +26,7 @@
#include "expressions/aggregation/AggregationHandle.hpp"
#include "expressions/aggregation/AggregationHandleSum.hpp"
#include "expressions/aggregation/AggregationID.hpp"
+#include "storage/StorageManager.hpp"
#include "types/CharType.hpp"
#include "types/DatetimeIntervalType.hpp"
#include "types/DoubleType.hpp"
@@ -237,6 +238,7 @@ class AggregationHandleSumTest : public::testing::Test {
std::unique_ptr<AggregationHandle> aggregation_handle_sum_;
std::unique_ptr<AggregationState> aggregation_handle_sum_state_;
+ std::unique_ptr<StorageManager> storage_manager_;
};
const int AggregationHandleSumTest::kNumSamples;
@@ -425,4 +427,126 @@ TEST_F(AggregationHandleSumTest, ResultTypeForArgumentTypeTest) {
EXPECT_TRUE(ResultTypeForArgumentTypeTest(kYearMonthInterval, kYearMonthInterval));
}
+TEST_F(AggregationHandleSumTest, GroupByTableMergeTest) {
+ const Type &long_non_null_type = LongType::Instance(false);
+ initializeHandle(long_non_null_type);
+ storage_manager_.reset(new StorageManager("./test_sum_data"));
+ std::unique_ptr<AggregationStateHashTableBase> source_hash_table(
+ aggregation_handle_sum_->createGroupByHashTable(
+ HashTableImplType::kSimpleScalarSeparateChaining,
+ std::vector<const Type *>(1, &long_non_null_type),
+ 10,
+ storage_manager_.get()));
+ std::unique_ptr<AggregationStateHashTableBase> destination_hash_table(
+ aggregation_handle_sum_->createGroupByHashTable(
+ HashTableImplType::kSimpleScalarSeparateChaining,
+ std::vector<const Type *>(1, &long_non_null_type),
+ 10,
+ storage_manager_.get()));
+
+ AggregationStateHashTable<AggregationStateSum> *destination_hash_table_derived =
+ static_cast<AggregationStateHashTable<AggregationStateSum> *>(
+ destination_hash_table.get());
+
+ AggregationStateHashTable<AggregationStateSum> *source_hash_table_derived =
+ static_cast<AggregationStateHashTable<AggregationStateSum> *>(
+ source_hash_table.get());
+
+ AggregationHandleSum *aggregation_handle_sum_derived =
+ static_cast<AggregationHandleSum *>(aggregation_handle_sum_.get());
+ // We create three keys: first is present in both the hash tables, second key
+ // is present only in the source hash table while the third key is present
+ // the destination hash table only.
+ std::vector<TypedValue> common_key;
+ common_key.emplace_back(static_cast<std::int64_t>(0));
+ std::vector<TypedValue> exclusive_source_key, exclusive_destination_key;
+ exclusive_source_key.emplace_back(static_cast<std::int64_t>(1));
+ exclusive_destination_key.emplace_back(static_cast<std::int64_t>(2));
+
+ const std::int64_t common_key_source_sum = 3000;
+ TypedValue common_key_source_sum_val(common_key_source_sum);
+
+ const std::int64_t common_key_destination_sum = 4000;
+ TypedValue common_key_destination_sum_val(common_key_destination_sum);
+
+ const std::int64_t merged_common_key = common_key_source_sum + common_key_destination_sum;
+ TypedValue common_key_merged_val(merged_common_key);
+
+ const std::int64_t exclusive_key_source_sum = 100;
+ TypedValue exclusive_key_source_sum_val(exclusive_key_source_sum);
+
+ const std::int64_t exclusive_key_destination_sum = 200;
+ TypedValue exclusive_key_destination_sum_val(exclusive_key_destination_sum);
+
+ std::unique_ptr<AggregationStateSum> common_key_source_state(
+ static_cast<AggregationStateSum *>(
+ aggregation_handle_sum_->createInitialState()));
+ std::unique_ptr<AggregationStateSum> common_key_destination_state(
+ static_cast<AggregationStateSum *>(
+ aggregation_handle_sum_->createInitialState()));
+ std::unique_ptr<AggregationStateSum> exclusive_key_source_state(
+ static_cast<AggregationStateSum *>(
+ aggregation_handle_sum_->createInitialState()));
+ std::unique_ptr<AggregationStateSum> exclusive_key_destination_state(
+ static_cast<AggregationStateSum *>(
+ aggregation_handle_sum_->createInitialState()));
+
+ // Create sum value states for keys.
+ aggregation_handle_sum_derived->iterateUnaryInl(common_key_source_state.get(),
+ common_key_source_sum_val);
+ std::int64_t actual_val = aggregation_handle_sum_->finalize(*common_key_source_state)
+ .getLiteral<std::int64_t>();
+ EXPECT_EQ(common_key_source_sum_val.getLiteral<std::int64_t>(), actual_val);
+
+ aggregation_handle_sum_derived->iterateUnaryInl(
+ common_key_destination_state.get(), common_key_destination_sum_val);
+ actual_val = aggregation_handle_sum_->finalize(*common_key_destination_state)
+ .getLiteral<std::int64_t>();
+ EXPECT_EQ(common_key_destination_sum_val.getLiteral<std::int64_t>(), actual_val);
+
+ aggregation_handle_sum_derived->iterateUnaryInl(
+ exclusive_key_destination_state.get(), exclusive_key_destination_sum_val);
+ actual_val =
+ aggregation_handle_sum_->finalize(*exclusive_key_destination_state)
+ .getLiteral<std::int64_t>();
+ EXPECT_EQ(exclusive_key_destination_sum_val.getLiteral<std::int64_t>(), actual_val);
+
+ aggregation_handle_sum_derived->iterateUnaryInl(
+ exclusive_key_source_state.get(), exclusive_key_source_sum_val);
+ actual_val = aggregation_handle_sum_->finalize(*exclusive_key_source_state)
+ .getLiteral<std::int64_t>();
+ EXPECT_EQ(exclusive_key_source_sum_val.getLiteral<std::int64_t>(), actual_val);
+
+ // Add the key-state pairs to the hash tables.
+ source_hash_table_derived->putCompositeKey(common_key,
+ *common_key_source_state);
+ destination_hash_table_derived->putCompositeKey(
+ common_key, *common_key_destination_state);
+ source_hash_table_derived->putCompositeKey(exclusive_source_key,
+ *exclusive_key_source_state);
+ destination_hash_table_derived->putCompositeKey(
+ exclusive_destination_key, *exclusive_key_destination_state);
+
+ EXPECT_EQ(2u, destination_hash_table_derived->numEntries());
+ EXPECT_EQ(2u, source_hash_table_derived->numEntries());
+
+ aggregation_handle_sum_->mergeGroupByHashTables(*source_hash_table,
+ destination_hash_table.get());
+
+ EXPECT_EQ(3u, destination_hash_table_derived->numEntries());
+
+ CheckSumValue<std::int64_t>(
+ common_key_merged_val.getLiteral<std::int64_t>(),
+ *aggregation_handle_sum_derived,
+ *(destination_hash_table_derived->getSingleCompositeKey(common_key)));
+ CheckSumValue<std::int64_t>(exclusive_key_destination_sum_val.getLiteral<std::int64_t>(),
+ *aggregation_handle_sum_derived,
+ *(destination_hash_table_derived->getSingleCompositeKey(
+ exclusive_destination_key)));
+ CheckSumValue<std::int64_t>(exclusive_key_source_sum_val.getLiteral<std::int64_t>(),
+ *aggregation_handle_sum_derived,
+ *(source_hash_table_derived->getSingleCompositeKey(
+ exclusive_source_key)));
+}
+
} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2ddb67bf/query_execution/QueryContext.hpp
----------------------------------------------------------------------
diff --git a/query_execution/QueryContext.hpp b/query_execution/QueryContext.hpp
index 9440fae..7d5628d 100644
--- a/query_execution/QueryContext.hpp
+++ b/query_execution/QueryContext.hpp
@@ -216,7 +216,7 @@ class QueryContext {
*
* @param id The BloomFilter id.
*
- * @return The constant pointer to BloomFilter that is
+ * @return The constant pointer to BloomFilter that is
* already created in the constructor.
**/
inline const BloomFilter* getBloomFilter(const bloom_filter_id id) const {
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2ddb67bf/storage/AggregationOperationState.cpp
----------------------------------------------------------------------
diff --git a/storage/AggregationOperationState.cpp b/storage/AggregationOperationState.cpp
index d209ceb..4878cf1 100644
--- a/storage/AggregationOperationState.cpp
+++ b/storage/AggregationOperationState.cpp
@@ -92,11 +92,12 @@ AggregationOperationState::AggregationOperationState(
arguments_.push_back({});
is_distinct_.emplace_back(false);
- group_by_hashtables_.emplace_back(handles_.back()->createGroupByHashTable(
- hash_table_impl_type,
- group_by_types,
- estimated_num_entries,
- storage_manager_));
+ group_by_hashtable_pools_.emplace_back(std::unique_ptr<HashTablePool>(
+ new HashTablePool(estimated_num_entries,
+ hash_table_impl_type,
+ group_by_types,
+ handles_.back().get(),
+ storage_manager)));
} else {
// Set up each individual aggregate in this operation.
std::vector<const AggregateFunction*>::const_iterator agg_func_it
@@ -124,12 +125,13 @@ AggregationOperationState::AggregationOperationState(
handles_.emplace_back((*agg_func_it)->createHandle(argument_types));
if (!group_by_list_.empty()) {
- // Aggregation with GROUP BY: create a HashTable for per-group states.
- group_by_hashtables_.emplace_back(handles_.back()->createGroupByHashTable(
- hash_table_impl_type,
- group_by_types,
- estimated_num_entries,
- storage_manager_));
+ // Aggregation with GROUP BY: create a HashTable pool for per-group states.
+ group_by_hashtable_pools_.emplace_back(std::unique_ptr<HashTablePool>(
+ new HashTablePool(estimated_num_entries,
+ hash_table_impl_type,
+ group_by_types,
+ handles_.back().get(),
+ storage_manager)));
} else {
// Aggregation without GROUP BY: create a single global state.
single_states_.emplace_back(handles_.back()->createInitialState());
@@ -408,17 +410,17 @@ void AggregationOperationState::aggregateBlockHashTable(const block_id input_blo
// Call StorageBlock::aggregateGroupBy() to aggregate this block's values
// directly into the (threadsafe) shared global HashTable for this
// aggregate.
- //
- // TODO(shoban): Implement optional code path for using local hash table per
- // block, which can be merged with global hash table for all blocks
- // aggregated on.
+ DCHECK(group_by_hashtable_pools_[agg_idx] != nullptr);
+ AggregationStateHashTableBase *agg_hash_table = group_by_hashtable_pools_[agg_idx]->getHashTable();
+ DCHECK(agg_hash_table != nullptr);
block->aggregateGroupBy(*handles_[agg_idx],
arguments_[agg_idx],
group_by_list_,
predicate_.get(),
- group_by_hashtables_[agg_idx].get(),
+ agg_hash_table,
&reuse_matches,
&reuse_group_by_vectors);
+ group_by_hashtable_pools_[agg_idx]->returnHashTable(agg_hash_table);
}
}
}
@@ -447,19 +449,65 @@ void AggregationOperationState::finalizeHashTable(InsertDestination *output_dest
// group (which is also the prefix of the finalized Tuple for that group).
std::vector<std::vector<TypedValue>> group_by_keys;
+ // TODO(harshad) - The merge phase may be slower when each hash table contains
+ // large number of entries. We should find ways in which we can perform a
+ // parallel merge.
+
+ // TODO(harshad) - Find heuristics for faster merge, even in a single thread.
+ // e.g. Keep merging entries from smaller hash tables to larger.
+ for (std::size_t agg_idx = 0; agg_idx < handles_.size(); ++agg_idx) {
+ auto *hash_tables = group_by_hashtable_pools_[agg_idx]->getAllHashTables();
+ if (hash_tables->size() > 1) {
+ for (int hash_table_index = 0;
+ hash_table_index < static_cast<int>(hash_tables->size() - 1);
+ ++hash_table_index) {
+ // Merge each hash table to the last hash table.
+ handles_[agg_idx]->mergeGroupByHashTables(
+ (*(*hash_tables)[hash_table_index]),
+ hash_tables->back().get());
+ }
+ }
+ }
+
// Collect per-aggregate finalized values.
std::vector<std::unique_ptr<ColumnVector>> final_values;
for (std::size_t agg_idx = 0;
agg_idx < handles_.size();
++agg_idx) {
if (is_distinct_[agg_idx]) {
+ DCHECK(group_by_hashtable_pools_[agg_idx] != nullptr);
+ auto *hash_tables = group_by_hashtable_pools_[agg_idx]->getAllHashTables();
+ DCHECK(hash_tables != nullptr);
+ if (hash_tables->empty()) {
+ // We may have a case where hash_tables is empty, e.g. no input blocks.
+ // However for aggregateOnDistinctifyHashTableForGroupBy to work
+ // correctly, we should create an empty group by hash table.
+ AggregationStateHashTableBase *new_hash_table = group_by_hashtable_pools_[agg_idx]->getHashTable();
+ group_by_hashtable_pools_[agg_idx]->returnHashTable(new_hash_table);
+ hash_tables = group_by_hashtable_pools_[agg_idx]->getAllHashTables();
+ }
+ DCHECK(hash_tables->back() != nullptr);
+ AggregationStateHashTableBase *agg_hash_table = hash_tables->back().get();
+ DCHECK(agg_hash_table != nullptr);
handles_[agg_idx]->aggregateOnDistinctifyHashTableForGroupBy(
*distinctify_hashtables_[agg_idx],
- group_by_hashtables_[agg_idx].get());
+ agg_hash_table);
}
+ auto *hash_tables = group_by_hashtable_pools_[agg_idx]->getAllHashTables();
+ DCHECK(hash_tables != nullptr);
+ if (hash_tables->empty()) {
+ // We may have a case where hash_tables is empty, e.g. no input blocks.
+ // However for aggregateOnDistinctifyHashTableForGroupBy to work
+ // correctly, we should create an empty group by hash table.
+ AggregationStateHashTableBase *new_hash_table = group_by_hashtable_pools_[agg_idx]->getHashTable();
+ group_by_hashtable_pools_[agg_idx]->returnHashTable(new_hash_table);
+ hash_tables = group_by_hashtable_pools_[agg_idx]->getAllHashTables();
+ }
+ AggregationStateHashTableBase *agg_hash_table = hash_tables->back().get();
+ DCHECK(agg_hash_table != nullptr);
ColumnVector* agg_result_col =
- handles_[agg_idx]->finalizeHashTable(*group_by_hashtables_[agg_idx],
+ handles_[agg_idx]->finalizeHashTable(*agg_hash_table,
&group_by_keys);
if (agg_result_col != nullptr) {
final_values.emplace_back(agg_result_col);
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2ddb67bf/storage/AggregationOperationState.hpp
----------------------------------------------------------------------
diff --git a/storage/AggregationOperationState.hpp b/storage/AggregationOperationState.hpp
index c3a1278..0199749 100644
--- a/storage/AggregationOperationState.hpp
+++ b/storage/AggregationOperationState.hpp
@@ -31,6 +31,7 @@
#include "expressions/scalar/Scalar.hpp"
#include "storage/AggregationOperationState.pb.h"
#include "storage/HashTableBase.hpp"
+#include "storage/HashTablePool.hpp"
#include "storage/StorageBlockInfo.hpp"
#include "utility/Macros.hpp"
@@ -209,6 +210,9 @@ class AggregationOperationState {
// hash table to prevent multiple lookups.
std::vector<std::unique_ptr<AggregationStateHashTableBase>> group_by_hashtables_;
+ // A vector of group by hash table pools, one for each group by clause.
+ std::vector<std::unique_ptr<HashTablePool>> group_by_hashtable_pools_;
+
StorageManager *storage_manager_;
DISALLOW_COPY_AND_ASSIGN(AggregationOperationState);
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2ddb67bf/storage/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/storage/CMakeLists.txt b/storage/CMakeLists.txt
index a3093df..87a5e54 100644
--- a/storage/CMakeLists.txt
+++ b/storage/CMakeLists.txt
@@ -187,6 +187,7 @@ add_library(quickstep_storage_HashTable_proto ${storage_HashTable_proto_srcs})
add_library(quickstep_storage_HashTableBase ../empty_src.cpp HashTableBase.hpp)
add_library(quickstep_storage_HashTableFactory HashTableFactory.cpp HashTableFactory.hpp)
add_library(quickstep_storage_HashTableKeyManager ../empty_src.cpp HashTableKeyManager.hpp)
+add_library(quickstep_storage_HashTablePool ../empty_src.cpp HashTablePool.hpp)
add_library(quickstep_storage_IndexSubBlock ../empty_src.cpp IndexSubBlock.hpp)
add_library(quickstep_storage_IndexSubBlockDescriptionFactory ../empty_src.cpp IndexSubBlockDescriptionFactory.hpp)
add_library(quickstep_storage_InsertDestination InsertDestination.cpp InsertDestination.hpp)
@@ -252,6 +253,7 @@ target_link_libraries(quickstep_storage_AggregationOperationState
quickstep_storage_HashTable
quickstep_storage_HashTableBase
quickstep_storage_HashTableFactory
+ quickstep_storage_HashTablePool
quickstep_storage_InsertDestination
quickstep_storage_StorageBlock
quickstep_storage_StorageBlockInfo
@@ -662,6 +664,13 @@ target_link_libraries(quickstep_storage_HashTableKeyManager
quickstep_types_TypedValue
quickstep_types_operations_comparisons_ComparisonUtil
quickstep_utility_Macros)
+target_link_libraries(quickstep_storage_HashTablePool
+ glog
+ quickstep_expressions_aggregation_AggregationHandle
+ quickstep_storage_HashTableBase
+ quickstep_threading_SpinMutex
+ quickstep_utility_Macros
+ quickstep_utility_StringUtil)
target_link_libraries(quickstep_storage_IndexSubBlock
quickstep_catalog_CatalogTypedefs
quickstep_expressions_predicate_PredicateCost
@@ -1012,6 +1021,7 @@ target_link_libraries(quickstep_storage
quickstep_storage_HashTableBase
quickstep_storage_HashTableFactory
quickstep_storage_HashTableKeyManager
+ quickstep_storage_HashTablePool
quickstep_storage_IndexSubBlock
quickstep_storage_IndexSubBlockDescriptionFactory
quickstep_storage_InsertDestination
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2ddb67bf/storage/HashTablePool.hpp
----------------------------------------------------------------------
diff --git a/storage/HashTablePool.hpp b/storage/HashTablePool.hpp
new file mode 100644
index 0000000..c16d0f1
--- /dev/null
+++ b/storage/HashTablePool.hpp
@@ -0,0 +1,166 @@
+/**
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#ifndef QUICKSTEP_STORAGE_HASH_TABLE_POOL_HPP_
+#define QUICKSTEP_STORAGE_HASH_TABLE_POOL_HPP_
+
+#include <chrono>
+#include <memory>
+#include <utility>
+#include <vector>
+
+#include "expressions/aggregation/AggregationHandle.hpp"
+#include "storage/HashTableBase.hpp"
+#include "threading/SpinMutex.hpp"
+#include "utility/Macros.hpp"
+#include "utility/StringUtil.hpp"
+
+#include "glog/logging.h"
+
+namespace quickstep {
+
+class StorageManager;
+class Type;
+
+/** \addtogroup Storage
+ * @{
+ */
+
+/**
+ * @brief A pool of HashTables used for a single aggregation handle. This class
+ * has similar functionality as InsertDestination, but for checking out
+ * HashTables. A worker thread can check out a hash table for insertion,
+ * perform the insertions and return the hash table to the pool. While
+ * one thread is using a hash table, no other thread can access it.
+ **/
+class HashTablePool {
+ public:
+ /**
+ * @brief Constructor.
+ *
+ * @param estimated_num_entries The maximum number of entries in a hash table.
+ * @param hash_table_impl_type The type of hash table implementation.
+ * @param group_by_types A vector of pointer of types which form the group by
+ * key.
+ * @param agg_handle The aggregation handle.
+ * @param storage_manager A pointer to the storage manager.
+ *
+ * @note The estimate of number of entries is quite inaccurate at this time.
+ * If we go by the current estimate, each hash table demands much
+ * larger space than it actually needs, which causes the system to
+ * either trigger evictions or worse - run out of memory. To fix this
+ * issue, we divide the estimate by 100. The division will not affect
+ * correctness, however it may allocate some hash tables smaller space
+ * than their requirement, causing them to be resized during build
+ * phase, which has a performance penalty.
+ **/
+ HashTablePool(const std::size_t estimated_num_entries,
+ const HashTableImplType hash_table_impl_type,
+ const std::vector<const Type *> &group_by_types,
+ AggregationHandle *agg_handle,
+ StorageManager *storage_manager)
+ : estimated_num_entries_(reduceEstimatedCardinality(estimated_num_entries)),
+ hash_table_impl_type_(hash_table_impl_type),
+ group_by_types_(group_by_types),
+ agg_handle_(DCHECK_NOTNULL(agg_handle)),
+ storage_manager_(DCHECK_NOTNULL(storage_manager)) {}
+
+ /**
+ * @brief Check out a hash table for insertion.
+ *
+ * @return A hash table pointer.
+ **/
+ AggregationStateHashTableBase* getHashTable() {
+ {
+ SpinMutexLock lock(mutex_);
+ if (!hash_tables_.empty()) {
+ std::unique_ptr<AggregationStateHashTableBase> ret_hash_table(
+ std::move(hash_tables_.back()));
+ hash_tables_.pop_back();
+ DCHECK(ret_hash_table != nullptr);
+ return ret_hash_table.release();
+ }
+ }
+ return createNewHashTable();
+ }
+
+ /**
+ * @brief Return a previously checked out hash table.
+ *
+ * @param hash_table A pointer to the checked out hash table.
+ **/
+ void returnHashTable(AggregationStateHashTableBase *hash_table) {
+ SpinMutexLock lock(mutex_);
+ hash_tables_.push_back(
+ std::unique_ptr<AggregationStateHashTableBase>(hash_table));
+ }
+
+ /**
+ * @brief Get all the hash tables from the pool.
+ *
+ * @warning The caller should ensure that this call is made when no hash table
+ * is being checked in or checked out from the pool. In other words
+ * the hash table pool is in read-only state.
+ *
+ * @param All the hash tables in the pool.
+ *
+ **/
+ const std::vector<std::unique_ptr<AggregationStateHashTableBase>>*
+ getAllHashTables() {
+ return &hash_tables_;
+ }
+
+ private:
+ AggregationStateHashTableBase* createNewHashTable() {
+ return agg_handle_->createGroupByHashTable(hash_table_impl_type_,
+ group_by_types_,
+ estimated_num_entries_,
+ storage_manager_);
+ }
+
+ inline std::size_t reduceEstimatedCardinality(
+ const std::size_t original_estimate) const {
+ if (original_estimate < kEstimateReductionFactor) {
+ return original_estimate;
+ } else {
+ DCHECK_GT(kEstimateReductionFactor, 0u);
+ return original_estimate / kEstimateReductionFactor;
+ }
+ }
+
+ static constexpr std::size_t kEstimateReductionFactor = 100;
+
+ std::vector<std::unique_ptr<AggregationStateHashTableBase>> hash_tables_;
+
+ const std::size_t estimated_num_entries_;
+ const HashTableImplType hash_table_impl_type_;
+
+ const std::vector<const Type *> group_by_types_;
+
+ AggregationHandle *agg_handle_;
+ StorageManager *storage_manager_;
+
+ SpinMutex mutex_;
+
+ DISALLOW_COPY_AND_ASSIGN(HashTablePool);
+};
+
+/** @} */
+
+} // namespace quickstep
+
+#endif // QUICKSTEP_STORAGE_HASH_TABLE_POOL_HPP_
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2ddb67bf/storage/StorageManager.cpp
----------------------------------------------------------------------
diff --git a/storage/StorageManager.cpp b/storage/StorageManager.cpp
index dfc95b8..5d91052 100644
--- a/storage/StorageManager.cpp
+++ b/storage/StorageManager.cpp
@@ -183,7 +183,8 @@ StorageManager::~StorageManager() {
it != blocks_.end();
++it) {
if (it->second.block->isDirty()) {
- LOG(WARNING) << "Block with ID " << BlockIdUtil::ToString(it->first)
+ LOG(WARNING) << (it->second.block->isBlob() ? "Blob " : "Block ")
+ << "with ID " << BlockIdUtil::ToString(it->first)
<< " is dirty during StorageManager shutdown";
}
delete it->second.block;
[10/50] [abbrv] incubator-quickstep git commit: Fixes bug which
disallows the creation of BITWEAVING_H indices. (#204)
Posted by zu...@apache.org.
Fixes bug which disallows the creation of BITWEAVING_H indices. (#204)
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/29a71ac2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/29a71ac2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/29a71ac2
Branch: refs/heads/work-order-serialization
Commit: 29a71ac21f746e74dc7a544c8d8e09f71854158c
Parents: 89acf23
Author: Marc S <cr...@users.noreply.github.com>
Authored: Sun May 1 17:14:20 2016 -0500
Committer: Jignesh Patel <pa...@users.noreply.github.com>
Committed: Sun May 1 17:14:20 2016 -0500
----------------------------------------------------------------------
parser/ParseIndexProperties.hpp | 2 +-
parser/tests/Index.test | 11 +++++++++++
2 files changed, 12 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/29a71ac2/parser/ParseIndexProperties.hpp
----------------------------------------------------------------------
diff --git a/parser/ParseIndexProperties.hpp b/parser/ParseIndexProperties.hpp
index 9ca1419..2cb5df9 100644
--- a/parser/ParseIndexProperties.hpp
+++ b/parser/ParseIndexProperties.hpp
@@ -220,7 +220,7 @@ class BitWeavingIndexProperties : public IndexProperties {
}
const std::string key = ToLower(key_value.key()->value());
const std::string value = ToLower(
- static_cast<const ParseKeyStringValue&>(key_value).key()->value());
+ static_cast<const ParseKeyStringValue&>(key_value).value()->value());
if (key.compare(kBitWeavingType) == 0) {
if (value.compare("h") == 0) {
index_sub_block_description_->set_sub_block_type(IndexSubBlockDescription::BITWEAVING_H);
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/29a71ac2/parser/tests/Index.test
----------------------------------------------------------------------
diff --git a/parser/tests/Index.test b/parser/tests/Index.test
index 605768b..39cd523 100644
--- a/parser/tests/Index.test
+++ b/parser/tests/Index.test
@@ -66,3 +66,14 @@ CREATE INDEX bwIndex ON test(int_col) USING bitweaving;
CreateIndexStatement[index_name=bwIndex,relation_name=test,index_type=bitweaving]
+-attribute_list=
+-AttributeReference[attribute_name=int_col]
+==
+CREATE INDEX bwhIndex ON test(int_col) USING bitweaving (TYPE H);
+--
+CreateIndexStatement[index_name=bwhIndex,relation_name=test,
+ index_type=bitweaving]
++-attribute_list=
+| +-AttributeReference[attribute_name=int_col]
++-index_property_list=
+ +-IndexProperties
+ +-index_property=KeyStringValue[key=TYPE]
+ +-value=String[value=H]
[45/50] [abbrv] incubator-quickstep git commit: Explicitly specify
where tcmalloc comes from (#235)
Posted by zu...@apache.org.
Explicitly specify where tcmalloc comes from (#235)
This change adds a declaration that `libtcmalloc_minimal.a` is generated
by the external project. This is needed by the Ninja generator to
properly build.
To reproduce:
run `cmake -GNinja path/to/quickstep`
then `cmake --build path/to/build/dir` and see it fails.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/908d3675
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/908d3675
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/908d3675
Branch: refs/heads/work-order-serialization
Commit: 908d367510052a68ed245ef0ce3dd3c834250850
Parents: 5a81b80
Author: Jesse Zhang <jz...@pivotallabs.com>
Authored: Mon May 23 13:38:59 2016 -0700
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Mon May 30 15:47:53 2016 -0700
----------------------------------------------------------------------
CMakeLists.txt | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/908d3675/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 87a8f7c..2ab0f57 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -490,6 +490,7 @@ if(USE_TCMALLOC)
CXXFLAGS=${THIRD_PARTY_CXX_FLAGS}
BUILD_COMMAND make
BUILD_IN_SOURCE 0
+ BUILD_BYPRODUCTS <INSTALL_DIR>/lib/libtcmalloc_minimal.a
)
# Static libtcmalloc_minimal.a
add_library(libtcmalloc_minimal STATIC IMPORTED)
[19/50] [abbrv] incubator-quickstep git commit: Added support for the
substring function. (#211)
Posted by zu...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/767b2ef1/parser/preprocessed/SqlParser_gen.hpp
----------------------------------------------------------------------
diff --git a/parser/preprocessed/SqlParser_gen.hpp b/parser/preprocessed/SqlParser_gen.hpp
index 72fa9ef..71e4332 100644
--- a/parser/preprocessed/SqlParser_gen.hpp
+++ b/parser/preprocessed/SqlParser_gen.hpp
@@ -104,67 +104,69 @@ extern int quickstep_yydebug;
TOKEN_FALSE = 314,
TOKEN_FIRST = 315,
TOKEN_FLOAT = 316,
- TOKEN_FOREIGN = 317,
- TOKEN_FROM = 318,
- TOKEN_FULL = 319,
- TOKEN_GROUP = 320,
- TOKEN_HASH = 321,
- TOKEN_HAVING = 322,
- TOKEN_HOUR = 323,
- TOKEN_IN = 324,
- TOKEN_INDEX = 325,
- TOKEN_INNER = 326,
- TOKEN_INSERT = 327,
- TOKEN_INTEGER = 328,
- TOKEN_INTERVAL = 329,
- TOKEN_INTO = 330,
- TOKEN_JOIN = 331,
- TOKEN_KEY = 332,
- TOKEN_LAST = 333,
- TOKEN_LEFT = 334,
- TOKEN_LIMIT = 335,
- TOKEN_LONG = 336,
- TOKEN_MINUTE = 337,
- TOKEN_MONTH = 338,
- TOKEN_NULL = 339,
- TOKEN_NULLS = 340,
- TOKEN_OFF = 341,
- TOKEN_ON = 342,
- TOKEN_ORDER = 343,
- TOKEN_OUTER = 344,
- TOKEN_PARTITION = 345,
- TOKEN_PARTITIONS = 346,
- TOKEN_PERCENT = 347,
- TOKEN_PRIMARY = 348,
- TOKEN_QUIT = 349,
- TOKEN_RANGE = 350,
- TOKEN_REAL = 351,
- TOKEN_REFERENCES = 352,
- TOKEN_RIGHT = 353,
- TOKEN_ROW_DELIMITER = 354,
- TOKEN_SECOND = 355,
- TOKEN_SELECT = 356,
- TOKEN_SET = 357,
- TOKEN_SMA = 358,
- TOKEN_SMALLINT = 359,
- TOKEN_TABLE = 360,
- TOKEN_THEN = 361,
- TOKEN_TIME = 362,
- TOKEN_TIMESTAMP = 363,
- TOKEN_TRUE = 364,
- TOKEN_TUPLESAMPLE = 365,
- TOKEN_UNIQUE = 366,
- TOKEN_UPDATE = 367,
- TOKEN_USING = 368,
- TOKEN_VALUES = 369,
- TOKEN_VARCHAR = 370,
- TOKEN_WHEN = 371,
- TOKEN_WHERE = 372,
- TOKEN_WITH = 373,
- TOKEN_YEAR = 374,
- TOKEN_YEARMONTH = 375,
- TOKEN_EOF = 376,
- TOKEN_LEX_ERROR = 377
+ TOKEN_FOR = 317,
+ TOKEN_FOREIGN = 318,
+ TOKEN_FROM = 319,
+ TOKEN_FULL = 320,
+ TOKEN_GROUP = 321,
+ TOKEN_HASH = 322,
+ TOKEN_HAVING = 323,
+ TOKEN_HOUR = 324,
+ TOKEN_IN = 325,
+ TOKEN_INDEX = 326,
+ TOKEN_INNER = 327,
+ TOKEN_INSERT = 328,
+ TOKEN_INTEGER = 329,
+ TOKEN_INTERVAL = 330,
+ TOKEN_INTO = 331,
+ TOKEN_JOIN = 332,
+ TOKEN_KEY = 333,
+ TOKEN_LAST = 334,
+ TOKEN_LEFT = 335,
+ TOKEN_LIMIT = 336,
+ TOKEN_LONG = 337,
+ TOKEN_MINUTE = 338,
+ TOKEN_MONTH = 339,
+ TOKEN_NULL = 340,
+ TOKEN_NULLS = 341,
+ TOKEN_OFF = 342,
+ TOKEN_ON = 343,
+ TOKEN_ORDER = 344,
+ TOKEN_OUTER = 345,
+ TOKEN_PARTITION = 346,
+ TOKEN_PARTITIONS = 347,
+ TOKEN_PERCENT = 348,
+ TOKEN_PRIMARY = 349,
+ TOKEN_QUIT = 350,
+ TOKEN_RANGE = 351,
+ TOKEN_REAL = 352,
+ TOKEN_REFERENCES = 353,
+ TOKEN_RIGHT = 354,
+ TOKEN_ROW_DELIMITER = 355,
+ TOKEN_SECOND = 356,
+ TOKEN_SELECT = 357,
+ TOKEN_SET = 358,
+ TOKEN_SMA = 359,
+ TOKEN_SMALLINT = 360,
+ TOKEN_SUBSTRING = 361,
+ TOKEN_TABLE = 362,
+ TOKEN_THEN = 363,
+ TOKEN_TIME = 364,
+ TOKEN_TIMESTAMP = 365,
+ TOKEN_TRUE = 366,
+ TOKEN_TUPLESAMPLE = 367,
+ TOKEN_UNIQUE = 368,
+ TOKEN_UPDATE = 369,
+ TOKEN_USING = 370,
+ TOKEN_VALUES = 371,
+ TOKEN_VARCHAR = 372,
+ TOKEN_WHEN = 373,
+ TOKEN_WHERE = 374,
+ TOKEN_WITH = 375,
+ TOKEN_YEAR = 376,
+ TOKEN_YEARMONTH = 377,
+ TOKEN_EOF = 378,
+ TOKEN_LEX_ERROR = 379
};
#endif
@@ -265,7 +267,7 @@ union YYSTYPE
quickstep::PtrVector<quickstep::ParseSubqueryTableReference> *with_list_;
quickstep::ParseSubqueryTableReference *with_list_element_;
-#line 269 "SqlParser_gen.hpp" /* yacc.c:1915 */
+#line 271 "SqlParser_gen.hpp" /* yacc.c:1915 */
};
typedef union YYSTYPE YYSTYPE;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/767b2ef1/query_optimizer/resolver/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_optimizer/resolver/CMakeLists.txt b/query_optimizer/resolver/CMakeLists.txt
index db2a8af..5959879 100644
--- a/query_optimizer/resolver/CMakeLists.txt
+++ b/query_optimizer/resolver/CMakeLists.txt
@@ -120,6 +120,7 @@ target_link_libraries(quickstep_queryoptimizer_resolver_Resolver
quickstep_types_operations_comparisons_ComparisonFactory
quickstep_types_operations_comparisons_ComparisonID
quickstep_types_operations_unaryoperations_DateExtractOperation
+ quickstep_types_operations_unaryoperations_SubstringOperation
quickstep_types_operations_unaryoperations_UnaryOperation
quickstep_utility_Macros
quickstep_utility_PtrList
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/767b2ef1/query_optimizer/resolver/Resolver.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/resolver/Resolver.cpp b/query_optimizer/resolver/Resolver.cpp
index 2667ee9..78985a0 100644
--- a/query_optimizer/resolver/Resolver.cpp
+++ b/query_optimizer/resolver/Resolver.cpp
@@ -115,6 +115,7 @@
#include "types/operations/comparisons/ComparisonFactory.hpp"
#include "types/operations/comparisons/ComparisonID.hpp"
#include "types/operations/unary_operations/DateExtractOperation.hpp"
+#include "types/operations/unary_operations/SubstringOperation.hpp"
#include "types/operations/unary_operations/UnaryOperation.hpp"
#include "utility/PtrList.hpp"
#include "utility/PtrVector.hpp"
@@ -2068,6 +2069,37 @@ E::ScalarPtr Resolver::resolveExpression(
return E::UnaryExpression::Create(op, argument);
}
+ case ParseExpression::kSubstring: {
+ const ParseSubstringFunction &parse_substring =
+ static_cast<const ParseSubstringFunction&>(parse_expression);
+
+ // Validate start position and substring length.
+ if (parse_substring.start_position() <= 0) {
+ THROW_SQL_ERROR_AT(&parse_expression)
+ << "The start position must be greater than 0";
+ }
+ if (parse_substring.length() <= 0) {
+ THROW_SQL_ERROR_AT(&parse_expression)
+ << "The substring length must be greater than 0";
+ }
+
+ // Convert 1-base position to 0-base position
+ const std::size_t zero_base_start_position = parse_substring.start_position() - 1;
+ const SubstringOperation &op =
+ SubstringOperation::Instance(zero_base_start_position,
+ parse_substring.length());
+
+ const E::ScalarPtr argument =
+ resolveExpression(*parse_substring.operand(),
+ op.pushDownTypeHint(type_hint),
+ expression_resolution_info);
+ if (!op.canApplyToType(argument->getValueType())) {
+ THROW_SQL_ERROR_AT(&parse_substring)
+ << "Can not apply substring function to argument of type "
+ << argument->getValueType().getName();
+ }
+ return E::UnaryExpression::Create(op, argument);
+ }
default:
LOG(FATAL) << "Unknown scalar type: "
<< parse_expression.getExpressionType();
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/767b2ef1/query_optimizer/tests/execution_generator/Select.test
----------------------------------------------------------------------
diff --git a/query_optimizer/tests/execution_generator/Select.test b/query_optimizer/tests/execution_generator/Select.test
index 390b7b6..023ad2e 100644
--- a/query_optimizer/tests/execution_generator/Select.test
+++ b/query_optimizer/tests/execution_generator/Select.test
@@ -765,6 +765,30 @@ WHERE
+-----------+
==
+# SUBSTRING function
+SELECT char_col,
+ SUBSTRING(char_col FROM 1 FOR 2) AS negative_value
+FROM test
+WHERE SUBSTRING(char_col FROM 1 FOR 1) = '-'
+--
++--------------------+--------------+
+|char_col |negative_value|
++--------------------+--------------+
+| -1 1.000000| -1|
+| -3 1.732051| -3|
+| -5 2.236068| -5|
+| -7 2.645751| -7|
+| -9 3.000000| -9|
+| -11 3.316625| -1|
+| -13 3.605551| -1|
+| -15 3.872983| -1|
+| -17 4.123106| -1|
+| -19 4.358899| -1|
+| -21 4.582576| -2|
+| -23 4.795832| -2|
++--------------------+--------------+
+==
+
# IN predicate
SELECT *
FROM generate_series(1, 5) AS gs(i)
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/767b2ef1/query_optimizer/tests/resolver/Select.test
----------------------------------------------------------------------
diff --git a/query_optimizer/tests/resolver/Select.test b/query_optimizer/tests/resolver/Select.test
index 9897934..141bfa0 100644
--- a/query_optimizer/tests/resolver/Select.test
+++ b/query_optimizer/tests/resolver/Select.test
@@ -3048,3 +3048,81 @@ TopLevelPlan
+-output_attributes=
+-AttributeReference[id=5,name=x,relation=,type=Int]
+-AttributeReference[id=6,name=y,relation=,type=Int]
+==
+
+SELECT SUBSTRING(char_col FROM 1 FOR 2)
+FROM test;
+--
+TopLevelPlan
++-plan=Project
+| +-input=TableReference[relation_name=Test,relation_alias=test]
+| | +-AttributeReference[id=0,name=int_col,relation=test,type=Int NULL]
+| | +-AttributeReference[id=1,name=long_col,relation=test,type=Long]
+| | +-AttributeReference[id=2,name=float_col,relation=test,type=Float]
+| | +-AttributeReference[id=3,name=double_col,relation=test,type=Double NULL]
+| | +-AttributeReference[id=4,name=char_col,relation=test,type=Char(20)]
+| | +-AttributeReference[id=5,name=vchar_col,relation=test,type=VarChar(20) NULL]
+| +-project_list=
+| +-Alias[id=6,name=,alias=SUBSTRING(char_col FROM 1 FOR 2),relation=,
+| type=Char(2)]
+| +-Substring
+| +-Operand=AttributeReference[id=4,name=char_col,relation=test,
+| type=Char(20)]
++-output_attributes=
+ +-AttributeReference[id=6,name=,alias=SUBSTRING(char_col FROM 1 FOR 2),
+ relation=,type=Char(2)]
+==
+
+SELECT *
+FROM test
+WHERE SUBSTRING(vchar_col FROM 1 FOR 2) IN ('12', '34', '56');
+--
+TopLevelPlan
++-plan=Project
+| +-input=Filter
+| | +-input=TableReference[relation_name=Test,relation_alias=test]
+| | | +-AttributeReference[id=0,name=int_col,relation=test,type=Int NULL]
+| | | +-AttributeReference[id=1,name=long_col,relation=test,type=Long]
+| | | +-AttributeReference[id=2,name=float_col,relation=test,type=Float]
+| | | +-AttributeReference[id=3,name=double_col,relation=test,type=Double NULL]
+| | | +-AttributeReference[id=4,name=char_col,relation=test,type=Char(20)]
+| | | +-AttributeReference[id=5,name=vchar_col,relation=test,
+| | | type=VarChar(20) NULL]
+| | +-filter_predicate=InValueList
+| | +-test_expression=Substring
+| | | +-Operand=AttributeReference[id=5,name=vchar_col,relation=test,
+| | | type=VarChar(20) NULL]
+| | +-match_expressions=
+| | +-Literal[value=12,type=Char(2)]
+| | +-Literal[value=34,type=Char(2)]
+| | +-Literal[value=56,type=Char(2)]
+| +-project_list=
+| +-AttributeReference[id=0,name=int_col,relation=test,type=Int NULL]
+| +-AttributeReference[id=1,name=long_col,relation=test,type=Long]
+| +-AttributeReference[id=2,name=float_col,relation=test,type=Float]
+| +-AttributeReference[id=3,name=double_col,relation=test,type=Double NULL]
+| +-AttributeReference[id=4,name=char_col,relation=test,type=Char(20)]
+| +-AttributeReference[id=5,name=vchar_col,relation=test,type=VarChar(20) NULL]
++-output_attributes=
+ +-AttributeReference[id=0,name=int_col,relation=test,type=Int NULL]
+ +-AttributeReference[id=1,name=long_col,relation=test,type=Long]
+ +-AttributeReference[id=2,name=float_col,relation=test,type=Float]
+ +-AttributeReference[id=3,name=double_col,relation=test,type=Double NULL]
+ +-AttributeReference[id=4,name=char_col,relation=test,type=Char(20)]
+ +-AttributeReference[id=5,name=vchar_col,relation=test,type=VarChar(20) NULL]
+==
+
+SELECT SUBSTRING(char_col FROM 0 FOR 2)
+FROM test;
+--
+ERROR: The start position must be greater than 0 (1 : 8)
+SELECT SUBSTRING(char_col FROM 0 FOR ...
+ ^
+==
+
+SELECT SUBSTRING(char_col FROM 1 FOR 0)
+FROM test;
+--
+ERROR: The substring length must be greater than 0 (1 : 8)
+SELECT SUBSTRING(char_col FROM 1 FOR ...
+ ^
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/767b2ef1/types/operations/Operation.proto
----------------------------------------------------------------------
diff --git a/types/operations/Operation.proto b/types/operations/Operation.proto
index fb44b1d..33e6b09 100644
--- a/types/operations/Operation.proto
+++ b/types/operations/Operation.proto
@@ -43,6 +43,7 @@ message UnaryOperation {
NEGATE = 0;
CAST = 1;
DATE_EXTRACT = 2;
+ SUBSTRING = 3;
}
required UnaryOperationID operation_id = 1;
@@ -73,6 +74,14 @@ message DateExtractOperation {
}
}
+message SubstringOperation {
+ extend UnaryOperation {
+ // Required when operation_id = SUBSTRING.
+ optional int64 start_position = 100;
+ optional int64 substring_length = 101;
+ }
+}
+
message BinaryOperation {
enum BinaryOperationID {
ADD = 0;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/767b2ef1/types/operations/unary_operations/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/types/operations/unary_operations/CMakeLists.txt b/types/operations/unary_operations/CMakeLists.txt
index 3a50445..5c54d9c 100644
--- a/types/operations/unary_operations/CMakeLists.txt
+++ b/types/operations/unary_operations/CMakeLists.txt
@@ -1,5 +1,7 @@
# Copyright 2011-2015 Quickstep Technologies LLC.
# Copyright 2015 Pivotal Software, Inc.
+# Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+# University of Wisconsin\u2014Madison.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -18,6 +20,7 @@ add_library(quickstep_types_operations_unaryoperations_ArithmeticUnaryOperations
add_library(quickstep_types_operations_unaryoperations_ArithmeticUnaryOperators ../../../empty_src.cpp ArithmeticUnaryOperators.hpp)
add_library(quickstep_types_operations_unaryoperations_DateExtractOperation DateExtractOperation.cpp DateExtractOperation.hpp)
add_library(quickstep_types_operations_unaryoperations_NumericCastOperation ../../../empty_src.cpp NumericCastOperation.hpp)
+add_library(quickstep_types_operations_unaryoperations_SubstringOperation SubstringOperation.cpp SubstringOperation.hpp)
add_library(quickstep_types_operations_unaryoperations_UnaryOperation UnaryOperation.cpp UnaryOperation.hpp)
add_library(quickstep_types_operations_unaryoperations_UnaryOperationFactory UnaryOperationFactory.cpp UnaryOperationFactory.hpp)
add_library(quickstep_types_operations_unaryoperations_UnaryOperationID UnaryOperationID.cpp UnaryOperationID.hpp)
@@ -85,6 +88,23 @@ target_link_libraries(quickstep_types_operations_unaryoperations_NumericCastOper
quickstep_types_operations_unaryoperations_UnaryOperationID
quickstep_utility_Macros
quickstep_utility_PtrMap)
+target_link_libraries(quickstep_types_operations_unaryoperations_SubstringOperation
+ quickstep_catalog_CatalogTypedefs
+ quickstep_storage_ValueAccessor
+ quickstep_storage_ValueAccessorUtil
+ quickstep_types_Type
+ quickstep_types_TypeFactory
+ quickstep_types_TypeID
+ quickstep_types_TypedValue
+ quickstep_types_containers_ColumnVector
+ quickstep_types_containers_ColumnVectorUtil
+ quickstep_types_operations_Operation_proto
+ quickstep_types_operations_unaryoperations_UnaryOperation
+ quickstep_types_operations_unaryoperations_UnaryOperationID
+ quickstep_types_port_strnlen
+ quickstep_utility_HashPair
+ quickstep_utility_Macros
+ quickstep_utility_TemplateUtil)
target_link_libraries(quickstep_types_operations_unaryoperations_UnaryOperation
quickstep_catalog_CatalogTypedefs
quickstep_storage_StorageBlockInfo
@@ -100,6 +120,7 @@ target_link_libraries(quickstep_types_operations_unaryoperations_UnaryOperationF
quickstep_types_operations_unaryoperations_ArithmeticUnaryOperations
quickstep_types_operations_unaryoperations_DateExtractOperation
quickstep_types_operations_unaryoperations_NumericCastOperation
+ quickstep_types_operations_unaryoperations_SubstringOperation
quickstep_types_operations_unaryoperations_UnaryOperationID
quickstep_utility_Macros)
@@ -110,6 +131,7 @@ target_link_libraries(quickstep_types_operations_unaryoperations
quickstep_types_operations_unaryoperations_ArithmeticUnaryOperators
quickstep_types_operations_unaryoperations_DateExtractOperation
quickstep_types_operations_unaryoperations_NumericCastOperation
+ quickstep_types_operations_unaryoperations_SubstringOperation
quickstep_types_operations_unaryoperations_UnaryOperation
quickstep_types_operations_unaryoperations_UnaryOperationFactory
quickstep_types_operations_unaryoperations_UnaryOperationID)
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/767b2ef1/types/operations/unary_operations/SubstringOperation.cpp
----------------------------------------------------------------------
diff --git a/types/operations/unary_operations/SubstringOperation.cpp b/types/operations/unary_operations/SubstringOperation.cpp
new file mode 100644
index 0000000..463cd33
--- /dev/null
+++ b/types/operations/unary_operations/SubstringOperation.cpp
@@ -0,0 +1,214 @@
+/**
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#include "types/operations/unary_operations/SubstringOperation.hpp"
+
+#include <algorithm>
+#include <tuple>
+#include <utility>
+#include <vector>
+
+#include "catalog/CatalogTypedefs.hpp"
+#include "storage/ValueAccessor.hpp"
+#include "storage/ValueAccessorUtil.hpp"
+#include "types/Type.hpp"
+#include "types/TypeID.hpp"
+#include "types/TypedValue.hpp"
+#include "types/containers/ColumnVector.hpp"
+#include "types/containers/ColumnVectorUtil.hpp"
+#include "types/operations/Operation.pb.h"
+#include "types/port/strnlen.hpp"
+#include "utility/TemplateUtil.hpp"
+
+#include "glog/logging.h"
+
+namespace quickstep {
+
+serialization::UnaryOperation SubstringOperation::getProto() const {
+ serialization::UnaryOperation proto;
+ proto.set_operation_id(serialization::UnaryOperation::SUBSTRING);
+ proto.SetExtension(serialization::SubstringOperation::start_position,
+ start_position_);
+ proto.SetExtension(serialization::SubstringOperation::substring_length,
+ substring_length_);
+ return proto;
+}
+
+UncheckedUnaryOperator* SubstringOperation::makeUncheckedUnaryOperatorForType(
+ const Type &type) const {
+ DCHECK(type.getSuperTypeID() == Type::kAsciiString);
+
+ const std::size_t input_maximum_length =
+ static_cast<const AsciiStringSuperType&>(type).getStringLength();
+ const bool input_null_terminated = (type.getTypeID() == TypeID::kVarChar);
+
+ const Type *result_type = resultTypeForArgumentType(type);
+ DCHECK(result_type != nullptr);
+
+ return CreateBoolInstantiatedInstance<SubstringUncheckedOperator, UncheckedUnaryOperator>(
+ std::forward_as_tuple(start_position_,
+ computeMaximumSubstringLength(type),
+ input_maximum_length,
+ *result_type),
+ input_null_terminated, type.isNullable());
+}
+
+template <bool null_terminated, bool input_nullable>
+inline void SubstringUncheckedOperator<null_terminated, input_nullable>
+ ::computeSubstring(const char *input,
+ char *output) const {
+ std::size_t string_length =
+ (null_terminated ? strlen(input) : strnlen(input, maximum_input_length_));
+
+ if (start_position_ >= string_length) {
+ *output = '\0';
+ return;
+ }
+
+ const std::size_t actual_substring_length =
+ std::min(string_length - start_position_, substring_length_);
+ std::memcpy(output, input + start_position_, actual_substring_length);
+
+ if (actual_substring_length < substring_length_) {
+ output[actual_substring_length] = '\0';
+ }
+}
+
+template <bool null_terminated, bool input_nullable>
+TypedValue SubstringUncheckedOperator<null_terminated,
+ input_nullable>
+ ::applyToTypedValue(const TypedValue& argument) const {
+ if (input_nullable && argument.isNull()) {
+ return TypedValue(result_type_.getTypeID());
+ }
+
+ char *output_ptr = static_cast<char*>(std::malloc(substring_length_));
+ computeSubstring(static_cast<const char*>(argument.getOutOfLineData()),
+ output_ptr);
+
+ return TypedValue::CreateWithOwnedData(result_type_.getTypeID(),
+ output_ptr,
+ substring_length_);
+}
+
+template <bool null_terminated, bool input_nullable>
+TypedValue SubstringUncheckedOperator<null_terminated,
+ input_nullable>
+ ::applyToDataPtr(const void *argument) const {
+ if (input_nullable && argument == nullptr) {
+ return TypedValue(result_type_.getTypeID());
+ }
+
+ char *output_ptr = static_cast<char*>(std::malloc(substring_length_));
+ computeSubstring(static_cast<const char*>(argument),
+ output_ptr);
+
+ return TypedValue::CreateWithOwnedData(result_type_.getTypeID(),
+ output_ptr,
+ substring_length_);
+}
+
+template <bool null_terminated, bool input_nullable>
+ColumnVector* SubstringUncheckedOperator<null_terminated,
+ input_nullable>
+ ::applyToColumnVector(const ColumnVector &argument) const {
+ return InvokeOnColumnVector(
+ argument,
+ [&](const auto &column_vector) -> ColumnVector* { // NOLINT(build/c++11)
+ NativeColumnVector *result =
+ new NativeColumnVector(result_type_, column_vector.size());
+
+ for (std::size_t cv_pos = 0;
+ cv_pos < column_vector.size();
+ ++cv_pos) {
+ const char *input_ptr = static_cast<const char *>(
+ column_vector.template getUntypedValue<input_nullable>(cv_pos));
+
+ if (input_nullable && input_ptr == nullptr) {
+ result->appendNullValue();
+ } else {
+ this->computeSubstring(input_ptr,
+ static_cast<char *>(result->getPtrForDirectWrite()));
+ }
+ }
+ return result;
+ });
+}
+
+#ifdef QUICKSTEP_ENABLE_VECTOR_COPY_ELISION_SELECTION
+template <bool null_terminated, bool input_nullable>
+ColumnVector* SubstringUncheckedOperator<null_terminated,
+ input_nullable>
+ ::applyToValueAccessor(ValueAccessor *accessor,
+ const attribute_id argument_attr_id) const {
+ return InvokeOnValueAccessorMaybeTupleIdSequenceAdapter(
+ accessor,
+ [&](auto *accessor) -> ColumnVector* { // NOLINT(build/c++11)
+ NativeColumnVector *result =
+ new NativeColumnVector(result_type_, accessor->getNumTuples());
+
+ accessor->beginIteration();
+ while (accessor->next()) {
+ const char *input_ptr = static_cast<const char *>(
+ accessor->template getUntypedValue<input_nullable>(argument_attr_id));
+
+ if (input_nullable && (input_ptr == nullptr)) {
+ result->appendNullValue();
+ } else {
+ this->computeSubstring(input_ptr,
+ static_cast<char *>(result->getPtrForDirectWrite()));
+ }
+ }
+ return result;
+ });
+}
+#endif
+
+#ifdef QUICKSTEP_ENABLE_VECTOR_COPY_ELISION_JOIN
+template <bool null_terminated, bool input_nullable>
+ColumnVector* SubstringUncheckedOperator<null_terminated,
+ input_nullable>
+ ::applyToValueAccessorForJoin(
+ ValueAccessor *accessor,
+ const bool use_left_relation,
+ const attribute_id argument_attr_id,
+ const std::vector<std::pair<tuple_id, tuple_id>> &joined_tuple_ids) const {
+ return InvokeOnValueAccessorNotAdapter(
+ accessor,
+ [&](auto *accessor) -> ColumnVector* { // NOLINT(build/c++11)
+ NativeColumnVector *result =
+ new NativeColumnVector(result_type_, accessor->getNumTuples());
+
+ for (const std::pair<tuple_id, tuple_id> &joined_pair : joined_tuple_ids) {
+ const char *input_ptr = static_cast<const char *>(
+ accessor->template getUntypedValueAtAbsolutePosition<input_nullable>(
+ argument_attr_id,
+ use_left_relation ? joined_pair.first : joined_pair.second));
+
+ if (input_nullable && input_ptr == nullptr) {
+ result->appendNullValue();
+ } else {
+ this->computeSubstring(input_ptr,
+ static_cast<char *>(result->getPtrForDirectWrite()));
+ }
+ }
+ return result;
+ });
+}
+#endif
+
+} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/767b2ef1/types/operations/unary_operations/SubstringOperation.hpp
----------------------------------------------------------------------
diff --git a/types/operations/unary_operations/SubstringOperation.hpp b/types/operations/unary_operations/SubstringOperation.hpp
new file mode 100644
index 0000000..d215eae
--- /dev/null
+++ b/types/operations/unary_operations/SubstringOperation.hpp
@@ -0,0 +1,234 @@
+/**
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#ifndef QUICKSTEP_TYPES_OPERATIONS_UNARY_OPERATIONS_SUBSTRING_OPERATION_HPP_
+#define QUICKSTEP_TYPES_OPERATIONS_UNARY_OPERATIONS_SUBSTRING_OPERATION_HPP_
+
+#include <algorithm>
+#include <cstddef>
+#include <cstdlib>
+#include <cstring>
+#include <memory>
+#include <unordered_map>
+#include <utility>
+#include <vector>
+
+#include "catalog/CatalogTypedefs.hpp"
+#include "types/Type.hpp"
+#include "types/TypeFactory.hpp"
+#include "types/TypeID.hpp"
+#include "types/TypedValue.hpp"
+#include "types/operations/Operation.pb.h"
+#include "types/operations/unary_operations/UnaryOperation.hpp"
+#include "types/operations/unary_operations/UnaryOperationID.hpp"
+#include "utility/HashPair.hpp"
+#include "utility/Macros.hpp"
+
+#include "glog/logging.h"
+
+namespace quickstep {
+
+class ColumnVector;
+class ValueAccessor;
+
+/**
+ * @brief Operation that extracts a number of characters from a string
+ * at a given starting position.
+ */
+class SubstringOperation : public UnaryOperation {
+ public:
+ /**
+ * @brief Get a reference to the singleton instance of this Operation for
+ * the given (start_position, substring_length) pair.
+ **/
+ static const SubstringOperation& Instance(const std::size_t start_position,
+ const std::size_t substring_length) {
+ // TODO(jianqiao): This is a temporary solution that creates a new instance
+ // for each distinct pair of start_position and substring_length arguments.
+ // The number of instances may be unbounded if quickstep continuously accepts
+ // queries that call SUBSTRING with different arguments. It still remains to
+ // design a better long-term solution.
+ const auto hash = [](const auto &pair) {
+ return hash_combine_detail::HashCombiner<std::size_t>::CombineHashes(pair.first, pair.second);
+ };
+ static std::unordered_map<std::pair<std::size_t, std::size_t>,
+ std::unique_ptr<const SubstringOperation>,
+ decltype(hash)> instance_map(10, hash);
+
+ const std::pair<std::size_t, std::size_t> key_pair =
+ std::make_pair(start_position, substring_length);
+ auto imit = instance_map.find(key_pair);
+ if (imit != instance_map.end()) {
+ return *imit->second;
+ } else {
+ const SubstringOperation *instance =
+ new SubstringOperation(start_position, substring_length);
+ instance_map.emplace(key_pair,
+ std::unique_ptr<const SubstringOperation>(instance));
+ return *instance;
+ }
+ }
+
+ serialization::UnaryOperation getProto() const override;
+
+ bool canApplyToType(const Type &type) const override {
+ return (type.getSuperTypeID() == Type::kAsciiString);
+ }
+
+ const Type *resultTypeForArgumentType(const Type &type) const override {
+ if (type.getSuperTypeID() == Type::kAsciiString) {
+ // Result is a Char string.
+ return &TypeFactory::GetType(TypeID::kChar,
+ computeMaximumSubstringLength(type),
+ type.isNullable());
+ }
+ return nullptr;
+ }
+
+ const Type* fixedNullableResultType() const override {
+ // Result type is not fixed (i.e. can have various lengths).
+ return nullptr;
+ }
+
+ bool resultTypeIsPlausible(const Type &result_type) const override {
+ // Result can be coerced to Char or VarChar.
+ return (result_type.getSuperTypeID() == Type::kAsciiString);
+ }
+
+ const Type* pushDownTypeHint(const Type *type_hint) const override {
+ // Input can only be a string, but we don't know the length.
+ return nullptr;
+ }
+
+ TypedValue applyToChecked(const TypedValue &argument,
+ const Type &argument_type) const override {
+ DCHECK(canApplyToType(argument_type));
+
+ const Type *result_type = resultTypeForArgumentType(argument_type);
+ DCHECK(result_type != nullptr);
+
+ if (argument_type.isNullable() && argument.isNull()) {
+ return result_type->makeNullValue();
+ } else {
+ const std::size_t result_length = computeMaximumSubstringLength(argument_type);
+ char *output_ptr = static_cast<char*>(std::malloc(result_length));
+ const char *input_ptr = static_cast<const char*>(argument.getOutOfLineData());
+
+ const std::size_t string_length = argument.getAsciiStringLength();
+ if (start_position_ >= string_length) {
+ *output_ptr = '\0';
+ } else {
+ const std::size_t actual_substring_length =
+ std::min(string_length - start_position_, substring_length_);
+ std::memcpy(output_ptr, input_ptr + start_position_, actual_substring_length);
+ if (actual_substring_length < result_length) {
+ output_ptr[actual_substring_length] = '\0';
+ }
+ }
+
+ return TypedValue::CreateWithOwnedData(result_type->getTypeID(),
+ output_ptr,
+ result_length);
+ }
+ }
+
+ UncheckedUnaryOperator* makeUncheckedUnaryOperatorForType(const Type &type) const override;
+
+ private:
+ /**
+ * @brief Constructor.
+ *
+ * @param input_type The data type of the input argument for substring.
+ * @param start_position The 0-base starting position of the substring.
+ * @param substring_length The substring length.
+ */
+ SubstringOperation(const std::size_t start_position,
+ const std::size_t substring_length)
+ : UnaryOperation(UnaryOperationID::kSubstring),
+ start_position_(start_position),
+ substring_length_(substring_length) {
+ }
+
+ /**
+ * @brief Compute an upper bound for the substring length regarding the input
+ * type and the substring_length_ field.
+ *
+ * @param type The type of the input, must be either CharType or VarCharType.
+ */
+ inline std::size_t computeMaximumSubstringLength(const Type& type) const {
+ DCHECK(type.getSuperTypeID() == Type::kAsciiString);
+
+ // Substring result should have length no greater than the minimum of
+ // (1) the input string length subtract the start position, and
+ // (2) the specified substring length.
+ return std::min(static_cast<const AsciiStringSuperType&>(type).getStringLength() - start_position_,
+ substring_length_);
+ }
+
+ const std::size_t start_position_;
+ const std::size_t substring_length_;
+
+ private:
+ DISALLOW_COPY_AND_ASSIGN(SubstringOperation);
+};
+
+template <bool null_terminated, bool input_nullable>
+class SubstringUncheckedOperator : public UncheckedUnaryOperator {
+ public:
+ SubstringUncheckedOperator(const std::size_t start_position,
+ const std::size_t substring_length,
+ const std::size_t maximum_input_length,
+ const Type &result_type)
+ : start_position_(start_position),
+ substring_length_(substring_length),
+ maximum_input_length_(maximum_input_length),
+ result_type_(result_type) {
+ }
+
+ TypedValue applyToTypedValue(const TypedValue& argument) const override;
+
+ TypedValue applyToDataPtr(const void *argument) const override;
+
+ ColumnVector* applyToColumnVector(const ColumnVector &argument) const override;
+
+#ifdef QUICKSTEP_ENABLE_VECTOR_COPY_ELISION_SELECTION
+ ColumnVector* applyToValueAccessor(ValueAccessor *accessor,
+ const attribute_id argument_attr_id) const override;
+#endif // QUICKSTEP_ENABLE_VECTOR_COPY_ELISION_SELECTION
+
+#ifdef QUICKSTEP_ENABLE_VECTOR_COPY_ELISION_JOIN
+ ColumnVector* applyToValueAccessorForJoin(
+ ValueAccessor *accessor,
+ const bool use_left_relation,
+ const attribute_id argument_attr_id,
+ const std::vector<std::pair<tuple_id, tuple_id>> &joined_tuple_ids) const override;
+#endif // QUICKSTEP_ENABLE_VECTOR_COPY_ELISION_JOIN
+
+ private:
+ inline void computeSubstring(const char *input, char *output) const;
+
+ const std::size_t start_position_;
+ const std::size_t substring_length_;
+ const std::size_t maximum_input_length_;
+ const Type &result_type_;
+
+ DISALLOW_COPY_AND_ASSIGN(SubstringUncheckedOperator);
+};
+
+} // namespace quickstep
+
+#endif /* QUICKSTEP_TYPES_OPERATIONS_UNARY_OPERATIONS_SUBSTRING_OPERATION_HPP_ */
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/767b2ef1/types/operations/unary_operations/UnaryOperation.cpp
----------------------------------------------------------------------
diff --git a/types/operations/unary_operations/UnaryOperation.cpp b/types/operations/unary_operations/UnaryOperation.cpp
index 0f1f2c7..6721bd6 100644
--- a/types/operations/unary_operations/UnaryOperation.cpp
+++ b/types/operations/unary_operations/UnaryOperation.cpp
@@ -1,6 +1,8 @@
/**
* Copyright 2011-2015 Quickstep Technologies LLC.
* Copyright 2015 Pivotal Software, Inc.
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -33,6 +35,8 @@ serialization::UnaryOperation UnaryOperation::getProto() const {
FATAL_ERROR("Must use the overridden NumericCastOperation::getProto");
case UnaryOperationID::kDateExtract:
FATAL_ERROR("Must use the overridden DateExtractOperation::getProto");
+ case UnaryOperationID::kSubstring:
+ FATAL_ERROR("Must use the overridden SubstringOperation::getProto");
default:
FATAL_ERROR("Unrecognized UnaryOperationID in UnaryOperation::getProto");
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/767b2ef1/types/operations/unary_operations/UnaryOperationFactory.cpp
----------------------------------------------------------------------
diff --git a/types/operations/unary_operations/UnaryOperationFactory.cpp b/types/operations/unary_operations/UnaryOperationFactory.cpp
index 9d79cb0..13cb76e 100644
--- a/types/operations/unary_operations/UnaryOperationFactory.cpp
+++ b/types/operations/unary_operations/UnaryOperationFactory.cpp
@@ -1,6 +1,8 @@
/**
* Copyright 2011-2015 Quickstep Technologies LLC.
* Copyright 2015 Pivotal Software, Inc.
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -24,6 +26,7 @@
#include "types/operations/unary_operations/ArithmeticUnaryOperations.hpp"
#include "types/operations/unary_operations/NumericCastOperation.hpp"
#include "types/operations/unary_operations/DateExtractOperation.hpp"
+#include "types/operations/unary_operations/SubstringOperation.hpp"
#include "types/operations/unary_operations/UnaryOperationID.hpp"
#include "utility/Macros.hpp"
@@ -39,6 +42,8 @@ const UnaryOperation& UnaryOperationFactory::GetUnaryOperation(const UnaryOperat
FATAL_ERROR("Getting a CastOperation through GetUnaryOperation is not supported");
case UnaryOperationID::kDateExtract:
FATAL_ERROR("Getting a DateExtractOperation through GetUnaryOperation is not supported");
+ case UnaryOperationID::kSubstring:
+ FATAL_ERROR("Getting a SubstringOperation through GetUnaryOperation is not supported");
default:
FATAL_ERROR("Unknown UnaryOperationID");
}
@@ -64,6 +69,9 @@ bool UnaryOperationFactory::ProtoIsValid(const serialization::UnaryOperation &pr
case serialization::UnaryOperation::DATE_EXTRACT:
return proto.HasExtension(serialization::DateExtractOperation::unit)
&& DateExtractOperation_Unit_IsValid(proto.GetExtension(serialization::DateExtractOperation::unit));
+ case serialization::UnaryOperation::SUBSTRING:
+ return proto.HasExtension(serialization::SubstringOperation::start_position)
+ && proto.HasExtension(serialization::SubstringOperation::substring_length);
default:
return false;
}
@@ -100,6 +108,10 @@ const UnaryOperation& UnaryOperationFactory::ReconstructFromProto(
default:
FATAL_ERROR("Unrecognized DateExtractOperation unit in UnaryOperation::ReconstructFromProto");
}
+ case serialization::UnaryOperation::SUBSTRING:
+ return SubstringOperation::Instance(
+ proto.GetExtension(serialization::SubstringOperation::start_position),
+ proto.GetExtension(serialization::SubstringOperation::substring_length));
default:
FATAL_ERROR("Unrecognized UnaryOperationID in UnaryOperation::ReconstructFromProto");
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/767b2ef1/types/operations/unary_operations/UnaryOperationID.cpp
----------------------------------------------------------------------
diff --git a/types/operations/unary_operations/UnaryOperationID.cpp b/types/operations/unary_operations/UnaryOperationID.cpp
index c45a0a9..dae64d3 100644
--- a/types/operations/unary_operations/UnaryOperationID.cpp
+++ b/types/operations/unary_operations/UnaryOperationID.cpp
@@ -1,6 +1,8 @@
/**
* Copyright 2011-2015 Quickstep Technologies LLC.
* Copyright 2015 Pivotal Software, Inc.
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -20,11 +22,11 @@
namespace quickstep {
const char *kUnaryOperationNames[] = {
- "Negate", "Cast", "DateExtract"
+ "Negate", "Cast", "DateExtract", "Substring"
};
const char *kUnaryOperationShortNames[] = {
- "-", "Cast", "DateExtract"
+ "-", "Cast", "DateExtract", "Substring"
};
} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/767b2ef1/types/operations/unary_operations/UnaryOperationID.hpp
----------------------------------------------------------------------
diff --git a/types/operations/unary_operations/UnaryOperationID.hpp b/types/operations/unary_operations/UnaryOperationID.hpp
index ba3e0ef..9cbb6e4 100644
--- a/types/operations/unary_operations/UnaryOperationID.hpp
+++ b/types/operations/unary_operations/UnaryOperationID.hpp
@@ -1,6 +1,8 @@
/**
* Copyright 2011-2015 Quickstep Technologies LLC.
* Copyright 2015 Pivotal Software, Inc.
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -33,6 +35,7 @@ enum class UnaryOperationID {
kNegate = 0,
kCast,
kDateExtract,
+ kSubstring,
kNumUnaryOperationIDs // Not a real UnaryOperationID, exists for counting purposes.
};
[39/50] [abbrv] incubator-quickstep git commit: Reverting the PR that
gets the number of rows (#233)
Posted by zu...@apache.org.
Reverting the PR that gets the number of rows (#233)
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/1d8c6822
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/1d8c6822
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/1d8c6822
Branch: refs/heads/work-order-serialization
Commit: 1d8c6822896ea7032b1f9f623f481177e534d7e4
Parents: 1605fd8
Author: Rogers Jeffrey Leo John <ro...@gmail.com>
Authored: Sat May 21 07:36:06 2016 -0500
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Mon May 30 15:47:52 2016 -0700
----------------------------------------------------------------------
cli/CommandExecutor.cpp | 23 +++--------------------
cli/tests/command_executor/D.test | 14 +++++++-------
cli/tests/command_executor/Dt.test | 20 ++++++++++----------
3 files changed, 20 insertions(+), 37 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1d8c6822/cli/CommandExecutor.cpp
----------------------------------------------------------------------
diff --git a/cli/CommandExecutor.cpp b/cli/CommandExecutor.cpp
index 3cb3f86..ddcd38f 100644
--- a/cli/CommandExecutor.cpp
+++ b/cli/CommandExecutor.cpp
@@ -72,7 +72,6 @@ void executeDescribeDatabase(
// Column width initialized to 6 to take into account the header name
// and the column value table
int max_column_width = C::kInitMaxColumnWidth;
- vector<std::size_t> num_tuples;
vector<std::size_t> num_blocks;
const CatalogRelation *relation = nullptr;
if (arguments->size() == 0) {
@@ -80,8 +79,6 @@ void executeDescribeDatabase(
max_column_width =
std::max(static_cast<int>(rel.getName().length()), max_column_width);
num_blocks.push_back(rel.size_blocks());
- num_tuples.push_back(
- PrintToScreen::GetNumTuplesInRelation(rel, storage_manager));
}
} else {
const ParseString &table_name = arguments->front();
@@ -94,49 +91,35 @@ void executeDescribeDatabase(
max_column_width = std::max(static_cast<int>(relation->getName().length()),
max_column_width);
num_blocks.push_back(relation->size_blocks());
- num_tuples.push_back(PrintToScreen::GetNumTuplesInRelation(
- *relation,
- storage_manager));
}
// Only if we have relations work on the printing logic.
if (catalog_database.size() > 0) {
const std::size_t max_num_blocks = *std::max_element(num_blocks.begin(), num_blocks.end());
- const std::size_t max_num_rows = *std::max_element(num_tuples.begin(), num_tuples.end());
- const int max_num_rows_digits = std::max(PrintToScreen::GetNumberOfDigits(max_num_rows),
- C::kInitMaxColumnWidth);
const int max_num_blocks_digits = std::max(PrintToScreen::GetNumberOfDigits(max_num_blocks),
C::kInitMaxColumnWidth+2);
-
vector<int> column_widths;
column_widths.push_back(max_column_width +1);
column_widths.push_back(C::kInitMaxColumnWidth + 1);
column_widths.push_back(max_num_blocks_digits + 1);
- column_widths.push_back(max_num_rows_digits + 1);
fputs(" List of relations\n\n", out);
fprintf(out, "%-*s |", max_column_width+1, " Name");
fprintf(out, "%-*s |", C::kInitMaxColumnWidth, " Type");
- fprintf(out, "%-*s |", max_num_blocks_digits, " Blocks");
- fprintf(out, "%-*s\n", max_num_rows_digits, " Rows");
+ fprintf(out, "%-*s\n", max_num_blocks_digits, " Blocks");
PrintToScreen::printHBar(column_widths, out);
// If there are no argument print the entire list of tables
// else print the particular table only.
- vector<std::size_t>::const_iterator num_tuples_it = num_tuples.begin();
vector<std::size_t>::const_iterator num_blocks_it = num_blocks.begin();
if (arguments->size() == 0) {
for (const CatalogRelation &rel : catalog_database) {
fprintf(out, " %-*s |", max_column_width, rel.getName().c_str());
fprintf(out, " %-*s |", C::kInitMaxColumnWidth - 1, "table");
- fprintf(out, " %-*lu |", max_num_blocks_digits - 1, *num_blocks_it);
- fprintf(out, " %-*lu\n", max_num_rows_digits - 1, *num_tuples_it);
- ++num_tuples_it;
+ fprintf(out, " %-*lu\n", max_num_blocks_digits - 1, *num_blocks_it);
++num_blocks_it;
}
} else {
fprintf(out, " %-*s |", max_column_width, relation->getName().c_str());
fprintf(out, " %-*s |", C::kInitMaxColumnWidth -1, "table");
- fprintf(out, " %-*lu |", max_num_blocks_digits - 1, *num_blocks_it);
- fprintf(out, " %-*lu\n", max_num_rows_digits - 1, *num_tuples_it);
- ++num_tuples_it;
+ fprintf(out, " %-*lu\n", max_num_blocks_digits - 1, *num_blocks_it);
++num_blocks_it;
}
fputc('\n', out);
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1d8c6822/cli/tests/command_executor/D.test
----------------------------------------------------------------------
diff --git a/cli/tests/command_executor/D.test b/cli/tests/command_executor/D.test
index 45f8d8b..d18ab47 100644
--- a/cli/tests/command_executor/D.test
+++ b/cli/tests/command_executor/D.test
@@ -104,13 +104,13 @@ INSERT INTO foo3 values(5, 1, 1.0, 1.0, 'XYZZ');
--
List of relations
- Name | Type | Blocks | Rows
-+--------------------------------------+-------+---------+-------+
- foo | table | 1 | 5
- foo2 | table | 1 | 2
- foo3 | table | 1 | 1
- foo4 | table | 0 | 0
- averylongtablenamethatseemstoneverend | table | 1 | 3
+ Name | Type | Blocks
++--------------------------------------+-------+---------+
+ foo | table | 1
+ foo2 | table | 1
+ foo3 | table | 1
+ foo4 | table | 0
+ averylongtablenamethatseemstoneverend | table | 1
==
\d invalidtable
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1d8c6822/cli/tests/command_executor/Dt.test
----------------------------------------------------------------------
diff --git a/cli/tests/command_executor/Dt.test b/cli/tests/command_executor/Dt.test
index 1de6360..65c6be3 100644
--- a/cli/tests/command_executor/Dt.test
+++ b/cli/tests/command_executor/Dt.test
@@ -52,22 +52,22 @@ INSERT INTO foo3 values(5, 1, 1.0, 1.0, 'XYZZ');
--
List of relations
- Name | Type | Blocks | Rows
-+--------------------------------------+-------+---------+-------+
- foo | table | 1 | 5
- foo2 | table | 1 | 2
- foo3 | table | 1 | 1
- foo4 | table | 0 | 0
- averylongtablenamethatseemstoneverend | table | 1 | 3
+ Name | Type | Blocks
++--------------------------------------+-------+---------+
+ foo | table | 1
+ foo2 | table | 1
+ foo3 | table | 1
+ foo4 | table | 0
+ averylongtablenamethatseemstoneverend | table | 1
==
\dt foo
--
List of relations
- Name | Type | Blocks | Rows
-+-------+-------+---------+-------+
- foo | table | 1 | 5
+ Name | Type | Blocks
++-------+-------+---------+
+ foo | table | 1
==
\dt invalidtable
[38/50] [abbrv] incubator-quickstep git commit: Revert "Quickstep gen
stats" (#231)
Posted by zu...@apache.org.
Revert "Quickstep gen stats" (#231)
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/d136e1dc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/d136e1dc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/d136e1dc
Branch: refs/heads/work-order-serialization
Commit: d136e1dc955969b9d6ab65e69913ca78634c49b3
Parents: 30f0981
Author: Rogers Jeffrey Leo John <ro...@gmail.com>
Authored: Fri May 20 14:51:35 2016 -0500
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Mon May 30 15:47:52 2016 -0700
----------------------------------------------------------------------
catalog/Catalog.proto | 2 +-
query_optimizer/CMakeLists.txt | 1 -
query_optimizer/ExecutionGenerator.cpp | 10 ---
relational_operators/CMakeLists.txt | 11 ---
.../GenerateNumRowsStatsOperator.cpp | 42 -----------
.../GenerateNumRowsStatsOperator.hpp | 79 --------------------
6 files changed, 1 insertion(+), 144 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/d136e1dc/catalog/Catalog.proto
----------------------------------------------------------------------
diff --git a/catalog/Catalog.proto b/catalog/Catalog.proto
index 8e44181..ce4bc2e 100644
--- a/catalog/Catalog.proto
+++ b/catalog/Catalog.proto
@@ -82,7 +82,7 @@ message IndexScheme {
message CatalogRelationStatistics {
optional fixed64 num_tuples = 1;
-
+
message NumDistinctValuesEntry {
required int32 attr_id = 1;
required fixed64 num_distinct_values = 2;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/d136e1dc/query_optimizer/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_optimizer/CMakeLists.txt b/query_optimizer/CMakeLists.txt
index 1cc38d1..aa2873e 100644
--- a/query_optimizer/CMakeLists.txt
+++ b/query_optimizer/CMakeLists.txt
@@ -111,7 +111,6 @@ target_link_libraries(quickstep_queryoptimizer_ExecutionGenerator
quickstep_relationaloperators_DestroyHashOperator
quickstep_relationaloperators_DropTableOperator
quickstep_relationaloperators_FinalizeAggregationOperator
- quickstep_relationaloperators_GenerateNumRowsStatsOperator
quickstep_relationaloperators_HashJoinOperator
quickstep_relationaloperators_InsertOperator
quickstep_relationaloperators_NestedLoopsJoinOperator
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/d136e1dc/query_optimizer/ExecutionGenerator.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/ExecutionGenerator.cpp b/query_optimizer/ExecutionGenerator.cpp
index 612efd9..c590b6e 100644
--- a/query_optimizer/ExecutionGenerator.cpp
+++ b/query_optimizer/ExecutionGenerator.cpp
@@ -91,7 +91,6 @@
#include "relational_operators/DestroyHashOperator.hpp"
#include "relational_operators/DropTableOperator.hpp"
#include "relational_operators/FinalizeAggregationOperator.hpp"
-#include "relational_operators/GenerateNumRowsStatsOperator.hpp"
#include "relational_operators/HashJoinOperator.hpp"
#include "relational_operators/InsertOperator.hpp"
#include "relational_operators/NestedLoopsJoinOperator.hpp"
@@ -948,15 +947,6 @@ void ExecutionGenerator::convertCopyFrom(
execution_plan_->addDirectDependency(save_blocks_operator_index,
scan_operator_index,
false /* is_pipeline_breaker */);
-
- const QueryPlan::DAGNodeIndex num_rows_operator_index =
- execution_plan_->addRelationalOperator(new GenerateNumRowsStatsOperator(
- optimizer_context_->catalog_database()->getRelationByIdMutable(
- output_relation->getID())));
- insert_destination_proto->set_relational_op_index(num_rows_operator_index);
- execution_plan_->addDirectDependency(num_rows_operator_index,
- scan_operator_index,
- true /* is_pipeline_breaker */);
}
void ExecutionGenerator::convertCreateIndex(
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/d136e1dc/relational_operators/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/relational_operators/CMakeLists.txt b/relational_operators/CMakeLists.txt
index e211630..eec5300 100644
--- a/relational_operators/CMakeLists.txt
+++ b/relational_operators/CMakeLists.txt
@@ -34,9 +34,6 @@ add_library(quickstep_relationaloperators_DropTableOperator DropTableOperator.cp
add_library(quickstep_relationaloperators_FinalizeAggregationOperator
FinalizeAggregationOperator.cpp
FinalizeAggregationOperator.hpp)
-add_library(quickstep_relationaloperators_GenerateNumRowsStatsOperator
- GenerateNumRowsStatsOperator.cpp
- GenerateNumRowsStatsOperator.hpp)
add_library(quickstep_relationaloperators_HashJoinOperator HashJoinOperator.cpp HashJoinOperator.hpp)
add_library(quickstep_relationaloperators_InsertOperator InsertOperator.cpp InsertOperator.hpp)
add_library(quickstep_relationaloperators_NestedLoopsJoinOperator
@@ -162,13 +159,6 @@ target_link_libraries(quickstep_relationaloperators_FinalizeAggregationOperator
quickstep_storage_AggregationOperationState
quickstep_utility_Macros
tmb)
-target_link_libraries(quickstep_relationaloperators_GenerateNumRowsStatsOperator
- glog
- quickstep_catalog_CatalogRelation
- quickstep_cli_PrintToScreen
- quickstep_relationaloperators_RelationalOperator
- quickstep_utility_Macros
- tmb)
target_link_libraries(quickstep_relationaloperators_HashJoinOperator
gflags_nothreads-static
glog
@@ -456,7 +446,6 @@ target_link_libraries(quickstep_relationaloperators
quickstep_relationaloperators_DestroyHashOperator
quickstep_relationaloperators_DropTableOperator
quickstep_relationaloperators_FinalizeAggregationOperator
- quickstep_relationaloperators_GenerateNumRowsStatsOperator
quickstep_relationaloperators_HashJoinOperator
quickstep_relationaloperators_InsertOperator
quickstep_relationaloperators_NestedLoopsJoinOperator
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/d136e1dc/relational_operators/GenerateNumRowsStatsOperator.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/GenerateNumRowsStatsOperator.cpp b/relational_operators/GenerateNumRowsStatsOperator.cpp
deleted file mode 100644
index 074e1ca..0000000
--- a/relational_operators/GenerateNumRowsStatsOperator.cpp
+++ /dev/null
@@ -1,42 +0,0 @@
-/**
- * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
- * University of Wisconsin\u2014Madison.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- **/
-
-#include "relational_operators/GenerateNumRowsStatsOperator.hpp"
-
-#include <memory>
-
-#include "catalog/CatalogRelation.hpp"
-#include "cli/PrintToScreen.hpp"
-
-#include "tmb/id_typedefs.h"
-
-namespace quickstep {
-
-bool GenerateNumRowsStatsOperator::getAllWorkOrders(
- WorkOrdersContainer *container,
- QueryContext *query_context,
- StorageManager *storage_manager,
- const tmb::client_id scheduler_client_id,
- tmb::MessageBus *bus) {
- std::size_t num_tuples =
- PrintToScreen::GetNumTuplesInRelation(*relation_, storage_manager);
- relation_->getStatisticsMutable()->setNumTuples(num_tuples);
- return true;
-}
-
-} // namespace quickstep
-
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/d136e1dc/relational_operators/GenerateNumRowsStatsOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/GenerateNumRowsStatsOperator.hpp b/relational_operators/GenerateNumRowsStatsOperator.hpp
deleted file mode 100644
index 8622a63..0000000
--- a/relational_operators/GenerateNumRowsStatsOperator.hpp
+++ /dev/null
@@ -1,79 +0,0 @@
-/**
- * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
- * University of Wisconsin\u2014Madison.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- **/
-
-#ifndef QUICKSTEP_RELATIONAL_OPERATORS_GENERATE_NUM_ROWS_STATS_OPERATOR_HPP_
-#define QUICKSTEP_RELATIONAL_OPERATORS_GENERATE_NUM_ROWS_STATS_OPERATOR_HPP_
-
-#include <memory>
-
-#include "catalog/CatalogRelation.hpp"
-#include "relational_operators/RelationalOperator.hpp"
-#include "utility/Macros.hpp"
-
-#include "glog/logging.h"
-
-#include "tmb/id_typedefs.h"
-
-namespace tmb { class MessageBus; }
-
-namespace quickstep {
-
-class CatalogRelation;
-class QueryContext;
-class StorageManager;
-class WorkOrdersContainer;
-
-/** \addtogroup RelationalOperators
- * @{
- */
-
-/**
- * @brief An operator that gets the number of rows after loading a relation.
- **/
-class GenerateNumRowsStatsOperator : public RelationalOperator {
- public:
- /**
- * @brief Constructor.
- *
- * @param relation The relation to get the number of rows from.
- * This GenNumRowStatsOperator owns relation until
- * the WorkOrder it produces is successfully executed.
- **/
- explicit GenerateNumRowsStatsOperator(CatalogRelation *relation)
- : relation_(relation) {}
- ~GenerateNumRowsStatsOperator() override {}
-
- /**
- * @note no WorkOrder is generated for this operator.
- **/
- bool getAllWorkOrders(WorkOrdersContainer *container,
- QueryContext *query_context,
- StorageManager *storage_manager,
- const tmb::client_id scheduler_client_id,
- tmb::MessageBus *bus) override;
-
- private:
- CatalogRelation *relation_;
-
- DISALLOW_COPY_AND_ASSIGN(GenerateNumRowsStatsOperator);
-};
-
-/** @} */
-
-} // namespace quickstep
-
-#endif // QUICKSTEP_RELATIONAL_OPERATORS_GENERATE_NUM_ROWS_STATS_OPERATOR_HPP_
[04/50] [abbrv] incubator-quickstep git commit: Fixes loss of alias
problem with PullUpProjectExpressions (#197)
Posted by zu...@apache.org.
Fixes loss of alias problem with PullUpProjectExpressions (#197)
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/07ca1e73
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/07ca1e73
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/07ca1e73
Branch: refs/heads/work-order-serialization
Commit: 07ca1e73ebce7b9df32af2b3c7721df5941f25a9
Parents: 3bd1586
Author: Jianqiao Zhu <ji...@cs.wisc.edu>
Authored: Thu Apr 28 01:00:50 2016 -0500
Committer: Jignesh Patel <pa...@users.noreply.github.com>
Committed: Thu Apr 28 01:00:50 2016 -0500
----------------------------------------------------------------------
query_optimizer/rules/RuleHelper.cpp | 18 +++++++++++-------
.../tests/execution_generator/Select.test | 20 ++++++++++++++++++++
.../tests/physical_generator/Select.test | 16 ++++++++++------
3 files changed, 41 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/07ca1e73/query_optimizer/rules/RuleHelper.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/rules/RuleHelper.cpp b/query_optimizer/rules/RuleHelper.cpp
index ce9ef9a..a273004 100644
--- a/query_optimizer/rules/RuleHelper.cpp
+++ b/query_optimizer/rules/RuleHelper.cpp
@@ -1,6 +1,8 @@
/**
* Copyright 2011-2015 Quickstep Technologies LLC.
* Copyright 2015 Pivotal Software, Inc.
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -75,19 +77,21 @@ void PullUpProjectExpressions(
E::ExpressionPtr updated_expression = rule.apply(expression_to_update);
E::NamedExpressionPtr updated_named_expression;
- // If the updated expression is not a NamedExpression or we have changed
- // the ID, restore the ID by adding an Alias.
- if (!E::SomeNamedExpression::MatchesWithConditionalCast(updated_expression,
- &updated_named_expression) ||
- expression_to_update->id() != updated_named_expression->id()) {
+ if (E::SomeNamedExpression::MatchesWithConditionalCast(updated_expression,
+ &updated_named_expression) &&
+ expression_to_update->id() == updated_named_expression->id() &&
+ expression_to_update->attribute_alias() == updated_named_expression->attribute_alias()) {
+ // Pull up directly if the updated expression has the same ExprId and
+ // attribute_alias as the original project expression.
+ (*project_expressions)[i] = updated_named_expression;
+ } else {
+ // Otherwise create an Alias to wrap the updated expression.
(*project_expressions)[i] =
E::Alias::Create(expression_to_update->id(),
updated_expression,
expression_to_update->attribute_name(),
expression_to_update->attribute_alias(),
expression_to_update->relation_name());
- } else {
- (*project_expressions)[i] = updated_named_expression;
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/07ca1e73/query_optimizer/tests/execution_generator/Select.test
----------------------------------------------------------------------
diff --git a/query_optimizer/tests/execution_generator/Select.test b/query_optimizer/tests/execution_generator/Select.test
index 438546c..3a64c9b 100644
--- a/query_optimizer/tests/execution_generator/Select.test
+++ b/query_optimizer/tests/execution_generator/Select.test
@@ -871,6 +871,26 @@ FROM generate_series(1, 5) AS gs(i);
+----------------------+
==
+# This query is to test that the output columns have the correct alias name as specified.
+SELECT *
+FROM (
+ SELECT i, SUM(i) AS sum
+ FROM generate_series(1, 2) AS gs(i)
+ GROUP BY i
+) t1 JOIN (
+ SELECT j, AVG(j) AS avg
+ FROM generate_series(1, 2) AS gs(j)
+ GROUP BY j
+) t2 ON i = j;
+--
++-----------+--------------------+-----------+------------------------+
+|i |sum |j |avg |
++-----------+--------------------+-----------+------------------------+
+| 1| 1| 1| 1|
+| 2| 2| 2| 2|
++-----------+--------------------+-----------+------------------------+
+==
+
# TODO(team): Fix Issue #9 to enable COUNT(*).
SELECT COUNT(long_col)
FROM test,
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/07ca1e73/query_optimizer/tests/physical_generator/Select.test
----------------------------------------------------------------------
diff --git a/query_optimizer/tests/physical_generator/Select.test b/query_optimizer/tests/physical_generator/Select.test
index b405bc9..62d09f5 100644
--- a/query_optimizer/tests/physical_generator/Select.test
+++ b/query_optimizer/tests/physical_generator/Select.test
@@ -1166,7 +1166,8 @@ TopLevelPlan
| | | | +-Add
| | | | +-AttributeReference[id=1,name=long_col,relation=test,type=Long]
| | | | +-AttributeReference[id=2,name=float_col,relation=test,type=Float]
-| | | +-AttributeReference[id=1,name=long_col,relation=test,type=Long]
+| | | +-Alias[id=1,name=col1,relation=,type=Long]
+| | | | +-AttributeReference[id=1,name=long_col,relation=test,type=Long]
| | | +-Alias[id=8,name=,alias=$groupby2,relation=$groupby,type=Float]
| | | +-Add
| | | +-AttributeReference[id=2,name=float_col,relation=test,type=Float]
@@ -1304,7 +1305,8 @@ TopLevelPlan
| | | +-Literal[value=2,type=Int]
| | +-grouping_expressions=
| | | +-AttributeReference[id=0,name=int_col,relation=test,type=Int NULL]
-| | | +-AttributeReference[id=4,name=char_col,relation=test,type=Char(20)]
+| | | +-Alias[id=4,name=subquery_col3,relation=subquery,type=Char(20)]
+| | | +-AttributeReference[id=4,name=char_col,relation=test,type=Char(20)]
| | +-aggregate_expressions=
| | +-Alias[id=8,name=,alias=$aggregate0,relation=$aggregate,type=Long]
| | | +-AggregateFunction[function=COUNT]
@@ -2609,10 +2611,12 @@ TopLevelPlan
| | | +-AttributeReference[id=5,name=y,relation=d,type=Int]
| | +-join_predicate=Literal[value=true]
| | +-project_expressions=
-| | +-AttributeReference[id=4,name=,alias=$aggregate0,relation=$aggregate,
-| | | type=Long NULL]
-| | +-AttributeReference[id=7,name=,alias=$aggregate0,relation=$aggregate,
-| | type=Double NULL]
+| | +-Alias[id=4,name=,alias=SUM(y),relation=,type=Long NULL]
+| | | +-AttributeReference[id=4,name=,alias=$aggregate0,relation=$aggregate,
+| | | type=Long NULL]
+| | +-Alias[id=7,name=,alias=AVG(y),relation=,type=Double NULL]
+| | +-AttributeReference[id=7,name=,alias=$aggregate0,relation=$aggregate,
+| | type=Double NULL]
| +-join_predicate=Literal[value=true]
| +-project_expressions=
| +-Alias[id=8,name=,alias=((x*SubqueryExpression)+SubqueryExpression),
[48/50] [abbrv] incubator-quickstep git commit: Added Async
DataExchange Service.
Posted by zu...@apache.org.
Added Async DataExchange Service.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/50b4e55a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/50b4e55a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/50b4e55a
Branch: refs/heads/work-order-serialization
Commit: 50b4e55a48c97fd5ce9800d8db92d64ee59ad2e3
Parents: 2221b7e
Author: Zuyu Zhang <zz...@pivotal.io>
Authored: Sat May 28 22:55:05 2016 -0700
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Mon May 30 15:47:54 2016 -0700
----------------------------------------------------------------------
CMakeLists.txt | 11 +-
storage/CMakeLists.txt | 69 +++++++-
storage/DataExchange.proto | 31 ++++
storage/DataExchangerAsync.cpp | 165 ++++++++++++++++++
storage/DataExchangerAsync.hpp | 97 +++++++++++
storage/StorageManager.cpp | 113 ++++++++++++-
storage/StorageManager.hpp | 61 ++++++-
storage/tests/DataExchange_unittest.cpp | 240 +++++++++++++++++++++++++++
third_party/iwyu/iwyu_helper.py | 3 +-
validate_cmakelists.py | 5 +-
10 files changed, 787 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/50b4e55a/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/CMakeLists.txt b/CMakeLists.txt
index dc51ca6..ef7fd50 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -662,7 +662,12 @@ set(ENABLE_PUREMEMORY ON CACHE BOOL "Enable PureMemory TMB")
set(ENABLE_LEVELDB OFF CACHE BOOL "Enable LevelDB TMB")
set(ENABLE_MEMORYMIRROR OFF CACHE BOOL "Enable MemoryMirror TMB")
set(ENABLE_NATIVELOG OFF CACHE BOOL "Enable NativeLog TMB")
-set(ENABLE_NATIVENET OFF CACHE BOOL "Enable NativeNet TMB")
+
+# The distributed version requires to use the NativeNet implementation.
+if (NOT ENABLE_DISTRIBUTED)
+ set(ENABLE_NATIVENET OFF CACHE BOOL "Enable NativeNet TMB")
+endif()
+
set(ENABLE ENABLE_SQLITE OFF CACHE BOOL "Enable SQLite TMB")
set(ENABLE_VOLTDB OFF CACHE BOOL "Enable VoltDB TMB")
set(ENABLE_ZOOKEEPER OFF CACHE BOOL "Enable Zookeeper TMB")
@@ -670,6 +675,10 @@ set(ENABLE_ZOOKEEPER OFF CACHE BOOL "Enable Zookeeper TMB")
add_subdirectory("${THIRD_PARTY_SOURCE_DIR}/tmb" "${CMAKE_CURRENT_BINARY_DIR}/third_party/tmb")
include_directories(${TMB_INCLUDE_DIRS})
+if (ENABLE_DISTRIBUTED)
+ include_directories(${CMAKE_CURRENT_BINARY_DIR}/third_party/tmb/include)
+endif()
+
# Add all of the module subdirectories. CMakeLists.txt in each of the subdirectories
# defines how to build that module's libraries.
add_subdirectory(catalog)
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/50b4e55a/storage/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/storage/CMakeLists.txt b/storage/CMakeLists.txt
index 4da16ea..a77976a 100644
--- a/storage/CMakeLists.txt
+++ b/storage/CMakeLists.txt
@@ -126,6 +126,13 @@ QS_PROTOBUF_GENERATE_CPP(storage_StorageBlockLayout_proto_srcs
storage_StorageBlockLayout_proto_hdrs
StorageBlockLayout.proto)
+if (ENABLE_DISTRIBUTED)
+ GRPC_GENERATE_CPP(storage_DataExchange_proto_srcs
+ storage_DataExchange_proto_hdrs
+ .
+ DataExchange.proto)
+endif()
+
# Declare micro-libs:
add_library(quickstep_storage_AggregationOperationState
AggregationOperationState.cpp
@@ -171,6 +178,14 @@ add_library(quickstep_storage_CompressedTupleStorageSubBlock
CompressedTupleStorageSubBlock.hpp)
add_library(quickstep_storage_CountedReference ../empty_src.cpp CountedReference.hpp)
add_library(quickstep_storage_CSBTreeIndexSubBlock CSBTreeIndexSubBlock.cpp CSBTreeIndexSubBlock.hpp)
+
+if (ENABLE_DISTRIBUTED)
+ add_library(quickstep_storage_DataExchange_proto
+ ${storage_DataExchange_proto_srcs}
+ ${storage_DataExchange_proto_hdrs})
+ add_library(quickstep_storage_DataExchangerAsync DataExchangerAsync.cpp DataExchangerAsync.hpp)
+endif()
+
add_library(quickstep_storage_EvictionPolicy EvictionPolicy.cpp EvictionPolicy.hpp)
add_library(quickstep_storage_FileManager ../empty_src.cpp FileManager.hpp)
if (QUICKSTEP_HAVE_FILE_MANAGER_HDFS)
@@ -575,6 +590,19 @@ target_link_libraries(quickstep_storage_CSBTreeIndexSubBlock
quickstep_utility_Macros
quickstep_utility_PtrVector
quickstep_utility_ScopedBuffer)
+
+if (ENABLE_DISTRIBUTED)
+ target_link_libraries(quickstep_storage_DataExchange_proto
+ ${PROTOBUF3_LIBRARY})
+ target_link_libraries(quickstep_storage_DataExchangerAsync
+ glog
+ quickstep_storage_DataExchange_proto
+ quickstep_storage_StorageManager
+ quickstep_threading_Thread
+ quickstep_utility_Macros
+ ${GRPCPLUSPLUS_LIBRARIES})
+endif()
+
target_link_libraries(quickstep_storage_EvictionPolicy
quickstep_storage_StorageBlockInfo
quickstep_storage_StorageConstants
@@ -925,6 +953,7 @@ target_link_libraries(quickstep_storage_StorageManager
gflags_nothreads-static
glog
gtest
+ quickstep_catalog_CatalogTypedefs
quickstep_storage_CountedReference
quickstep_storage_EvictionPolicy
quickstep_storage_FileManager
@@ -955,7 +984,9 @@ if (ENABLE_DISTRIBUTED)
target_link_libraries(quickstep_storage_StorageManager
quickstep_queryexecution_QueryExecutionMessages_proto
quickstep_queryexecution_QueryExecutionTypedefs
- quickstep_queryexecution_QueryExecutionUtil)
+ quickstep_queryexecution_QueryExecutionUtil
+ quickstep_storage_DataExchange_proto
+ ${GRPCPLUSPLUS_LIBRARIES})
endif(ENABLE_DISTRIBUTED)
target_link_libraries(quickstep_storage_SubBlockTypeRegistry
glog
@@ -1071,6 +1102,11 @@ elseif (QUICKSTEP_HAVE_FILE_MANAGER_WINDOWS)
target_link_libraries(quickstep_storage
quickstep_storage_FileManagerWindows)
endif()
+if (ENABLE_DISTRIBUTED)
+ target_link_libraries(quickstep_storage
+ quickstep_storage_DataExchange_proto
+ quickstep_storage_DataExchangerAsync)
+endif()
# CMAKE_VALIDATE_IGNORE_BEGIN
if(QUICKSTEP_HAVE_BITWEAVING)
target_link_libraries(quickstep_storage
@@ -1340,6 +1376,37 @@ target_link_libraries(CompressedPackedRowStoreTupleStorageSubBlock_unittest
${LIBS})
add_test(CompressedPackedRowStoreTupleStorageSubBlock_unittest CompressedPackedRowStoreTupleStorageSubBlock_unittest)
+if (ENABLE_DISTRIBUTED)
+ add_executable(DataExchange_unittest
+ "${CMAKE_CURRENT_SOURCE_DIR}/tests/DataExchange_unittest.cpp")
+ target_link_libraries(DataExchange_unittest
+ gflags_nothreads-static
+ glog
+ gtest
+ quickstep_catalog_CatalogAttribute
+ quickstep_catalog_CatalogRelation
+ quickstep_catalog_CatalogTypedefs
+ quickstep_queryexecution_BlockLocator
+ quickstep_queryexecution_QueryExecutionMessages_proto
+ quickstep_queryexecution_QueryExecutionTypedefs
+ quickstep_queryexecution_QueryExecutionUtil
+ quickstep_storage_CountedReference
+ quickstep_storage_DataExchangerAsync
+ quickstep_storage_StorageBlob
+ quickstep_storage_StorageBlock
+ quickstep_storage_StorageBlockInfo
+ quickstep_storage_StorageConstants
+ quickstep_storage_StorageManager
+ quickstep_storage_TupleStorageSubBlock
+ quickstep_types_TypeFactory
+ quickstep_types_TypeID
+ quickstep_types_TypedValue
+ quickstep_types_containers_Tuple
+ tmb
+ ${LIBS})
+ add_test(DataExchange_unittest DataExchange_unittest)
+endif(ENABLE_DISTRIBUTED)
+
add_executable(CSBTreeIndexSubBlock_unittest "${CMAKE_CURRENT_SOURCE_DIR}/tests/CSBTreeIndexSubBlock_unittest.cpp")
target_link_libraries(CSBTreeIndexSubBlock_unittest
gtest
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/50b4e55a/storage/DataExchange.proto
----------------------------------------------------------------------
diff --git a/storage/DataExchange.proto b/storage/DataExchange.proto
new file mode 100644
index 0000000..a2636e5
--- /dev/null
+++ b/storage/DataExchange.proto
@@ -0,0 +1,31 @@
+// Copyright 2016 Pivotal Software, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto3";
+
+package quickstep;
+
+service DataExchange {
+ rpc Pull (PullRequest) returns (PullResponse) {}
+}
+
+message PullRequest {
+ fixed64 block_id = 1;
+}
+
+message PullResponse {
+ bool is_valid = 1;
+ uint64 num_slots = 2;
+ bytes block = 3;
+}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/50b4e55a/storage/DataExchangerAsync.cpp
----------------------------------------------------------------------
diff --git a/storage/DataExchangerAsync.cpp b/storage/DataExchangerAsync.cpp
new file mode 100644
index 0000000..78c6565
--- /dev/null
+++ b/storage/DataExchangerAsync.cpp
@@ -0,0 +1,165 @@
+/**
+ * Copyright 2016 Pivotal Software, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#include "storage/DataExchangerAsync.hpp"
+
+#include <grpc++/grpc++.h>
+
+#include <iostream>
+#include <memory>
+#include <string>
+
+#include "storage/DataExchange.grpc.pb.h"
+#include "storage/DataExchange.pb.h"
+#include "storage/StorageManager.hpp"
+
+#include "glog/logging.h"
+
+using grpc::ServerCompletionQueue;
+
+namespace quickstep {
+namespace {
+
+/**
+ * @brief RPC Request Context Instance.
+ **/
+class CallContext {
+ public:
+ /**
+ * @brief Constructor.
+ *
+ * @param service The async service.
+ * @param queue The RPC request queue.
+ * @param storage_manager The StorageManager to use.
+ **/
+ CallContext(DataExchange::AsyncService *service,
+ ServerCompletionQueue *queue,
+ StorageManager *storage_manager)
+ : service_(service),
+ queue_(queue),
+ storage_manager_(DCHECK_NOTNULL(storage_manager)),
+ responder_(&context_),
+ status_(CallStatus::CREATE) {
+ Proceed();
+ }
+
+ /**
+ * @brief Process the RPC request.
+ **/
+ void Proceed();
+
+ private:
+ DataExchange::AsyncService *service_;
+ ServerCompletionQueue *queue_;
+
+ StorageManager *storage_manager_;
+
+ grpc::ServerContext context_;
+
+ PullRequest request_;
+ PullResponse response_;
+
+ grpc::ServerAsyncResponseWriter<PullResponse> responder_;
+
+ enum class CallStatus {
+ CREATE = 0,
+ PROCESS,
+ FINISH,
+ };
+ CallStatus status_;
+};
+
+void CallContext::Proceed() {
+ switch (status_) {
+ case CallStatus::CREATE: {
+ // Change this instance progress to the PROCESS state.
+ status_ = CallStatus::PROCESS;
+
+ // As part of the initial CREATE state, we *request* that the system
+ // start processing Pull requests. In this request, "this" acts are
+ // the tag uniquely identifying the request (so that different CallContext
+ // instances can serve different requests concurrently), in this case
+ // the memory address of this CallContext instance.
+ service_->RequestPull(&context_, &request_, &responder_, queue_, queue_, this);
+ break;
+ }
+ case CallStatus::PROCESS: {
+ // Spawn a new CallContext instance to serve new clients while we process
+ // the one for this CallContext. The instance will deallocate itself as
+ // part of its FINISH state.
+ new CallContext(service_, queue_, storage_manager_);
+
+ // The actual processing.
+ storage_manager_->pullBlockOrBlob(request_.block_id(), &response_);
+
+ // And we are done! Let the gRPC runtime know we've finished, using the
+ // memory address of this instance as the uniquely identifying tag for
+ // the event.
+ status_ = CallStatus::FINISH;
+ responder_.Finish(response_, grpc::Status::OK, this);
+ break;
+ }
+ case CallStatus::FINISH: {
+ // Once in the FINISH state, deallocate ourselves (CallContext).
+ delete this;
+ break;
+ }
+ default:
+ LOG(FATAL) << "Unknown call status.";
+ }
+}
+
+} // namespace
+
+const char *DataExchangerAsync::kLocalNetworkAddress = "0.0.0.0:";
+
+DataExchangerAsync::DataExchangerAsync() {
+ grpc::ServerBuilder builder;
+ builder.AddListeningPort(kLocalNetworkAddress, grpc::InsecureServerCredentials(), &port_);
+ builder.RegisterService(&service_);
+
+ queue_ = builder.AddCompletionQueue();
+ server_ = builder.BuildAndStart();
+
+ DCHECK_GT(port_, 0);
+ server_address_ = kLocalNetworkAddress + std::to_string(port_);
+ LOG(INFO) << "DataExchangerAsync Service listening on " << server_address_;
+}
+
+void DataExchangerAsync::run() {
+ // Self-destruct upon success.
+ new CallContext(&service_, queue_.get(), storage_manager_);
+
+ void *tag = nullptr; // Uniquely identify a request.
+ bool ok = false;
+
+ while (true) {
+ if (queue_->Next(&tag, &ok)) {
+ CallContext *call_context = static_cast<CallContext*>(tag);
+ if (ok) {
+ call_context->Proceed();
+ } else {
+ LOG(WARNING) << "Not ok\n";
+ delete call_context;
+ }
+ } else {
+ LOG(INFO) << "Shutdown\n";
+ return;
+ }
+ }
+}
+
+} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/50b4e55a/storage/DataExchangerAsync.hpp
----------------------------------------------------------------------
diff --git a/storage/DataExchangerAsync.hpp b/storage/DataExchangerAsync.hpp
new file mode 100644
index 0000000..75a4e4d
--- /dev/null
+++ b/storage/DataExchangerAsync.hpp
@@ -0,0 +1,97 @@
+/**
+ * Copyright 2016 Pivotal Software, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#ifndef QUICKSTEP_STORAGE_DATA_EXCHANGER_ASYNC_HPP_
+#define QUICKSTEP_STORAGE_DATA_EXCHANGER_ASYNC_HPP_
+
+#include <grpc++/grpc++.h>
+
+#include <memory>
+#include <string>
+
+#include "storage/DataExchange.grpc.pb.h"
+#include "threading/Thread.hpp"
+#include "utility/Macros.hpp"
+
+#include "glog/logging.h"
+
+namespace quickstep {
+
+class StorageManager;
+
+/**
+ * @brief A class which exchanges data from a StorageManager to its peer.
+ **/
+class DataExchangerAsync final : public Thread {
+ public:
+ /**
+ * @brief Constructor.
+ **/
+ DataExchangerAsync();
+
+ ~DataExchangerAsync() override {}
+
+ /**
+ * @brief Set the local StorageManager.
+ *
+ * @param storage_manager The StorageManager to use.
+ **/
+ void set_storage_manager(StorageManager *storage_manager) {
+ storage_manager_ = storage_manager;
+ }
+
+ /**
+ * @brief Return its network address for peers to connect.
+ *
+ * @return Its network address.
+ **/
+ std::string network_address() const {
+ DCHECK(!server_address_.empty());
+ return server_address_;
+ }
+
+ /**
+ * @brief Shutdown itself.
+ **/
+ void shutdown() {
+ server_->Shutdown();
+ // Always shutdown the completion queue after the server.
+ queue_->Shutdown();
+ }
+
+ protected:
+ void run() override;
+
+ private:
+ static const char *kLocalNetworkAddress;
+
+ DataExchange::AsyncService service_;
+
+ int port_ = -1;
+ // Format IP:port, i.e., "0.0.0.0:0".
+ std::string server_address_;
+
+ std::unique_ptr<grpc::ServerCompletionQueue> queue_;
+ std::unique_ptr<grpc::Server> server_;
+
+ StorageManager *storage_manager_ = nullptr;
+
+ DISALLOW_COPY_AND_ASSIGN(DataExchangerAsync);
+};
+
+} // namespace quickstep
+
+#endif // QUICKSTEP_STORAGE_DATA_EXCHANGER_ASYNC_HPP_
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/50b4e55a/storage/StorageManager.cpp
----------------------------------------------------------------------
diff --git a/storage/StorageManager.cpp b/storage/StorageManager.cpp
index 15e2503..8fc1224 100644
--- a/storage/StorageManager.cpp
+++ b/storage/StorageManager.cpp
@@ -41,6 +41,10 @@
#include <numaif.h>
#endif
+#ifdef QUICKSTEP_DISTRIBUTED
+#include <grpc++/grpc++.h>
+#endif
+
#include <atomic>
#include <cerrno>
#include <cstddef>
@@ -53,6 +57,8 @@
#include <unordered_map>
#include <vector>
+#include "catalog/CatalogTypedefs.hpp"
+
#ifdef QUICKSTEP_DISTRIBUTED
#include "query_execution/QueryExecutionMessages.pb.h"
#include "query_execution/QueryExecutionTypedefs.hpp"
@@ -60,6 +66,12 @@
#endif
#include "storage/CountedReference.hpp"
+
+#ifdef QUICKSTEP_DISTRIBUTED
+#include "storage/DataExchange.grpc.pb.h"
+#include "storage/DataExchange.pb.h"
+#endif
+
#include "storage/EvictionPolicy.hpp"
#include "storage/FileManagerLocal.hpp"
#include "storage/StorageBlob.hpp"
@@ -456,6 +468,80 @@ block_id StorageManager::allocateNewBlockOrBlob(const std::size_t num_slots,
}
#ifdef QUICKSTEP_DISTRIBUTED
+void StorageManager::pullBlockOrBlob(const block_id block,
+ PullResponse *response) const {
+ SpinSharedMutexSharedLock<false> read_lock(blocks_shared_mutex_);
+ std::unordered_map<block_id, BlockHandle>::const_iterator cit = blocks_.find(block);
+ if (cit != blocks_.end()) {
+ response->set_is_valid(true);
+
+ const BlockHandle &block_handle = cit->second;
+ const std::size_t num_slots = block_handle.block_memory_size;
+
+ response->set_num_slots(num_slots);
+ response->set_block(block_handle.block_memory,
+ num_slots * kSlotSizeBytes);
+ } else {
+ response->set_is_valid(false);
+ }
+}
+
+StorageManager::DataExchangerClientAsync::DataExchangerClientAsync(const std::shared_ptr<grpc::Channel> &channel,
+ StorageManager *storage_manager)
+ : stub_(DataExchange::NewStub(channel)),
+ storage_manager_(storage_manager) {
+}
+
+bool StorageManager::DataExchangerClientAsync::Pull(const block_id block,
+ const numa_node_id numa_node,
+ BlockHandle *block_handle) {
+ grpc::ClientContext context;
+
+ PullRequest request;
+ request.set_block_id(block);
+
+ grpc::CompletionQueue queue;
+
+ unique_ptr<grpc::ClientAsyncResponseReader<PullResponse>> rpc(
+ stub_->AsyncPull(&context, request, &queue));
+
+ PullResponse response;
+ grpc::Status status;
+
+ rpc->Finish(&response, &status, reinterpret_cast<void*>(1));
+
+ void *got_tag;
+ bool ok = false;
+
+ queue.Next(&got_tag, &ok);
+ CHECK(got_tag == reinterpret_cast<void*>(1));
+ CHECK(ok);
+
+ if (!status.ok()) {
+ LOG(ERROR) << "DataExchangerClientAsync Pull error: RPC failed";
+ return false;
+ }
+
+ if (!response.is_valid()) {
+ LOG(INFO) << "The pulling block not found in all the peers";
+ return false;
+ }
+
+ const size_t num_slots = response.num_slots();
+ DCHECK_NE(num_slots, 0u);
+
+ const string &block_content = response.block();
+ DCHECK_EQ(kSlotSizeBytes * num_slots, block_content.size());
+
+ void *block_buffer = storage_manager_->allocateSlots(num_slots, numa_node);
+
+ block_handle->block_memory =
+ std::memcpy(block_buffer, block_content.c_str(), block_content.size());
+ block_handle->block_memory_size = num_slots;
+
+ return true;
+}
+
vector<string> StorageManager::getPeerDomainNetworkAddresses(const block_id block) {
serialization::BlockMessage proto;
proto.set_block_id(block);
@@ -541,14 +627,37 @@ StorageManager::BlockHandle StorageManager::loadBlockOrBlob(
// The caller of this function holds an exclusive lock on this block/blob's
// mutex in the lock manager. The caller has ensured that the block is not
// already loaded before this function gets called.
- size_t num_slots = file_manager_->numSlots(block);
+ BlockHandle loaded_handle;
+
+#ifdef QUICKSTEP_DISTRIBUTED
+ // TODO(quickstep-team): Use a cost model to determine whether to load from
+ // a remote peer or the disk.
+ if (BlockIdUtil::Domain(block) != block_domain_) {
+ LOG(INFO) << "Pulling Block " << BlockIdUtil::ToString(block) << " from a remote peer";
+ const vector<string> peer_domain_network_addresses = getPeerDomainNetworkAddresses(block);
+ for (const string &peer_domain_network_address : peer_domain_network_addresses) {
+ DataExchangerClientAsync client(
+ grpc::CreateChannel(peer_domain_network_address, grpc::InsecureChannelCredentials()),
+ this);
+
+ if (client.Pull(block, numa_node, &loaded_handle)) {
+ sendBlockLocationMessage(block, kAddBlockLocationMessage);
+ return loaded_handle;
+ }
+ }
+
+ LOG(INFO) << "Failed to pull Block " << BlockIdUtil::ToString(block)
+ << " from remote peers, so try to load from disk.";
+ }
+#endif
+
+ const size_t num_slots = file_manager_->numSlots(block);
DEBUG_ASSERT(num_slots != 0);
void *block_buffer = allocateSlots(num_slots, numa_node);
const bool status = file_manager_->readBlockOrBlob(block, block_buffer, kSlotSizeBytes * num_slots);
CHECK(status) << "Failed to read block from persistent storage: " << block;
- BlockHandle loaded_handle;
loaded_handle.block_memory = block_buffer;
loaded_handle.block_memory_size = num_slots;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/50b4e55a/storage/StorageManager.hpp
----------------------------------------------------------------------
diff --git a/storage/StorageManager.hpp b/storage/StorageManager.hpp
index 55a011e..50ddb0f 100644
--- a/storage/StorageManager.hpp
+++ b/storage/StorageManager.hpp
@@ -26,9 +26,14 @@
#include <unordered_map>
#include <vector>
+#include "catalog/CatalogTypedefs.hpp"
#include "query_optimizer/QueryOptimizerConfig.h" // For QUICKSTEP_DISTRIBUTED
-
#include "storage/CountedReference.hpp"
+
+#ifdef QUICKSTEP_DISTRIBUTED
+#include "storage/DataExchange.grpc.pb.h"
+#endif
+
#include "storage/EvictionPolicy.hpp"
#include "storage/FileManager.hpp"
#include "storage/StorageBlob.hpp"
@@ -45,6 +50,10 @@
#include "tmb/id_typedefs.h"
+#ifdef QUICKSTEP_DISTRIBUTED
+namespace grpc { class Channel; }
+#endif
+
namespace tmb { class MessageBus; }
namespace quickstep {
@@ -58,6 +67,10 @@ DECLARE_bool(use_hdfs);
class CatalogRelationSchema;
+#ifdef QUICKSTEP_DISTRIBUTED
+class PullResponse;
+#endif
+
class StorageBlockLayout;
/** \addtogroup Storage
@@ -365,6 +378,16 @@ class StorageManager {
**/
bool blockOrBlobIsLoadedAndDirty(const block_id block);
+#ifdef QUICKSTEP_DISTRIBUTED
+ /**
+ * @brief Pull a block or a blob. Used by DataExchangerAsync.
+ *
+ * @param block The id of the block or blob.
+ * @param response Where to store the pulled block content.
+ **/
+ void pullBlockOrBlob(const block_id block, PullResponse *response) const;
+#endif
+
private:
struct BlockHandle {
void *block_memory;
@@ -374,6 +397,42 @@ class StorageManager {
#ifdef QUICKSTEP_DISTRIBUTED
/**
+ * @brief A class which connects to DataExchangerAsync to exchange data from
+ * remote peers.
+ **/
+ class DataExchangerClientAsync {
+ public:
+ /**
+ * @brief Constructor.
+ *
+ * @param channel The RPC channel to connect DataExchangerAsync.
+ * @param storage_manager The StorageManager to use.
+ */
+ DataExchangerClientAsync(const std::shared_ptr<grpc::Channel> &channel,
+ StorageManager *storage_manager);
+
+ /**
+ * @brief Pull a block or blob from a remote StorageManager.
+ *
+ * @param block The block or blob to pull.
+ * @param numa_node The NUMA node for placing this block.
+ * @param block_handle Where the pulled block or blob stores.
+ *
+ * @return Whether the pull operation is successful or not.
+ */
+ bool Pull(const block_id block,
+ const numa_node_id numa_node,
+ BlockHandle *block_handle);
+
+ private:
+ std::unique_ptr<DataExchange::Stub> stub_;
+
+ StorageManager *storage_manager_;
+
+ DISALLOW_COPY_AND_ASSIGN(DataExchangerClientAsync);
+ };
+
+ /**
* @brief Get the network info of all the remote StorageManagers which may
* load the given block in the buffer pool.
*
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/50b4e55a/storage/tests/DataExchange_unittest.cpp
----------------------------------------------------------------------
diff --git a/storage/tests/DataExchange_unittest.cpp b/storage/tests/DataExchange_unittest.cpp
new file mode 100644
index 0000000..38d12f6
--- /dev/null
+++ b/storage/tests/DataExchange_unittest.cpp
@@ -0,0 +1,240 @@
+/**
+ * Copyright 2016 Pivotal Software, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#include <cstdlib>
+#include <memory>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "catalog/CatalogAttribute.hpp"
+#include "catalog/CatalogRelation.hpp"
+#include "catalog/CatalogTypedefs.hpp"
+#include "query_execution/BlockLocator.hpp"
+#include "query_execution/QueryExecutionMessages.pb.h"
+#include "query_execution/QueryExecutionTypedefs.hpp"
+#include "query_execution/QueryExecutionUtil.hpp"
+#include "storage/CountedReference.hpp"
+#include "storage/DataExchangerAsync.hpp"
+#include "storage/StorageBlob.hpp"
+#include "storage/StorageBlock.hpp"
+#include "storage/StorageBlockInfo.hpp"
+#include "storage/StorageConstants.hpp"
+#include "storage/StorageManager.hpp"
+#include "storage/TupleStorageSubBlock.hpp"
+#include "types/TypeFactory.hpp"
+#include "types/TypeID.hpp"
+#include "types/TypedValue.hpp"
+#include "types/containers/Tuple.hpp"
+
+#include "gflags/gflags.h"
+#include "glog/logging.h"
+#include "gtest/gtest.h"
+
+#include "tmb/id_typedefs.h"
+#include "tmb/message_bus.h"
+#include "tmb/tagged_message.h"
+
+using std::free;
+using std::malloc;
+using std::move;
+using std::string;
+using std::unique_ptr;
+using std::vector;
+
+using tmb::MessageBus;
+using tmb::TaggedMessage;
+
+namespace quickstep {
+
+class DataExchangeTest : public ::testing::Test {
+ protected:
+ static const char kStoragePath[];
+ static const char kCheckedDomainNetworkAddress[];
+
+ ~DataExchangeTest() {
+ data_exchanger_expected_.join();
+ locator_->join();
+ }
+
+ virtual void SetUp() {
+ bus_.Initialize();
+
+ locator_.reset(new BlockLocator(&bus_));
+ locator_client_id_ = locator_->getBusClientID();
+ locator_->start();
+
+ worker_client_id_ = bus_.Connect();
+ bus_.RegisterClientAsSender(worker_client_id_, kBlockDomainRegistrationMessage);
+ bus_.RegisterClientAsReceiver(worker_client_id_, kBlockDomainRegistrationResponseMessage);
+
+ bus_.RegisterClientAsSender(worker_client_id_, kPoisonMessage);
+
+ storage_manager_expected_.reset(new StorageManager(
+ kStoragePath,
+ getBlockDomain(data_exchanger_expected_.network_address()),
+ locator_client_id_,
+ &bus_));
+
+ data_exchanger_expected_.set_storage_manager(storage_manager_expected_.get());
+ data_exchanger_expected_.start();
+
+ storage_manager_checked_.reset(new StorageManager(
+ kStoragePath,
+ getBlockDomain(kCheckedDomainNetworkAddress),
+ locator_client_id_,
+ &bus_));
+ }
+
+ virtual void TearDown() {
+ storage_manager_checked_.reset();
+
+ data_exchanger_expected_.shutdown();
+ storage_manager_expected_.reset();
+
+ serialization::EmptyMessage proto;
+
+ const int proto_length = proto.ByteSize();
+ char *proto_bytes = static_cast<char*>(malloc(proto_length));
+ CHECK(proto.SerializeToArray(proto_bytes, proto_length));
+
+ TaggedMessage message(static_cast<const void*>(proto_bytes),
+ proto_length,
+ kPoisonMessage);
+ free(proto_bytes);
+
+ LOG(INFO) << "Worker (id '" << worker_client_id_
+ << "') sent PoisonMessage (typed '" << kPoisonMessage
+ << "') to BlockLocator (id '" << locator_client_id_ << "')";
+ CHECK(MessageBus::SendStatus::kOK ==
+ QueryExecutionUtil::SendTMBMessage(&bus_,
+ worker_client_id_,
+ locator_client_id_,
+ move(message)));
+ }
+
+ unique_ptr<StorageManager> storage_manager_expected_, storage_manager_checked_;
+
+ private:
+ block_id_domain getBlockDomain(const string &network_address) {
+ serialization::BlockDomainRegistrationMessage proto;
+ proto.set_domain_network_address(network_address);
+
+ const int proto_length = proto.ByteSize();
+ char *proto_bytes = static_cast<char*>(malloc(proto_length));
+ CHECK(proto.SerializeToArray(proto_bytes, proto_length));
+
+ TaggedMessage message(static_cast<const void*>(proto_bytes),
+ proto_length,
+ kBlockDomainRegistrationMessage);
+ free(proto_bytes);
+
+ LOG(INFO) << "Worker (id '" << worker_client_id_
+ << "') sent BlockDomainRegistrationMessage (typed '" << kBlockDomainRegistrationMessage
+ << "') to BlockLocator (id '" << locator_client_id_ << "')";
+
+ CHECK(MessageBus::SendStatus::kOK ==
+ QueryExecutionUtil::SendTMBMessage(&bus_,
+ worker_client_id_,
+ locator_client_id_,
+ move(message)));
+
+ const tmb::AnnotatedMessage annotated_message(bus_.Receive(worker_client_id_, 0, true));
+ const TaggedMessage &tagged_message = annotated_message.tagged_message;
+ EXPECT_EQ(locator_client_id_, annotated_message.sender);
+ EXPECT_EQ(kBlockDomainRegistrationResponseMessage, tagged_message.message_type());
+
+ LOG(INFO) << "Worker (id '" << worker_client_id_
+ << "') received BlockDomainRegistrationResponseMessage (typed '"
+ << kBlockDomainRegistrationResponseMessage
+ << "') from BlockLocator";
+
+ serialization::BlockDomainMessage response_proto;
+ CHECK(response_proto.ParseFromArray(tagged_message.message(), tagged_message.message_bytes()));
+
+ return static_cast<block_id_domain>(response_proto.block_domain());
+ }
+
+ MessageBusImpl bus_;
+
+ unique_ptr<BlockLocator> locator_;
+ tmb::client_id locator_client_id_;
+
+ tmb::client_id worker_client_id_;
+
+ DataExchangerAsync data_exchanger_expected_;
+};
+
+const char DataExchangeTest::kStoragePath[] = "./data_exchange_test_data/";
+const char DataExchangeTest::kCheckedDomainNetworkAddress[] = "ip:port";
+
+TEST_F(DataExchangeTest, BlockPull) {
+ CatalogRelation relation(nullptr, "rel");
+ const attribute_id attr_id =
+ relation.addAttribute(new CatalogAttribute(nullptr, "attr_int", TypeFactory::GetType(kInt)));
+
+ const block_id block =
+ storage_manager_expected_->createBlock(relation, relation.getDefaultStorageBlockLayout());
+
+ {
+ MutableBlockReference block_expected = storage_manager_expected_->getBlockMutable(block, relation);
+
+ // Insert a tuple.
+ const int value_expected = -1;
+ {
+ vector<TypedValue> attrs(1, TypedValue(value_expected));
+ const Tuple tuple(move(attrs));
+
+ EXPECT_TRUE(block_expected->insertTuple(tuple));
+ }
+
+ const BlockReference block_checked = storage_manager_checked_->getBlock(block, relation);
+ EXPECT_FALSE(block_checked->isBlob());
+
+ const TupleStorageSubBlock &tuple_store_checked = block_checked->getTupleStorageSubBlock();
+
+ EXPECT_EQ(1, tuple_store_checked.numTuples());
+ EXPECT_EQ(value_expected, tuple_store_checked.getAttributeValueTyped(0 /* tuple_id */, attr_id).getLiteral<int>());
+ }
+
+ storage_manager_checked_->deleteBlockOrBlobFile(block);
+ storage_manager_expected_->deleteBlockOrBlobFile(block);
+}
+
+TEST_F(DataExchangeTest, BlobPull) {
+ const block_id blob = storage_manager_expected_->createBlob(kDefaultBlockSizeInSlots);
+ {
+ const BlobReference blob_expected = storage_manager_expected_->getBlob(blob);
+ const BlobReference blob_checked = storage_manager_checked_->getBlob(blob);
+ EXPECT_TRUE(blob_checked->isBlob());
+ EXPECT_EQ(blob, blob_checked->getID());
+ }
+
+ storage_manager_checked_->deleteBlockOrBlobFile(blob);
+ storage_manager_expected_->deleteBlockOrBlobFile(blob);
+}
+
+} // namespace quickstep
+
+int main(int argc, char **argv) {
+ google::InitGoogleLogging(argv[0]);
+ // Honor FLAGS_buffer_pool_slots in StorageManager.
+ gflags::ParseCommandLineFlags(&argc, &argv, true);
+
+ ::testing::InitGoogleTest(&argc, argv);
+
+ return RUN_ALL_TESTS();
+}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/50b4e55a/third_party/iwyu/iwyu_helper.py
----------------------------------------------------------------------
diff --git a/third_party/iwyu/iwyu_helper.py b/third_party/iwyu/iwyu_helper.py
index a204c50..13697be 100755
--- a/third_party/iwyu/iwyu_helper.py
+++ b/third_party/iwyu/iwyu_helper.py
@@ -19,8 +19,9 @@ import sys
QUICKSTEP_INCLUDES = [ '.',
'./build',
'./build/third_party',
- './build/third_party/protobuf/include',
'./build/third_party/gflags/include',
+ './build/third_party/protobuf/include',
+ './build/third_party/tmb/include',
'./third_party/benchmark/include',
'./third_party/glog/src',
'./third_party/googletest/googletest/include',
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/50b4e55a/validate_cmakelists.py
----------------------------------------------------------------------
diff --git a/validate_cmakelists.py b/validate_cmakelists.py
index c7b5883..7dd6fc5 100755
--- a/validate_cmakelists.py
+++ b/validate_cmakelists.py
@@ -17,7 +17,7 @@ TODO List / Known Issues & Limitations:
"""
# Copyright 2011-2015 Quickstep Technologies LLC.
-# Copyright 2015 Pivotal Software, Inc.
+# Copyright 2015-2016 Pivotal Software, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -40,7 +40,8 @@ EXCLUDED_TOP_LEVEL_DIRS = ["build", "third_party"]
# Explicitly ignored dependencies (special headers with no other quickstep
# dependencies).
IGNORED_DEPENDENCIES = frozenset(
- ["quickstep_threading_WinThreadsAPI",
+ ["quickstep_storage_DataExchange.grpc_proto",
+ "quickstep_threading_WinThreadsAPI",
"quickstep_utility_textbasedtest_TextBasedTest",
"quickstep_utility_textbasedtest_TextBasedTestDriver",
"quickstep_storage_bitweaving_BitWeavingHIndexSubBlock",
[09/50] [abbrv] incubator-quickstep git commit: Partition aware
selection using NUMA-awareness (#175)
Posted by zu...@apache.org.
Partition aware selection using NUMA-awareness (#175)
Link: https://github.com/pivotalsoftware/quickstep/pull/175
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/89acf230
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/89acf230
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/89acf230
Branch: refs/heads/work-order-serialization
Commit: 89acf230e85f5b3d1bdf01bb5decdf76f6dec128
Parents: 5ea898b
Author: Adalbert Gerald Soosai Raj <ad...@gmail.com>
Authored: Sun May 1 15:38:38 2016 -0500
Committer: Harshad Deshmukh <d....@gmail.com>
Committed: Sun May 1 15:38:38 2016 -0500
----------------------------------------------------------------------
catalog/CMakeLists.txt | 5 +-
catalog/CatalogRelation.hpp | 9 ++
relational_operators/CMakeLists.txt | 5 ++
relational_operators/SelectOperator.cpp | 129 +++++++++++++++++++++------
relational_operators/SelectOperator.hpp | 115 +++++++++++++++++++++---
5 files changed, 226 insertions(+), 37 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/89acf230/catalog/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/catalog/CMakeLists.txt b/catalog/CMakeLists.txt
index 8c89d7e..94da838 100644
--- a/catalog/CMakeLists.txt
+++ b/catalog/CMakeLists.txt
@@ -175,9 +175,12 @@ target_link_libraries(quickstep_catalog
quickstep_catalog_CatalogRelationSchema
quickstep_catalog_CatalogTypedefs
quickstep_catalog_IndexScheme
- quickstep_catalog_NUMAPlacementScheme
quickstep_catalog_PartitionScheme
quickstep_catalog_PartitionSchemeHeader)
+if(QUICKSTEP_HAVE_LIBNUMA)
+target_link_libraries(quickstep_catalog
+ quickstep_catalog_NUMAPlacementScheme)
+endif()
# Tests:
add_executable(Catalog_unittest "${CMAKE_CURRENT_SOURCE_DIR}/tests/Catalog_unittest.cpp")
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/89acf230/catalog/CatalogRelation.hpp
----------------------------------------------------------------------
diff --git a/catalog/CatalogRelation.hpp b/catalog/CatalogRelation.hpp
index 4cc8d79..3701090 100644
--- a/catalog/CatalogRelation.hpp
+++ b/catalog/CatalogRelation.hpp
@@ -165,6 +165,15 @@ class CatalogRelation : public CatalogRelationSchema {
}
/**
+ * @brief Get the NUMA placement scheme of the relation.
+ *
+ * @return A pointer to a const NUMA placement scheme.
+ **/
+ const NUMAPlacementScheme* getNUMAPlacementSchemePtr() const {
+ return placement_scheme_.get();
+ }
+
+ /**
* @brief Set the NUMA placement scheme for the catalog relation.
*
* @param placement_scheme The NUMA placement scheme object for the relation,
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/89acf230/relational_operators/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/relational_operators/CMakeLists.txt b/relational_operators/CMakeLists.txt
index a4600e6..eec5300 100644
--- a/relational_operators/CMakeLists.txt
+++ b/relational_operators/CMakeLists.txt
@@ -266,6 +266,7 @@ target_link_libraries(quickstep_relationaloperators_SelectOperator
glog
quickstep_catalog_CatalogRelation
quickstep_catalog_CatalogTypedefs
+ quickstep_catalog_PartitionSchemeHeader
quickstep_queryexecution_QueryContext
quickstep_queryexecution_WorkOrdersContainer
quickstep_relationaloperators_RelationalOperator
@@ -276,6 +277,10 @@ target_link_libraries(quickstep_relationaloperators_SelectOperator
quickstep_storage_StorageManager
quickstep_utility_Macros
tmb)
+if(QUICKSTEP_HAVE_LIBNUMA)
+target_link_libraries(quickstep_relationaloperators_SelectOperator
+ quickstep_catalog_NUMAPlacementScheme)
+endif()
target_link_libraries(quickstep_relationaloperators_SortMergeRunOperator
glog
quickstep_catalog_CatalogRelation
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/89acf230/relational_operators/SelectOperator.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/SelectOperator.cpp b/relational_operators/SelectOperator.cpp
index 3cac199..69bb434 100644
--- a/relational_operators/SelectOperator.cpp
+++ b/relational_operators/SelectOperator.cpp
@@ -35,6 +35,93 @@ namespace quickstep {
class Predicate;
+void SelectOperator::addWorkOrders(WorkOrdersContainer *container,
+ StorageManager *storage_manager,
+ const Predicate *predicate,
+ const std::vector<std::unique_ptr<const Scalar>> *selection,
+ InsertDestination *output_destination) {
+ if (input_relation_is_stored_) {
+ for (const block_id input_block_id : input_relation_block_ids_) {
+ container->addNormalWorkOrder(
+ new SelectWorkOrder(input_relation_,
+ input_block_id,
+ predicate,
+ simple_projection_,
+ simple_selection_,
+ selection,
+ output_destination,
+ storage_manager),
+ op_index_);
+ }
+ } else {
+ while (num_workorders_generated_ < input_relation_block_ids_.size()) {
+ container->addNormalWorkOrder(
+ new SelectWorkOrder(
+ input_relation_,
+ input_relation_block_ids_[num_workorders_generated_],
+ predicate,
+ simple_projection_,
+ simple_selection_,
+ selection,
+ output_destination,
+ storage_manager),
+ op_index_);
+ ++num_workorders_generated_;
+ }
+ }
+}
+
+#ifdef QUICKSTEP_HAVE_LIBNUMA
+void SelectOperator::addPartitionAwareWorkOrders(WorkOrdersContainer *container,
+ StorageManager *storage_manager,
+ const Predicate *predicate,
+ const std::vector<std::unique_ptr<const Scalar>> *selection,
+ InsertDestination *output_destination) {
+ DCHECK(placement_scheme_ != nullptr);
+ const std::size_t num_partitions = input_relation_.getPartitionScheme().getPartitionSchemeHeader().getNumPartitions();
+ if (input_relation_is_stored_) {
+ for (std::size_t part_id = 0; part_id < num_partitions; ++part_id) {
+ for (const block_id input_block_id :
+ input_relation_block_ids_in_partition_[part_id]) {
+ container->addNormalWorkOrder(
+ new SelectWorkOrder(
+ input_relation_,
+ input_block_id,
+ predicate,
+ simple_projection_,
+ simple_selection_,
+ selection,
+ output_destination,
+ storage_manager,
+ placement_scheme_->getNUMANodeForBlock(input_block_id)),
+ op_index_);
+ }
+ }
+ } else {
+ for (std::size_t part_id = 0; part_id < num_partitions; ++part_id) {
+ while (num_workorders_generated_in_partition_[part_id] <
+ input_relation_block_ids_in_partition_[part_id].size()) {
+ block_id block_in_partition
+ = input_relation_block_ids_in_partition_[part_id][num_workorders_generated_in_partition_[part_id]];
+ container->addNormalWorkOrder(
+ new SelectWorkOrder(
+ input_relation_,
+ block_in_partition,
+ predicate,
+ simple_projection_,
+ simple_selection_,
+ selection,
+ output_destination,
+ storage_manager,
+ placement_scheme_->getNUMANodeForBlock(block_in_partition)),
+ op_index_);
+ ++num_workorders_generated_in_partition_[part_id];
+ }
+ }
+ }
+}
+#endif
+
bool SelectOperator::getAllWorkOrders(
WorkOrdersContainer *container,
QueryContext *query_context,
@@ -54,35 +141,27 @@ bool SelectOperator::getAllWorkOrders(
if (input_relation_is_stored_) {
if (!started_) {
- for (const block_id input_block_id : input_relation_block_ids_) {
- container->addNormalWorkOrder(
- new SelectWorkOrder(input_relation_,
- input_block_id,
- predicate,
- simple_projection_,
- simple_selection_,
- selection,
- output_destination,
- storage_manager),
- op_index_);
+ if (input_relation_.hasPartitionScheme()) {
+#ifdef QUICKSTEP_HAVE_LIBNUMA
+ if (input_relation_.hasNUMAPlacementScheme()) {
+ addPartitionAwareWorkOrders(container, storage_manager, predicate, selection, output_destination);
+ }
+#endif
+ } else {
+ addWorkOrders(container, storage_manager, predicate, selection, output_destination);
}
started_ = true;
}
return started_;
} else {
- while (num_workorders_generated_ < input_relation_block_ids_.size()) {
- container->addNormalWorkOrder(
- new SelectWorkOrder(
- input_relation_,
- input_relation_block_ids_[num_workorders_generated_],
- predicate,
- simple_projection_,
- simple_selection_,
- selection,
- output_destination,
- storage_manager),
- op_index_);
- ++num_workorders_generated_;
+ if (input_relation_.hasPartitionScheme()) {
+#ifdef QUICKSTEP_HAVE_LIBNUMA
+ if (input_relation_.hasNUMAPlacementScheme()) {
+ addPartitionAwareWorkOrders(container, storage_manager, predicate, selection, output_destination);
+ }
+#endif
+ } else {
+ addWorkOrders(container, storage_manager, predicate, selection, output_destination);
}
return done_feeding_input_relation_;
}
@@ -90,7 +169,7 @@ bool SelectOperator::getAllWorkOrders(
void SelectWorkOrder::execute() {
BlockReference block(
- storage_manager_->getBlock(input_block_id_, input_relation_));
+ storage_manager_->getBlock(input_block_id_, input_relation_, getPreferredNUMANodes()[0]));
if (simple_projection_) {
block->selectSimple(simple_selection_,
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/89acf230/relational_operators/SelectOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/SelectOperator.hpp b/relational_operators/SelectOperator.hpp
index 3da496c..76f4cb6 100644
--- a/relational_operators/SelectOperator.hpp
+++ b/relational_operators/SelectOperator.hpp
@@ -24,6 +24,12 @@
#include "catalog/CatalogRelation.hpp"
#include "catalog/CatalogTypedefs.hpp"
+
+#ifdef QUICKSTEP_HAVE_LIBNUMA
+#include "catalog/NUMAPlacementScheme.hpp"
+#endif
+
+#include "catalog/PartitionSchemeHeader.hpp"
#include "query_execution/QueryContext.hpp"
#include "relational_operators/RelationalOperator.hpp"
#include "relational_operators/WorkOrder.hpp"
@@ -87,7 +93,28 @@ class SelectOperator : public RelationalOperator {
num_workorders_generated_(0),
simple_projection_(false),
input_relation_is_stored_(input_relation_is_stored),
- started_(false) {}
+ started_(false) {
+#ifdef QUICKSTEP_HAVE_LIBNUMA
+ placement_scheme_ = input_relation.getNUMAPlacementSchemePtr();
+#endif
+ if (input_relation.hasPartitionScheme()) {
+ const PartitionScheme &part_scheme = input_relation.getPartitionScheme();
+ const PartitionSchemeHeader &part_scheme_header = part_scheme.getPartitionSchemeHeader();
+ const std::size_t num_partitions = part_scheme_header.getNumPartitions();
+ input_relation_block_ids_in_partition_.resize(num_partitions);
+ num_workorders_generated_in_partition_.resize(num_partitions);
+ num_workorders_generated_in_partition_.assign(num_partitions, 0);
+ for (std::size_t part_id = 0; part_id < num_partitions; ++part_id) {
+ if (input_relation_is_stored) {
+ input_relation_block_ids_in_partition_[part_id] =
+ part_scheme.getBlocksInPartition(part_id);
+ } else {
+ input_relation_block_ids_in_partition_[part_id] =
+ std::vector<block_id>();
+ }
+ }
+ }
+ }
/**
* @brief Constructor for selection with simple projection of attributes.
@@ -124,7 +151,28 @@ class SelectOperator : public RelationalOperator {
num_workorders_generated_(0),
simple_projection_(true),
input_relation_is_stored_(input_relation_is_stored),
- started_(false) {}
+ started_(false) {
+#ifdef QUICKSTEP_HAVE_LIBNUMA
+ placement_scheme_ = input_relation.getNUMAPlacementSchemePtr();
+#endif
+ if (input_relation.hasPartitionScheme()) {
+ const PartitionScheme &part_scheme = input_relation.getPartitionScheme();
+ const PartitionSchemeHeader &part_scheme_header = part_scheme.getPartitionSchemeHeader();
+ const std::size_t num_partitions = part_scheme_header.getNumPartitions();
+ input_relation_block_ids_in_partition_.resize(num_partitions);
+ num_workorders_generated_in_partition_.resize(num_partitions);
+ num_workorders_generated_in_partition_.assign(num_partitions, 0);
+ for (std::size_t part_id = 0; part_id < num_partitions; ++part_id) {
+ if (input_relation_is_stored) {
+ input_relation_block_ids_in_partition_[part_id] =
+ part_scheme.getBlocksInPartition(part_id);
+ } else {
+ input_relation_block_ids_in_partition_[part_id] =
+ std::vector<block_id>();
+ }
+ }
+ }
+ }
~SelectOperator() override {}
@@ -135,13 +183,33 @@ class SelectOperator : public RelationalOperator {
tmb::MessageBus *bus) override;
void feedInputBlock(const block_id input_block_id, const relation_id input_relation_id) override {
- input_relation_block_ids_.push_back(input_block_id);
+ if (input_relation_.hasPartitionScheme()) {
+ const partition_id part_id =
+ input_relation_.getPartitionScheme().getPartitionForBlock(input_block_id);
+ input_relation_block_ids_in_partition_[part_id].push_back(input_block_id);
+ } else {
+ input_relation_block_ids_.push_back(input_block_id);
+ }
}
+ // TODO(gerald): Each call to getPartitionForBlock() involves grabbing shared
+ // locks on each partition's mutex, checking if the block belongs to the
+ // partition. Instead, we can provide a method getPartitionsForBlocks() which
+ // accepts a list of blocks and returns corresponding list of their partition IDs.
+ // Therefore, once we grab a lock for a partition, we search for all the blocks
+ // and then release the lock.
void feedInputBlocks(const relation_id rel_id, std::vector<block_id> *partially_filled_blocks) override {
- input_relation_block_ids_.insert(input_relation_block_ids_.end(),
- partially_filled_blocks->begin(),
- partially_filled_blocks->end());
+ if (input_relation_.hasPartitionScheme()) {
+ for (auto it = partially_filled_blocks->begin(); it != partially_filled_blocks->end(); ++it) {
+ const partition_id part_id = input_relation_.getPartitionScheme().getPartitionForBlock((*it));
+ input_relation_block_ids_in_partition_[part_id].insert(input_relation_block_ids_in_partition_[part_id].end(),
+ *it);
+ }
+ } else {
+ input_relation_block_ids_.insert(input_relation_block_ids_.end(),
+ partially_filled_blocks->begin(),
+ partially_filled_blocks->end());
+ }
}
QueryContext::insert_destination_id getInsertDestinationID() const override {
@@ -152,9 +220,20 @@ class SelectOperator : public RelationalOperator {
return output_relation_.getID();
}
+ void addWorkOrders(WorkOrdersContainer *container,
+ StorageManager *storage_manager,
+ const Predicate *predicate,
+ const std::vector<std::unique_ptr<const Scalar>> *selection,
+ InsertDestination *output_destination);
+
+ void addPartitionAwareWorkOrders(WorkOrdersContainer *container,
+ StorageManager *storage_manager,
+ const Predicate *predicate,
+ const std::vector<std::unique_ptr<const Scalar>> *selection,
+ InsertDestination *output_destination);
+
private:
const CatalogRelation &input_relation_;
-
const CatalogRelation &output_relation_;
const QueryContext::insert_destination_id output_destination_index_;
const QueryContext::predicate_id predicate_index_;
@@ -163,12 +242,20 @@ class SelectOperator : public RelationalOperator {
const std::vector<attribute_id> simple_selection_;
std::vector<block_id> input_relation_block_ids_;
+ // A vector of vectors V where V[i] indicates the list of block IDs of the
+ // input relation that belong to the partition i.
+ std::vector<std::vector<block_id>> input_relation_block_ids_in_partition_;
// A single workorder is generated for each block of input relation.
std::vector<block_id>::size_type num_workorders_generated_;
+ // A single workorder is generated for each block in each partition of input relation.
+ std::vector<std::size_t> num_workorders_generated_in_partition_;
const bool simple_projection_;
const bool input_relation_is_stored_;
+#ifdef QUICKSTEP_HAVE_LIBNUMA
+ const NUMAPlacementScheme *placement_scheme_;
+#endif
bool started_;
DISALLOW_COPY_AND_ASSIGN(SelectOperator);
@@ -205,7 +292,8 @@ class SelectWorkOrder : public WorkOrder {
const std::vector<attribute_id> &simple_selection,
const std::vector<std::unique_ptr<const Scalar>> *selection,
InsertDestination *output_destination,
- StorageManager *storage_manager)
+ StorageManager *storage_manager,
+ const numa_node_id numa_node = 0)
: input_relation_(input_relation),
input_block_id_(input_block_id),
predicate_(predicate),
@@ -213,7 +301,9 @@ class SelectWorkOrder : public WorkOrder {
simple_selection_(simple_selection),
selection_(selection),
output_destination_(DCHECK_NOTNULL(output_destination)),
- storage_manager_(DCHECK_NOTNULL(storage_manager)) {}
+ storage_manager_(DCHECK_NOTNULL(storage_manager)) {
+ preferred_numa_nodes_.push_back(numa_node);
+ }
/**
* @brief Constructor for the distributed version.
@@ -241,7 +331,8 @@ class SelectWorkOrder : public WorkOrder {
std::vector<attribute_id> &&simple_selection,
const std::vector<std::unique_ptr<const Scalar>> *selection,
InsertDestination *output_destination,
- StorageManager *storage_manager)
+ StorageManager *storage_manager,
+ const numa_node_id numa_node = 0)
: input_relation_(input_relation),
input_block_id_(input_block_id),
predicate_(predicate),
@@ -249,7 +340,9 @@ class SelectWorkOrder : public WorkOrder {
simple_selection_(std::move(simple_selection)),
selection_(selection),
output_destination_(DCHECK_NOTNULL(output_destination)),
- storage_manager_(DCHECK_NOTNULL(storage_manager)) {}
+ storage_manager_(DCHECK_NOTNULL(storage_manager)) {
+ preferred_numa_nodes_.push_back(numa_node);
+ }
~SelectWorkOrder() override {}
[26/50] [abbrv] incubator-quickstep git commit: Added a method for
double parsing with significant number argument. (#216)
Posted by zu...@apache.org.
Added a method for double parsing with significant number argument. (#216)
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/4937e684
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/4937e684
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/4937e684
Branch: refs/heads/work-order-serialization
Commit: 4937e6840d8a6fbb1034bd9d4ad9ae50b8446e9d
Parents: ab79cf6
Author: Hakan Memisoglu <ha...@gmail.com>
Authored: Sun May 8 08:22:17 2016 -0500
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Mon May 30 15:46:32 2016 -0700
----------------------------------------------------------------------
utility/StringUtil.cpp | 14 ++++++++++++++
utility/StringUtil.hpp | 18 ++++++++++++++++++
2 files changed, 32 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4937e684/utility/StringUtil.cpp
----------------------------------------------------------------------
diff --git a/utility/StringUtil.cpp b/utility/StringUtil.cpp
index 77b0a3a..b38e8f4 100644
--- a/utility/StringUtil.cpp
+++ b/utility/StringUtil.cpp
@@ -22,10 +22,13 @@
#include <algorithm>
#include <cctype>
#include <cinttypes>
+#include <cmath>
#include <cstddef>
#include <cstdint>
#include <cstdio>
#include <cstdlib>
+#include <iomanip>
+#include <sstream>
#include <string>
#include <vector>
@@ -128,4 +131,15 @@ string ToZeroPaddedString(std::uint64_t val, int pad_width) {
return result;
}
+std::string
+DoubleToStringWithSignificantDigits(double val,
+ std::uint64_t significant_digits) {
+ std::uint64_t precision_needed =
+ (val >= 1.0) ? significant_digits
+ : significant_digits + -(std::trunc(std::log10(val)));
+ std::stringstream string_buffer;
+ string_buffer << std::fixed << std::setprecision(precision_needed) << val;
+ return string_buffer.str();
+}
+
} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4937e684/utility/StringUtil.hpp
----------------------------------------------------------------------
diff --git a/utility/StringUtil.hpp b/utility/StringUtil.hpp
index 26c04a4..bd138bb 100644
--- a/utility/StringUtil.hpp
+++ b/utility/StringUtil.hpp
@@ -70,6 +70,24 @@ extern bool ParseIntString(const std::string &int_string,
*/
extern std::string ToZeroPaddedString(std::uint64_t val, int pad_width);
+
+/**
+ * @brief Parse double with significant digits.
+ *
+ * @param val The value that will be parsed.
+ * @param significant_digit The number of non-zero digits that will be parsed
+ * in the fraction part of double.
+ * @return Corresponding string representation of parsed double.
+ * Ex:
+ * DoubleToStringWithSignificantDigits(3.124355123, 3)
+ * => 3.124
+ * DoubleToStringWithSignificantDigits(0.00000000323411, 3)
+ * => 0.00000000323
+ **/
+extern std::string
+DoubleToStringWithSignificantDigits(double val,
+ std::uint64_t significant_digits);
+
/** @} */
} // namespace quickstep
[23/50] [abbrv] incubator-quickstep git commit: Enable semi-join
optimization for left-deep trees through bloom filters (#195)
Posted by zu...@apache.org.
Enable semi-join optimization for left-deep trees through bloom filters (#195)
Link: https://github.com/pivotalsoftware/quickstep/pull/195
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/21b85088
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/21b85088
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/21b85088
Branch: refs/heads/work-order-serialization
Commit: 21b85088f5f2b177ade45d3bc86612501b7a92d4
Parents: 6f18495
Author: Saket Saurabh <sa...@users.noreply.github.com>
Authored: Mon May 9 12:29:47 2016 -0500
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Mon May 30 15:46:32 2016 -0700
----------------------------------------------------------------------
query_execution/CMakeLists.txt | 2 +
query_execution/QueryContext.cpp | 16 +-
query_execution/QueryContext.hpp | 57 +++-
query_execution/QueryContext.proto | 18 +-
query_optimizer/CMakeLists.txt | 11 +
query_optimizer/ExecutionGenerator.cpp | 55 +++-
query_optimizer/ExecutionGenerator.hpp | 5 +-
query_optimizer/ExecutionHeuristics.cpp | 127 ++++++++
query_optimizer/ExecutionHeuristics.hpp | 155 ++++++++++
query_optimizer/tests/CMakeLists.txt | 16 +
.../tests/ExecutionHeuristics_unittest.cpp | 301 +++++++++++++++++++
storage/CMakeLists.txt | 4 +-
storage/HashTable.hpp | 104 +++++++
storage/HashTable.proto | 8 +
storage/HashTableFactory.hpp | 44 ++-
utility/BloomFilter.hpp | 198 +++++++++++-
utility/BloomFilter.proto | 30 ++
utility/CMakeLists.txt | 15 +
18 files changed, 1135 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/21b85088/query_execution/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_execution/CMakeLists.txt b/query_execution/CMakeLists.txt
index 5887237..04a0348 100644
--- a/query_execution/CMakeLists.txt
+++ b/query_execution/CMakeLists.txt
@@ -90,9 +90,11 @@ target_link_libraries(quickstep_queryexecution_QueryContext
quickstep_storage_InsertDestination_proto
quickstep_types_TypedValue
quickstep_types_containers_Tuple
+ quickstep_utility_BloomFilter
quickstep_utility_Macros
quickstep_utility_SortConfiguration)
target_link_libraries(quickstep_queryexecution_QueryContext_proto
+ quickstep_utility_BloomFilter_proto
quickstep_expressions_Expressions_proto
quickstep_expressions_tablegenerator_GeneratorFunction_proto
quickstep_storage_AggregationOperationState_proto
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/21b85088/query_execution/QueryContext.cpp
----------------------------------------------------------------------
diff --git a/query_execution/QueryContext.cpp b/query_execution/QueryContext.cpp
index b0e9cae..3bfce17 100644
--- a/query_execution/QueryContext.cpp
+++ b/query_execution/QueryContext.cpp
@@ -1,6 +1,8 @@
/**
* Copyright 2011-2015 Quickstep Technologies LLC.
* Copyright 2015-2016 Pivotal Software, Inc.
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -37,6 +39,7 @@
#include "storage/InsertDestination.pb.h"
#include "types/TypedValue.hpp"
#include "types/containers/Tuple.hpp"
+#include "utility/BloomFilter.hpp"
#include "utility/SortConfiguration.hpp"
#include "glog/logging.h"
@@ -65,6 +68,10 @@ QueryContext::QueryContext(const serialization::QueryContext &proto,
storage_manager));
}
+ for (int i = 0; i < proto.bloom_filters_size(); ++i) {
+ bloom_filters_.emplace_back(new BloomFilter(proto.bloom_filters(i)));
+ }
+
for (int i = 0; i < proto.generator_functions_size(); ++i) {
const GeneratorFunctionHandle *func_handle =
GeneratorFunctionFactory::Instance().reconstructFromProto(proto.generator_functions(i));
@@ -76,7 +83,8 @@ QueryContext::QueryContext(const serialization::QueryContext &proto,
for (int i = 0; i < proto.join_hash_tables_size(); ++i) {
join_hash_tables_.emplace_back(
JoinHashTableFactory::CreateResizableFromProto(proto.join_hash_tables(i),
- storage_manager));
+ storage_manager,
+ bloom_filters_));
}
for (int i = 0; i < proto.insert_destinations_size(); ++i) {
@@ -142,6 +150,12 @@ bool QueryContext::ProtoIsValid(const serialization::QueryContext &proto,
}
}
+ for (int i = 0; i < proto.bloom_filters_size(); ++i) {
+ if (!BloomFilter::ProtoIsValid(proto.bloom_filters(i))) {
+ return false;
+ }
+ }
+
// Each GeneratorFunctionHandle object is serialized as a function name with
// a list of arguments. Here checks that the arguments are valid TypedValue's.
for (int i = 0; i < proto.generator_functions_size(); ++i) {
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/21b85088/query_execution/QueryContext.hpp
----------------------------------------------------------------------
diff --git a/query_execution/QueryContext.hpp b/query_execution/QueryContext.hpp
index 0e9e21c..9440fae 100644
--- a/query_execution/QueryContext.hpp
+++ b/query_execution/QueryContext.hpp
@@ -1,6 +1,8 @@
/**
* Copyright 2011-2015 Quickstep Technologies LLC.
* Copyright 2015-2016 Pivotal Software, Inc.
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -32,6 +34,7 @@
#include "storage/HashTable.hpp"
#include "storage/InsertDestination.hpp"
#include "types/containers/Tuple.hpp"
+#include "utility/BloomFilter.hpp"
#include "utility/Macros.hpp"
#include "utility/SortConfiguration.hpp"
@@ -63,6 +66,11 @@ class QueryContext {
typedef std::uint32_t aggregation_state_id;
/**
+ * @brief A unique identifier for a BloomFilter per query.
+ **/
+ typedef std::uint32_t bloom_filter_id;
+
+ /**
* @brief A unique identifier for a GeneratorFunctionHandle per query.
**/
typedef std::uint32_t generator_function_id;
@@ -181,6 +189,52 @@ class QueryContext {
}
/**
+ * @brief Whether the given BloomFilter id is valid.
+ *
+ * @param id The BloomFilter id.
+ *
+ * @return True if valid, otherwise false.
+ **/
+ bool isValidBloomFilterId(const bloom_filter_id id) const {
+ return id < bloom_filters_.size();
+ }
+
+ /**
+ * @brief Get a mutable reference to the BloomFilter.
+ *
+ * @param id The BloomFilter id.
+ *
+ * @return The BloomFilter, already created in the constructor.
+ **/
+ inline BloomFilter* getBloomFilterMutable(const bloom_filter_id id) {
+ DCHECK_LT(id, bloom_filters_.size());
+ return bloom_filters_[id].get();
+ }
+
+ /**
+ * @brief Get a constant pointer to the BloomFilter.
+ *
+ * @param id The BloomFilter id.
+ *
+ * @return The constant pointer to BloomFilter that is
+ * already created in the constructor.
+ **/
+ inline const BloomFilter* getBloomFilter(const bloom_filter_id id) const {
+ DCHECK_LT(id, bloom_filters_.size());
+ return bloom_filters_[id].get();
+ }
+
+ /**
+ * @brief Destory the given BloomFilter.
+ *
+ * @param id The id of the BloomFilter to destroy.
+ **/
+ inline void destroyBloomFilter(const bloom_filter_id id) {
+ DCHECK_LT(id, bloom_filters_.size());
+ bloom_filters_[id].reset();
+ }
+
+ /**
* @brief Whether the given GeneratorFunctionHandle id is valid.
*
* @param id The GeneratorFunctionHandle id.
@@ -257,7 +311,7 @@ class QueryContext {
*
* @param id The JoinHashTable id in the query.
*
- * @return The JoinHashTable, alreadly created in the constructor.
+ * @return The JoinHashTable, already created in the constructor.
**/
inline JoinHashTable* getJoinHashTable(const join_hash_table_id id) {
DCHECK_LT(id, join_hash_tables_.size());
@@ -408,6 +462,7 @@ class QueryContext {
private:
std::vector<std::unique_ptr<AggregationOperationState>> aggregation_states_;
+ std::vector<std::unique_ptr<BloomFilter>> bloom_filters_;
std::vector<std::unique_ptr<const GeneratorFunctionHandle>> generator_functions_;
std::vector<std::unique_ptr<InsertDestination>> insert_destinations_;
std::vector<std::unique_ptr<JoinHashTable>> join_hash_tables_;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/21b85088/query_execution/QueryContext.proto
----------------------------------------------------------------------
diff --git a/query_execution/QueryContext.proto b/query_execution/QueryContext.proto
index a7c2a25..b37286c 100644
--- a/query_execution/QueryContext.proto
+++ b/query_execution/QueryContext.proto
@@ -23,6 +23,7 @@ import "storage/AggregationOperationState.proto";
import "storage/HashTable.proto";
import "storage/InsertDestination.proto";
import "types/containers/Tuple.proto";
+import "utility/BloomFilter.proto";
import "utility/SortConfiguration.proto";
message QueryContext {
@@ -42,14 +43,15 @@ message QueryContext {
}
repeated AggregationOperationState aggregation_states = 1;
- repeated HashTable join_hash_tables = 2;
- repeated InsertDestination insert_destinations = 3;
- repeated Predicate predicates = 4;
- repeated ScalarGroup scalar_groups = 5;
- repeated SortConfiguration sort_configs = 6;
- repeated Tuple tuples = 7;
- repeated GeneratorFunctionHandle generator_functions = 8;
+ repeated BloomFilter bloom_filters = 2;
+ repeated GeneratorFunctionHandle generator_functions = 3;
+ repeated HashTable join_hash_tables = 4;
+ repeated InsertDestination insert_destinations = 5;
+ repeated Predicate predicates = 6;
+ repeated ScalarGroup scalar_groups = 7;
+ repeated SortConfiguration sort_configs = 8;
+ repeated Tuple tuples = 9;
// NOTE(zuyu): For UpdateWorkOrder only.
- repeated UpdateGroup update_groups = 9;
+ repeated UpdateGroup update_groups = 10;
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/21b85088/query_optimizer/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_optimizer/CMakeLists.txt b/query_optimizer/CMakeLists.txt
index 2d09bee..feaecb3 100644
--- a/query_optimizer/CMakeLists.txt
+++ b/query_optimizer/CMakeLists.txt
@@ -35,6 +35,7 @@ add_subdirectory(tests)
# Declare micro-libs:
add_library(quickstep_queryoptimizer_ExecutionGenerator ExecutionGenerator.cpp ExecutionGenerator.hpp)
+add_library(quickstep_queryoptimizer_ExecutionHeuristics ExecutionHeuristics.cpp ExecutionHeuristics.hpp)
add_library(quickstep_queryoptimizer_LogicalGenerator LogicalGenerator.cpp LogicalGenerator.hpp)
add_library(quickstep_queryoptimizer_LogicalToPhysicalMapper
../empty_src.cpp
@@ -64,6 +65,7 @@ target_link_libraries(quickstep_queryoptimizer_ExecutionGenerator
quickstep_expressions_scalar_ScalarAttribute
quickstep_queryexecution_QueryContext
quickstep_queryexecution_QueryContext_proto
+ quickstep_queryoptimizer_ExecutionHeuristics
quickstep_queryoptimizer_OptimizerContext
quickstep_queryoptimizer_QueryHandle
quickstep_queryoptimizer_QueryPlan
@@ -139,6 +141,14 @@ if (ENABLE_DISTRIBUTED)
target_link_libraries(quickstep_queryoptimizer_ExecutionGenerator
quickstep_catalog_Catalog_proto)
endif()
+target_link_libraries(quickstep_queryoptimizer_ExecutionHeuristics
+ glog
+ quickstep_catalog_CatalogRelation
+ quickstep_catalog_CatalogTypedefs
+ quickstep_queryexecution_QueryContext
+ quickstep_queryexecution_QueryContext_proto
+ quickstep_queryoptimizer_QueryPlan
+ quickstep_utility_Macros)
target_link_libraries(quickstep_queryoptimizer_LogicalGenerator
glog
quickstep_parser_ParseStatement
@@ -211,6 +221,7 @@ target_link_libraries(quickstep_queryoptimizer_Validator
add_library(quickstep_queryoptimizer ../empty_src.cpp QueryOptimizerModule.hpp)
target_link_libraries(quickstep_queryoptimizer
quickstep_queryoptimizer_ExecutionGenerator
+ quickstep_queryoptimizer_ExecutionHeuristics
quickstep_queryoptimizer_LogicalGenerator
quickstep_queryoptimizer_LogicalToPhysicalMapper
quickstep_queryoptimizer_Optimizer
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/21b85088/query_optimizer/ExecutionGenerator.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/ExecutionGenerator.cpp b/query_optimizer/ExecutionGenerator.cpp
index 077d35d..7f26e85 100644
--- a/query_optimizer/ExecutionGenerator.cpp
+++ b/query_optimizer/ExecutionGenerator.cpp
@@ -50,6 +50,7 @@
#include "expressions/scalar/ScalarAttribute.hpp"
#include "query_execution/QueryContext.hpp"
#include "query_execution/QueryContext.pb.h"
+#include "query_optimizer/ExecutionHeuristics.hpp"
#include "query_optimizer/OptimizerContext.hpp"
#include "query_optimizer/QueryHandle.hpp"
#include "query_optimizer/QueryPlan.hpp"
@@ -144,6 +145,9 @@ static const volatile bool aggregate_hashtable_type_dummy
DEFINE_bool(parallelize_load, true, "Parallelize loading data files.");
+DEFINE_bool(optimize_joins, false,
+ "Enable post execution plan generation optimizations for joins.");
+
namespace E = ::quickstep::optimizer::expressions;
namespace P = ::quickstep::optimizer::physical;
namespace S = ::quickstep::serialization;
@@ -198,6 +202,11 @@ void ExecutionGenerator::generatePlan(const P::PhysicalPtr &physical_plan) {
temporary_relation_info.producer_operator_index);
}
+ // Optimize execution plan based on heuristics captured during execution plan generation, if enabled.
+ if (FLAGS_optimize_joins) {
+ execution_heuristics_->optimizeExecutionPlan(execution_plan_, query_context_proto_);
+ }
+
#ifdef QUICKSTEP_DISTRIBUTED
catalog_database_cache_proto_->set_name(optimizer_context_->catalog_database()->getName());
@@ -576,12 +585,32 @@ void ExecutionGenerator::convertHashJoin(const P::HashJoinPtr &physical_plan) {
std::vector<attribute_id> probe_attribute_ids;
std::vector<attribute_id> build_attribute_ids;
+ std::vector<attribute_id> probe_original_attribute_ids;
+ std::vector<attribute_id> build_original_attribute_ids;
+
+ const CatalogRelation *referenced_stored_probe_relation;
+ const CatalogRelation *referenced_stored_build_relation;
+
bool any_probe_attributes_nullable = false;
bool any_build_attributes_nullable = false;
+ bool skip_hash_join_optimization = false;
+
const std::vector<E::AttributeReferencePtr> &left_join_attributes =
physical_plan->left_join_attributes();
for (const E::AttributeReferencePtr &left_join_attribute : left_join_attributes) {
+ // Try to determine the original stored relation referenced in the Hash Join.
+ referenced_stored_probe_relation =
+ optimizer_context_->catalog_database()->getRelationByName(left_join_attribute->relation_name());
+ if (referenced_stored_probe_relation == nullptr) {
+ // Hash Join optimizations are not possible, if the referenced relation cannot be determined.
+ skip_hash_join_optimization = true;
+ } else {
+ const attribute_id probe_operator_attribute_id =
+ referenced_stored_probe_relation->getAttributeByName(left_join_attribute->attribute_name())->getID();
+ probe_original_attribute_ids.emplace_back(probe_operator_attribute_id);
+ }
+
const CatalogAttribute *probe_catalog_attribute
= attribute_substitution_map_[left_join_attribute->id()];
probe_attribute_ids.emplace_back(probe_catalog_attribute->getID());
@@ -594,6 +623,18 @@ void ExecutionGenerator::convertHashJoin(const P::HashJoinPtr &physical_plan) {
const std::vector<E::AttributeReferencePtr> &right_join_attributes =
physical_plan->right_join_attributes();
for (const E::AttributeReferencePtr &right_join_attribute : right_join_attributes) {
+ // Try to determine the original stored relation referenced in the Hash Join.
+ referenced_stored_build_relation =
+ optimizer_context_->catalog_database()->getRelationByName(right_join_attribute->relation_name());
+ if (referenced_stored_build_relation == nullptr) {
+ // Hash Join optimizations are not possible, if the referenced relation cannot be determined.
+ skip_hash_join_optimization = true;
+ } else {
+ const attribute_id build_operator_attribute_id =
+ referenced_stored_build_relation->getAttributeByName(right_join_attribute->attribute_name())->getID();
+ build_original_attribute_ids.emplace_back(build_operator_attribute_id);
+ }
+
const CatalogAttribute *build_catalog_attribute
= attribute_substitution_map_[right_join_attribute->id()];
build_attribute_ids.emplace_back(build_catalog_attribute->getID());
@@ -629,6 +670,8 @@ void ExecutionGenerator::convertHashJoin(const P::HashJoinPtr &physical_plan) {
std::swap(probe_cardinality, build_cardinality);
std::swap(probe_attribute_ids, build_attribute_ids);
std::swap(any_probe_attributes_nullable, any_build_attributes_nullable);
+ std::swap(probe_original_attribute_ids, build_original_attribute_ids);
+ std::swap(referenced_stored_probe_relation, referenced_stored_build_relation);
}
}
@@ -783,6 +826,17 @@ void ExecutionGenerator::convertHashJoin(const P::HashJoinPtr &physical_plan) {
std::forward_as_tuple(join_operator_index,
output_relation));
temporary_relation_info_vec_.emplace_back(join_operator_index, output_relation);
+
+ // Add heuristics for the Hash Join, if enabled.
+ if (FLAGS_optimize_joins && !skip_hash_join_optimization) {
+ execution_heuristics_->addHashJoinInfo(build_operator_index,
+ join_operator_index,
+ referenced_stored_build_relation,
+ referenced_stored_probe_relation,
+ std::move(build_original_attribute_ids),
+ std::move(probe_original_attribute_ids),
+ join_hash_table_index);
+ }
}
void ExecutionGenerator::convertNestedLoopsJoin(
@@ -895,7 +949,6 @@ void ExecutionGenerator::convertCopyFrom(
false /* is_pipeline_breaker */);
}
-
void ExecutionGenerator::convertCreateIndex(
const P::CreateIndexPtr &physical_plan) {
// CreateIndex is converted to a CreateIndex operator.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/21b85088/query_optimizer/ExecutionGenerator.hpp
----------------------------------------------------------------------
diff --git a/query_optimizer/ExecutionGenerator.hpp b/query_optimizer/ExecutionGenerator.hpp
index 7c563d4..0630bca 100644
--- a/query_optimizer/ExecutionGenerator.hpp
+++ b/query_optimizer/ExecutionGenerator.hpp
@@ -33,6 +33,7 @@
#include "catalog/CatalogTypedefs.hpp"
#include "query_execution/QueryContext.hpp"
#include "query_execution/QueryContext.pb.h"
+#include "query_optimizer/ExecutionHeuristics.hpp"
#include "query_optimizer/QueryHandle.hpp"
#include "query_optimizer/QueryPlan.hpp"
#include "query_optimizer/cost_model/CostModel.hpp"
@@ -102,7 +103,8 @@ class ExecutionGenerator {
: optimizer_context_(DCHECK_NOTNULL(optimizer_context)),
query_handle_(DCHECK_NOTNULL(query_handle)),
execution_plan_(DCHECK_NOTNULL(query_handle->getQueryPlanMutable())),
- query_context_proto_(DCHECK_NOTNULL(query_handle->getQueryContextProtoMutable())) {
+ query_context_proto_(DCHECK_NOTNULL(query_handle->getQueryContextProtoMutable())),
+ execution_heuristics_(new ExecutionHeuristics()) {
#ifdef QUICKSTEP_DISTRIBUTED
catalog_database_cache_proto_ = DCHECK_NOTNULL(query_handle->getCatalogDatabaseCacheProtoMutable());
#endif
@@ -376,6 +378,7 @@ class ExecutionGenerator {
QueryHandle *query_handle_;
QueryPlan *execution_plan_; // A part of QueryHandle.
serialization::QueryContext *query_context_proto_; // A part of QueryHandle.
+ std::unique_ptr<ExecutionHeuristics> execution_heuristics_;
#ifdef QUICKSTEP_DISTRIBUTED
serialization::CatalogDatabase *catalog_database_cache_proto_; // A part of QueryHandle.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/21b85088/query_optimizer/ExecutionHeuristics.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/ExecutionHeuristics.cpp b/query_optimizer/ExecutionHeuristics.cpp
new file mode 100644
index 0000000..fc31c53
--- /dev/null
+++ b/query_optimizer/ExecutionHeuristics.cpp
@@ -0,0 +1,127 @@
+/**
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#include "query_optimizer/ExecutionHeuristics.hpp"
+
+#include <cstddef>
+#include <utility>
+#include <unordered_map>
+#include <vector>
+
+#include "catalog/CatalogTypedefs.hpp"
+#include "query_execution/QueryContext.pb.h"
+#include "query_optimizer/QueryPlan.hpp"
+#include "utility/Macros.hpp"
+
+#include "glog/logging.h"
+
+namespace quickstep {
+namespace optimizer {
+
+void ExecutionHeuristics::optimizeExecutionPlan(QueryPlan *query_plan,
+ serialization::QueryContext *query_context_proto) {
+ // Currently this only optimizes left deep joins using bloom filters.
+ // It uses a simple algorithm to discover the left deep joins.
+ // It starts with the first hash join in the plan and keeps on iterating
+ // over the next hash joins, till a probe on a different relation id is found.
+ // The set of hash joins found in this way forms a chain and can be recognized
+ // as a left deep join. It becomes a candidate for optimization.
+
+ // The optimization is done by modifying each of the build operators in the chain
+ // to generate a bloom filter on the build key during their hash table creation.
+ // The leaf-level probe operator is then modified to query all the bloom
+ // filters generated from all the build operators in the chain. These
+ // bloom filters are queried to test the membership of the probe key
+ // just prior to probing the hash table.
+
+ QueryPlan::DAGNodeIndex origin_node = 0;
+ while (origin_node < hash_joins_.size() - 1) {
+ std::vector<std::size_t> chained_nodes;
+ chained_nodes.push_back(origin_node);
+ for (std::size_t i = origin_node + 1; i < hash_joins_.size(); ++i) {
+ const relation_id checked_relation_id = hash_joins_[origin_node].referenced_stored_probe_relation_->getID();
+ const relation_id expected_relation_id = hash_joins_[i].referenced_stored_probe_relation_->getID();
+ if (checked_relation_id == expected_relation_id) {
+ chained_nodes.push_back(i);
+ } else {
+ break;
+ }
+ }
+
+ // Only chains of length greater than one are suitable candidates for semi-join optimization.
+ if (chained_nodes.size() > 1) {
+ std::unordered_map<QueryContext::bloom_filter_id, std::vector<attribute_id>> probe_bloom_filter_info;
+ for (const std::size_t node : chained_nodes) {
+ // Provision for a new bloom filter to be used by the build operator.
+ const QueryContext::bloom_filter_id bloom_filter_id = query_context_proto->bloom_filters_size();
+ serialization::BloomFilter *bloom_filter_proto = query_context_proto->add_bloom_filters();
+
+ // Modify the bloom filter properties based on the statistics of the relation.
+ setBloomFilterProperties(bloom_filter_proto, hash_joins_[node].referenced_stored_build_relation_);
+
+ // Add build-side bloom filter information to the corresponding hash table proto.
+ query_context_proto->mutable_join_hash_tables(hash_joins_[node].join_hash_table_id_)
+ ->add_build_side_bloom_filter_id(bloom_filter_id);
+
+ probe_bloom_filter_info.insert(std::make_pair(bloom_filter_id, hash_joins_[node].probe_attributes_));
+ }
+
+ // Add probe-side bloom filter information to the corresponding hash table proto for each build-side bloom filter.
+ for (const std::pair<QueryContext::bloom_filter_id, std::vector<attribute_id>>
+ &bloom_filter_info : probe_bloom_filter_info) {
+ auto *probe_side_bloom_filter =
+ query_context_proto->mutable_join_hash_tables(hash_joins_[origin_node].join_hash_table_id_)
+ ->add_probe_side_bloom_filters();
+ probe_side_bloom_filter->set_probe_side_bloom_filter_id(bloom_filter_info.first);
+ for (const attribute_id &probe_attribute_id : bloom_filter_info.second) {
+ probe_side_bloom_filter->add_probe_side_attr_ids(probe_attribute_id);
+ }
+ }
+
+ // Add node dependencies from chained build nodes to origin node probe.
+ for (std::size_t i = 1; i < chained_nodes.size(); ++i) { // Note: It starts from index 1.
+ query_plan->addDirectDependency(hash_joins_[origin_node].join_operator_index_,
+ hash_joins_[origin_node + i].build_operator_index_,
+ true /* is_pipeline_breaker */);
+ }
+ }
+
+ // Update the origin node.
+ origin_node = chained_nodes.back() + 1;
+ }
+}
+
+void ExecutionHeuristics::setBloomFilterProperties(serialization::BloomFilter *bloom_filter_proto,
+ const CatalogRelation *relation) {
+ const std::size_t cardinality = relation->estimateTupleCardinality();
+ if (cardinality < kOneThousand) {
+ bloom_filter_proto->set_bloom_filter_size(kOneThousand / kCompressionFactor);
+ bloom_filter_proto->set_number_of_hashes(kVeryLowSparsityHash);
+ } else if (cardinality < kTenThousand) {
+ bloom_filter_proto->set_bloom_filter_size(kTenThousand / kCompressionFactor);
+ bloom_filter_proto->set_number_of_hashes(kLowSparsityHash);
+ } else if (cardinality < kHundredThousand) {
+ bloom_filter_proto->set_bloom_filter_size(kHundredThousand / kCompressionFactor);
+ bloom_filter_proto->set_number_of_hashes(kMediumSparsityHash);
+ } else {
+ bloom_filter_proto->set_bloom_filter_size(kMillion / kCompressionFactor);
+ bloom_filter_proto->set_number_of_hashes(kHighSparsityHash);
+ }
+}
+
+} // namespace optimizer
+} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/21b85088/query_optimizer/ExecutionHeuristics.hpp
----------------------------------------------------------------------
diff --git a/query_optimizer/ExecutionHeuristics.hpp b/query_optimizer/ExecutionHeuristics.hpp
new file mode 100644
index 0000000..92a7fe8
--- /dev/null
+++ b/query_optimizer/ExecutionHeuristics.hpp
@@ -0,0 +1,155 @@
+/**
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#ifndef QUICKSTEP_QUERY_OPTIMIZER_EXECUTION_HEURISTICS_HPP_
+#define QUICKSTEP_QUERY_OPTIMIZER_EXECUTION_HEURISTICS_HPP_
+
+#include <vector>
+
+#include "catalog/CatalogRelation.hpp"
+#include "catalog/CatalogTypedefs.hpp"
+#include "query_execution/QueryContext.hpp"
+#include "query_execution/QueryContext.pb.h"
+#include "query_optimizer/QueryPlan.hpp"
+#include "utility/Macros.hpp"
+
+#include "glog/logging.h"
+
+namespace quickstep {
+namespace optimizer {
+
+/** \addtogroup QueryOptimizer
+ * @{
+ */
+
+/**
+ * @brief The ExecutionHeuristics compiles certain heuristics for an execution plan
+ * as it is being converted to a physical plan. These heuristics can then be
+ * used to optimize the execution plan after it has been generated.
+ **/
+class ExecutionHeuristics {
+ public:
+ static const std::size_t kOneHundred = 100;
+ static const std::size_t kOneThousand = 1000;
+ static const std::size_t kTenThousand = 10000;
+ static const std::size_t kHundredThousand = 100000;
+ static const std::size_t kMillion = 1000000;
+
+ static const std::size_t kCompressionFactor = 10;
+
+ static const std::size_t kVeryLowSparsityHash = 1;
+ static const std::size_t kLowSparsityHash = 2;
+ static const std::size_t kMediumSparsityHash = 5;
+ static const std::size_t kHighSparsityHash = 10;
+
+ /**
+ * @brief A simple internal class that holds information about various
+ * hash joins within the execution plan for a query.
+ **/
+ struct HashJoinInfo {
+ HashJoinInfo(const QueryPlan::DAGNodeIndex build_operator_index,
+ const QueryPlan::DAGNodeIndex join_operator_index,
+ const CatalogRelation *referenced_stored_build_relation,
+ const CatalogRelation *referenced_stored_probe_relation,
+ std::vector<attribute_id> &&build_attributes,
+ std::vector<attribute_id> &&probe_attributes,
+ const QueryContext::join_hash_table_id join_hash_table_id)
+ : build_operator_index_(build_operator_index),
+ join_operator_index_(join_operator_index),
+ referenced_stored_build_relation_(referenced_stored_build_relation),
+ referenced_stored_probe_relation_(referenced_stored_probe_relation),
+ build_attributes_(std::move(build_attributes)),
+ probe_attributes_(std::move(probe_attributes)),
+ join_hash_table_id_(join_hash_table_id) {
+ }
+
+ const QueryPlan::DAGNodeIndex build_operator_index_;
+ const QueryPlan::DAGNodeIndex join_operator_index_;
+ const CatalogRelation *referenced_stored_build_relation_;
+ const CatalogRelation *referenced_stored_probe_relation_;
+ const std::vector<attribute_id> build_attributes_;
+ const std::vector<attribute_id> probe_attributes_;
+ const QueryContext::join_hash_table_id join_hash_table_id_;
+ };
+
+
+ /**
+ * @brief Constructor.
+ **/
+ ExecutionHeuristics() {}
+
+ /**
+ * @brief Saves information about a hash join used within the execution plan
+ * for a query.
+ *
+ * @param build_operator_index Index of the build operator of the hash join.
+ * @param join_operator_index Index of the join operator of the hash join.
+ * @param build_relation_id Id of the relation on which hash table is being built.
+ * @param probe_relation_id Id of the relation on which hash table is being probed.
+ * @param build_attributes List of attributes on which hash table is being built.
+ * @param probe_attributes List of attributes on which hash table is being probed.
+ * @param join_hash_table_id Id of the hash table which refers to the actual hash
+ * table within the query context.
+ **/
+ inline void addHashJoinInfo(const QueryPlan::DAGNodeIndex build_operator_index,
+ const QueryPlan::DAGNodeIndex join_operator_index,
+ const CatalogRelation *referenced_stored_build_relation,
+ const CatalogRelation *referenced_stored_probe_relation,
+ std::vector<attribute_id> &&build_attributes,
+ std::vector<attribute_id> &&probe_attributes,
+ const QueryContext::join_hash_table_id join_hash_table_id) {
+ hash_joins_.push_back(HashJoinInfo(build_operator_index,
+ join_operator_index,
+ referenced_stored_build_relation,
+ referenced_stored_probe_relation,
+ std::move(build_attributes),
+ std::move(probe_attributes),
+ join_hash_table_id));
+ }
+
+ /**
+ * @brief Optimize the execution plan based on heuristics generated
+ * during physical plan to execution plan conversion.
+ *
+ * @param query_plan A mutable reference to the query execution plan.
+ * @param query_context_proto A mutable reference to the protobuf representation
+ * of the query context.
+ **/
+ void optimizeExecutionPlan(QueryPlan *query_plan, serialization::QueryContext *query_context_proto);
+
+ /**
+ * @brief Set the properties of the bloom filter proto based on the statistics
+ * of the given relation.
+ *
+ * @param bloom_filter_proto A mutable reference to the bloom filter protobuf representation.
+ * @param relation The catalog relation on which bloom filter is being built.
+ **/
+ void setBloomFilterProperties(serialization::BloomFilter *bloom_filter_proto,
+ const CatalogRelation *relation);
+
+ private:
+ std::vector<HashJoinInfo> hash_joins_;
+
+ DISALLOW_COPY_AND_ASSIGN(ExecutionHeuristics);
+};
+
+/** @} */
+
+} // namespace optimizer
+} // namespace quickstep
+
+#endif /* QUICKSTEP_QUERY_OPTIMIZER_EXECUTION_HEURISTICS_HPP_ */
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/21b85088/query_optimizer/tests/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_optimizer/tests/CMakeLists.txt b/query_optimizer/tests/CMakeLists.txt
index 1d2fa10..5647bfd 100644
--- a/query_optimizer/tests/CMakeLists.txt
+++ b/query_optimizer/tests/CMakeLists.txt
@@ -78,6 +78,22 @@ add_executable(quickstep_queryoptimizer_tests_ExecutionGeneratorTest
ExecutionGeneratorTestRunner.hpp
"${PROJECT_SOURCE_DIR}/utility/textbased_test/TextBasedTest.cpp"
"${PROJECT_SOURCE_DIR}/utility/textbased_test/TextBasedTest.hpp")
+add_executable(ExecutionHeuristics_unittest ExecutionHeuristics_unittest.cpp)
+target_link_libraries(ExecutionHeuristics_unittest
+ gtest
+ gtest_main
+ quickstep_catalog_Catalog
+ quickstep_catalog_CatalogDatabase
+ quickstep_catalog_CatalogTypedefs
+ quickstep_queryexecution_QueryContext
+ quickstep_queryexecution_QueryContext_proto
+ quickstep_queryoptimizer_ExecutionHeuristics
+ quickstep_queryoptimizer_QueryPlan
+ quickstep_relationaloperators_BuildHashOperator
+ quickstep_relationaloperators_HashJoinOperator
+ quickstep_utility_Macros)
+add_test(ExecutionHeuristics_unittest ExecutionHeuristics_unittest)
+
add_executable(quickstep_queryoptimizer_tests_OptimizerTextTest
OptimizerTextTest.cpp
OptimizerTextTestRunner.cpp
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/21b85088/query_optimizer/tests/ExecutionHeuristics_unittest.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/tests/ExecutionHeuristics_unittest.cpp b/query_optimizer/tests/ExecutionHeuristics_unittest.cpp
new file mode 100644
index 0000000..12acaff
--- /dev/null
+++ b/query_optimizer/tests/ExecutionHeuristics_unittest.cpp
@@ -0,0 +1,301 @@
+/**
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "catalog/Catalog.hpp"
+#include "catalog/CatalogDatabase.hpp"
+#include "catalog/CatalogTypedefs.hpp"
+#include "query_execution/QueryContext.hpp"
+#include "query_execution/QueryContext.pb.h"
+#include "query_optimizer/ExecutionHeuristics.hpp"
+#include "query_optimizer/QueryPlan.hpp"
+#include "relational_operators/BuildHashOperator.hpp"
+#include "relational_operators/HashJoinOperator.hpp"
+#include "utility/Macros.hpp"
+
+#include "glog/logging.h"
+#include "gtest/gtest.h"
+
+namespace quickstep {
+namespace optimizer {
+
+class ExecutionHeuristicsTest : public ::testing::Test {
+ protected:
+ virtual void SetUp() {
+ db_ = cat_.getDatabaseByIdMutable(cat_.addDatabase(new CatalogDatabase(nullptr, "db")));
+ execution_heuristics_.reset(new ExecutionHeuristics());
+ query_plan_.reset(new QueryPlan());
+ query_context_proto_.reset(new serialization::QueryContext());
+ }
+
+ CatalogRelation* createCatalogRelation(const std::string &name, bool temporary = false) {
+ return db_->getRelationByIdMutable(db_->addRelation(new CatalogRelation(nullptr, name, -1, temporary)));
+ }
+
+ void addDummyHashJoinInfo(ExecutionHeuristics *execution_heuristics,
+ const QueryPlan::DAGNodeIndex build_operator_index,
+ const QueryPlan::DAGNodeIndex join_operator_index,
+ const CatalogRelation *build_relation,
+ const CatalogRelation *probe_relation,
+ const attribute_id build_attribute_id,
+ const attribute_id probe_attribute_id,
+ const QueryContext::join_hash_table_id join_hash_table_id) {
+ std::vector<attribute_id> build_attribute_ids(1, build_attribute_id);
+ std::vector<attribute_id> probe_attribute_ids(1, probe_attribute_id);
+ execution_heuristics->addHashJoinInfo(build_operator_index,
+ join_operator_index,
+ build_relation,
+ probe_relation,
+ std::move(build_attribute_ids),
+ std::move(probe_attribute_ids),
+ join_hash_table_id);
+ }
+
+ QueryPlan::DAGNodeIndex createDummyBuildHashOperator(QueryPlan *query_plan,
+ const CatalogRelation *build_relation,
+ const attribute_id build_attribute_id,
+ const QueryContext::join_hash_table_id join_hash_table_index) {
+ std::vector<attribute_id> build_attribute_ids;
+ build_attribute_ids.push_back(build_attribute_id);
+ QueryPlan::DAGNodeIndex build_operator_index =
+ query_plan->addRelationalOperator(new BuildHashOperator(*build_relation,
+ true,
+ build_attribute_ids,
+ false,
+ join_hash_table_index));
+ return build_operator_index;
+ }
+
+ QueryPlan::DAGNodeIndex createDummyHashJoinOperator(QueryPlan *query_plan,
+ const CatalogRelation *build_relation,
+ const CatalogRelation *probe_relation,
+ const attribute_id probe_attribute_id,
+ const QueryContext::join_hash_table_id join_hash_table_index) {
+ std::vector<attribute_id> probe_attribute_ids;
+ probe_attribute_ids.push_back(probe_attribute_id);
+ QueryPlan::DAGNodeIndex join_operator_index =
+ query_plan->addRelationalOperator(new HashJoinOperator(*build_relation,
+ *probe_relation,
+ true,
+ probe_attribute_ids,
+ false,
+ *probe_relation,
+ 0,
+ join_hash_table_index,
+ 0,
+ 0));
+ return join_operator_index;
+ }
+
+ Catalog cat_;
+ CatalogDatabase *db_; // db_ is owned by cat_.
+ std::unique_ptr<QueryPlan> query_plan_;
+ std::unique_ptr<serialization::QueryContext> query_context_proto_;
+ std::unique_ptr<ExecutionHeuristics> execution_heuristics_;
+};
+
+TEST_F(ExecutionHeuristicsTest, HashJoinOptimizedTest) {
+ // This test case creates three hash joins, all of which are being probed on the same relation.
+ // Since the probe are being made on the same relation, ExecutionHeuristics should optimize
+ // these hash joins using bloom filters.
+
+ const CatalogRelation *build_relation_1 = createCatalogRelation("build_relation_1");
+ const CatalogRelation *build_relation_2 = createCatalogRelation("build_relation_2");
+ const CatalogRelation *build_relation_3 = createCatalogRelation("build_relation_3");
+ const CatalogRelation *probe_relation_1 = createCatalogRelation("probe_relation_1");
+
+ const attribute_id build_attribute_id_1 = 0;
+ const attribute_id build_attribute_id_2 = 0;
+ const attribute_id build_attribute_id_3 = 0;
+ const attribute_id probe_attribute_id_1 = 1;
+ const attribute_id probe_attribute_id_2 = 2;
+ const attribute_id probe_attribute_id_3 = 3;
+
+ const QueryContext::join_hash_table_id join_hash_table_index_1 = 0;
+ const QueryContext::join_hash_table_id join_hash_table_index_2 = 1;
+ const QueryContext::join_hash_table_id join_hash_table_index_3 = 2;
+ query_context_proto_->add_join_hash_tables();
+ query_context_proto_->add_join_hash_tables();
+ query_context_proto_->add_join_hash_tables();
+
+ const QueryPlan::DAGNodeIndex build_operator_index_1 = createDummyBuildHashOperator(query_plan_.get(),
+ build_relation_1,
+ build_attribute_id_1,
+ join_hash_table_index_1);
+ const QueryPlan::DAGNodeIndex probe_operator_index_1 = createDummyHashJoinOperator(query_plan_.get(),
+ build_relation_1,
+ probe_relation_1,
+ probe_attribute_id_1,
+ join_hash_table_index_1);
+ const QueryPlan::DAGNodeIndex build_operator_index_2 = createDummyBuildHashOperator(query_plan_.get(),
+ build_relation_2,
+ build_attribute_id_2,
+ join_hash_table_index_2);
+ const QueryPlan::DAGNodeIndex probe_operator_index_2 = createDummyHashJoinOperator(query_plan_.get(),
+ build_relation_2,
+ probe_relation_1,
+ probe_attribute_id_2,
+ join_hash_table_index_2);
+ const QueryPlan::DAGNodeIndex build_operator_index_3 = createDummyBuildHashOperator(query_plan_.get(),
+ build_relation_3,
+ build_attribute_id_3,
+ join_hash_table_index_3);
+ const QueryPlan::DAGNodeIndex probe_operator_index_3 = createDummyHashJoinOperator(query_plan_.get(),
+ build_relation_3,
+ probe_relation_1,
+ probe_attribute_id_3,
+ join_hash_table_index_3);
+
+ addDummyHashJoinInfo(execution_heuristics_.get(),
+ build_operator_index_1,
+ probe_operator_index_1,
+ build_relation_1,
+ probe_relation_1,
+ build_attribute_id_1,
+ probe_attribute_id_1,
+ join_hash_table_index_1);
+ addDummyHashJoinInfo(execution_heuristics_.get(),
+ build_operator_index_2,
+ probe_operator_index_2,
+ build_relation_2,
+ probe_relation_1,
+ build_attribute_id_2,
+ probe_attribute_id_2,
+ join_hash_table_index_2);
+ addDummyHashJoinInfo(execution_heuristics_.get(),
+ build_operator_index_3,
+ probe_operator_index_3,
+ build_relation_3,
+ probe_relation_1,
+ build_attribute_id_3,
+ probe_attribute_id_3,
+ join_hash_table_index_3);
+
+ execution_heuristics_->optimizeExecutionPlan(query_plan_.get(), query_context_proto_.get());
+
+ // Test whether correct number of bloom filters were added.
+ EXPECT_EQ(1, query_context_proto_->join_hash_tables(0).build_side_bloom_filter_id_size());
+ EXPECT_EQ(1, query_context_proto_->join_hash_tables(1).build_side_bloom_filter_id_size());
+ EXPECT_EQ(1, query_context_proto_->join_hash_tables(2).build_side_bloom_filter_id_size());
+ EXPECT_EQ(3, query_context_proto_->join_hash_tables(0).probe_side_bloom_filters_size());
+
+ // Test that the DAG was modified correctly or not.
+ // Probe operator 1 should have now build operator 1 and build operator 2 added as dependencies.
+ auto const probe_node_dependencies = query_plan_->getQueryPlanDAG().getDependencies(probe_operator_index_1);
+ EXPECT_EQ(1u, probe_node_dependencies.count(build_operator_index_2));
+ EXPECT_EQ(1u, probe_node_dependencies.count(build_operator_index_3));
+}
+
+TEST_F(ExecutionHeuristicsTest, HashJoinNotOptimizedTest) {
+ // This test case creates three hash joins, all of which are being probed on different relations.
+ // Since the probe are being made on the different relations, ExecutionHeuristics should optimize
+ // these hash joins using bloom filters.
+
+ const CatalogRelation *build_relation_1 = createCatalogRelation("build_relation_1");
+ const CatalogRelation *build_relation_2 = createCatalogRelation("build_relation_2");
+ const CatalogRelation *build_relation_3 = createCatalogRelation("build_relation_3");
+ const CatalogRelation *probe_relation_1 = createCatalogRelation("probe_relation_1");
+ const CatalogRelation *probe_relation_2 = createCatalogRelation("probe_relation_2");
+ const CatalogRelation *probe_relation_3 = createCatalogRelation("probe_relation_3");
+
+ const attribute_id build_attribute_id_1 = 0;
+ const attribute_id build_attribute_id_2 = 0;
+ const attribute_id build_attribute_id_3 = 0;
+ const attribute_id probe_attribute_id_1 = 1;
+ const attribute_id probe_attribute_id_2 = 2;
+ const attribute_id probe_attribute_id_3 = 3;
+
+ const QueryContext::join_hash_table_id join_hash_table_index_1 = 0;
+ const QueryContext::join_hash_table_id join_hash_table_index_2 = 1;
+ const QueryContext::join_hash_table_id join_hash_table_index_3 = 2;
+ query_context_proto_->add_join_hash_tables();
+ query_context_proto_->add_join_hash_tables();
+ query_context_proto_->add_join_hash_tables();
+
+ const QueryPlan::DAGNodeIndex build_operator_index_1 = createDummyBuildHashOperator(query_plan_.get(),
+ build_relation_1,
+ build_attribute_id_1,
+ join_hash_table_index_1);
+ const QueryPlan::DAGNodeIndex probe_operator_index_1 = createDummyHashJoinOperator(query_plan_.get(),
+ build_relation_1,
+ probe_relation_1,
+ probe_attribute_id_1,
+ join_hash_table_index_1);
+ const QueryPlan::DAGNodeIndex build_operator_index_2 = createDummyBuildHashOperator(query_plan_.get(),
+ build_relation_2,
+ build_attribute_id_2,
+ join_hash_table_index_2);
+ const QueryPlan::DAGNodeIndex probe_operator_index_2 = createDummyHashJoinOperator(query_plan_.get(),
+ build_relation_2,
+ probe_relation_2,
+ probe_attribute_id_2,
+ join_hash_table_index_2);
+ const QueryPlan::DAGNodeIndex build_operator_index_3 = createDummyBuildHashOperator(query_plan_.get(),
+ build_relation_3,
+ build_attribute_id_3,
+ join_hash_table_index_3);
+ const QueryPlan::DAGNodeIndex probe_operator_index_3 = createDummyHashJoinOperator(query_plan_.get(),
+ build_relation_3,
+ probe_relation_3,
+ probe_attribute_id_3,
+ join_hash_table_index_3);
+
+ addDummyHashJoinInfo(execution_heuristics_.get(),
+ build_operator_index_1,
+ probe_operator_index_1,
+ build_relation_1,
+ probe_relation_1,
+ build_attribute_id_1,
+ probe_attribute_id_1,
+ join_hash_table_index_1);
+ addDummyHashJoinInfo(execution_heuristics_.get(),
+ build_operator_index_2,
+ probe_operator_index_2,
+ build_relation_2,
+ probe_relation_2,
+ build_attribute_id_2,
+ probe_attribute_id_2,
+ join_hash_table_index_2);
+ addDummyHashJoinInfo(execution_heuristics_.get(),
+ build_operator_index_3,
+ probe_operator_index_3,
+ build_relation_3,
+ probe_relation_3,
+ build_attribute_id_3,
+ probe_attribute_id_3,
+ join_hash_table_index_3);
+
+ execution_heuristics_->optimizeExecutionPlan(query_plan_.get(), query_context_proto_.get());
+
+ // Test that no bloom filters were added.
+ EXPECT_EQ(0, query_context_proto_->join_hash_tables(0).build_side_bloom_filter_id_size());
+ EXPECT_EQ(0, query_context_proto_->join_hash_tables(1).build_side_bloom_filter_id_size());
+ EXPECT_EQ(0, query_context_proto_->join_hash_tables(2).build_side_bloom_filter_id_size());
+ EXPECT_EQ(0, query_context_proto_->join_hash_tables(0).probe_side_bloom_filters_size());
+
+ // Test that the DAG was not modified at all.
+ // Probe operator 1 should not have build operator 1 and build operator 2 added as dependencies.
+ auto probe_node_dependencies = query_plan_->getQueryPlanDAG().getDependencies(probe_operator_index_1);
+ EXPECT_EQ(0u, probe_node_dependencies.count(build_operator_index_2));
+ EXPECT_EQ(0u, probe_node_dependencies.count(build_operator_index_3));
+}
+
+} // namespace optimizer
+} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/21b85088/storage/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/storage/CMakeLists.txt b/storage/CMakeLists.txt
index dacacfa..115248c 100644
--- a/storage/CMakeLists.txt
+++ b/storage/CMakeLists.txt
@@ -629,6 +629,7 @@ target_link_libraries(quickstep_storage_HashTable
quickstep_threading_SpinSharedMutex
quickstep_types_Type
quickstep_types_TypedValue
+ quickstep_utility_BloomFilter
quickstep_utility_HashPair
quickstep_utility_Macros)
target_link_libraries(quickstep_storage_HashTableBase
@@ -648,6 +649,7 @@ target_link_libraries(quickstep_storage_HashTableFactory
quickstep_types_Type
quickstep_types_TypeFactory
quickstep_types_TypedValue
+ quickstep_utility_BloomFilter
quickstep_utility_Macros)
target_link_libraries(quickstep_storage_HashTableKeyManager
glog
@@ -1196,7 +1198,7 @@ target_link_libraries(BloomFilterIndexSubBlock_unittest
add_test(BloomFilterIndexSubBlock_unittest BloomFilterIndexSubBlock_unittest)
if(QUICKSTEP_HAVE_BITWEAVING)
- add_executable(BitWeavingIndexSubBlock_unittest
+ add_executable(BitWeavingIndexSubBlock_unittest
"${CMAKE_CURRENT_SOURCE_DIR}/bitweaving/tests/BitWeavingIndexSubBlock_unittest.cpp")
target_link_libraries(BitWeavingIndexSubBlock_unittest
glog
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/21b85088/storage/HashTable.hpp
----------------------------------------------------------------------
diff --git a/storage/HashTable.hpp b/storage/HashTable.hpp
index 667848e..be31fd9 100644
--- a/storage/HashTable.hpp
+++ b/storage/HashTable.hpp
@@ -38,6 +38,7 @@
#include "threading/SpinSharedMutex.hpp"
#include "types/Type.hpp"
#include "types/TypedValue.hpp"
+#include "utility/BloomFilter.hpp"
#include "utility/HashPair.hpp"
#include "utility/Macros.hpp"
@@ -990,6 +991,61 @@ class HashTable : public HashTableBase<resizable,
template <typename FunctorT>
std::size_t forEachCompositeKey(FunctorT *functor) const;
+ /**
+ * @brief A call to this function will cause a bloom filter to be built
+ * during the build phase of this hash table.
+ **/
+ inline void enableBuildSideBloomFilter() {
+ has_build_side_bloom_filter_ = true;
+ }
+
+ /**
+ * @brief A call to this function will cause a set of bloom filters to be
+ * probed during the probe phase of this hash table.
+ **/
+ inline void enableProbeSideBloomFilter() {
+ has_probe_side_bloom_filter_ = true;
+ }
+
+ /**
+ * @brief This function sets the pointer to the bloom filter to be
+ * used during the build phase of this hash table.
+ * @warning Should call enable_build_side_bloom_filter() first to enable
+ * bloom filter usage during build phase.
+ * @note The ownership of the bloom filter lies with the caller.
+ *
+ * @param bloom_filter The pointer to the bloom filter.
+ **/
+ inline void setBuildSideBloomFilter(BloomFilter *bloom_filter) {
+ build_bloom_filter_ = bloom_filter;
+ }
+
+ /**
+ * @brief This function adds a pointer to the list of bloom filters to be
+ * used during the probe phase of this hash table.
+ * @warning Should call enable_probe_side_bloom_filter() first to enable
+ * bloom filter usage during probe phase.
+ * @note The ownership of the bloom filter lies with the caller.
+ *
+ * @param bloom_filter The pointer to the bloom filter.
+ **/
+ inline void addProbeSideBloomFilter(const BloomFilter *bloom_filter) {
+ probe_bloom_filters_.emplace_back(bloom_filter);
+ }
+
+ /**
+ * @brief This function adds a vector of attribute ids corresponding to a
+ * bloom filter used during the probe phase of this hash table.
+ * @warning Should call enable_probe_side_bloom_filter() first to enable
+ * bloom filter usage during probe phase.
+ *
+ * @param probe_attribute_ids The vector of attribute ids to use for probing
+ * the bloom filter.
+ **/
+ inline void addProbeSideAttributeIds(std::vector<attribute_id> &&probe_attribute_ids) {
+ probe_attribute_ids_.push_back(probe_attribute_ids);
+ }
+
protected:
/**
* @brief Constructor for new resizable hash table.
@@ -1270,6 +1326,13 @@ class HashTable : public HashTableBase<resizable,
const attribute_id key_attr_id,
FunctorT *functor) const;
+ // Data structures used for bloom filter optimized semi-joins.
+ bool has_build_side_bloom_filter_ = false;
+ bool has_probe_side_bloom_filter_ = false;
+ BloomFilter *build_bloom_filter_;
+ std::vector<const BloomFilter*> probe_bloom_filters_;
+ std::vector<std::vector<attribute_id>> probe_attribute_ids_;
+
DISALLOW_COPY_AND_ASSIGN(HashTable);
};
@@ -1414,6 +1477,12 @@ HashTablePutResult HashTable<ValueT, resizable, serializable, force_key_copy, al
&prealloc_state);
}
}
+ std::unique_ptr<BloomFilter> thread_local_bloom_filter;
+ if (has_build_side_bloom_filter_) {
+ thread_local_bloom_filter.reset(new BloomFilter(build_bloom_filter_->getRandomSeed(),
+ build_bloom_filter_->getNumberOfHashes(),
+ build_bloom_filter_->getBitArraySize()));
+ }
if (resizable) {
while (result == HashTablePutResult::kOutOfSpace) {
{
@@ -1429,6 +1498,11 @@ HashTablePutResult HashTable<ValueT, resizable, serializable, force_key_copy, al
variable_size,
(*functor)(*accessor),
using_prealloc ? &prealloc_state : nullptr);
+ // Insert into bloom filter, if enabled.
+ if (has_build_side_bloom_filter_) {
+ thread_local_bloom_filter->insertUnSafe(static_cast<const std::uint8_t *>(key.getDataPtr()),
+ key.getDataSize());
+ }
if (result == HashTablePutResult::kDuplicateKey) {
DEBUG_ASSERT(!using_prealloc);
return result;
@@ -1454,11 +1528,20 @@ HashTablePutResult HashTable<ValueT, resizable, serializable, force_key_copy, al
variable_size,
(*functor)(*accessor),
using_prealloc ? &prealloc_state : nullptr);
+ // Insert into bloom filter, if enabled.
+ if (has_build_side_bloom_filter_) {
+ thread_local_bloom_filter->insertUnSafe(static_cast<const std::uint8_t *>(key.getDataPtr()),
+ key.getDataSize());
+ }
if (result != HashTablePutResult::kOK) {
return result;
}
}
}
+ // Update the build side bloom filter with thread local copy, if available.
+ if (has_build_side_bloom_filter_) {
+ build_bloom_filter_->bitwiseOr(thread_local_bloom_filter.get());
+ }
return HashTablePutResult::kOK;
});
@@ -2164,6 +2247,27 @@ void HashTable<ValueT, resizable, serializable, force_key_copy, allow_duplicate_
accessor,
[&](auto *accessor) -> void { // NOLINT(build/c++11)
while (accessor->next()) {
+ // Probe any bloom filters, if enabled.
+ if (has_probe_side_bloom_filter_) {
+ DCHECK_EQ(probe_bloom_filters_.size(), probe_attribute_ids_.size());
+ // Check if the key is contained in the BloomFilters or not.
+ bool bloom_miss = false;
+ for (std::size_t i = 0; i < probe_bloom_filters_.size() && !bloom_miss; ++i) {
+ const BloomFilter *bloom_filter = probe_bloom_filters_[i];
+ for (const attribute_id &attr_id : probe_attribute_ids_[i]) {
+ TypedValue bloom_key = accessor->getTypedValue(attr_id);
+ if (!bloom_filter->contains(static_cast<const std::uint8_t*>(bloom_key.getDataPtr()),
+ bloom_key.getDataSize())) {
+ bloom_miss = true;
+ break;
+ }
+ }
+ }
+ if (bloom_miss) {
+ continue; // On a bloom filter miss, probing the hash table can be skipped.
+ }
+ }
+
TypedValue key = accessor->getTypedValue(key_attr_id);
if (check_for_null_keys && key.isNull()) {
continue;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/21b85088/storage/HashTable.proto
----------------------------------------------------------------------
diff --git a/storage/HashTable.proto b/storage/HashTable.proto
index 653c3a7..7f00f29 100644
--- a/storage/HashTable.proto
+++ b/storage/HashTable.proto
@@ -1,5 +1,7 @@
// Copyright 2011-2015 Quickstep Technologies LLC.
// Copyright 2015-2016 Pivotal Software, Inc.
+// Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+// University of Wisconsin\u2014Madison.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -32,4 +34,10 @@ message HashTable {
required HashTableImplType hash_table_impl_type = 1;
repeated Type key_types = 2;
required uint64 estimated_num_entries = 3;
+ repeated uint32 build_side_bloom_filter_id = 4;
+ message ProbeSideBloomFilter {
+ required uint32 probe_side_bloom_filter_id = 1;
+ repeated uint32 probe_side_attr_ids = 2;
+ }
+ repeated ProbeSideBloomFilter probe_side_bloom_filters = 6;
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/21b85088/storage/HashTableFactory.hpp
----------------------------------------------------------------------
diff --git a/storage/HashTableFactory.hpp b/storage/HashTableFactory.hpp
index 94a0721..34baaeb 100644
--- a/storage/HashTableFactory.hpp
+++ b/storage/HashTableFactory.hpp
@@ -29,6 +29,7 @@
#include "storage/SimpleScalarSeparateChainingHashTable.hpp"
#include "storage/TupleReference.hpp"
#include "types/TypeFactory.hpp"
+#include "utility/BloomFilter.hpp"
#include "utility/Macros.hpp"
#include "glog/logging.h"
@@ -291,11 +292,14 @@ class HashTableFactory {
* @param proto A protobuf description of a resizable HashTable.
* @param storage_manager The StorageManager to use (a StorageBlob will be
* allocated to hold the HashTable's contents).
+ * @param bloom_filters A vector of pointers to bloom filters that may be used
+ * during hash table construction in build/probe phase.
* @return A new resizable HashTable with parameters specified by proto.
**/
static HashTable<ValueT, resizable, serializable, force_key_copy, allow_duplicate_keys>*
CreateResizableFromProto(const serialization::HashTable &proto,
- StorageManager *storage_manager) {
+ StorageManager *storage_manager,
+ const std::vector<std::unique_ptr<BloomFilter>> &bloom_filters) {
DCHECK(ProtoIsValid(proto))
<< "Attempted to create HashTable from invalid proto description:\n"
<< proto.DebugString();
@@ -305,10 +309,40 @@ class HashTableFactory {
key_types.emplace_back(&TypeFactory::ReconstructFromProto(proto.key_types(i)));
}
- return CreateResizable(HashTableImplTypeFromProto(proto.hash_table_impl_type()),
- key_types,
- proto.estimated_num_entries(),
- storage_manager);
+ auto hash_table = CreateResizable(HashTableImplTypeFromProto(proto.hash_table_impl_type()),
+ key_types,
+ proto.estimated_num_entries(),
+ storage_manager);
+
+ // TODO(ssaurabh): These lazy initializations can be moved from here and pushed to the
+ // individual implementations of the hash table constructors.
+
+ // Check if there are any build side bloom filter defined on the hash table.
+ if (proto.build_side_bloom_filter_id_size() > 0) {
+ hash_table->enableBuildSideBloomFilter();
+ hash_table->setBuildSideBloomFilter(bloom_filters[proto.build_side_bloom_filter_id(0)].get());
+ }
+
+ // Check if there are any probe side bloom filters defined on the hash table.
+ if (proto.probe_side_bloom_filters_size() > 0) {
+ hash_table->enableProbeSideBloomFilter();
+ // Add as many probe bloom filters as defined by the proto.
+ for (int j = 0; j < proto.probe_side_bloom_filters_size(); ++j) {
+ // Add the pointer to the probe bloom filter within the list of probe bloom filters to use.
+ const auto probe_side_bloom_filter = proto.probe_side_bloom_filters(j);
+ hash_table->addProbeSideBloomFilter(bloom_filters[probe_side_bloom_filter.probe_side_bloom_filter_id()].get());
+
+ // Add the attribute ids corresponding to this probe bloom filter.
+ std::vector<attribute_id> probe_attribute_ids;
+ for (int k = 0; k < probe_side_bloom_filter.probe_side_attr_ids_size(); ++k) {
+ const attribute_id probe_attribute_id = probe_side_bloom_filter.probe_side_attr_ids(k);
+ probe_attribute_ids.push_back(probe_attribute_id);
+ }
+ hash_table->addProbeSideAttributeIds(std::move(probe_attribute_ids));
+ }
+ }
+
+ return hash_table;
}
private:
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/21b85088/utility/BloomFilter.hpp
----------------------------------------------------------------------
diff --git a/utility/BloomFilter.hpp b/utility/BloomFilter.hpp
index 1d4fdc7..b93df84 100644
--- a/utility/BloomFilter.hpp
+++ b/utility/BloomFilter.hpp
@@ -26,8 +26,15 @@
#include <algorithm>
#include <cstddef>
#include <cstdint>
+#include <memory>
+#include <utility>
#include <vector>
+#include "storage/StorageConstants.hpp"
+#include "threading/Mutex.hpp"
+#include "threading/SharedMutex.hpp"
+#include "threading/SpinSharedMutex.hpp"
+#include "utility/BloomFilter.pb.h"
#include "utility/Macros.hpp"
#include "glog/logging.h"
@@ -47,7 +54,30 @@ class BloomFilter {
/**
* @brief Constructor.
- * @note The ownership of the bit array lies with the caller.
+ * @note When no bit_array is being passed to the constructor,
+ * then the bit_array is owned and managed by this class.
+ *
+ * @param random_seed A random_seed that generates unique hash functions.
+ * @param hash_fn_count The number of hash functions used by this bloom filter.
+ * @param bit_array_size_in_bytes Size of the bit array.
+ **/
+ BloomFilter(const std::uint64_t random_seed,
+ const std::size_t hash_fn_count,
+ const std::uint64_t bit_array_size_in_bytes)
+ : random_seed_(random_seed),
+ hash_fn_count_(hash_fn_count),
+ array_size_in_bytes_(bit_array_size_in_bytes),
+ array_size_(array_size_in_bytes_ * kNumBitsPerByte),
+ bit_array_(new std::uint8_t[array_size_in_bytes_]),
+ is_bit_array_owner_(true) {
+ reset();
+ generate_unique_hash_fn();
+ }
+
+ /**
+ * @brief Constructor.
+ * @note When a bit_array is passed as an argument to the constructor,
+ * then the ownership of the bit array lies with the caller.
*
* @param random_seed A random_seed that generates unique hash functions.
* @param hash_fn_count The number of hash functions used by this bloom filter.
@@ -61,11 +91,12 @@ class BloomFilter {
const std::uint64_t bit_array_size_in_bytes,
std::uint8_t *bit_array,
const bool is_initialized)
- : hash_fn_count_(hash_fn_count),
- random_seed_(random_seed) {
- array_size_ = bit_array_size_in_bytes * kNumBitsPerByte;
- array_size_in_bytes_ = bit_array_size_in_bytes;
- bit_array_ = bit_array; // Owned by the calling method.
+ : random_seed_(random_seed),
+ hash_fn_count_(hash_fn_count),
+ array_size_in_bytes_(bit_array_size_in_bytes),
+ array_size_(bit_array_size_in_bytes * kNumBitsPerByte),
+ bit_array_(bit_array), // Owned by the calling method.
+ is_bit_array_owner_(false) {
if (!is_initialized) {
reset();
}
@@ -73,27 +104,149 @@ class BloomFilter {
}
/**
+ * @brief Constructor.
+ * @note When a bloom filter proto is passed as an initializer,
+ * then the bit_array is owned and managed by this class.
+ *
+ * @param bloom_filter_proto The protobuf representation of a
+ * bloom filter configuration.
+ **/
+ explicit BloomFilter(const serialization::BloomFilter &bloom_filter_proto)
+ : random_seed_(bloom_filter_proto.bloom_filter_seed()),
+ hash_fn_count_(bloom_filter_proto.number_of_hashes()),
+ array_size_in_bytes_(bloom_filter_proto.bloom_filter_size()),
+ array_size_(array_size_in_bytes_ * kNumBitsPerByte),
+ bit_array_(new std::uint8_t[array_size_in_bytes_]),
+ is_bit_array_owner_(true) {
+ reset();
+ generate_unique_hash_fn();
+ }
+
+ /**
+ * @brief Destructor.
+ **/
+ ~BloomFilter() {
+ if (is_bit_array_owner_) {
+ bit_array_.reset();
+ } else {
+ bit_array_.release();
+ }
+ }
+
+ static bool ProtoIsValid(const serialization::BloomFilter &bloom_filter_proto) {
+ return bloom_filter_proto.IsInitialized();
+ }
+
+ /**
* @brief Zeros out the contents of the bit array.
**/
inline void reset() {
// Initialize the bit_array with all zeros.
- std::fill_n(bit_array_, array_size_in_bytes_, 0x00);
+ std::fill_n(bit_array_.get(), array_size_in_bytes_, 0x00);
inserted_element_count_ = 0;
}
/**
+ * @brief Get the random seed that was used to initialize this bloom filter.
+ *
+ * @return Returns the random seed.
+ **/
+ inline std::uint64_t getRandomSeed() const {
+ return random_seed_;
+ }
+
+ /**
+ * @brief Get the number of hash functions used in this bloom filter.
+ *
+ * @return Returns the number of hash functions.
+ **/
+ inline std::uint32_t getNumberOfHashes() const {
+ return hash_fn_count_;
+ }
+
+ /**
+ * @brief Get the size of the bit array in bytes for this bloom filter.
+ *
+ * @return Returns the bit array size (in bytes).
+ **/
+ inline std::uint64_t getBitArraySize() const {
+ return array_size_in_bytes_;
+ }
+
+ /**
+ * @brief Get the constant pointer to the bit array for this bloom filter
+ *
+ * @return Returns constant pointer to the bit array.
+ **/
+ inline const std::uint8_t* getBitArray() const {
+ return bit_array_.get();
+ }
+
+ /**
+ * @brief Inserts a given value into the bloom filter in a thread-safe manner.
+ *
+ * @param key_begin A pointer to the value being inserted.
+ * @param length Size of the value being inserted in bytes.
+ */
+ inline void insert(const std::uint8_t *key_begin, const std::size_t length) {
+ // Locks are needed during insertion, when multiple workers may be modifying the
+ // bloom filter concurrently. However, locks are not required during membership test.
+ std::size_t bit_index = 0;
+ std::size_t bit = 0;
+ std::vector<std::pair<std::size_t, std::size_t>> modified_bit_positions;
+ std::vector<bool> is_bit_position_correct;
+
+ // Determine all the bit positions that are required to be set.
+ for (std::size_t i = 0; i < hash_fn_count_; ++i) {
+ compute_indices(hash_ap(key_begin, length, hash_fn_[i]), &bit_index, &bit);
+ modified_bit_positions.push_back(std::make_pair(bit_index, bit));
+ }
+
+ // Acquire a reader lock and check which of the bit positions are already set.
+ {
+ SpinSharedMutexSharedLock<false> shared_reader_lock(bloom_filter_insert_mutex_);
+ for (std::size_t i = 0; i < hash_fn_count_; ++i) {
+ bit_index = modified_bit_positions[i].first;
+ bit = modified_bit_positions[i].second;
+ if (((bit_array_.get())[bit_index / kNumBitsPerByte] & (1 << bit)) != (1 << bit)) {
+ is_bit_position_correct.push_back(false);
+ } else {
+ is_bit_position_correct.push_back(true);
+ }
+ }
+ }
+
+ // Acquire a writer lock and set the bit positions are which are not set.
+ {
+ SpinSharedMutexExclusiveLock<false> exclusive_writer_lock(bloom_filter_insert_mutex_);
+ for (std::size_t i = 0; i < hash_fn_count_; ++i) {
+ if (!is_bit_position_correct[i]) {
+ bit_index = modified_bit_positions[i].first;
+ bit = modified_bit_positions[i].second;
+ (bit_array_.get())[bit_index / kNumBitsPerByte] |= (1 << bit);
+ }
+ }
+ }
+ ++inserted_element_count_;
+ }
+
+ /**
* @brief Inserts a given value into the bloom filter.
+ * @Warning This is a faster thread-unsafe version of the insert() function.
+ * The caller needs to ensure the thread safety.
*
* @param key_begin A pointer to the value being inserted.
* @param length Size of the value being inserted in bytes.
*/
- inline void insert(const std::uint8_t *key_begin, const std::size_t &length) {
+ inline void insertUnSafe(const std::uint8_t *key_begin, const std::size_t length) {
std::size_t bit_index = 0;
std::size_t bit = 0;
+
for (std::size_t i = 0; i < hash_fn_count_; ++i) {
compute_indices(hash_ap(key_begin, length, hash_fn_[i]), &bit_index, &bit);
- bit_array_[bit_index / kNumBitsPerByte] |= (1 << bit);
+ (bit_array_.get())[bit_index / kNumBitsPerByte] |= (1 << bit);
}
+
++inserted_element_count_;
}
@@ -102,6 +255,9 @@ class BloomFilter {
* If true is returned, then a value may or may not be present in the bloom filter.
* If false is returned, a value is certainly not present in the bloom filter.
*
+ * @note The membersip test does not require any locks, because the assumption is that
+ * the bloom filter will only be used after it has been built.
+ *
* @param key_begin A pointer to the value being tested for membership.
* @param length Size of the value being inserted in bytes.
*/
@@ -110,7 +266,7 @@ class BloomFilter {
std::size_t bit = 0;
for (std::size_t i = 0; i < hash_fn_count_; ++i) {
compute_indices(hash_ap(key_begin, length, hash_fn_[i]), &bit_index, &bit);
- if ((bit_array_[bit_index / kNumBitsPerByte] & (1 << bit)) != (1 << bit)) {
+ if (((bit_array_.get())[bit_index / kNumBitsPerByte] & (1 << bit)) != (1 << bit)) {
return false;
}
}
@@ -118,6 +274,19 @@ class BloomFilter {
}
/**
+ * @brief Perform a bitwise-OR of the given Bloom filter with this bloom filter.
+ * Essentially, it does a union of this bloom filter with the passed bloom filter.
+ *
+ * @param bloom_filter A const pointer to the bloom filter object to do bitwise-OR with.
+ */
+ inline void bitwiseOr(const BloomFilter *bloom_filter) {
+ SpinSharedMutexExclusiveLock<false> exclusive_writer_lock(bloom_filter_insert_mutex_);
+ for (std::size_t byte_index = 0; byte_index < bloom_filter->getBitArraySize(); ++byte_index) {
+ (bit_array_.get())[byte_index] |= bloom_filter->getBitArray()[byte_index];
+ }
+ }
+
+ /**
* @brief Return the number of elements currently inserted into bloom filter.
*
* @return The number of elements inserted into bloom filter.
@@ -219,13 +388,16 @@ class BloomFilter {
}
private:
+ const std::uint64_t random_seed_;
std::vector<std::uint32_t> hash_fn_;
- std::uint8_t *bit_array_;
const std::uint32_t hash_fn_count_;
- std::uint64_t array_size_;
std::uint64_t array_size_in_bytes_;
+ std::uint64_t array_size_;
+ std::unique_ptr<std::uint8_t> bit_array_;
std::uint32_t inserted_element_count_;
- const std::uint64_t random_seed_;
+ const bool is_bit_array_owner_;
+
+ alignas(kCacheLineBytes) mutable SpinSharedMutex<false> bloom_filter_insert_mutex_;
DISALLOW_COPY_AND_ASSIGN(BloomFilter);
};
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/21b85088/utility/BloomFilter.proto
----------------------------------------------------------------------
diff --git a/utility/BloomFilter.proto b/utility/BloomFilter.proto
new file mode 100644
index 0000000..8dd9163
--- /dev/null
+++ b/utility/BloomFilter.proto
@@ -0,0 +1,30 @@
+// Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+// University of Wisconsin\u2014Madison.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package quickstep.serialization;
+
+message BloomFilter {
+ // The default values were determined from empirical experiments.
+ // These values control the amount of false positivity that
+ // is expected from Bloom Filter.
+ // - Default seed for initializing family of hashes = 0xA5A5A5A55A5A5A5A.
+ // - Default bloom filter size = 10 KB.
+ // - Default number of hash functions used in bloom filter = 5.
+ optional fixed64 bloom_filter_seed = 1 [default = 0xA5A5A5A55A5A5A5A];
+ optional uint32 bloom_filter_size = 2 [default = 10000];
+ optional uint32 number_of_hashes = 3 [default = 5];
+}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/21b85088/utility/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/utility/CMakeLists.txt b/utility/CMakeLists.txt
index bb59f65..6d1eeab 100644
--- a/utility/CMakeLists.txt
+++ b/utility/CMakeLists.txt
@@ -146,6 +146,10 @@ configure_file (
"${CMAKE_CURRENT_BINARY_DIR}/UtilityConfig.h"
)
+QS_PROTOBUF_GENERATE_CPP(quickstep_utility_BloomFilter_proto_srcs
+ quickstep_utility_BloomFilter_proto_hdrs
+ BloomFilter.proto)
+
QS_PROTOBUF_GENERATE_CPP(quickstep_utility_SortConfiguration_proto_srcs
quickstep_utility_SortConfiguration_proto_hdrs
SortConfiguration.proto)
@@ -155,6 +159,9 @@ add_library(quickstep_utility_Alignment ../empty_src.cpp Alignment.hpp)
add_library(quickstep_utility_BitManipulation ../empty_src.cpp BitManipulation.hpp)
add_library(quickstep_utility_BitVector ../empty_src.cpp BitVector.hpp)
add_library(quickstep_utility_BloomFilter ../empty_src.cpp BloomFilter.hpp)
+add_library(quickstep_utility_BloomFilter_proto
+ ${quickstep_utility_BloomFilter_proto_srcs}
+ ${quickstep_utility_BloomFilter_proto_hdrs})
add_library(quickstep_utility_CalculateInstalledMemory CalculateInstalledMemory.cpp CalculateInstalledMemory.hpp)
add_library(quickstep_utility_Cast ../empty_src.cpp Cast.hpp)
add_library(quickstep_utility_CheckSnprintf ../empty_src.cpp CheckSnprintf.hpp)
@@ -202,7 +209,14 @@ target_link_libraries(quickstep_utility_BitVector
quickstep_utility_Macros)
target_link_libraries(quickstep_utility_BloomFilter
glog
+ quickstep_storage_StorageConstants
+ quickstep_threading_Mutex
+ quickstep_threading_SharedMutex
+ quickstep_threading_SpinSharedMutex
+ quickstep_utility_BloomFilter_proto
quickstep_utility_Macros)
+target_link_libraries(quickstep_utility_BloomFilter_proto
+ ${PROTOBUF_LIBRARY})
target_link_libraries(quickstep_utility_CalculateInstalledMemory
glog)
target_link_libraries(quickstep_utility_CheckSnprintf
@@ -271,6 +285,7 @@ target_link_libraries(quickstep_utility
quickstep_utility_BitManipulation
quickstep_utility_BitVector
quickstep_utility_BloomFilter
+ quickstep_utility_BloomFilter_proto
quickstep_utility_CalculateInstalledMemory
quickstep_utility_Cast
quickstep_utility_CheckSnprintf
[03/50] [abbrv] incubator-quickstep git commit: Refactored
makeRoomForBlock. (#192)
Posted by zu...@apache.org.
Refactored makeRoomForBlock. (#192)
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/3bd1586e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/3bd1586e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/3bd1586e
Branch: refs/heads/work-order-serialization
Commit: 3bd1586ecfa343f467398892a8887b3ea3c7a5f5
Parents: 7ac1d22
Author: Zuyu ZHANG <zu...@users.noreply.github.com>
Authored: Wed Apr 27 23:00:23 2016 -0700
Committer: Jignesh Patel <pa...@users.noreply.github.com>
Committed: Thu Apr 28 01:00:23 2016 -0500
----------------------------------------------------------------------
storage/StorageManager.cpp | 89 ++++++++++++++++++++---------------------
storage/StorageManager.hpp | 10 +++--
2 files changed, 50 insertions(+), 49 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3bd1586e/storage/StorageManager.cpp
----------------------------------------------------------------------
diff --git a/storage/StorageManager.cpp b/storage/StorageManager.cpp
index b98a28c..b990a22 100644
--- a/storage/StorageManager.cpp
+++ b/storage/StorageManager.cpp
@@ -397,7 +397,7 @@ void* StorageManager::allocateSlots(const std::size_t num_slots,
= MAP_PRIVATE | MAP_ANONYMOUS | MAP_ALIGNED_SUPER;
#endif
- makeRoomForBlock(num_slots);
+ makeRoomForBlockOrBlob(num_slots);
void *slots = nullptr;
#if defined(QUICKSTEP_HAVE_MMAP_LINUX_HUGETLB) || defined(QUICKSTEP_HAVE_MMAP_BSD_SUPERPAGE)
@@ -568,65 +568,62 @@ MutableBlobReference StorageManager::getBlobInternal(const block_id blob,
return ret;
}
-void StorageManager::makeRoomForBlock(const size_t slots) {
+void StorageManager::makeRoomForBlockOrBlob(const size_t slots) {
+ block_id block_to_evict;
while (total_memory_usage_ + slots > max_memory_usage_) {
- block_id block_index;
- EvictionPolicy::Status status = eviction_policy_->chooseBlockToEvict(&block_index);
-
- if (status == EvictionPolicy::Status::kOk) {
- bool has_collision = false;
- SpinSharedMutexExclusiveLock<false> eviction_lock(*lock_manager_.get(block_index, &has_collision));
- if (has_collision) {
- // We have a collision in the shared lock manager, where some callers
- // of this function (i.e., getBlockInternal or getBlobInternal) has
- // acquired an exclusive lock, and we are trying to evict a block that
- // hashes to the same location. This will cause a deadlock.
-
- // For now simply treat this situation as the case where there is not
- // enough memory and we temporarily go over the memory limit.
- break;
- }
+ const EvictionPolicy::Status status = eviction_policy_->chooseBlockToEvict(&block_to_evict);
+ if (status != EvictionPolicy::Status::kOk) {
+ // If status was not ok, then we must not have been able to evict enough
+ // blocks; therefore, we return anyway, temporarily going over the memory
+ // limit.
+ break;
+ }
- StorageBlockBase* block;
- {
- SpinSharedMutexSharedLock<false> read_lock(blocks_shared_mutex_);
- if (blocks_.find(block_index) == blocks_.end()) {
- // another thread must have jumped in and evicted it before us
-
- // NOTE(zuyu): It is ok to release the shard for a block or blob,
- // before 'eviction_lock' destructs, because we will never encounter a
- // self-deadlock in a single thread, and in multiple-thread case some
- // thread will block but not deadlock if there is a shard collision.
- lock_manager_.release(block_index);
- continue;
- }
- block = blocks_[block_index].block;
- }
- if (eviction_policy_->getRefCount(block->getID()) > 0) {
- // Someone sneaked in and referenced the block before we could evict it.
+ bool has_collision = false;
+ SpinSharedMutexExclusiveLock<false> eviction_lock(*lock_manager_.get(block_to_evict, &has_collision));
+ if (has_collision) {
+ // We have a collision in the shared lock manager, where some callers
+ // of this function (i.e., getBlockInternal or getBlobInternal) has
+ // acquired an exclusive lock, and we are trying to evict a block that
+ // hashes to the same location. This will cause a deadlock.
+
+ // For now simply treat this situation as the case where there is not
+ // enough memory and we temporarily go over the memory limit.
+ break;
+ }
- // NOTE(zuyu): It is ok to release the shard for a block or blob, before
+ {
+ SpinSharedMutexSharedLock<false> read_lock(blocks_shared_mutex_);
+ if (blocks_.find(block_to_evict) == blocks_.end()) {
+ // another thread must have jumped in and evicted it before us
+
+ // NOTE(zuyu): It is ok to release the shard for a block or blob,
// before 'eviction_lock' destructs, because we will never encounter a
// self-deadlock in a single thread, and in multiple-thread case some
// thread will block but not deadlock if there is a shard collision.
- lock_manager_.release(block_index);
+ lock_manager_.release(block_to_evict);
continue;
}
- if (saveBlockOrBlob(block->getID())) {
- evictBlockOrBlob(block->getID());
- } // else : Someone sneaked in and evicted the block before we could.
+ }
+ if (eviction_policy_->getRefCount(block_to_evict) > 0) {
+ // Someone sneaked in and referenced the block before we could evict it.
// NOTE(zuyu): It is ok to release the shard for a block or blob, before
// before 'eviction_lock' destructs, because we will never encounter a
// self-deadlock in a single thread, and in multiple-thread case some
// thread will block but not deadlock if there is a shard collision.
- lock_manager_.release(block_index);
- } else {
- // If status was not ok, then we must not have been able to evict enough
- // blocks; therefore, we return anyway, temporarily going over the memory
- // limit.
- break;
+ lock_manager_.release(block_to_evict);
+ continue;
}
+ if (saveBlockOrBlob(block_to_evict)) {
+ evictBlockOrBlob(block_to_evict);
+ } // else : Someone sneaked in and evicted the block before we could.
+
+ // NOTE(zuyu): It is ok to release the shard for a block or blob, before
+ // before 'eviction_lock' destructs, because we will never encounter a
+ // self-deadlock in a single thread, and in multiple-thread case some
+ // thread will block but not deadlock if there is a shard collision.
+ lock_manager_.release(block_to_evict);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3bd1586e/storage/StorageManager.hpp
----------------------------------------------------------------------
diff --git a/storage/StorageManager.hpp b/storage/StorageManager.hpp
index dab33f6..0b68b76 100644
--- a/storage/StorageManager.hpp
+++ b/storage/StorageManager.hpp
@@ -402,12 +402,16 @@ class StorageManager {
const int numa_node);
/**
- * @brief Evict blocks until there is enough space for a new block of the
- * requested size.
+ * @brief Evict blocks or blobs until there is enough space for a new block
+ * or blob of the requested size.
+ *
+ * @note This non-blocking method gives up evictions if there is a shard
+ * collision, and thus the buffer pool size may temporarily go beyond
+ * the memory limit.
*
* @param slots Number of slots to make room for.
*/
- void makeRoomForBlock(const std::size_t slots);
+ void makeRoomForBlockOrBlob(const std::size_t slots);
/**
* @brief Load a block from the persistent storage into memory.
[31/50] [abbrv] incubator-quickstep git commit: Quickstep print
catalog (#222)
Posted by zu...@apache.org.
Quickstep print catalog (#222)
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/3c845376
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/3c845376
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/3c845376
Branch: refs/heads/work-order-serialization
Commit: 3c8453766249d070b034b7a16e48aeff6a840e42
Parents: 2c0722e
Author: Rogers Jeffrey Leo John <ro...@gmail.com>
Authored: Tue May 17 18:43:02 2016 -0500
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Mon May 30 15:47:45 2016 -0700
----------------------------------------------------------------------
cli/CMakeLists.txt | 3 +-
cli/CommandExecutor.cpp | 50 +++++++++++++++++++++++-----
cli/CommandExecutor.hpp | 8 +++--
cli/PrintToScreen.cpp | 11 ++++++
cli/PrintToScreen.hpp | 8 +++++
cli/QuickstepCli.cpp | 4 ++-
cli/tests/CommandExecutorTestRunner.cpp | 1 +
cli/tests/command_executor/D.test | 35 ++++++++++++-------
cli/tests/command_executor/Dt.test | 36 +++++++++++++++-----
9 files changed, 124 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3c845376/cli/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/cli/CMakeLists.txt b/cli/CMakeLists.txt
index a1989d5..761b6d8 100644
--- a/cli/CMakeLists.txt
+++ b/cli/CMakeLists.txt
@@ -73,8 +73,9 @@ target_link_libraries(quickstep_cli_CommandExecutor
quickstep_catalog_CatalogDatabase
quickstep_catalog_CatalogRelation
quickstep_catalog_CatalogRelationSchema
+ quickstep_cli_PrintToScreen
quickstep_parser_ParseStatement
- quickstep_cli_PrintToScreen
+ quickstep_storage_StorageBlockInfo
quickstep_utility_Macros
quickstep_utility_PtrVector
quickstep_utility_SqlError)
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3c845376/cli/CommandExecutor.cpp
----------------------------------------------------------------------
diff --git a/cli/CommandExecutor.cpp b/cli/CommandExecutor.cpp
index f38121f..026922a 100644
--- a/cli/CommandExecutor.cpp
+++ b/cli/CommandExecutor.cpp
@@ -30,6 +30,7 @@
#include "catalog/CatalogRelationSchema.hpp"
#include "cli/PrintToScreen.hpp"
#include "parser/ParseStatement.hpp"
+#include "storage/StorageBlockInfo.hpp"
#include "utility/PtrVector.hpp"
#include "utility/Macros.hpp"
#include "utility/SqlError.hpp"
@@ -52,15 +53,22 @@ namespace C = ::quickstep::cli;
void executeDescribeDatabase(
const PtrVector<ParseString> *arguments,
- const CatalogDatabase &catalog_database, FILE *out) {
+ const CatalogDatabase &catalog_database,
+ StorageManager *storage_manager,
+ FILE *out) {
// Column width initialized to 6 to take into account the header name
// and the column value table
int max_column_width = C::kInitMaxColumnWidth;
+ vector<std::size_t> num_tuples;
+ vector<std::size_t> num_blocks;
const CatalogRelation *relation = nullptr;
if (arguments->size() == 0) {
for (const CatalogRelation &rel : catalog_database) {
max_column_width =
std::max(static_cast<int>(rel.getName().length()), max_column_width);
+ num_blocks.push_back(rel.size_blocks());
+ num_tuples.push_back(
+ PrintToScreen::GetNumTuplesInRelation(rel, storage_manager));
}
} else {
const ParseString &table_name = arguments->front();
@@ -72,26 +80,51 @@ void executeDescribeDatabase(
}
max_column_width = std::max(static_cast<int>(relation->getName().length()),
max_column_width);
+ num_blocks.push_back(relation->size_blocks());
+ num_tuples.push_back(PrintToScreen::GetNumTuplesInRelation(
+ *relation,
+ storage_manager));
}
// Only if we have relations work on the printing logic.
if (catalog_database.size() > 0) {
+ const std::size_t max_num_blocks = *std::max_element(num_blocks.begin(), num_blocks.end());
+ const std::size_t max_num_rows = *std::max_element(num_tuples.begin(), num_tuples.end());
+ const int max_num_rows_digits = std::max(PrintToScreen::GetNumberOfDigits(max_num_rows),
+ C::kInitMaxColumnWidth);
+ const int max_num_blocks_digits = std::max(PrintToScreen::GetNumberOfDigits(max_num_blocks),
+ C::kInitMaxColumnWidth+2);
+
vector<int> column_widths;
- column_widths.push_back(max_column_width+1);
- column_widths.push_back(C::kInitMaxColumnWidth+1);
+ column_widths.push_back(max_column_width +1);
+ column_widths.push_back(C::kInitMaxColumnWidth + 1);
+ column_widths.push_back(max_num_blocks_digits + 1);
+ column_widths.push_back(max_num_rows_digits + 1);
fputs(" List of relations\n\n", out);
fprintf(out, "%-*s |", max_column_width+1, " Name");
- fprintf(out, "%-*s\n", C::kInitMaxColumnWidth, " Type");
+ fprintf(out, "%-*s |", C::kInitMaxColumnWidth, " Type");
+ fprintf(out, "%-*s |", max_num_blocks_digits, " Blocks");
+ fprintf(out, "%-*s\n", max_num_rows_digits, " Rows");
PrintToScreen::printHBar(column_widths, out);
// If there are no argument print the entire list of tables
// else print the particular table only.
+ vector<std::size_t>::const_iterator num_tuples_it = num_tuples.begin();
+ vector<std::size_t>::const_iterator num_blocks_it = num_blocks.begin();
if (arguments->size() == 0) {
for (const CatalogRelation &rel : catalog_database) {
fprintf(out, " %-*s |", max_column_width, rel.getName().c_str());
- fprintf(out, " %-*s\n", C::kInitMaxColumnWidth, "table");
+ fprintf(out, " %-*s |", C::kInitMaxColumnWidth - 1, "table");
+ fprintf(out, " %-*lu |", max_num_blocks_digits - 1, *num_blocks_it);
+ fprintf(out, " %-*lu\n", max_num_rows_digits - 1, *num_tuples_it);
+ ++num_tuples_it;
+ ++num_blocks_it;
}
} else {
fprintf(out, " %-*s |", max_column_width, relation->getName().c_str());
- fprintf(out, " %-*s\n", C::kInitMaxColumnWidth, "table");
+ fprintf(out, " %-*s |", C::kInitMaxColumnWidth -1, "table");
+ fprintf(out, " %-*lu |", max_num_blocks_digits - 1, *num_blocks_it);
+ fprintf(out, " %-*lu\n", max_num_rows_digits - 1, *num_tuples_it);
+ ++num_tuples_it;
+ ++num_blocks_it;
}
fputc('\n', out);
}
@@ -166,15 +199,16 @@ void executeDescribeTable(
void executeCommand(const ParseStatement &statement,
const CatalogDatabase &catalog_database,
+ StorageManager *storage_manager,
FILE *out) {
const ParseCommand &command = static_cast<const ParseCommand &>(statement);
const PtrVector<ParseString> *arguments = command.arguments();
const std::string &command_str = command.command()->value();
if (command_str == C::kDescribeDatabaseCommand) {
- executeDescribeDatabase(arguments, catalog_database, out);
+ executeDescribeDatabase(arguments, catalog_database, storage_manager, out);
} else if (command_str == C::kDescribeTableCommand) {
if (arguments->size() == 0) {
- executeDescribeDatabase(arguments, catalog_database, out);
+ executeDescribeDatabase(arguments, catalog_database, storage_manager, out);
} else {
executeDescribeTable(arguments, catalog_database, out);
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3c845376/cli/CommandExecutor.hpp
----------------------------------------------------------------------
diff --git a/cli/CommandExecutor.hpp b/cli/CommandExecutor.hpp
index 21eee6a..f367ca1 100644
--- a/cli/CommandExecutor.hpp
+++ b/cli/CommandExecutor.hpp
@@ -19,9 +19,11 @@
#define QUICKSTEP_CLI_COMMAND_COMMAND_EXECUTOR_HPP_
#include <cstdio>
+#include <limits>
#include <string>
#include "parser/ParseStatement.hpp"
+#include "storage/StorageBlockInfo.hpp"
#include "utility/Macros.hpp"
using std::fprintf;
@@ -33,6 +35,7 @@ namespace quickstep {
class CatalogDatabase;
class CatalogAttribute;
class CatalogRelation;
+class StorageManager;
namespace cli {
/** \addtogroup CLI
@@ -49,13 +52,14 @@ constexpr char kDescribeTableCommand[] = "\\d";
/**
* @brief Executes the command by calling the command handler.
- *
+ *
* @param statement The parsed statement from the cli.
* @param catalog_database The catalog information about the current database.
- * @param out The stream where the output of the command has to be redirected to.
+ * @param out The stream where the output of the command has to be redirected to.
*/
void executeCommand(const ParseStatement &statement,
const CatalogDatabase &catalog_database,
+ StorageManager *storage_manager,
FILE *out);
/** @} */
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3c845376/cli/PrintToScreen.cpp
----------------------------------------------------------------------
diff --git a/cli/PrintToScreen.cpp b/cli/PrintToScreen.cpp
index 227ff39..76e90eb 100644
--- a/cli/PrintToScreen.cpp
+++ b/cli/PrintToScreen.cpp
@@ -19,6 +19,7 @@
#include <cstddef>
#include <cstdio>
+#include <cmath>
#include <memory>
#include <vector>
@@ -47,6 +48,16 @@ DEFINE_bool(printing_enabled, true,
"If true, print query results to screen normally. If false, skip "
"printing output (e.g. for benchmarking).");
+int PrintToScreen::GetNumberOfDigits(int number) {
+ if (number > 0) {
+ return static_cast<int>(log10 (number)) + 1;
+ } else if (number < 0) {
+ return static_cast<int>(log10 ( abs(number) )) + 2;
+ } else {
+ return 1;
+ }
+}
+
void PrintToScreen::PrintRelation(const CatalogRelation &relation,
StorageManager *storage_manager,
FILE *out) {
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3c845376/cli/PrintToScreen.hpp
----------------------------------------------------------------------
diff --git a/cli/PrintToScreen.hpp b/cli/PrintToScreen.hpp
index 0b57b4b..6a29426 100644
--- a/cli/PrintToScreen.hpp
+++ b/cli/PrintToScreen.hpp
@@ -69,6 +69,14 @@ class PrintToScreen {
StorageManager *storage_manager,
FILE *out);
+ /**
+ * @brief Return the number of digits in a number
+ *
+ * @param number The input number.
+ * @param out The number of digits in the input number.
+ **/
+ static int GetNumberOfDigits(int number);
+
private:
// Undefined default constructor. Class is all-static and should not be
// instantiated.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3c845376/cli/QuickstepCli.cpp
----------------------------------------------------------------------
diff --git a/cli/QuickstepCli.cpp b/cli/QuickstepCli.cpp
index 5881b3e..b7b28ba 100644
--- a/cli/QuickstepCli.cpp
+++ b/cli/QuickstepCli.cpp
@@ -365,7 +365,9 @@ int main(int argc, char* argv[]) {
try {
quickstep::cli::executeCommand(
*result.parsed_statement,
- *(query_processor->getDefaultDatabase()), stdout);
+ *(query_processor->getDefaultDatabase()),
+ query_processor->getStorageManager(),
+ stdout);
} catch (const quickstep::SqlError &sql_error) {
fprintf(stderr, "%s",
sql_error.formatMessage(*command_string).c_str());
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3c845376/cli/tests/CommandExecutorTestRunner.cpp
----------------------------------------------------------------------
diff --git a/cli/tests/CommandExecutorTestRunner.cpp b/cli/tests/CommandExecutorTestRunner.cpp
index 49930e1..73d2092 100644
--- a/cli/tests/CommandExecutorTestRunner.cpp
+++ b/cli/tests/CommandExecutorTestRunner.cpp
@@ -87,6 +87,7 @@ void CommandExecutorTestRunner::runTestCase(
quickstep::cli::executeCommand(
*result.parsed_statement,
*(test_database_loader_.catalog_database()),
+ test_database_loader_.storage_manager(),
output_stream.file());
} else {
QueryHandle query_handle(optimizer_context.query_id());
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3c845376/cli/tests/command_executor/D.test
----------------------------------------------------------------------
diff --git a/cli/tests/command_executor/D.test b/cli/tests/command_executor/D.test
index 1d500df..45f8d8b 100644
--- a/cli/tests/command_executor/D.test
+++ b/cli/tests/command_executor/D.test
@@ -34,9 +34,21 @@ CREATE TABLE foo4 (col1 INT,
col3 DOUBLE,
col4 FLOAT,
col5 CHAR(5));
-CREATE INDEX foo4_index_1 ON foo4 (col1, col2) USING CSBTREE;
-CREATE INDEX foo4_index_2 ON foo4 (col3, col4) USING CSBTREE;
+CREATE INDEX foo4_index_1 ON foo4 (col1, col2) USING CSBTREE;
+CREATE INDEX foo4_index_2 ON foo4 (col3, col4) USING CSBTREE;
CREATE TABLE averylongtablenamethatseemstoneverend (col1 INT);
+DROP TABLE TEST;
+INSERT INTO averylongtablenamethatseemstoneverend VALUES (1);
+INSERT INTO averylongtablenamethatseemstoneverend VALUES (2);
+INSERT INTO averylongtablenamethatseemstoneverend VALUES (3);
+INSERT INTO foo values(1, 1, 1.0, 1.0, 'XYZ');
+INSERT INTO foo values(2, 1, 1.0, 1.0, 'XYZ');
+INSERT INTO foo values(3, 1, 1.0, 1.0, 'XYZ');
+INSERT INTO foo values(4, 1, 1.0, 1.0, 'XYZ');
+INSERT INTO foo values(5, 1, 1.0, 1.0, 'XYZ');
+INSERT INTO foo2 values(5, 1, 1.0, 1.0, 'XYZ');
+INSERT INTO foo2 values(5, 1, 1.0, 1.0, 'XYZ');
+INSERT INTO foo3 values(5, 1, 1.0, 1.0, 'XYZZ');
--
==
\d foo
@@ -50,7 +62,7 @@ CREATE TABLE averylongtablenamethatseemstoneverend (col1 INT);
col4 | Float
col5 | Char(5)
==
-\d foo2
+\d foo2
--
Table "foo2"
Column | Type
@@ -73,7 +85,7 @@ CREATE TABLE averylongtablenamethatseemstoneverend (col1 INT);
col5 | Char(5)
Indexes
"foo3_index_1" CSB_TREE (col1)
-==
+==
\d foo4
--
Table "foo4"
@@ -92,14 +104,13 @@ CREATE TABLE averylongtablenamethatseemstoneverend (col1 INT);
--
List of relations
- Name | Type
-+--------------------------------------+-------+
- Test | table
- foo | table
- foo2 | table
- foo3 | table
- foo4 | table
- averylongtablenamethatseemstoneverend | table
+ Name | Type | Blocks | Rows
++--------------------------------------+-------+---------+-------+
+ foo | table | 1 | 5
+ foo2 | table | 1 | 2
+ foo3 | table | 1 | 1
+ foo4 | table | 0 | 0
+ averylongtablenamethatseemstoneverend | table | 1 | 3
==
\d invalidtable
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3c845376/cli/tests/command_executor/Dt.test
----------------------------------------------------------------------
diff --git a/cli/tests/command_executor/Dt.test b/cli/tests/command_executor/Dt.test
index 6458e15..1de6360 100644
--- a/cli/tests/command_executor/Dt.test
+++ b/cli/tests/command_executor/Dt.test
@@ -33,21 +33,41 @@ CREATE TABLE foo4 (col1 INT,
col3 DOUBLE,
col4 FLOAT,
col5 CHAR(5));
+DROP TABLE TEST;
CREATE TABLE averylongtablenamethatseemstoneverend (col1 INT);
+INSERT INTO averylongtablenamethatseemstoneverend VALUES (1);
+INSERT INTO averylongtablenamethatseemstoneverend VALUES (2);
+INSERT INTO averylongtablenamethatseemstoneverend VALUES (3);
+INSERT INTO foo values(1, 1, 1.0, 1.0, 'XYZ');
+INSERT INTO foo values(2, 1, 1.0, 1.0, 'XYZ');
+INSERT INTO foo values(3, 1, 1.0, 1.0, 'XYZ');
+INSERT INTO foo values(4, 1, 1.0, 1.0, 'XYZ');
+INSERT INTO foo values(5, 1, 1.0, 1.0, 'XYZ');
+INSERT INTO foo2 values(5, 1, 1.0, 1.0, 'XYZ');
+INSERT INTO foo2 values(5, 1, 1.0, 1.0, 'XYZ');
+INSERT INTO foo3 values(5, 1, 1.0, 1.0, 'XYZZ');
--
==
\dt
--
List of relations
- Name | Type
-+--------------------------------------+-------+
- Test | table
- foo | table
- foo2 | table
- foo3 | table
- foo4 | table
- averylongtablenamethatseemstoneverend | table
+ Name | Type | Blocks | Rows
++--------------------------------------+-------+---------+-------+
+ foo | table | 1 | 5
+ foo2 | table | 1 | 2
+ foo3 | table | 1 | 1
+ foo4 | table | 0 | 0
+ averylongtablenamethatseemstoneverend | table | 1 | 3
+
+==
+\dt foo
+--
+ List of relations
+
+ Name | Type | Blocks | Rows
++-------+-------+---------+-------+
+ foo | table | 1 | 5
==
\dt invalidtable
[15/50] [abbrv] incubator-quickstep git commit: Transaction Part 4:
LockManager, CycleDetector and DeadLockDetector. (#187)
Posted by zu...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bbaff7a2/transaction/tests/StronglyConnectedComponents_unittest.cpp
----------------------------------------------------------------------
diff --git a/transaction/tests/StronglyConnectedComponents_unittest.cpp b/transaction/tests/StronglyConnectedComponents_unittest.cpp
index 79d6881..35ef842 100644
--- a/transaction/tests/StronglyConnectedComponents_unittest.cpp
+++ b/transaction/tests/StronglyConnectedComponents_unittest.cpp
@@ -35,21 +35,20 @@ namespace transaction {
class GraphConfiguration {
public:
GraphConfiguration(DirectedGraph *graph,
- std::size_t no_transaction,
+ const std::size_t num_transactions,
const std::vector<std::pair<transaction_id,
transaction_id>> &mapping)
: graph_(graph) {
- for (std::size_t index = 0; index < no_transaction; ++index) {
- std::unique_ptr<transaction_id> tid =
- std::make_unique<transaction_id>(transaction_id(index));
- transaction_list_.push_back(*tid);
- DirectedGraph::node_id nid = graph->addNodeUnchecked(tid.release());
+ for (std::size_t index = 0; index < num_transactions; ++index) {
+ const transaction_id transaction = static_cast<transaction_id>(index);
+ transaction_list_.push_back(transaction);
+ const DirectedGraph::node_id nid = graph->addNodeUnchecked(transaction);
node_id_list_.push_back(nid);
}
for (const std::pair<transaction_id, transaction_id> &edge : mapping) {
- transaction_id pending = edge.first;
- transaction_id owner = edge.second;
+ const transaction_id pending = edge.first;
+ const transaction_id owner = edge.second;
graph_->addEdgeUnchecked(pending, owner);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bbaff7a2/transaction/tests/TransactionTable_unittest.cpp
----------------------------------------------------------------------
diff --git a/transaction/tests/TransactionTable_unittest.cpp b/transaction/tests/TransactionTable_unittest.cpp
index f5b5bc9..cd47a2b 100644
--- a/transaction/tests/TransactionTable_unittest.cpp
+++ b/transaction/tests/TransactionTable_unittest.cpp
@@ -41,91 +41,101 @@ class TransactionTableTest : public ::testing::Test {
};
TEST_F(TransactionTableTest, NormalOperations) {
- EXPECT_EQ(transaction_table_.putOwnEntry(tid_1_,
+ const AccessMode is_lock_mode = AccessMode::IsLockMode();
+ const AccessMode x_lock_mode = AccessMode::XLockMode();
+
+ EXPECT_EQ(TransactionTableResult::kPlacedInOwned,
+ transaction_table_.putOwnEntry(tid_1_,
ResourceId(3),
- AccessMode(AccessModeType::kIsLock)),
- TransactionTableResult::kPlacedInOwned);
+ is_lock_mode));
+
- EXPECT_EQ(transaction_table_.putPendingEntry(tid_1_,
+ EXPECT_EQ(TransactionTableResult::kPlacedInPending,
+ transaction_table_.putPendingEntry(tid_1_,
ResourceId(5),
- AccessMode(AccessModeType::kXLock)),
- TransactionTableResult::kPlacedInPending);
+ x_lock_mode));
}
TEST_F(TransactionTableTest, DeleteEntryOperations) {
- EXPECT_EQ(transaction_table_.deleteOwnEntry(tid_2_,
+ const AccessMode s_lock_mode = AccessMode::SLockMode();
+ const AccessMode x_lock_mode = AccessMode::XLockMode();
+
+ EXPECT_EQ(TransactionTableResult::kDelError,
+ transaction_table_.deleteOwnEntry(tid_2_,
ResourceId(5),
- AccessMode(AccessModeType::kSLock)),
- TransactionTableResult::kDelError);
+ s_lock_mode));
- EXPECT_EQ(transaction_table_.putOwnEntry(tid_2_,
+ EXPECT_EQ(TransactionTableResult::kPlacedInOwned,
+ transaction_table_.putOwnEntry(tid_2_,
ResourceId(5),
- AccessMode(AccessModeType::kSLock)),
- TransactionTableResult::kPlacedInOwned);
+ s_lock_mode));
// Tring to delete a lock with different acces mode on same resource id
// will result in an error.
- EXPECT_EQ(transaction_table_.deleteOwnEntry(tid_2_,
+ EXPECT_EQ(TransactionTableResult::kDelError,
+ transaction_table_.deleteOwnEntry(tid_2_,
ResourceId(5),
- AccessMode(AccessModeType::kXLock)),
- TransactionTableResult::kDelError);
+ x_lock_mode));
// Transaction 3 does not have a lock on this resource id.
- EXPECT_EQ(transaction_table_.deleteOwnEntry(tid_3_,
+ EXPECT_EQ(TransactionTableResult::kDelError,
+ transaction_table_.deleteOwnEntry(tid_3_,
ResourceId(5),
- AccessMode(AccessModeType::kSLock)),
- TransactionTableResult::kDelError);
+ s_lock_mode));
// This will result in success since transaction 2 have acquired the lock on
// this resource with the corresponding mode.
- EXPECT_EQ(transaction_table_.deleteOwnEntry(tid_2_,
+ EXPECT_EQ(TransactionTableResult::kDelFromOwned,
+ transaction_table_.deleteOwnEntry(tid_2_,
ResourceId(5),
- AccessMode(AccessModeType::kSLock)),
- TransactionTableResult::kDelFromOwned);
+ s_lock_mode));
// Repeat the previous sequence, with pending list.
- EXPECT_EQ(transaction_table_.deletePendingEntry(tid_2_,
+ EXPECT_EQ(TransactionTableResult::kDelError,
+ transaction_table_.deletePendingEntry(tid_2_,
ResourceId(5),
- AccessMode(AccessModeType::kSLock)),
- TransactionTableResult::kDelError);
+ s_lock_mode));
- EXPECT_EQ(transaction_table_.putPendingEntry(tid_2_,
+ EXPECT_EQ(TransactionTableResult::kPlacedInPending,
+ transaction_table_.putPendingEntry(tid_2_,
ResourceId(5),
- AccessMode(AccessModeType::kSLock)),
- TransactionTableResult::kPlacedInPending);
+ s_lock_mode));
- EXPECT_EQ(transaction_table_.deletePendingEntry(tid_2_,
+
+ EXPECT_EQ(TransactionTableResult::kDelError,
+ transaction_table_.deletePendingEntry(tid_2_,
ResourceId(5),
- AccessMode(AccessModeType::kXLock)),
- TransactionTableResult::kDelError);
+ x_lock_mode));
- EXPECT_EQ(transaction_table_.deletePendingEntry(tid_3_,
+ EXPECT_EQ(TransactionTableResult::kDelError,
+ transaction_table_.deletePendingEntry(tid_3_,
ResourceId(5),
- AccessMode(AccessModeType::kSLock)),
- TransactionTableResult::kDelError);
+ s_lock_mode));
- EXPECT_EQ(transaction_table_.deletePendingEntry(tid_2_,
+ EXPECT_EQ(TransactionTableResult::kDelFromPending,
+ transaction_table_.deletePendingEntry(tid_2_,
ResourceId(5),
- AccessMode(AccessModeType::kSLock)),
- TransactionTableResult::kDelFromPending);
+ s_lock_mode));
}
TEST_F(TransactionTableTest, TransactionEntries) {
- EXPECT_EQ(transaction_table_.deleteTransaction(tid_1_),
- TransactionTableResult::kTransactionDeleteError);
+ const AccessMode s_lock_mode = AccessMode::SLockMode();
+
+ EXPECT_EQ(TransactionTableResult::kTransactionDeleteError,
+ transaction_table_.deleteTransaction(tid_1_));
- EXPECT_EQ(transaction_table_.putOwnEntry(tid_1_,
+ EXPECT_EQ(TransactionTableResult::kPlacedInOwned,
+ transaction_table_.putOwnEntry(tid_1_,
ResourceId(4),
- AccessMode(AccessModeType::kSLock)),
- TransactionTableResult::kPlacedInOwned);
+ s_lock_mode));
- EXPECT_EQ(transaction_table_.deleteTransaction(tid_1_),
- TransactionTableResult::kTransactionDeleteOk);
+ EXPECT_EQ(TransactionTableResult::kTransactionDeleteOk,
+ transaction_table_.deleteTransaction(tid_1_));
- EXPECT_EQ(transaction_table_.deleteOwnEntry(tid_1_,
+ EXPECT_EQ(TransactionTableResult::kDelError,
+ transaction_table_.deleteOwnEntry(tid_1_,
ResourceId(4),
- AccessMode(AccessModeType::kSLock)),
- TransactionTableResult::kDelError);
+ s_lock_mode));
}
} // namespace transaction
[11/50] [abbrv] incubator-quickstep git commit: Fixes a bug where
numa settings were not properly set (not compiled in) (#205)
Posted by zu...@apache.org.
Fixes a bug where numa settings were not properly set (not compiled in) (#205)
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/456b4348
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/456b4348
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/456b4348
Branch: refs/heads/work-order-serialization
Commit: 456b4348ddf8d6cb1f8b4c125e561da7fe205322
Parents: 29a71ac
Author: Marc S <cr...@users.noreply.github.com>
Authored: Tue May 3 12:06:38 2016 -0500
Committer: Harshad Deshmukh <d....@gmail.com>
Committed: Tue May 3 12:06:38 2016 -0500
----------------------------------------------------------------------
cli/DefaultsConfigurator.hpp | 16 ++++++++++++++++
cli/InputParserUtil.cpp | 1 +
cli/QuickstepCli.cpp | 6 +++---
3 files changed, 20 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/456b4348/cli/DefaultsConfigurator.hpp
----------------------------------------------------------------------
diff --git a/cli/DefaultsConfigurator.hpp b/cli/DefaultsConfigurator.hpp
index b40ef87..4da05b2 100644
--- a/cli/DefaultsConfigurator.hpp
+++ b/cli/DefaultsConfigurator.hpp
@@ -22,6 +22,7 @@
#include <unordered_map>
#include <vector>
+#include "storage/StorageConfig.h"
#include "utility/Macros.hpp"
#ifdef QUICKSTEP_HAVE_LIBNUMA
@@ -51,6 +52,21 @@ class DefaultsConfigurator {
}
/**
+ * @brief Get the number of available numa sockets.
+ *
+ * @return Number of available numa sockets. Always 1 if the system doesn't
+ * have libnuma.
+ **/
+ static std::size_t GetNumNUMANodes() {
+ #ifdef QUICKSTEP_HAVE_LIBNUMA
+ // Id of the maximum node.
+ return numa_max_node() + 1;
+ #else
+ return 1;
+ #endif
+ }
+
+ /**
* @brief Get the number of NUMA nodes covered by the given worker affinities
* to the CPU cores.
*
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/456b4348/cli/InputParserUtil.cpp
----------------------------------------------------------------------
diff --git a/cli/InputParserUtil.cpp b/cli/InputParserUtil.cpp
index 328aaeb..352883e 100644
--- a/cli/InputParserUtil.cpp
+++ b/cli/InputParserUtil.cpp
@@ -24,6 +24,7 @@
#include <vector>
#include "catalog/CatalogConfig.h"
+#include "storage/StorageConfig.h"
#include "utility/StringUtil.hpp"
#include "glog/logging.h"
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/456b4348/cli/QuickstepCli.cpp
----------------------------------------------------------------------
diff --git a/cli/QuickstepCli.cpp b/cli/QuickstepCli.cpp
index 8dee1f7..ec195f7 100644
--- a/cli/QuickstepCli.cpp
+++ b/cli/QuickstepCli.cpp
@@ -258,8 +258,7 @@ int main(int argc, char* argv[]) {
InputParserUtil::ParseWorkerAffinities(real_num_workers,
quickstep::FLAGS_worker_affinities);
- const std::size_t num_numa_nodes_covered =
- DefaultsConfigurator::GetNumNUMANodesCoveredByWorkers(worker_cpu_affinities);
+ const std::size_t num_numa_nodes_system = DefaultsConfigurator::GetNumNUMANodes();
if (quickstep::FLAGS_preload_buffer_pool) {
std::chrono::time_point<std::chrono::steady_clock> preload_start, preload_end;
@@ -280,7 +279,8 @@ int main(int argc, char* argv[]) {
Foreman foreman(&bus,
query_processor->getDefaultDatabase(),
query_processor->getStorageManager(),
- num_numa_nodes_covered);
+ -1, /* CPU id to bind foreman. -1 is unbound. */
+ num_numa_nodes_system);
// Get the NUMA affinities for workers.
vector<int> cpu_numa_nodes = InputParserUtil::GetNUMANodesForCPUs();
[35/50] [abbrv] incubator-quickstep git commit: Added hash join order
optimization for star schema queries. (#229)
Posted by zu...@apache.org.
Added hash join order optimization for star schema queries. (#229)
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/0fffe505
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/0fffe505
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/0fffe505
Branch: refs/heads/work-order-serialization
Commit: 0fffe5057da6b6abc1d9cdf0f7dab0accc8a0fe1
Parents: df4a05d
Author: Jianqiao Zhu <ji...@cs.wisc.edu>
Authored: Thu May 19 18:19:28 2016 -0500
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Mon May 30 15:47:52 2016 -0700
----------------------------------------------------------------------
query_optimizer/CMakeLists.txt | 2 +
query_optimizer/PhysicalGenerator.cpp | 14 +
query_optimizer/cost_model/CMakeLists.txt | 32 +-
.../cost_model/StarSchemaSimpleCostModel.cpp | 258 ++++++++++++++++
.../cost_model/StarSchemaSimpleCostModel.hpp | 115 +++++++
query_optimizer/rules/CMakeLists.txt | 20 +-
.../StarSchemaHashJoinOrderOptimization.cpp | 309 +++++++++++++++++++
.../StarSchemaHashJoinOrderOptimization.hpp | 136 ++++++++
query_optimizer/tests/CMakeLists.txt | 1 +
query_optimizer/tests/OptimizerTextTest.cpp | 14 +
10 files changed, 899 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/0fffe505/query_optimizer/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_optimizer/CMakeLists.txt b/query_optimizer/CMakeLists.txt
index feaecb3..aa2873e 100644
--- a/query_optimizer/CMakeLists.txt
+++ b/query_optimizer/CMakeLists.txt
@@ -182,10 +182,12 @@ target_link_libraries(quickstep_queryoptimizer_OptimizerTree
quickstep_utility_Macros
quickstep_utility_TreeStringSerializable)
target_link_libraries(quickstep_queryoptimizer_PhysicalGenerator
+ gflags_nothreads-static
quickstep_queryoptimizer_LogicalToPhysicalMapper
quickstep_queryoptimizer_logical_Logical
quickstep_queryoptimizer_physical_Physical
quickstep_queryoptimizer_rules_PruneColumns
+ quickstep_queryoptimizer_rules_StarSchemaHashJoinOrderOptimization
quickstep_queryoptimizer_strategy_Aggregate
quickstep_queryoptimizer_strategy_Join
quickstep_queryoptimizer_strategy_OneToOne
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/0fffe505/query_optimizer/PhysicalGenerator.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/PhysicalGenerator.cpp b/query_optimizer/PhysicalGenerator.cpp
index ee52966..662236f 100644
--- a/query_optimizer/PhysicalGenerator.cpp
+++ b/query_optimizer/PhysicalGenerator.cpp
@@ -1,6 +1,8 @@
/**
* Copyright 2011-2015 Quickstep Technologies LLC.
* Copyright 2015 Pivotal Software, Inc.
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -25,17 +27,26 @@
#include "query_optimizer/logical/Logical.hpp"
#include "query_optimizer/physical/Physical.hpp"
#include "query_optimizer/rules/PruneColumns.hpp"
+#include "query_optimizer/rules/StarSchemaHashJoinOrderOptimization.hpp"
#include "query_optimizer/strategy/Aggregate.hpp"
#include "query_optimizer/strategy/Join.hpp"
#include "query_optimizer/strategy/OneToOne.hpp"
#include "query_optimizer/strategy/Selection.hpp"
#include "query_optimizer/strategy/Strategy.hpp"
+#include "gflags/gflags.h"
+
#include "glog/logging.h"
namespace quickstep {
namespace optimizer {
+DEFINE_bool(reorder_hash_joins, true,
+ "If true, apply hash join order optimization to each group of hash "
+ "joins. The optimization applies a greedy algorithm to favor smaller "
+ "cardinality and selective tables to be joined first, which is suitable "
+ "for queries on star-schema tables");
+
namespace L = ::quickstep::optimizer::logical;
namespace P = ::quickstep::optimizer::physical;
namespace S = ::quickstep::optimizer::strategy;
@@ -77,6 +88,9 @@ P::PhysicalPtr PhysicalGenerator::generateInitialPlan(
P::PhysicalPtr PhysicalGenerator::optimizePlan() {
std::vector<std::unique_ptr<Rule<P::Physical>>> rules;
+ if (FLAGS_reorder_hash_joins) {
+ rules.emplace_back(new StarSchemaHashJoinOrderOptimization());
+ }
rules.emplace_back(new PruneColumns());
for (std::unique_ptr<Rule<P::Physical>> &rule : rules) {
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/0fffe505/query_optimizer/cost_model/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_optimizer/cost_model/CMakeLists.txt b/query_optimizer/cost_model/CMakeLists.txt
index 6697d52..6bf5240 100644
--- a/query_optimizer/cost_model/CMakeLists.txt
+++ b/query_optimizer/cost_model/CMakeLists.txt
@@ -1,5 +1,7 @@
# Copyright 2011-2015 Quickstep Technologies LLC.
# Copyright 2015 Pivotal Software, Inc.
+# Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+# University of Wisconsin\u2014Madison.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -16,6 +18,9 @@
# Declare micro-libs:
add_library(quickstep_queryoptimizer_costmodel_CostModel ../../empty_src.cpp CostModel.hpp)
add_library(quickstep_queryoptimizer_costmodel_SimpleCostModel SimpleCostModel.cpp SimpleCostModel.hpp)
+add_library(quickstep_queryoptimizer_costmodel_StarSchemaSimpleCostModel
+ StarSchemaSimpleCostModel.cpp
+ StarSchemaSimpleCostModel.hpp)
# Link dependencies:
target_link_libraries(quickstep_queryoptimizer_costmodel_CostModel
@@ -36,9 +41,34 @@ target_link_libraries(quickstep_queryoptimizer_costmodel_SimpleCostModel
quickstep_queryoptimizer_physical_TableReference
quickstep_queryoptimizer_physical_TopLevelPlan
quickstep_utility_Macros)
+target_link_libraries(quickstep_queryoptimizer_costmodel_StarSchemaSimpleCostModel
+ glog
+ quickstep_catalog_CatalogRelation
+ quickstep_queryoptimizer_costmodel_CostModel
+ quickstep_queryoptimizer_expressions_AttributeReference
+ quickstep_queryoptimizer_expressions_ComparisonExpression
+ quickstep_queryoptimizer_expressions_ExprId
+ quickstep_queryoptimizer_expressions_ExpressionType
+ quickstep_queryoptimizer_expressions_LogicalAnd
+ quickstep_queryoptimizer_expressions_LogicalOr
+ quickstep_queryoptimizer_expressions_PatternMatcher
+ quickstep_queryoptimizer_expressions_Predicate
+ quickstep_queryoptimizer_physical_Aggregate
+ quickstep_queryoptimizer_physical_HashJoin
+ quickstep_queryoptimizer_physical_NestedLoopsJoin
+ quickstep_queryoptimizer_physical_Physical
+ quickstep_queryoptimizer_physical_PhysicalType
+ quickstep_queryoptimizer_physical_Selection
+ quickstep_queryoptimizer_physical_SharedSubplanReference
+ quickstep_queryoptimizer_physical_Sort
+ quickstep_queryoptimizer_physical_TableGenerator
+ quickstep_queryoptimizer_physical_TableReference
+ quickstep_queryoptimizer_physical_TopLevelPlan
+ quickstep_utility_Macros)
# Module all-in-one library:
add_library(quickstep_queryoptimizer_costmodel ../../empty_src.cpp CostModelModule.hpp)
target_link_libraries(quickstep_queryoptimizer_costmodel
quickstep_queryoptimizer_costmodel_CostModel
- quickstep_queryoptimizer_costmodel_SimpleCostModel)
+ quickstep_queryoptimizer_costmodel_SimpleCostModel
+ quickstep_queryoptimizer_costmodel_StarSchemaSimpleCostModel)
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/0fffe505/query_optimizer/cost_model/StarSchemaSimpleCostModel.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/cost_model/StarSchemaSimpleCostModel.cpp b/query_optimizer/cost_model/StarSchemaSimpleCostModel.cpp
new file mode 100644
index 0000000..eb9fcc1
--- /dev/null
+++ b/query_optimizer/cost_model/StarSchemaSimpleCostModel.cpp
@@ -0,0 +1,258 @@
+/**
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#include "query_optimizer/cost_model/StarSchemaSimpleCostModel.hpp"
+
+#include <algorithm>
+#include <memory>
+#include <unordered_map>
+#include <vector>
+
+#include "catalog/CatalogRelation.hpp"
+#include "query_optimizer/expressions/AttributeReference.hpp"
+#include "query_optimizer/expressions/ComparisonExpression.hpp"
+#include "query_optimizer/expressions/ExprId.hpp"
+#include "query_optimizer/expressions/ExpressionType.hpp"
+#include "query_optimizer/expressions/LogicalAnd.hpp"
+#include "query_optimizer/expressions/LogicalOr.hpp"
+#include "query_optimizer/expressions/Predicate.hpp"
+#include "query_optimizer/expressions/PatternMatcher.hpp"
+#include "query_optimizer/physical/Aggregate.hpp"
+#include "query_optimizer/physical/NestedLoopsJoin.hpp"
+#include "query_optimizer/physical/HashJoin.hpp"
+#include "query_optimizer/physical/Physical.hpp"
+#include "query_optimizer/physical/PhysicalType.hpp"
+#include "query_optimizer/physical/Selection.hpp"
+#include "query_optimizer/physical/SharedSubplanReference.hpp"
+#include "query_optimizer/physical/Sort.hpp"
+#include "query_optimizer/physical/TableGenerator.hpp"
+#include "query_optimizer/physical/TableReference.hpp"
+#include "query_optimizer/physical/TopLevelPlan.hpp"
+
+#include "glog/logging.h"
+
+namespace quickstep {
+namespace optimizer {
+namespace cost {
+
+namespace E = ::quickstep::optimizer::expressions;
+namespace P = ::quickstep::optimizer::physical;
+
+std::size_t StarSchemaSimpleCostModel::estimateCardinality(
+ const P::PhysicalPtr &physical_plan) {
+ switch (physical_plan->getPhysicalType()) {
+ case P::PhysicalType::kTopLevelPlan:
+ return estimateCardinalityForTopLevelPlan(
+ std::static_pointer_cast<const P::TopLevelPlan>(physical_plan));
+ case P::PhysicalType::kTableReference:
+ return estimateCardinalityForTableReference(
+ std::static_pointer_cast<const P::TableReference>(physical_plan));
+ case P::PhysicalType::kSelection:
+ return estimateCardinalityForSelection(
+ std::static_pointer_cast<const P::Selection>(physical_plan));
+ case P::PhysicalType::kTableGenerator:
+ return estimateCardinalityForTableGenerator(
+ std::static_pointer_cast<const P::TableGenerator>(physical_plan));
+ case P::PhysicalType::kHashJoin:
+ return estimateCardinalityForHashJoin(
+ std::static_pointer_cast<const P::HashJoin>(physical_plan));
+ case P::PhysicalType::kNestedLoopsJoin:
+ return estimateCardinalityForNestedLoopsJoin(
+ std::static_pointer_cast<const P::NestedLoopsJoin>(physical_plan));
+ case P::PhysicalType::kAggregate:
+ return estimateCardinalityForAggregate(
+ std::static_pointer_cast<const P::Aggregate>(physical_plan));
+ case P::PhysicalType::kSharedSubplanReference: {
+ const P::SharedSubplanReferencePtr shared_subplan_reference =
+ std::static_pointer_cast<const P::SharedSubplanReference>(physical_plan);
+ return estimateCardinality(
+ shared_subplans_[shared_subplan_reference->subplan_id()]);
+ }
+ case P::PhysicalType::kSort:
+ return estimateCardinality(
+ std::static_pointer_cast<const P::Sort>(physical_plan)->input());
+ default:
+ LOG(FATAL) << "Unsupported physical plan:" << physical_plan->toString();
+ }
+}
+
+std::size_t StarSchemaSimpleCostModel::estimateCardinalityForTopLevelPlan(
+ const P::TopLevelPlanPtr &physical_plan) {
+ return estimateCardinality(physical_plan->plan());
+}
+
+std::size_t StarSchemaSimpleCostModel::estimateCardinalityForTableReference(
+ const P::TableReferencePtr &physical_plan) {
+ std::size_t num_tuples = physical_plan->relation()->getStatistics().getNumTuples();
+ if (num_tuples == 0) {
+ num_tuples = physical_plan->relation()->estimateTupleCardinality();
+ }
+ return num_tuples;
+}
+
+std::size_t StarSchemaSimpleCostModel::estimateCardinalityForSelection(
+ const P::SelectionPtr &physical_plan) {
+ double selectivity = estimateSelectivityForSelection(physical_plan);
+ return std::max(static_cast<std::size_t>(estimateCardinality(physical_plan->input()) * selectivity),
+ static_cast<std::size_t>(1));
+}
+
+std::size_t StarSchemaSimpleCostModel::estimateCardinalityForTableGenerator(
+ const P::TableGeneratorPtr &physical_plan) {
+ return physical_plan->generator_function_handle()->getEstimatedCardinality();
+}
+
+std::size_t StarSchemaSimpleCostModel::estimateCardinalityForHashJoin(
+ const P::HashJoinPtr &physical_plan) {
+ std::size_t left_cardinality = estimateCardinality(physical_plan->left());
+ std::size_t right_cardinality = estimateCardinality(physical_plan->right());
+ double left_selectivity = estimateSelectivity(physical_plan->left());
+ double right_selectivity = estimateSelectivity(physical_plan->right());
+ return std::max(static_cast<std::size_t>(left_cardinality * right_selectivity) + 1,
+ static_cast<std::size_t>(right_cardinality * left_selectivity) + 1);
+}
+
+std::size_t StarSchemaSimpleCostModel::estimateCardinalityForNestedLoopsJoin(
+ const P::NestedLoopsJoinPtr &physical_plan) {
+ return std::max(estimateCardinality(physical_plan->left()),
+ estimateCardinality(physical_plan->right()));
+}
+
+std::size_t StarSchemaSimpleCostModel::estimateCardinalityForAggregate(
+ const P::AggregatePtr &physical_plan) {
+ if (physical_plan->grouping_expressions().empty()) {
+ return 1;
+ }
+ return std::max(static_cast<std::size_t>(1),
+ estimateCardinality(physical_plan->input()) / 10);
+}
+
+double StarSchemaSimpleCostModel::estimateSelectivity(
+ const physical::PhysicalPtr &physical_plan) {
+ switch (physical_plan->getPhysicalType()) {
+ case P::PhysicalType::kSelection: {
+ return estimateSelectivityForSelection(
+ std::static_pointer_cast<const P::Selection>(physical_plan));
+ }
+ case P::PhysicalType::kHashJoin: {
+ const P::HashJoinPtr &hash_join =
+ std::static_pointer_cast<const P::HashJoin>(physical_plan);
+ return std::min(estimateSelectivity(hash_join->left()),
+ estimateSelectivity(hash_join->right()));
+ }
+ case P::PhysicalType::kNestedLoopsJoin: {
+ const P::NestedLoopsJoinPtr &nested_loop_join =
+ std::static_pointer_cast<const P::NestedLoopsJoin>(physical_plan);
+ return std::min(estimateSelectivity(nested_loop_join->left()),
+ estimateSelectivity(nested_loop_join->right()));
+ }
+ case P::PhysicalType::kSharedSubplanReference: {
+ const P::SharedSubplanReferencePtr shared_subplan_reference =
+ std::static_pointer_cast<const P::SharedSubplanReference>(physical_plan);
+ return estimateSelectivity(
+ shared_subplans_[shared_subplan_reference->subplan_id()]);
+ }
+ default:
+ return 1.0;
+ }
+}
+
+double StarSchemaSimpleCostModel::estimateSelectivityForSelection(
+ const physical::SelectionPtr &physical_plan) {
+ const E::PredicatePtr &filter_predicate = physical_plan->filter_predicate();
+
+ // If the subplan is a table reference, gather the number of distinct values
+ // statistics for each column (attribute).
+ std::unordered_map<E::ExprId, std::size_t> num_distinct_values_map;
+ if (physical_plan->input()->getPhysicalType() == P::PhysicalType::kTableReference) {
+ const P::TableReferencePtr &table_reference =
+ std::static_pointer_cast<const P::TableReference>(physical_plan->input());
+ const CatalogRelation &relation = *table_reference->relation();
+ const std::vector<E::AttributeReferencePtr> &attributes = table_reference->attribute_list();
+ for (std::size_t i = 0; i < attributes.size(); ++i) {
+ std::size_t num_distinct_values = relation.getStatistics().getNumDistinctValues(i);
+ if (num_distinct_values > 0) {
+ num_distinct_values_map[attributes[i]->id()] = num_distinct_values;
+ }
+ }
+ }
+
+ return estimateSelectivityForPredicate(num_distinct_values_map, filter_predicate);
+}
+
+double StarSchemaSimpleCostModel::estimateSelectivityForPredicate(
+ const std::unordered_map<expressions::ExprId, std::size_t> &num_distinct_values_map,
+ const expressions::PredicatePtr &filter_predicate) {
+ if (filter_predicate == nullptr) {
+ return 1.0;
+ }
+
+ switch (filter_predicate->getExpressionType()) {
+ case E::ExpressionType::kComparisonExpression: {
+ // Case 1 - Number of distinct values statistics available
+ // Case 1.1 - Equality comparison: 1.0 / num_distinct_values
+ // Case 1.2 - Otherwise: 5.0 / num_distinct_values
+ // Case 2 - Number of distinct values statistics not available
+ // Case 2.1 - Equality comparison: 0.1
+ // Case 2.2 - Otherwise: 0.5
+ const E::ComparisonExpressionPtr &comparison_expression =
+ std::static_pointer_cast<const E::ComparisonExpression>(filter_predicate);
+ E::AttributeReferencePtr attr;
+ if ((E::SomeAttributeReference::MatchesWithConditionalCast(comparison_expression->left(), &attr) &&
+ E::SomeScalarLiteral::Matches(comparison_expression->right())) ||
+ (E::SomeAttributeReference::MatchesWithConditionalCast(comparison_expression->right(), &attr) &&
+ E::SomeScalarLiteral::Matches(comparison_expression->left()))) {
+ const auto it = num_distinct_values_map.find(attr->id());
+ if (it != num_distinct_values_map.end() && it->second > 0) {
+ double unit_selectivity = 1.0 / it->second;
+ return comparison_expression->isEqualityComparisonPredicate()
+ ? unit_selectivity
+ : std::min(0.5, unit_selectivity * 5.0);
+ }
+ }
+
+ return comparison_expression->isEqualityComparisonPredicate() ? 0.1 : 0.5;
+ }
+ case E::ExpressionType::kLogicalAnd: {
+ const E::LogicalAndPtr &logical_and =
+ std::static_pointer_cast<const E::LogicalAnd>(filter_predicate);
+ double selectivity = 1.0;
+ for (const auto &predicate : logical_and->operands()) {
+ selectivity =
+ std::min(selectivity,
+ estimateSelectivityForPredicate(num_distinct_values_map, predicate));
+ }
+ return selectivity;
+ }
+ case E::ExpressionType::kLogicalOr: {
+ const E::LogicalOrPtr &logical_or =
+ std::static_pointer_cast<const E::LogicalOr>(filter_predicate);
+ double selectivity = 0;
+ for (const auto &predicate : logical_or->operands()) {
+ selectivity += estimateSelectivityForPredicate(num_distinct_values_map, predicate);
+ }
+ return std::min(selectivity, 1.0);
+ }
+ default:
+ break;
+ }
+ return 1.0;
+}
+
+} // namespace cost
+} // namespace optimizer
+} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/0fffe505/query_optimizer/cost_model/StarSchemaSimpleCostModel.hpp
----------------------------------------------------------------------
diff --git a/query_optimizer/cost_model/StarSchemaSimpleCostModel.hpp b/query_optimizer/cost_model/StarSchemaSimpleCostModel.hpp
new file mode 100644
index 0000000..c63e55a
--- /dev/null
+++ b/query_optimizer/cost_model/StarSchemaSimpleCostModel.hpp
@@ -0,0 +1,115 @@
+/**
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#ifndef QUERY_OPTIMIZER_COST_MODEL_STAR_SCHEMA_SIMPLE_COST_MODEL_HPP_
+#define QUERY_OPTIMIZER_COST_MODEL_STAR_SCHEMA_SIMPLE_COST_MODEL_HPP_
+
+#include <cstddef>
+#include <unordered_map>
+#include <vector>
+
+#include "query_optimizer/cost_model/CostModel.hpp"
+#include "query_optimizer/expressions/ExprId.hpp"
+#include "query_optimizer/expressions/Predicate.hpp"
+#include "query_optimizer/physical/Aggregate.hpp"
+#include "query_optimizer/physical/NestedLoopsJoin.hpp"
+#include "query_optimizer/physical/HashJoin.hpp"
+#include "query_optimizer/physical/Physical.hpp"
+#include "query_optimizer/physical/Selection.hpp"
+#include "query_optimizer/physical/TableGenerator.hpp"
+#include "query_optimizer/physical/TableReference.hpp"
+#include "query_optimizer/physical/TopLevelPlan.hpp"
+#include "utility/Macros.hpp"
+
+namespace quickstep {
+namespace optimizer {
+namespace cost {
+
+/** \addtogroup CostModel
+ * @{
+ */
+
+/**
+ * @brief A simple cost model for hash join planning.
+ */
+class StarSchemaSimpleCostModel : public CostModel {
+ public:
+ /**
+ * @brief Constructor.
+ */
+ explicit StarSchemaSimpleCostModel(const std::vector<physical::PhysicalPtr> &shared_subplans)
+ : shared_subplans_(shared_subplans) {}
+
+ /**
+ * @brief Estimate the cardinality of a physical plan.
+ *
+ * @param physical_plan The physical plan.
+ * @return The estimated cardinality.
+ */
+ std::size_t estimateCardinality(
+ const physical::PhysicalPtr &physical_plan) override;
+
+ /**
+ * @brief Estimate the "selectivity" of a physical plan under the assumption
+ * that it acts as a filtered dimension table in a hash join.
+ *
+ * @param phyiscal_plan The physical plan.
+ * @return The estimated selectivity.
+ */
+ double estimateSelectivity(const physical::PhysicalPtr &physical_plan);
+
+ private:
+ std::size_t estimateCardinalityForTopLevelPlan(
+ const physical::TopLevelPlanPtr &physical_plan);
+
+ std::size_t estimateCardinalityForTableReference(
+ const physical::TableReferencePtr &physical_plan);
+
+ std::size_t estimateCardinalityForSelection(
+ const physical::SelectionPtr &physical_plan);
+
+ std::size_t estimateCardinalityForTableGenerator(
+ const physical::TableGeneratorPtr &physical_plan);
+
+ std::size_t estimateCardinalityForHashJoin(
+ const physical::HashJoinPtr &physical_plan);
+
+ std::size_t estimateCardinalityForNestedLoopsJoin(
+ const physical::NestedLoopsJoinPtr &physical_plan);
+
+ std::size_t estimateCardinalityForAggregate(
+ const physical::AggregatePtr &physical_plan);
+
+ double estimateSelectivityForSelection(
+ const physical::SelectionPtr &physical_plan);
+
+ double estimateSelectivityForPredicate(
+ const std::unordered_map<expressions::ExprId, std::size_t> &num_distinct_values_map,
+ const expressions::PredicatePtr &filter_predicate);
+
+ const std::vector<physical::PhysicalPtr> &shared_subplans_;
+
+ DISALLOW_COPY_AND_ASSIGN(StarSchemaSimpleCostModel);
+};
+
+/** @} */
+
+} // namespace cost
+} // namespace optimizer
+} // namespace quickstep
+
+#endif /* QUERY_OPTIMIZER_COST_MODEL_STAR_SCHEMA_SIMPLE_COST_MODEL_HPP_ */
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/0fffe505/query_optimizer/rules/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_optimizer/rules/CMakeLists.txt b/query_optimizer/rules/CMakeLists.txt
index 7032af5..1990174 100644
--- a/query_optimizer/rules/CMakeLists.txt
+++ b/query_optimizer/rules/CMakeLists.txt
@@ -26,11 +26,14 @@ add_library(quickstep_queryoptimizer_rules_PushDownFilter PushDownFilter.cpp Pus
add_library(quickstep_queryoptimizer_rules_PushDownSemiAntiJoin PushDownSemiAntiJoin.cpp PushDownSemiAntiJoin.hpp)
add_library(quickstep_queryoptimizer_rules_Rule ../../empty_src.cpp Rule.hpp)
add_library(quickstep_queryoptimizer_rules_RuleHelper RuleHelper.cpp RuleHelper.hpp)
+add_library(quickstep_queryoptimizer_rules_StarSchemaHashJoinOrderOptimization
+ StarSchemaHashJoinOrderOptimization.cpp
+ StarSchemaHashJoinOrderOptimization.hpp)
add_library(quickstep_queryoptimizer_rules_TopDownRule ../../empty_src.cpp TopDownRule.hpp)
add_library(quickstep_queryoptimizer_rules_UpdateExpression UpdateExpression.cpp UpdateExpression.hpp)
add_library(quickstep_queryoptimizer_rules_UnnestSubqueries UnnestSubqueries.cpp UnnestSubqueries.hpp)
-
+
# Link dependencies:
target_link_libraries(quickstep_queryoptimizer_rules_BottomUpRule
glog
@@ -110,6 +113,20 @@ target_link_libraries(quickstep_queryoptimizer_rules_RuleHelper
quickstep_queryoptimizer_expressions_PatternMatcher
quickstep_queryoptimizer_expressions_Predicate
quickstep_queryoptimizer_rules_UpdateExpression)
+target_link_libraries(quickstep_queryoptimizer_rules_StarSchemaHashJoinOrderOptimization
+ quickstep_queryoptimizer_costmodel_StarSchemaSimpleCostModel
+ quickstep_queryoptimizer_expressions_AttributeReference
+ quickstep_queryoptimizer_expressions_ExprId
+ quickstep_queryoptimizer_expressions_NamedExpression
+ quickstep_queryoptimizer_expressions_PatternMatcher
+ quickstep_queryoptimizer_expressions_Predicate
+ quickstep_queryoptimizer_physical_HashJoin
+ quickstep_queryoptimizer_physical_PatternMatcher
+ quickstep_queryoptimizer_physical_Physical
+ quickstep_queryoptimizer_physical_PhysicalType
+ quickstep_queryoptimizer_physical_TopLevelPlan
+ quickstep_queryoptimizer_rules_Rule
+ quickstep_utility_Macros)
target_link_libraries(quickstep_queryoptimizer_rules_TopDownRule
quickstep_queryoptimizer_rules_Rule
quickstep_utility_Macros)
@@ -167,6 +184,7 @@ target_link_libraries(quickstep_queryoptimizer_rules
quickstep_queryoptimizer_rules_PushDownSemiAntiJoin
quickstep_queryoptimizer_rules_Rule
quickstep_queryoptimizer_rules_RuleHelper
+ quickstep_queryoptimizer_rules_StarSchemaHashJoinOrderOptimization
quickstep_queryoptimizer_rules_TopDownRule
quickstep_queryoptimizer_rules_UpdateExpression
quickstep_queryoptimizer_rules_UnnestSubqueries)
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/0fffe505/query_optimizer/rules/StarSchemaHashJoinOrderOptimization.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/rules/StarSchemaHashJoinOrderOptimization.cpp b/query_optimizer/rules/StarSchemaHashJoinOrderOptimization.cpp
new file mode 100644
index 0000000..9770606
--- /dev/null
+++ b/query_optimizer/rules/StarSchemaHashJoinOrderOptimization.cpp
@@ -0,0 +1,309 @@
+/**
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#include "query_optimizer/rules/StarSchemaHashJoinOrderOptimization.hpp"
+
+#include <memory>
+#include <set>
+#include <unordered_map>
+#include <vector>
+
+#include "query_optimizer/cost_model/StarSchemaSimpleCostModel.hpp"
+#include "query_optimizer/expressions/AttributeReference.hpp"
+#include "query_optimizer/expressions/NamedExpression.hpp"
+#include "query_optimizer/expressions/PatternMatcher.hpp"
+#include "query_optimizer/physical/HashJoin.hpp"
+#include "query_optimizer/physical/PatternMatcher.hpp"
+#include "query_optimizer/physical/Physical.hpp"
+#include "query_optimizer/physical/PhysicalType.hpp"
+#include "query_optimizer/physical/TopLevelPlan.hpp"
+
+#include "glog/logging.h"
+
+namespace quickstep {
+namespace optimizer {
+
+namespace E = ::quickstep::optimizer::expressions;
+namespace P = ::quickstep::optimizer::physical;
+
+P::PhysicalPtr StarSchemaHashJoinOrderOptimization::apply(const P::PhysicalPtr &input) {
+ DCHECK(input->getPhysicalType() == P::PhysicalType::kTopLevelPlan);
+ cost_model_.reset(
+ new cost::StarSchemaSimpleCostModel(
+ std::static_pointer_cast<const P::TopLevelPlan>(input)->shared_subplans()));
+
+ return applyInternal(input, nullptr);
+}
+
+P::PhysicalPtr StarSchemaHashJoinOrderOptimization::applyInternal(const P::PhysicalPtr &input,
+ JoinGroupInfo *parent_join_group) {
+ P::HashJoinPtr hash_join;
+ const bool is_hash_inner_join =
+ P::SomeHashJoin::MatchesWithConditionalCast(input, &hash_join)
+ && hash_join->join_type() == P::HashJoin::JoinType::kInnerJoin;
+
+ if (is_hash_inner_join) {
+ bool is_valid_cascading_hash_join = false;
+ if (hash_join->residual_predicate() == nullptr) {
+ is_valid_cascading_hash_join = true;
+ for (const E::NamedExpressionPtr expr : hash_join->project_expressions()) {
+ if (!E::SomeAttributeReference::Matches(expr)) {
+ is_valid_cascading_hash_join = false;
+ break;
+ }
+ }
+ }
+
+ std::unique_ptr<JoinGroupInfo> new_join_group;
+ JoinGroupInfo *join_group = nullptr;
+ if (parent_join_group == nullptr || !is_valid_cascading_hash_join) {
+ new_join_group.reset(new JoinGroupInfo());
+ join_group = new_join_group.get();
+ } else {
+ join_group = parent_join_group;
+ }
+
+ // Gather tables into the join group.
+ for (const P::PhysicalPtr &child : input->children()) {
+ applyInternal(child, join_group);
+ }
+
+ // Gather join attribute pairs.
+ for (std::size_t i = 0; i < hash_join->left_join_attributes().size(); ++i) {
+ const std::size_t left_attr_id = hash_join->left_join_attributes()[i]->id();
+ const std::size_t right_attr_id = hash_join->right_join_attributes()[i]->id();
+
+ join_group->join_attribute_pairs.emplace_back(left_attr_id, right_attr_id);
+ }
+
+ if (join_group != parent_join_group) {
+ // This node is the root node for a group of hash inner joins. Now plan the
+ // ordering of the joins.
+ P::PhysicalPtr output = generatePlan(*join_group,
+ hash_join->residual_predicate(),
+ hash_join->project_expressions());
+ if (parent_join_group == nullptr) {
+ return output;
+ } else {
+ parent_join_group->tables.emplace_back(output);
+ return nullptr;
+ }
+ } else {
+ return nullptr;
+ }
+ } else {
+ std::vector<P::PhysicalPtr> new_children;
+ bool has_changed_children = false;
+ for (const P::PhysicalPtr &child : input->children()) {
+ P::PhysicalPtr new_child = applyInternal(child, nullptr);
+ DCHECK(new_child != nullptr);
+ if (child != new_child && !has_changed_children) {
+ has_changed_children = true;
+ }
+ new_children.push_back(new_child);
+ }
+
+ P::PhysicalPtr output =
+ (has_changed_children ? input->copyWithNewChildren(new_children)
+ : input);
+
+ if (parent_join_group == nullptr) {
+ return output;
+ } else {
+ parent_join_group->tables.emplace_back(output);
+ return nullptr;
+ }
+ }
+}
+
+physical::PhysicalPtr StarSchemaHashJoinOrderOptimization::generatePlan(
+ const JoinGroupInfo &join_group,
+ const E::PredicatePtr &residual_predicate,
+ const std::vector<E::NamedExpressionPtr> &project_expressions) {
+ const std::size_t num_tables = join_group.tables.size();
+ DCHECK_GE(num_tables, 2u);
+
+ std::vector<TableInfo> table_info_storage;
+ const std::vector<P::PhysicalPtr> &tables = join_group.tables;
+ for (std::size_t i = 0; i < join_group.tables.size(); ++i) {
+ table_info_storage.emplace_back(
+ i,
+ tables[i],
+ cost_model_->estimateCardinality(tables[i]),
+ cost_model_->estimateSelectivity(tables[i]));
+ }
+
+ // Auxiliary mapping info.
+ std::unordered_map<E::ExprId, std::size_t> attribute_id_to_table_info_index_map;
+ std::unordered_map<E::ExprId, E::AttributeReferencePtr> attribute_id_to_reference_map;
+ for (std::size_t table_idx = 0; table_idx < num_tables; ++table_idx) {
+ for (const E::AttributeReferencePtr &attr :
+ table_info_storage[table_idx].table->getOutputAttributes()) {
+ DCHECK(attribute_id_to_table_info_index_map.find(attr->id())
+ == attribute_id_to_table_info_index_map.end());
+
+ attribute_id_to_table_info_index_map.emplace(attr->id(), table_idx);
+ attribute_id_to_reference_map.emplace(attr->id(), attr);
+ }
+ }
+
+ // Create a join graph where tables are vertices, and add an edge between vertices
+ // t1 and t2 for each join predicate t1.x = t2.y
+ std::vector<std::unordered_set<std::size_t>> join_graph(table_info_storage.size());
+ for (const auto &attr_id_pair : join_group.join_attribute_pairs) {
+ DCHECK(attribute_id_to_table_info_index_map.find(attr_id_pair.first)
+ != attribute_id_to_table_info_index_map.end());
+ DCHECK(attribute_id_to_table_info_index_map.find(attr_id_pair.second)
+ != attribute_id_to_table_info_index_map.end());
+
+ std::size_t first_table_idx =
+ attribute_id_to_table_info_index_map[attr_id_pair.first];
+ std::size_t second_table_idx =
+ attribute_id_to_table_info_index_map[attr_id_pair.second];
+ DCHECK_NE(first_table_idx, second_table_idx);
+
+ table_info_storage[first_table_idx].join_attribute_pairs.emplace(
+ attr_id_pair.first, attr_id_pair.second);
+ table_info_storage[second_table_idx].join_attribute_pairs.emplace(
+ attr_id_pair.second, attr_id_pair.first);
+
+ join_graph[first_table_idx].emplace(second_table_idx);
+ join_graph[second_table_idx].emplace(first_table_idx);
+ }
+
+ std::set<TableInfo*, TableInfoPtrLessComparator> table_info_ordered_by_priority;
+ for (std::size_t i = 0; i < table_info_storage.size(); ++i) {
+ table_info_ordered_by_priority.emplace(&table_info_storage[i]);
+ }
+
+ // Contruct hash join tree.
+ while (true) {
+ TableInfo *first_table_info = *table_info_ordered_by_priority.begin();
+ table_info_ordered_by_priority.erase(
+ table_info_ordered_by_priority.begin());
+ const std::size_t first_table_info_id = first_table_info->table_info_id;
+
+ TableInfo *second_table_info = nullptr;
+ std::set<TableInfo*, TableInfoPtrLessComparator>::iterator second_table_info_it;
+ for (auto candidate_table_info_it = table_info_ordered_by_priority.begin();
+ candidate_table_info_it != table_info_ordered_by_priority.end();
+ ++candidate_table_info_it) {
+ TableInfo *candidate_table_info = *candidate_table_info_it;
+ const std::size_t candidate_table_info_id = candidate_table_info->table_info_id;
+
+ if (join_graph[first_table_info_id].find(candidate_table_info_id)
+ == join_graph[first_table_info_id].end() &&
+ join_graph[candidate_table_info_id].find(first_table_info_id)
+ == join_graph[candidate_table_info_id].end()) {
+ continue;
+ } else if (second_table_info == nullptr) {
+ second_table_info = candidate_table_info;
+ second_table_info_it = candidate_table_info_it;
+ }
+
+ bool is_likely_many_to_many_join = false;
+ for (const auto join_attr_pair : first_table_info->join_attribute_pairs) {
+ if (candidate_table_info->joined_attribute_set.find(join_attr_pair.second)
+ != candidate_table_info->joined_attribute_set.end()) {
+ is_likely_many_to_many_join = true;
+ break;
+ }
+ }
+ for (const auto join_attr_pair : candidate_table_info->join_attribute_pairs) {
+ if (first_table_info->joined_attribute_set.find(join_attr_pair.second)
+ != first_table_info->joined_attribute_set.end()) {
+ is_likely_many_to_many_join = true;
+ break;
+ }
+ }
+ if (!is_likely_many_to_many_join) {
+ second_table_info = candidate_table_info;
+ second_table_info_it = candidate_table_info_it;
+ break;
+ }
+ }
+ DCHECK(second_table_info != nullptr);
+ table_info_ordered_by_priority.erase(second_table_info_it);
+
+ const P::PhysicalPtr &left_child = first_table_info->table;
+ const P::PhysicalPtr &right_child = second_table_info->table;
+ std::vector<E::NamedExpressionPtr> output_attributes;
+ for (const E::AttributeReferencePtr &left_attr : left_child->getOutputAttributes()) {
+ output_attributes.emplace_back(left_attr);
+ }
+ for (const E::AttributeReferencePtr &right_attr : right_child->getOutputAttributes()) {
+ output_attributes.emplace_back(right_attr);
+ }
+
+ std::vector<E::AttributeReferencePtr> left_join_attributes;
+ std::vector<E::AttributeReferencePtr> right_join_attributes;
+ std::unordered_set<expressions::ExprId> new_joined_attribute_set;
+ for (const auto &join_attr_pair : first_table_info->join_attribute_pairs) {
+ if (second_table_info->join_attribute_pairs.find(join_attr_pair.second)
+ != second_table_info->join_attribute_pairs.end()) {
+ left_join_attributes.emplace_back(
+ attribute_id_to_reference_map[join_attr_pair.first]);
+ right_join_attributes.emplace_back(
+ attribute_id_to_reference_map[join_attr_pair.second]);
+
+ new_joined_attribute_set.emplace(join_attr_pair.first);
+ new_joined_attribute_set.emplace(join_attr_pair.second);
+ }
+ }
+ DCHECK_GE(left_join_attributes.size(), static_cast<std::size_t>(1));
+
+ if (table_info_ordered_by_priority.size() > 0) {
+ P::PhysicalPtr output =
+ P::HashJoin::Create(left_child,
+ right_child,
+ left_join_attributes,
+ right_join_attributes,
+ nullptr,
+ output_attributes,
+ P::HashJoin::JoinType::kInnerJoin);
+
+ second_table_info->table = output;
+
+ // TODO(jianqiao): Cache the estimated cardinality for each plan in cost
+ // model to avoid duplicated estimation.
+ second_table_info->estimated_cardinality = cost_model_->estimateCardinality(output);
+
+ second_table_info->join_attribute_pairs.insert(first_table_info->join_attribute_pairs.begin(),
+ first_table_info->join_attribute_pairs.end());
+ second_table_info->joined_attribute_set.insert(first_table_info->joined_attribute_set.begin(),
+ first_table_info->joined_attribute_set.end());
+ second_table_info->joined_attribute_set.insert(new_joined_attribute_set.begin(),
+ new_joined_attribute_set.end());
+ table_info_ordered_by_priority.emplace(second_table_info);
+
+ join_graph[second_table_info->table_info_id].insert(join_graph[first_table_info_id].begin(),
+ join_graph[first_table_info_id].end());
+
+ } else {
+ return P::HashJoin::Create(left_child,
+ right_child,
+ left_join_attributes,
+ right_join_attributes,
+ residual_predicate,
+ project_expressions,
+ P::HashJoin::JoinType::kInnerJoin);
+ }
+ }
+}
+
+} // namespace optimizer
+} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/0fffe505/query_optimizer/rules/StarSchemaHashJoinOrderOptimization.hpp
----------------------------------------------------------------------
diff --git a/query_optimizer/rules/StarSchemaHashJoinOrderOptimization.hpp b/query_optimizer/rules/StarSchemaHashJoinOrderOptimization.hpp
new file mode 100644
index 0000000..deddffd
--- /dev/null
+++ b/query_optimizer/rules/StarSchemaHashJoinOrderOptimization.hpp
@@ -0,0 +1,136 @@
+/**
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#ifndef QUICKSTEP_QUERY_OPTIMIZER_RULES_STAR_SCHEMA_HASH_JOIN_ORDER_OPTIMIZATION_HPP_
+#define QUICKSTEP_QUERY_OPTIMIZER_RULES_STAR_SCHEMA_HASH_JOIN_ORDER_OPTIMIZATION_HPP_
+
+#include <algorithm>
+#include <cstddef>
+#include <memory>
+#include <string>
+#include <unordered_map>
+#include <unordered_set>
+#include <utility>
+#include <vector>
+
+#include "query_optimizer/cost_model/StarSchemaSimpleCostModel.hpp"
+#include "query_optimizer/expressions/ExprId.hpp"
+#include "query_optimizer/expressions/NamedExpression.hpp"
+#include "query_optimizer/expressions/Predicate.hpp"
+#include "query_optimizer/physical/Physical.hpp"
+#include "query_optimizer/rules/Rule.hpp"
+#include "utility/Macros.hpp"
+
+namespace quickstep {
+namespace optimizer {
+
+/** \addtogroup OptimizerRules
+ * @{
+ */
+
+/**
+ * @brief TODO
+ */
+class StarSchemaHashJoinOrderOptimization : public Rule<physical::Physical> {
+ public:
+ StarSchemaHashJoinOrderOptimization() {}
+
+ ~StarSchemaHashJoinOrderOptimization() override {}
+
+ std::string getName() const override {
+ return "StarSchemaHashJoinOrderOptimization";
+ }
+
+ physical::PhysicalPtr apply(const physical::PhysicalPtr &input) override;
+
+ private:
+ /**
+ * @brief A group of tables to form a hash join tree.
+ */
+ struct JoinGroupInfo {
+ std::vector<physical::PhysicalPtr> tables;
+ std::vector<std::pair<expressions::ExprId, expressions::ExprId>> join_attribute_pairs;
+ };
+
+ /**
+ * @brief Auxiliary information of a table for the optimizer.
+ */
+ struct TableInfo {
+ TableInfo(const std::size_t in_table_info_id,
+ const physical::PhysicalPtr &in_table,
+ const std::size_t in_estimated_cardinality,
+ const double in_estimated_selectivity)
+ : table_info_id(in_table_info_id),
+ table(in_table),
+ estimated_cardinality(in_estimated_cardinality),
+ estimated_selectivity(in_estimated_selectivity) {
+ }
+
+ const std::size_t table_info_id;
+ physical::PhysicalPtr table;
+ std::size_t estimated_cardinality;
+ double estimated_selectivity;
+ std::unordered_multimap<expressions::ExprId, expressions::ExprId> join_attribute_pairs;
+ std::unordered_set<expressions::ExprId> joined_attribute_set;
+ };
+
+ /**
+ * @brief Comparator that compares the join priorities between two tables.
+ */
+ struct TableInfoPtrLessComparator {
+ inline bool operator() (const TableInfo *lhs, const TableInfo *rhs) {
+ bool swapped = false;
+ if (lhs->estimated_cardinality > rhs->estimated_cardinality) {
+ std::swap(lhs, rhs);
+ swapped = true;
+ }
+
+ if (lhs->estimated_selectivity < rhs->estimated_selectivity) {
+ return !swapped;
+ } else if (lhs->estimated_cardinality < 1000u &&
+ rhs->estimated_cardinality > 10000u &&
+ lhs->estimated_selectivity < rhs->estimated_selectivity * 1.5) {
+ return !swapped;
+ } else if (lhs->estimated_selectivity > rhs->estimated_selectivity) {
+ return swapped;
+ } else if (lhs->estimated_cardinality != rhs->estimated_cardinality) {
+ return !swapped;
+ } else {
+ return swapped ^ (lhs->table < rhs->table);
+ }
+ }
+ };
+
+ physical::PhysicalPtr applyInternal(const physical::PhysicalPtr &input,
+ JoinGroupInfo *paret_join_group);
+
+ physical::PhysicalPtr generatePlan(
+ const JoinGroupInfo &join_group_info,
+ const expressions::PredicatePtr &residual_predicate,
+ const std::vector<expressions::NamedExpressionPtr> &project_expressions);
+
+ std::unique_ptr<cost::StarSchemaSimpleCostModel> cost_model_;
+
+ DISALLOW_COPY_AND_ASSIGN(StarSchemaHashJoinOrderOptimization);
+};
+
+/** @} */
+
+} // namespace optimizer
+} // namespace quickstep
+
+#endif /* QUICKSTEP_QUERY_OPTIMIZER_RULES_STAR_SCHEMA_HASH_JOIN_ORDER_OPTIMIZATION_HPP_ */
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/0fffe505/query_optimizer/tests/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_optimizer/tests/CMakeLists.txt b/query_optimizer/tests/CMakeLists.txt
index 5647bfd..07af404 100644
--- a/query_optimizer/tests/CMakeLists.txt
+++ b/query_optimizer/tests/CMakeLists.txt
@@ -132,6 +132,7 @@ target_link_libraries(quickstep_queryoptimizer_tests_ExecutionGeneratorTest
tmb
${LIBS})
target_link_libraries(quickstep_queryoptimizer_tests_OptimizerTextTest
+ gflags_nothreads-static
glog
gtest
gtest_main
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/0fffe505/query_optimizer/tests/OptimizerTextTest.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/tests/OptimizerTextTest.cpp b/query_optimizer/tests/OptimizerTextTest.cpp
index c35cfa3..f3c243b 100644
--- a/query_optimizer/tests/OptimizerTextTest.cpp
+++ b/query_optimizer/tests/OptimizerTextTest.cpp
@@ -22,8 +22,18 @@
#include "query_optimizer/tests/OptimizerTextTestRunner.hpp"
#include "utility/textbased_test/TextBasedTestDriver.hpp"
+#include "gflags/gflags.h"
+
#include "glog/logging.h"
+namespace quickstep {
+namespace optimizer {
+
+DECLARE_bool(reorder_hash_joins);
+
+}
+}
+
using quickstep::TextBasedTest;
QUICKSTEP_GENERATE_TEXT_TEST(OPTIMIZER_TEST);
@@ -45,6 +55,10 @@ int main(int argc, char** argv) {
test_driver->registerOptions(
quickstep::optimizer::OptimizerTextTestRunner::kTestOptions);
+ // Turn off join order optimization for optimizer test since it is up to change
+ // and affects a large number of test cases.
+ quickstep::optimizer::FLAGS_reorder_hash_joins = false;
+
::testing::InitGoogleTest(&argc, argv);
int success = RUN_ALL_TESTS();
if (success != 0) {
[43/50] [abbrv] incubator-quickstep git commit: QUICKSTEP-1: Add
incubation disclaimer to README.md.
Posted by zu...@apache.org.
QUICKSTEP-1: Add incubation disclaimer to README.md.
This closes #2
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/fe80c013
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/fe80c013
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/fe80c013
Branch: refs/heads/work-order-serialization
Commit: fe80c013a7a466e72e63bd2641b75a165028f4e4
Parents: 2ddb67b
Author: Zuyu Zhang <zz...@pivotal.io>
Authored: Tue Apr 19 12:23:51 2016 -0500
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Mon May 30 15:47:53 2016 -0700
----------------------------------------------------------------------
DISCLAIMER | 11 +++++++++++
README.md | 21 ++++++++++++++++++---
2 files changed, 29 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/fe80c013/DISCLAIMER
----------------------------------------------------------------------
diff --git a/DISCLAIMER b/DISCLAIMER
new file mode 100644
index 0000000..09d629d
--- /dev/null
+++ b/DISCLAIMER
@@ -0,0 +1,11 @@
+Apache Quickstep is an effort undergoing incubation at the Apache Software
+Foundation (ASF), sponsored by the Apache Incubator PMC.
+
+Incubation is required of all newly accepted projects until a further
+review indicates that the infrastructure, communications, and decision
+making process have stabilized in a manner consistent with other
+successful ASF projects.
+
+While incubation status is not necessarily a reflection of the
+completeness or stability of the code, it does indicate that the
+project has yet to be fully endorsed by the ASF.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/fe80c013/README.md
----------------------------------------------------------------------
diff --git a/README.md b/README.md
index c44e0b9..7cea80c 100644
--- a/README.md
+++ b/README.md
@@ -1,14 +1,16 @@
-# Quickstep
+# Apache Quickstep (Incubating)
[![Travis Widget]][Travis]
[Travis]: https://travis-ci.org/pivotalsoftware/quickstep
[Travis Widget]: https://travis-ci.org/pivotalsoftware/quickstep.svg?branch=master
-Quickstep is an experimental high-performance database engine designed with the
-aim of Data at Bare-Metal Speed. It began life in 2011 as a
+Apache Quickstep is an experimental high-performance database engine designed with the
+aim of Data at Bare-Metal Speed. It began life in 2011 as a
[research project at the University of Wisconsin](https://quickstep.cs.wisc.edu)
and was acquired by [Pivotal](https://pivotal.io) in 2015.
+Quickstep entered incubation at the
+[Apache Software Foundation](https://www.apache.org) in April, 2016.
## Getting Started (Building)
@@ -91,3 +93,16 @@ concerns of a database system. The main modules are:
## Licensing
Quickstep is licensed under the Apache License, Version 2.0. See [LICENSE](https://github.com/pivotalsoftware/quickstep/blob/master/LICENSE) for the full license text.
+
+## Disclaimer
+Apache Quickstep is an effort undergoing incubation at the Apache Software
+Foundation (ASF), sponsored by the Apache Incubator PMC.
+
+Incubation is required of all newly accepted projects until a further
+review indicates that the infrastructure, communications, and decision
+making process have stabilized in a manner consistent with other
+successful ASF projects.
+
+While incubation status is not necessarily a reflection of the
+completeness or stability of the code, it does indicate that the
+project has yet to be fully endorsed by the ASF.
[08/50] [abbrv] incubator-quickstep git commit: Refactored
HashJoinWorkOrder protos. (#203)
Posted by zu...@apache.org.
Refactored HashJoinWorkOrder protos. (#203)
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/5ea898b2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/5ea898b2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/5ea898b2
Branch: refs/heads/work-order-serialization
Commit: 5ea898b228df3a2166ed82d242f0a50a78f1c290
Parents: abfc5f2
Author: Zuyu ZHANG <zu...@users.noreply.github.com>
Authored: Fri Apr 29 21:11:23 2016 -0700
Committer: Jianqiao Zhu <ji...@cs.wisc.edu>
Committed: Fri Apr 29 23:11:23 2016 -0500
----------------------------------------------------------------------
relational_operators/CMakeLists.txt | 3 +
relational_operators/HashJoinOperator.cpp | 5 +-
relational_operators/WorkOrder.proto | 98 +++----
relational_operators/WorkOrderFactory.cpp | 362 +++++++++----------------
4 files changed, 162 insertions(+), 306 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5ea898b2/relational_operators/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/relational_operators/CMakeLists.txt b/relational_operators/CMakeLists.txt
index 759a233..a4600e6 100644
--- a/relational_operators/CMakeLists.txt
+++ b/relational_operators/CMakeLists.txt
@@ -181,6 +181,9 @@ target_link_libraries(quickstep_relationaloperators_HashJoinOperator
quickstep_storage_TupleReference
quickstep_storage_TupleStorageSubBlock
quickstep_storage_ValueAccessor
+ quickstep_types_Type
+ quickstep_types_TypedValue
+ quickstep_types_containers_ColumnVector
quickstep_types_containers_ColumnVectorsValueAccessor
quickstep_utility_Macros
tmb)
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5ea898b2/relational_operators/HashJoinOperator.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/HashJoinOperator.cpp b/relational_operators/HashJoinOperator.cpp
index 82f6b2a..104a02d 100644
--- a/relational_operators/HashJoinOperator.cpp
+++ b/relational_operators/HashJoinOperator.cpp
@@ -42,6 +42,9 @@
#include "storage/TupleReference.hpp"
#include "storage/TupleStorageSubBlock.hpp"
#include "storage/ValueAccessor.hpp"
+#include "types/Type.hpp"
+#include "types/TypedValue.hpp"
+#include "types/containers/ColumnVector.hpp"
#include "types/containers/ColumnVectorsValueAccessor.hpp"
#include "gflags/gflags.h"
@@ -820,7 +823,7 @@ void HashOuterJoinWorkOrder::execute() {
// where x is an attribute of the build relation.
// In that case, this HashOuterJoinWorkOrder needs to be updated to
// correctly handle the selections.
- const Type& column_type = selection_[i]->getType().getNullableVersion();
+ const Type &column_type = selection_[i]->getType().getNullableVersion();
if (NativeColumnVector::UsableForType(column_type)) {
NativeColumnVector *result = new NativeColumnVector(
column_type, num_tuples_without_matches);
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5ea898b2/relational_operators/WorkOrder.proto
----------------------------------------------------------------------
diff --git a/relational_operators/WorkOrder.proto b/relational_operators/WorkOrder.proto
index 8ed2080..5d0619a 100644
--- a/relational_operators/WorkOrder.proto
+++ b/relational_operators/WorkOrder.proto
@@ -29,21 +29,18 @@ enum WorkOrderType {
DESTROY_HASH = 6;
DROP_TABLE = 7;
FINALIZE_AGGREGATION = 8;
- HASH_ANTI_JOIN = 9;
- HASH_INNER_JOIN = 10;
- HASH_OUTER_JOIN = 11;
- HASH_SEMI_JOIN = 12;
- INSERT = 13;
- NESTED_LOOP_JOIN = 14;
- SAMPLE = 15;
- SAVE_BLOCKS = 16;
- SELECT = 17;
- SORT_MERGE_RUN = 18;
- SORT_RUN_GENERATION = 19;
- TABLE_GENERATOR = 20;
- TEXT_SCAN = 21;
- TEXT_SPLIT = 22;
- UPDATE = 23;
+ HASH_JOIN = 9;
+ INSERT = 10;
+ NESTED_LOOP_JOIN = 11;
+ SAMPLE = 12;
+ SAVE_BLOCKS = 13;
+ SELECT = 14;
+ SORT_MERGE_RUN = 15;
+ SORT_RUN_GENERATION = 16;
+ TABLE_GENERATOR = 17;
+ TEXT_SCAN = 18;
+ TEXT_SPLIT = 19;
+ UPDATE = 20;
}
message WorkOrder {
@@ -107,63 +104,30 @@ message FinalizeAggregationWorkOrder {
}
}
-message HashInnerJoinWorkOrder {
- extend WorkOrder {
- // All required.
- optional int32 build_relation_id = 160;
- optional int32 probe_relation_id = 161;
- repeated int32 join_key_attributes = 162;
- optional bool any_join_key_attributes_nullable = 163;
- optional int32 insert_destination_index = 164;
- optional uint32 join_hash_table_index = 165;
- optional int32 residual_predicate_index = 166;
- optional int32 selection_index = 167;
- optional fixed64 block_id = 168;
- }
-}
-
-message HashAntiJoinWorkOrder {
- extend WorkOrder {
- // All required.
- optional int32 build_relation_id = 350;
- optional int32 probe_relation_id = 351;
- repeated int32 join_key_attributes = 352;
- optional bool any_join_key_attributes_nullable = 353;
- optional int32 insert_destination_index = 354;
- optional uint32 join_hash_table_index = 355;
- optional int32 residual_predicate_index = 356;
- optional int32 selection_index = 357;
- optional fixed64 block_id = 358;
+message HashJoinWorkOrder {
+ enum HashJoinWorkOrderType {
+ HASH_ANTI_JOIN = 0;
+ HASH_INNER_JOIN = 1;
+ HASH_OUTER_JOIN = 2;
+ HASH_SEMI_JOIN = 3;
}
-}
-message HashSemiJoinWorkOrder {
extend WorkOrder {
// All required.
- optional int32 build_relation_id = 360;
- optional int32 probe_relation_id = 361;
- repeated int32 join_key_attributes = 362;
- optional bool any_join_key_attributes_nullable = 363;
- optional int32 insert_destination_index = 364;
- optional uint32 join_hash_table_index = 365;
- optional int32 residual_predicate_index = 366;
- optional int32 selection_index = 367;
- optional fixed64 block_id = 368;
- }
-}
+ optional HashJoinWorkOrderType hash_join_work_order_type = 160;
+ optional int32 build_relation_id = 161;
+ optional int32 probe_relation_id = 162;
+ repeated int32 join_key_attributes = 163;
+ optional bool any_join_key_attributes_nullable = 164;
+ optional int32 insert_destination_index = 165;
+ optional uint32 join_hash_table_index = 166;
+ optional int32 selection_index = 167;
+ optional fixed64 block_id = 168;
-message HashOuterJoinWorkOrder {
- extend WorkOrder {
- // All required.
- optional int32 build_relation_id = 370;
- optional int32 probe_relation_id = 371;
- repeated int32 join_key_attributes = 372;
- optional bool any_join_key_attributes_nullable = 373;
- optional int32 insert_destination_index = 374;
- optional uint32 join_hash_table_index = 375;
- optional int32 selection_index = 376;
- repeated bool is_selection_on_build = 377;
- optional fixed64 block_id = 378;
+ // Used by all but HashOuterJoinWorkOrder.
+ optional int32 residual_predicate_index = 169;
+ // Used by HashOuterJoinWorkOrder only.
+ repeated bool is_selection_on_build = 170;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5ea898b2/relational_operators/WorkOrderFactory.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/WorkOrderFactory.cpp b/relational_operators/WorkOrderFactory.cpp
index 964c11c..4157d0f 100644
--- a/relational_operators/WorkOrderFactory.cpp
+++ b/relational_operators/WorkOrderFactory.cpp
@@ -16,6 +16,7 @@
#include "relational_operators/WorkOrderFactory.hpp"
+#include <memory>
#include <utility>
#include <vector>
@@ -55,6 +56,10 @@ using std::vector;
namespace quickstep {
+class InsertDestination;
+class Predicate;
+class Scalar;
+
WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder &proto,
CatalogDatabaseLite *catalog_database,
QueryContext *query_context,
@@ -135,123 +140,115 @@ WorkOrder* WorkOrderFactory::ReconstructFromProto(const serialization::WorkOrder
query_context->getInsertDestination(
proto.GetExtension(serialization::FinalizeAggregationWorkOrder::insert_destination_index)));
}
- case serialization::HASH_ANTI_JOIN: {
- LOG(INFO) << "Creating HashAntiJoinWorkOrder";
- vector<attribute_id> join_key_attributes;
- const int join_key_attributes_size =
- proto.ExtensionSize(serialization::HashAntiJoinWorkOrder::join_key_attributes);
- for (int i = 0; i < join_key_attributes_size; ++i) {
- join_key_attributes.push_back(
- proto.GetExtension(serialization::HashAntiJoinWorkOrder::join_key_attributes, i));
- }
+ case serialization::HASH_JOIN: {
+ const auto hash_join_work_order_type =
+ proto.GetExtension(serialization::HashJoinWorkOrder::hash_join_work_order_type);
- return new HashAntiJoinWorkOrder(
+ const CatalogRelationSchema &build_relation =
catalog_database->getRelationSchemaById(
- proto.GetExtension(serialization::HashAntiJoinWorkOrder::build_relation_id)),
+ proto.GetExtension(serialization::HashJoinWorkOrder::build_relation_id));
+ const CatalogRelationSchema &probe_relation =
catalog_database->getRelationSchemaById(
- proto.GetExtension(serialization::HashAntiJoinWorkOrder::probe_relation_id)),
- move(join_key_attributes),
- proto.GetExtension(serialization::HashAntiJoinWorkOrder::any_join_key_attributes_nullable),
- proto.GetExtension(serialization::HashAntiJoinWorkOrder::block_id),
- query_context->getPredicate(
- proto.GetExtension(serialization::HashAntiJoinWorkOrder::residual_predicate_index)),
- query_context->getScalarGroup(
- proto.GetExtension(serialization::HashAntiJoinWorkOrder::selection_index)),
- *query_context->getJoinHashTable(
- proto.GetExtension(serialization::HashAntiJoinWorkOrder::join_hash_table_index)),
- query_context->getInsertDestination(
- proto.GetExtension(serialization::HashAntiJoinWorkOrder::insert_destination_index)),
- storage_manager);
- }
- case serialization::HASH_INNER_JOIN: {
- LOG(INFO) << "Creating HashInnerJoinWorkOrder";
- vector<attribute_id> join_key_attributes;
- const int join_key_attributes_size =
- proto.ExtensionSize(serialization::HashInnerJoinWorkOrder::join_key_attributes);
- for (int i = 0; i < join_key_attributes_size; ++i) {
- join_key_attributes.push_back(
- proto.GetExtension(serialization::HashInnerJoinWorkOrder::join_key_attributes, i));
- }
+ proto.GetExtension(serialization::HashJoinWorkOrder::probe_relation_id));
- return new HashInnerJoinWorkOrder(
- catalog_database->getRelationSchemaById(
- proto.GetExtension(serialization::HashInnerJoinWorkOrder::build_relation_id)),
- catalog_database->getRelationSchemaById(
- proto.GetExtension(serialization::HashInnerJoinWorkOrder::probe_relation_id)),
- move(join_key_attributes),
- proto.GetExtension(serialization::HashInnerJoinWorkOrder::any_join_key_attributes_nullable),
- proto.GetExtension(serialization::HashInnerJoinWorkOrder::block_id),
- query_context->getPredicate(
- proto.GetExtension(serialization::HashInnerJoinWorkOrder::residual_predicate_index)),
- query_context->getScalarGroup(
- proto.GetExtension(serialization::HashInnerJoinWorkOrder::selection_index)),
- *query_context->getJoinHashTable(
- proto.GetExtension(serialization::HashInnerJoinWorkOrder::join_hash_table_index)),
- query_context->getInsertDestination(
- proto.GetExtension(serialization::HashInnerJoinWorkOrder::insert_destination_index)),
- storage_manager);
- }
- case serialization::HASH_OUTER_JOIN: {
- LOG(INFO) << "Creating HashOuterJoinWorkOrder";
vector<attribute_id> join_key_attributes;
const int join_key_attributes_size =
- proto.ExtensionSize(serialization::HashOuterJoinWorkOrder::join_key_attributes);
+ proto.ExtensionSize(serialization::HashJoinWorkOrder::join_key_attributes);
for (int i = 0; i < join_key_attributes_size; ++i) {
join_key_attributes.push_back(
- proto.GetExtension(serialization::HashOuterJoinWorkOrder::join_key_attributes, i));
- }
- vector<bool> is_selection_on_build;
- const int is_selection_on_build_size =
- proto.ExtensionSize(serialization::HashOuterJoinWorkOrder::is_selection_on_build);
- for (int i = 0; i < is_selection_on_build_size; ++i) {
- is_selection_on_build.push_back(
- proto.GetExtension(serialization::HashOuterJoinWorkOrder::is_selection_on_build, i));
+ proto.GetExtension(serialization::HashJoinWorkOrder::join_key_attributes, i));
}
- return new HashOuterJoinWorkOrder(
- catalog_database->getRelationSchemaById(
- proto.GetExtension(serialization::HashOuterJoinWorkOrder::build_relation_id)),
- catalog_database->getRelationSchemaById(
- proto.GetExtension(serialization::HashOuterJoinWorkOrder::probe_relation_id)),
- move(join_key_attributes),
- proto.GetExtension(serialization::HashOuterJoinWorkOrder::any_join_key_attributes_nullable),
- proto.GetExtension(serialization::HashOuterJoinWorkOrder::block_id),
- query_context->getScalarGroup(
- proto.GetExtension(serialization::HashOuterJoinWorkOrder::selection_index)),
- move(is_selection_on_build),
- *query_context->getJoinHashTable(
- proto.GetExtension(serialization::HashOuterJoinWorkOrder::join_hash_table_index)),
- query_context->getInsertDestination(
- proto.GetExtension(serialization::HashOuterJoinWorkOrder::insert_destination_index)),
- storage_manager);
- }
- case serialization::HASH_SEMI_JOIN: {
- LOG(INFO) << "Creating HashSemiJoinWorkOrder";
- vector<attribute_id> join_key_attributes;
- const int join_key_attributes_size =
- proto.ExtensionSize(serialization::HashSemiJoinWorkOrder::join_key_attributes);
- for (int i = 0; i < join_key_attributes_size; ++i) {
- join_key_attributes.push_back(
- proto.GetExtension(serialization::HashSemiJoinWorkOrder::join_key_attributes, i));
+ const bool any_join_key_attributes_nullable =
+ proto.GetExtension(serialization::HashJoinWorkOrder::any_join_key_attributes_nullable);
+ const block_id lookup_block_id =
+ proto.GetExtension(serialization::HashJoinWorkOrder::block_id);
+
+ const Predicate *residual_predicate = nullptr;
+ if (hash_join_work_order_type != serialization::HashJoinWorkOrder::HASH_OUTER_JOIN) {
+ residual_predicate =
+ query_context->getPredicate(
+ proto.GetExtension(serialization::HashJoinWorkOrder::residual_predicate_index));
}
- return new HashSemiJoinWorkOrder(
- catalog_database->getRelationSchemaById(
- proto.GetExtension(serialization::HashSemiJoinWorkOrder::build_relation_id)),
- catalog_database->getRelationSchemaById(
- proto.GetExtension(serialization::HashSemiJoinWorkOrder::probe_relation_id)),
- move(join_key_attributes),
- proto.GetExtension(serialization::HashSemiJoinWorkOrder::any_join_key_attributes_nullable),
- proto.GetExtension(serialization::HashSemiJoinWorkOrder::block_id),
- query_context->getPredicate(
- proto.GetExtension(serialization::HashSemiJoinWorkOrder::residual_predicate_index)),
+ const std::vector<std::unique_ptr<const Scalar>> &selection =
query_context->getScalarGroup(
- proto.GetExtension(serialization::HashSemiJoinWorkOrder::selection_index)),
+ proto.GetExtension(serialization::HashJoinWorkOrder::selection_index));
+ const JoinHashTable &hash_table =
*query_context->getJoinHashTable(
- proto.GetExtension(serialization::HashSemiJoinWorkOrder::join_hash_table_index)),
+ proto.GetExtension(serialization::HashJoinWorkOrder::join_hash_table_index));
+ InsertDestination *output_destination =
query_context->getInsertDestination(
- proto.GetExtension(serialization::HashSemiJoinWorkOrder::insert_destination_index)),
- storage_manager);
+ proto.GetExtension(serialization::HashJoinWorkOrder::insert_destination_index));
+
+ switch (hash_join_work_order_type) {
+ case serialization::HashJoinWorkOrder::HASH_ANTI_JOIN: {
+ LOG(INFO) << "Creating HashAntiJoinWorkOrder";
+ return new HashAntiJoinWorkOrder(
+ build_relation,
+ probe_relation,
+ move(join_key_attributes),
+ any_join_key_attributes_nullable,
+ lookup_block_id,
+ residual_predicate,
+ selection,
+ hash_table,
+ output_destination,
+ storage_manager);
+ }
+ case serialization::HashJoinWorkOrder::HASH_INNER_JOIN: {
+ LOG(INFO) << "Creating HashInnerJoinWorkOrder";
+ return new HashInnerJoinWorkOrder(
+ build_relation,
+ probe_relation,
+ move(join_key_attributes),
+ any_join_key_attributes_nullable,
+ lookup_block_id,
+ residual_predicate,
+ selection,
+ hash_table,
+ output_destination,
+ storage_manager);
+ }
+ case serialization::HashJoinWorkOrder::HASH_OUTER_JOIN: {
+ vector<bool> is_selection_on_build;
+ const int is_selection_on_build_size =
+ proto.ExtensionSize(serialization::HashJoinWorkOrder::is_selection_on_build);
+ for (int i = 0; i < is_selection_on_build_size; ++i) {
+ is_selection_on_build.push_back(
+ proto.GetExtension(serialization::HashJoinWorkOrder::is_selection_on_build, i));
+ }
+
+ LOG(INFO) << "Creating HashOuterJoinWorkOrder";
+ return new HashOuterJoinWorkOrder(
+ build_relation,
+ probe_relation,
+ move(join_key_attributes),
+ any_join_key_attributes_nullable,
+ lookup_block_id,
+ selection,
+ move(is_selection_on_build),
+ hash_table,
+ output_destination,
+ storage_manager);
+ }
+ case serialization::HashJoinWorkOrder::HASH_SEMI_JOIN: {
+ LOG(INFO) << "Creating HashSemiJoinWorkOrder";
+ return new HashSemiJoinWorkOrder(
+ build_relation,
+ probe_relation,
+ move(join_key_attributes),
+ any_join_key_attributes_nullable,
+ lookup_block_id,
+ residual_predicate,
+ selection,
+ hash_table,
+ output_destination,
+ storage_manager);
+ }
+ default:
+ LOG(FATAL) << "Unknown HashJoinWorkOrder Type in WorkOrderFactory::ReconstructFromProto";
+ }
}
case serialization::INSERT: {
LOG(INFO) << "Creating InsertWorkOrder";
@@ -486,179 +483,68 @@ bool WorkOrderFactory::ProtoIsValid(const serialization::WorkOrder &proto,
query_context.isValidInsertDestinationId(
proto.GetExtension(serialization::FinalizeAggregationWorkOrder::insert_destination_index));
}
- case serialization::HASH_ANTI_JOIN: {
- if (!proto.HasExtension(serialization::HashAntiJoinWorkOrder::build_relation_id) ||
- !proto.HasExtension(serialization::HashAntiJoinWorkOrder::probe_relation_id)) {
+ case serialization::HASH_JOIN: {
+ if (!proto.HasExtension(serialization::HashJoinWorkOrder::hash_join_work_order_type)) {
return false;
}
- const relation_id build_relation_id =
- proto.GetExtension(serialization::HashAntiJoinWorkOrder::build_relation_id);
- if (!catalog_database.hasRelationWithId(build_relation_id)) {
+ const auto hash_join_work_order_type =
+ proto.GetExtension(serialization::HashJoinWorkOrder::hash_join_work_order_type);
+ if (!serialization::HashJoinWorkOrder_HashJoinWorkOrderType_IsValid(hash_join_work_order_type)) {
return false;
}
- const relation_id probe_relation_id =
- proto.GetExtension(serialization::HashAntiJoinWorkOrder::probe_relation_id);
- if (!catalog_database.hasRelationWithId(probe_relation_id)) {
- return false;
- }
-
- const CatalogRelationSchema &build_relation = catalog_database.getRelationSchemaById(build_relation_id);
- const CatalogRelationSchema &probe_relation = catalog_database.getRelationSchemaById(probe_relation_id);
- for (int i = 0; i < proto.ExtensionSize(serialization::HashAntiJoinWorkOrder::join_key_attributes); ++i) {
- const attribute_id attr_id =
- proto.GetExtension(serialization::HashAntiJoinWorkOrder::join_key_attributes, i);
- if (!build_relation.hasAttributeWithId(attr_id) ||
- !probe_relation.hasAttributeWithId(attr_id)) {
- return false;
- }
- }
-
- return proto.HasExtension(serialization::HashAntiJoinWorkOrder::any_join_key_attributes_nullable) &&
- proto.HasExtension(serialization::HashAntiJoinWorkOrder::insert_destination_index) &&
- query_context.isValidInsertDestinationId(
- proto.GetExtension(serialization::HashAntiJoinWorkOrder::insert_destination_index)) &&
- proto.HasExtension(serialization::HashAntiJoinWorkOrder::join_hash_table_index) &&
- query_context.isValidJoinHashTableId(
- proto.GetExtension(serialization::HashAntiJoinWorkOrder::join_hash_table_index)) &&
- proto.HasExtension(serialization::HashAntiJoinWorkOrder::residual_predicate_index) &&
- query_context.isValidPredicate(
- proto.GetExtension(serialization::HashAntiJoinWorkOrder::residual_predicate_index)) &&
- proto.HasExtension(serialization::HashAntiJoinWorkOrder::selection_index) &&
- query_context.isValidScalarGroupId(
- proto.GetExtension(serialization::HashAntiJoinWorkOrder::selection_index)) &&
- proto.HasExtension(serialization::HashAntiJoinWorkOrder::block_id);
- }
- case serialization::HASH_INNER_JOIN: {
- if (!proto.HasExtension(serialization::HashInnerJoinWorkOrder::build_relation_id) ||
- !proto.HasExtension(serialization::HashInnerJoinWorkOrder::probe_relation_id)) {
+ if (!proto.HasExtension(serialization::HashJoinWorkOrder::build_relation_id) ||
+ !proto.HasExtension(serialization::HashJoinWorkOrder::probe_relation_id)) {
return false;
}
const relation_id build_relation_id =
- proto.GetExtension(serialization::HashInnerJoinWorkOrder::build_relation_id);
+ proto.GetExtension(serialization::HashJoinWorkOrder::build_relation_id);
if (!catalog_database.hasRelationWithId(build_relation_id)) {
return false;
}
const relation_id probe_relation_id =
- proto.GetExtension(serialization::HashInnerJoinWorkOrder::probe_relation_id);
+ proto.GetExtension(serialization::HashJoinWorkOrder::probe_relation_id);
if (!catalog_database.hasRelationWithId(probe_relation_id)) {
return false;
}
const CatalogRelationSchema &build_relation = catalog_database.getRelationSchemaById(build_relation_id);
const CatalogRelationSchema &probe_relation = catalog_database.getRelationSchemaById(probe_relation_id);
- for (int i = 0; i < proto.ExtensionSize(serialization::HashInnerJoinWorkOrder::join_key_attributes); ++i) {
+ for (int i = 0; i < proto.ExtensionSize(serialization::HashJoinWorkOrder::join_key_attributes); ++i) {
const attribute_id attr_id =
- proto.GetExtension(serialization::HashInnerJoinWorkOrder::join_key_attributes, i);
+ proto.GetExtension(serialization::HashJoinWorkOrder::join_key_attributes, i);
if (!build_relation.hasAttributeWithId(attr_id) ||
!probe_relation.hasAttributeWithId(attr_id)) {
return false;
}
}
- return proto.HasExtension(serialization::HashInnerJoinWorkOrder::any_join_key_attributes_nullable) &&
- proto.HasExtension(serialization::HashInnerJoinWorkOrder::insert_destination_index) &&
- query_context.isValidInsertDestinationId(
- proto.GetExtension(serialization::HashInnerJoinWorkOrder::insert_destination_index)) &&
- proto.HasExtension(serialization::HashInnerJoinWorkOrder::join_hash_table_index) &&
- query_context.isValidJoinHashTableId(
- proto.GetExtension(serialization::HashInnerJoinWorkOrder::join_hash_table_index)) &&
- proto.HasExtension(serialization::HashInnerJoinWorkOrder::residual_predicate_index) &&
- query_context.isValidPredicate(
- proto.GetExtension(serialization::HashInnerJoinWorkOrder::residual_predicate_index)) &&
- proto.HasExtension(serialization::HashInnerJoinWorkOrder::selection_index) &&
- query_context.isValidScalarGroupId(
- proto.GetExtension(serialization::HashInnerJoinWorkOrder::selection_index)) &&
- proto.HasExtension(serialization::HashInnerJoinWorkOrder::block_id);
- }
- case serialization::HASH_OUTER_JOIN: {
- if (!proto.HasExtension(serialization::HashOuterJoinWorkOrder::build_relation_id) ||
- !proto.HasExtension(serialization::HashOuterJoinWorkOrder::probe_relation_id)) {
- return false;
- }
-
- const relation_id build_relation_id =
- proto.GetExtension(serialization::HashOuterJoinWorkOrder::build_relation_id);
- if (!catalog_database.hasRelationWithId(build_relation_id)) {
- return false;
- }
-
- const relation_id probe_relation_id =
- proto.GetExtension(serialization::HashOuterJoinWorkOrder::probe_relation_id);
- if (!catalog_database.hasRelationWithId(probe_relation_id)) {
- return false;
- }
-
- const CatalogRelationSchema &build_relation = catalog_database.getRelationSchemaById(build_relation_id);
- const CatalogRelationSchema &probe_relation = catalog_database.getRelationSchemaById(probe_relation_id);
- for (int i = 0; i < proto.ExtensionSize(serialization::HashOuterJoinWorkOrder::join_key_attributes); ++i) {
- const attribute_id attr_id =
- proto.GetExtension(serialization::HashOuterJoinWorkOrder::join_key_attributes, i);
- if (!build_relation.hasAttributeWithId(attr_id) ||
- !probe_relation.hasAttributeWithId(attr_id)) {
+ if (hash_join_work_order_type == serialization::HashJoinWorkOrder::HASH_OUTER_JOIN) {
+ if (!proto.HasExtension(serialization::HashJoinWorkOrder::is_selection_on_build)) {
return false;
}
- }
-
- return proto.HasExtension(serialization::HashOuterJoinWorkOrder::any_join_key_attributes_nullable) &&
- proto.HasExtension(serialization::HashOuterJoinWorkOrder::insert_destination_index) &&
- query_context.isValidInsertDestinationId(
- proto.GetExtension(serialization::HashOuterJoinWorkOrder::insert_destination_index)) &&
- proto.HasExtension(serialization::HashOuterJoinWorkOrder::join_hash_table_index) &&
- query_context.isValidJoinHashTableId(
- proto.GetExtension(serialization::HashOuterJoinWorkOrder::join_hash_table_index)) &&
- proto.HasExtension(serialization::HashOuterJoinWorkOrder::selection_index) &&
- query_context.isValidScalarGroupId(
- proto.GetExtension(serialization::HashOuterJoinWorkOrder::selection_index)) &&
- proto.HasExtension(serialization::HashOuterJoinWorkOrder::is_selection_on_build) &&
- proto.HasExtension(serialization::HashOuterJoinWorkOrder::block_id);
- }
- case serialization::HASH_SEMI_JOIN: {
- if (!proto.HasExtension(serialization::HashSemiJoinWorkOrder::build_relation_id) ||
- !proto.HasExtension(serialization::HashSemiJoinWorkOrder::probe_relation_id)) {
- return false;
- }
-
- const relation_id build_relation_id =
- proto.GetExtension(serialization::HashSemiJoinWorkOrder::build_relation_id);
- if (!catalog_database.hasRelationWithId(build_relation_id)) {
- return false;
- }
-
- const relation_id probe_relation_id =
- proto.GetExtension(serialization::HashSemiJoinWorkOrder::probe_relation_id);
- if (!catalog_database.hasRelationWithId(probe_relation_id)) {
- return false;
- }
-
- const CatalogRelationSchema &build_relation = catalog_database.getRelationSchemaById(build_relation_id);
- const CatalogRelationSchema &probe_relation = catalog_database.getRelationSchemaById(probe_relation_id);
- for (int i = 0; i < proto.ExtensionSize(serialization::HashSemiJoinWorkOrder::join_key_attributes); ++i) {
- const attribute_id attr_id =
- proto.GetExtension(serialization::HashSemiJoinWorkOrder::join_key_attributes, i);
- if (!build_relation.hasAttributeWithId(attr_id) ||
- !probe_relation.hasAttributeWithId(attr_id)) {
+ } else {
+ if (!proto.HasExtension(serialization::HashJoinWorkOrder::residual_predicate_index) ||
+ !query_context.isValidPredicate(
+ proto.GetExtension(serialization::HashJoinWorkOrder::residual_predicate_index))) {
return false;
}
}
- return proto.HasExtension(serialization::HashSemiJoinWorkOrder::any_join_key_attributes_nullable) &&
- proto.HasExtension(serialization::HashSemiJoinWorkOrder::insert_destination_index) &&
+ return proto.HasExtension(serialization::HashJoinWorkOrder::any_join_key_attributes_nullable) &&
+ proto.HasExtension(serialization::HashJoinWorkOrder::insert_destination_index) &&
query_context.isValidInsertDestinationId(
- proto.GetExtension(serialization::HashSemiJoinWorkOrder::insert_destination_index)) &&
- proto.HasExtension(serialization::HashSemiJoinWorkOrder::join_hash_table_index) &&
+ proto.GetExtension(serialization::HashJoinWorkOrder::insert_destination_index)) &&
+ proto.HasExtension(serialization::HashJoinWorkOrder::join_hash_table_index) &&
query_context.isValidJoinHashTableId(
- proto.GetExtension(serialization::HashSemiJoinWorkOrder::join_hash_table_index)) &&
- proto.HasExtension(serialization::HashSemiJoinWorkOrder::residual_predicate_index) &&
- query_context.isValidPredicate(
- proto.GetExtension(serialization::HashSemiJoinWorkOrder::residual_predicate_index)) &&
- proto.HasExtension(serialization::HashSemiJoinWorkOrder::selection_index) &&
+ proto.GetExtension(serialization::HashJoinWorkOrder::join_hash_table_index)) &&
+ proto.HasExtension(serialization::HashJoinWorkOrder::selection_index) &&
query_context.isValidScalarGroupId(
- proto.GetExtension(serialization::HashSemiJoinWorkOrder::selection_index)) &&
- proto.HasExtension(serialization::HashSemiJoinWorkOrder::block_id);
+ proto.GetExtension(serialization::HashJoinWorkOrder::selection_index)) &&
+ proto.HasExtension(serialization::HashJoinWorkOrder::block_id);
}
case serialization::INSERT: {
return proto.HasExtension(serialization::InsertWorkOrder::insert_destination_index) &&
[41/50] [abbrv] incubator-quickstep git commit: Revert "Explicitly
specify where tcmalloc comes from" (#237)
Posted by zu...@apache.org.
Revert "Explicitly specify where tcmalloc comes from" (#237)
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/5bda90ea
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/5bda90ea
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/5bda90ea
Branch: refs/heads/work-order-serialization
Commit: 5bda90ea8cb478e37e9a5de31a46c70a85c732cc
Parents: 908d367
Author: Jignesh Patel <pa...@users.noreply.github.com>
Authored: Mon May 23 22:56:22 2016 -0500
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Mon May 30 15:47:53 2016 -0700
----------------------------------------------------------------------
CMakeLists.txt | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5bda90ea/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 2ab0f57..dc51ca6 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -490,7 +490,9 @@ if(USE_TCMALLOC)
CXXFLAGS=${THIRD_PARTY_CXX_FLAGS}
BUILD_COMMAND make
BUILD_IN_SOURCE 0
- BUILD_BYPRODUCTS <INSTALL_DIR>/lib/libtcmalloc_minimal.a
+ # Uncomment the next line to change the path of the build by products
+ # as some generators, e.g. Ninja, may need it to build properly
+ # BUILD_BYPRODUCTS <INSTALL_DIR>/lib/libtcmalloc_minimal.a
)
# Static libtcmalloc_minimal.a
add_library(libtcmalloc_minimal STATIC IMPORTED)
[02/50] [abbrv] incubator-quickstep git commit: Fixed the deadlock
when loading while evicting. (#196)
Posted by zu...@apache.org.
Fixed the deadlock when loading while evicting. (#196)
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/7ac1d22e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/7ac1d22e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/7ac1d22e
Branch: refs/heads/work-order-serialization
Commit: 7ac1d22e0ae156730d14b86855df76bc10cefc47
Parents: d353a64
Author: Zuyu ZHANG <zu...@users.noreply.github.com>
Authored: Wed Apr 27 22:59:25 2016 -0700
Committer: Jignesh Patel <pa...@users.noreply.github.com>
Committed: Thu Apr 28 00:59:25 2016 -0500
----------------------------------------------------------------------
storage/StorageManager.cpp | 20 ++++++++++++------
utility/CMakeLists.txt | 2 ++
utility/ShardedLockManager.hpp | 41 +++++++++++++++++++++++++------------
3 files changed, 44 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/7ac1d22e/storage/StorageManager.cpp
----------------------------------------------------------------------
diff --git a/storage/StorageManager.cpp b/storage/StorageManager.cpp
index a3f265d..b98a28c 100644
--- a/storage/StorageManager.cpp
+++ b/storage/StorageManager.cpp
@@ -494,12 +494,16 @@ MutableBlockReference StorageManager::getBlockInternal(
// To be safe, release the block's shard after 'eviction_lock' destructs.
lock_manager_.release(block);
+ if (ret.valid()) {
+ return ret;
+ }
+
// Note that there is no way for the block to be evicted between the call to
// loadBlock and the call to EvictionPolicy::blockReferenced from
// MutableBlockReference's constructor; this is because EvictionPolicy
// doesn't know about the block until blockReferenced is called, so
// chooseBlockToEvict shouldn't return the block.
- if (!ret.valid()) {
+ do {
SpinSharedMutexExclusiveLock<false> io_lock(*lock_manager_.get(block));
{
// Check one more time if the block got loaded in memory by someone else.
@@ -508,12 +512,12 @@ MutableBlockReference StorageManager::getBlockInternal(
if (it != blocks_.end()) {
DEBUG_ASSERT(!it->second.block->isBlob());
ret = MutableBlockReference(static_cast<StorageBlock*>(it->second.block), eviction_policy_.get());
- return ret;
+ break;
}
}
// No other thread loaded the block before us.
ret = MutableBlockReference(loadBlock(block, relation, numa_node), eviction_policy_.get());
- }
+ } while (false);
// To be safe, release the block's shard after 'io_lock' destructs.
lock_manager_.release(block);
@@ -535,7 +539,11 @@ MutableBlobReference StorageManager::getBlobInternal(const block_id blob,
// To be safe, release the blob's shard after 'eviction_lock' destructs.
lock_manager_.release(blob);
- if (!ret.valid()) {
+ if (ret.valid()) {
+ return ret;
+ }
+
+ do {
SpinSharedMutexExclusiveLock<false> io_lock(*lock_manager_.get(blob));
// Note that there is no way for the block to be evicted between the call to
// loadBlob and the call to EvictionPolicy::blockReferenced from
@@ -548,12 +556,12 @@ MutableBlobReference StorageManager::getBlobInternal(const block_id blob,
if (it != blocks_.end()) {
DEBUG_ASSERT(it->second.block->isBlob());
ret = MutableBlobReference(static_cast<StorageBlob*>(it->second.block), eviction_policy_.get());
- return ret;
+ break;
}
}
// No other thread loaded the blob before us.
ret = MutableBlobReference(loadBlob(blob, numa_node), eviction_policy_.get());
- }
+ } while (false);
// To be safe, release the blob's shard after 'io_lock' destructs.
lock_manager_.release(blob);
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/7ac1d22e/utility/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/utility/CMakeLists.txt b/utility/CMakeLists.txt
index 4ff9254..bb59f65 100644
--- a/utility/CMakeLists.txt
+++ b/utility/CMakeLists.txt
@@ -244,7 +244,9 @@ target_link_libraries(quickstep_utility_SortConfiguration_proto
${PROTOBUF_LIBRARY})
target_link_libraries(quickstep_utility_ShardedLockManager
quickstep_storage_StorageConstants
+ quickstep_threading_Mutex
quickstep_threading_SharedMutex
+ quickstep_threading_SpinSharedMutex
quickstep_utility_Macros)
target_link_libraries(quickstep_utility_StringUtil
glog)
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/7ac1d22e/utility/ShardedLockManager.hpp
----------------------------------------------------------------------
diff --git a/utility/ShardedLockManager.hpp b/utility/ShardedLockManager.hpp
index 1d59acb..e3eba85 100644
--- a/utility/ShardedLockManager.hpp
+++ b/utility/ShardedLockManager.hpp
@@ -21,9 +21,12 @@
#include <array>
#include <cstddef>
#include <functional>
+#include <unordered_map>
#include "storage/StorageConstants.hpp"
+#include "threading/Mutex.hpp"
#include "threading/SharedMutex.hpp"
+#include "threading/SpinSharedMutex.hpp"
#include "utility/Macros.hpp"
namespace quickstep {
@@ -64,24 +67,29 @@ class ShardedLockManager {
if (has_collision != nullptr) {
// In StorageManager::makeRoomForBlock, check whether the evicting block
// or blob has a shard collision with existing referenced shards.
- SpinSharedMutexSharedLock<false> read_lock(shards_mutex_);
- if (shards_.find(shard) != shards_.end()) {
+ SpinSharedMutexSharedLock<false> read_lock(shard_count_mutex_);
+ if (shard_count_.find(shard) != shard_count_.end()) {
*has_collision = true;
return &collision_mutex_;
}
}
{
- SpinSharedMutexExclusiveLock<false> write_lock(shards_mutex_);
+ SpinSharedMutexExclusiveLock<false> write_lock(shard_count_mutex_);
// Check one more time for the evicting block or blob if there is a shard
// collision.
- if (has_collision != nullptr && shards_.find(shard) != shards_.end()) {
- *has_collision = true;
- return &collision_mutex_;
+ auto it = shard_count_.find(shard);
+ if (it != shard_count_.end()) {
+ if (has_collision != nullptr) {
+ *has_collision = true;
+ return &collision_mutex_;
+ }
+
+ ++it->second;
+ } else {
+ shard_count_.emplace(shard, 1);
}
-
- shards_.insert(shard);
}
return &sharded_mutexes_[shard];
}
@@ -91,8 +99,13 @@ class ShardedLockManager {
* @param key The key to compute the shard.
*/
void release(const T key) {
- SpinSharedMutexExclusiveLock<false> write_lock(shards_mutex_);
- shards_.erase(hash_(key) % N);
+ SpinSharedMutexExclusiveLock<false> write_lock(shard_count_mutex_);
+ auto it = shard_count_.find(hash_(key) % N);
+ DCHECK(it != shard_count_.end());
+
+ if (--it->second == 0) {
+ shard_count_.erase(it);
+ }
}
private:
@@ -102,9 +115,11 @@ class ShardedLockManager {
// The placeholder mutex used whenever there is a hash collision.
SharedMutexT collision_mutex_;
- // Bookkeep all shards referenced by StorageManager in multiple threads.
- std::unordered_set<std::size_t> shards_;
- alignas(kCacheLineBytes) mutable SpinSharedMutex<false> shards_mutex_;
+ // Count all shards referenced by StorageManager in multiple threads.
+ // The key is the shard, while the value is the count. If the count equals to
+ // zero, we delete the shard entry.
+ std::unordered_map<std::size_t, std::size_t> shard_count_;
+ alignas(kCacheLineBytes) mutable SpinSharedMutex<false> shard_count_mutex_;
DISALLOW_COPY_AND_ASSIGN(ShardedLockManager);
};
[13/50] [abbrv] incubator-quickstep git commit: Change default
aggregate_hashtable_type from LinearOpenAddressing to SeparateChaining (#207)
Posted by zu...@apache.org.
Change default aggregate_hashtable_type from LinearOpenAddressing to SeparateChaining (#207)
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/8444e2dc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/8444e2dc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/8444e2dc
Branch: refs/heads/work-order-serialization
Commit: 8444e2dc85739170f2c00d378513ab71573fb4c1
Parents: 0f261ea
Author: Jianqiao Zhu <ji...@cs.wisc.edu>
Authored: Wed May 4 13:54:55 2016 -0500
Committer: Jignesh Patel <pa...@users.noreply.github.com>
Committed: Wed May 4 13:54:55 2016 -0500
----------------------------------------------------------------------
query_optimizer/ExecutionGenerator.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/8444e2dc/query_optimizer/ExecutionGenerator.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/ExecutionGenerator.cpp b/query_optimizer/ExecutionGenerator.cpp
index c34f084..3698701 100644
--- a/query_optimizer/ExecutionGenerator.cpp
+++ b/query_optimizer/ExecutionGenerator.cpp
@@ -135,7 +135,7 @@ static const volatile bool join_hashtable_type_dummy
= gflags::RegisterFlagValidator(&FLAGS_join_hashtable_type,
&ValidateHashTableImplTypeString);
-DEFINE_string(aggregate_hashtable_type, "LinearOpenAddressing",
+DEFINE_string(aggregate_hashtable_type, "SeparateChaining",
"HashTable implementation to use for aggregates with GROUP BY "
"(valid options are SeparateChaining or LinearOpenAddressing)");
static const volatile bool aggregate_hashtable_type_dummy
[05/50] [abbrv] incubator-quickstep git commit: Adds support for
multiple SharedSubplanReference. (#199)
Posted by zu...@apache.org.
Adds support for multiple SharedSubplanReference. (#199)
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/c5460f40
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/c5460f40
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/c5460f40
Branch: refs/heads/work-order-serialization
Commit: c5460f40e82b9e2c81e14380f17609a6e7230eb4
Parents: 07ca1e7
Author: Jianqiao Zhu <ji...@cs.wisc.edu>
Authored: Thu Apr 28 16:09:32 2016 -0500
Committer: Jignesh Patel <pa...@users.noreply.github.com>
Committed: Thu Apr 28 16:09:32 2016 -0500
----------------------------------------------------------------------
query_optimizer/ExecutionGenerator.cpp | 13 ++
query_optimizer/ExecutionGenerator.hpp | 11 +-
query_optimizer/cost_model/CMakeLists.txt | 1 +
query_optimizer/cost_model/SimpleCostModel.cpp | 9 +
query_optimizer/cost_model/SimpleCostModel.hpp | 8 +-
.../logical/SharedSubplanReference.cpp | 7 +-
.../logical/SharedSubplanReference.hpp | 34 +++-
.../physical/SharedSubplanReference.cpp | 5 +
.../physical/SharedSubplanReference.hpp | 34 +++-
query_optimizer/resolver/Resolver.cpp | 25 ++-
query_optimizer/strategy/OneToOne.cpp | 1 +
.../tests/execution_generator/Select.test | 25 +++
.../tests/logical_generator/Select.test | 106 ++++++++++-
.../tests/physical_generator/Select.test | 165 ++++++++++++++++-
query_optimizer/tests/resolver/Select.test | 181 +++++++++++++++----
15 files changed, 542 insertions(+), 83 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c5460f40/query_optimizer/ExecutionGenerator.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/ExecutionGenerator.cpp b/query_optimizer/ExecutionGenerator.cpp
index abcdd6c..c34f084 100644
--- a/query_optimizer/ExecutionGenerator.cpp
+++ b/query_optimizer/ExecutionGenerator.cpp
@@ -154,6 +154,9 @@ void ExecutionGenerator::generatePlan(const P::PhysicalPtr &physical_plan) {
CHECK(P::SomeTopLevelPlan::MatchesWithConditionalCast(physical_plan, &top_level_physical_plan_))
<< "The physical plan must be rooted by a TopLevelPlan";
+ cost_model_.reset(
+ new cost::SimpleCostModel(top_level_physical_plan_->shared_subplans()));
+
const CatalogRelation *result_relation = nullptr;
try {
@@ -550,6 +553,16 @@ void ExecutionGenerator::convertSharedSubplanReference(const physical::SharedSub
top_level_physical_plan_->shared_subplan_at(physical_plan->subplan_id()));
if (found_it != physical_to_output_relation_map_.end()) {
physical_to_output_relation_map_.emplace(physical_plan, found_it->second);
+
+ // Propagate the (ExprId -> CatalogAttribute) mapping.
+ const std::vector<E::AttributeReferencePtr> &referenced_attributes =
+ physical_plan->referenced_attributes();
+ const std::vector<E::AttributeReferencePtr> &output_attributes =
+ physical_plan->output_attributes();
+ for (std::size_t i = 0; i < referenced_attributes.size(); ++i) {
+ attribute_substitution_map_[output_attributes[i]->id()] =
+ attribute_substitution_map_[referenced_attributes[i]->id()];
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c5460f40/query_optimizer/ExecutionGenerator.hpp
----------------------------------------------------------------------
diff --git a/query_optimizer/ExecutionGenerator.hpp b/query_optimizer/ExecutionGenerator.hpp
index df47b31..7c563d4 100644
--- a/query_optimizer/ExecutionGenerator.hpp
+++ b/query_optimizer/ExecutionGenerator.hpp
@@ -1,6 +1,8 @@
/**
* Copyright 2011-2015 Quickstep Technologies LLC.
* Copyright 2015-2016 Pivotal Software, Inc.
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -104,8 +106,6 @@ class ExecutionGenerator {
#ifdef QUICKSTEP_DISTRIBUTED
catalog_database_cache_proto_ = DCHECK_NOTNULL(query_handle->getCatalogDatabaseCacheProtoMutable());
#endif
-
- setupCostModel();
}
/**
@@ -163,13 +163,6 @@ class ExecutionGenerator {
void generatePlanInternal(const physical::PhysicalPtr &physical_plan);
/**
- * @brief Sets up the cost model.
- */
- void setupCostModel() {
- cost_model_.reset(new cost::SimpleCostModel());
- }
-
- /**
* @brief Finds the CatalogRelationInfo from <physical_to_output_relation_map_>
* by the physical node \p physical. Returns NULL if not found.
*
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c5460f40/query_optimizer/cost_model/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_optimizer/cost_model/CMakeLists.txt b/query_optimizer/cost_model/CMakeLists.txt
index e561fc8..6697d52 100644
--- a/query_optimizer/cost_model/CMakeLists.txt
+++ b/query_optimizer/cost_model/CMakeLists.txt
@@ -31,6 +31,7 @@ target_link_libraries(quickstep_queryoptimizer_costmodel_SimpleCostModel
quickstep_queryoptimizer_physical_Physical
quickstep_queryoptimizer_physical_PhysicalType
quickstep_queryoptimizer_physical_Selection
+ quickstep_queryoptimizer_physical_SharedSubplanReference
quickstep_queryoptimizer_physical_TableGenerator
quickstep_queryoptimizer_physical_TableReference
quickstep_queryoptimizer_physical_TopLevelPlan
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c5460f40/query_optimizer/cost_model/SimpleCostModel.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/cost_model/SimpleCostModel.cpp b/query_optimizer/cost_model/SimpleCostModel.cpp
index 291fb9d..48f76fa 100644
--- a/query_optimizer/cost_model/SimpleCostModel.cpp
+++ b/query_optimizer/cost_model/SimpleCostModel.cpp
@@ -1,6 +1,8 @@
/**
* Copyright 2011-2015 Quickstep Technologies LLC.
* Copyright 2015 Pivotal Software, Inc.
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -27,6 +29,7 @@
#include "query_optimizer/physical/Physical.hpp"
#include "query_optimizer/physical/PhysicalType.hpp"
#include "query_optimizer/physical/Selection.hpp"
+#include "query_optimizer/physical/SharedSubplanReference.hpp"
#include "query_optimizer/physical/TableGenerator.hpp"
#include "query_optimizer/physical/TableReference.hpp"
#include "query_optimizer/physical/TopLevelPlan.hpp"
@@ -63,6 +66,12 @@ std::size_t SimpleCostModel::estimateCardinality(
case P::PhysicalType::kAggregate:
return estimateCardinalityForAggregate(
std::static_pointer_cast<const P::Aggregate>(physical_plan));
+ case P::PhysicalType::kSharedSubplanReference: {
+ const P::SharedSubplanReferencePtr shared_subplan_reference =
+ std::static_pointer_cast<const P::SharedSubplanReference>(physical_plan);
+ return estimateCardinality(
+ shared_subplans_[shared_subplan_reference->subplan_id()]);
+ }
default:
LOG(FATAL) << "Unsupported physical plan:" << physical_plan->toString();
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c5460f40/query_optimizer/cost_model/SimpleCostModel.hpp
----------------------------------------------------------------------
diff --git a/query_optimizer/cost_model/SimpleCostModel.hpp b/query_optimizer/cost_model/SimpleCostModel.hpp
index e53887d..9862198 100644
--- a/query_optimizer/cost_model/SimpleCostModel.hpp
+++ b/query_optimizer/cost_model/SimpleCostModel.hpp
@@ -1,6 +1,8 @@
/**
* Copyright 2011-2015 Quickstep Technologies LLC.
* Copyright 2015 Pivotal Software, Inc.
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -19,6 +21,7 @@
#define QUERY_OPTIMIZER_COST_MODEL_SIMPLE_COST_MODEL_HPP_
#include <cstddef>
+#include <vector>
#include "query_optimizer/cost_model/CostModel.hpp"
#include "query_optimizer/physical/Aggregate.hpp"
@@ -47,7 +50,8 @@ class SimpleCostModel : public CostModel {
/**
* @brief Constructor.
*/
- SimpleCostModel() {}
+ explicit SimpleCostModel(const std::vector<physical::PhysicalPtr> &shared_subplans)
+ : shared_subplans_(shared_subplans) {}
std::size_t estimateCardinality(
const physical::PhysicalPtr &physical_plan) override;
@@ -84,6 +88,8 @@ class SimpleCostModel : public CostModel {
std::size_t estimateCardinalityForAggregate(
const physical::AggregatePtr &physical_plan);
+ const std::vector<physical::PhysicalPtr> &shared_subplans_;
+
DISALLOW_COPY_AND_ASSIGN(SimpleCostModel);
};
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c5460f40/query_optimizer/logical/SharedSubplanReference.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/logical/SharedSubplanReference.cpp b/query_optimizer/logical/SharedSubplanReference.cpp
index 8a5bec9..b5d3143 100644
--- a/query_optimizer/logical/SharedSubplanReference.cpp
+++ b/query_optimizer/logical/SharedSubplanReference.cpp
@@ -1,6 +1,8 @@
/**
* Copyright 2011-2015 Quickstep Technologies LLC.
* Copyright 2015 Pivotal Software, Inc.
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -41,13 +43,16 @@ void SharedSubplanReference::getFieldStringItems(
inline_field_names->push_back("subplan_id");
inline_field_values->push_back(std::to_string(subplan_id_));
+ container_child_field_names->push_back("referenced_attributes");
+ container_child_fields->push_back(CastSharedPtrVector<OptimizerTreeBase>(referenced_attributes_));
+
container_child_field_names->push_back("output_attributes");
container_child_fields->push_back(CastSharedPtrVector<OptimizerTreeBase>(output_attributes_));
}
LogicalPtr SharedSubplanReference::copyWithNewChildren(const std::vector<LogicalPtr> &new_children) const {
DCHECK(new_children.empty());
- return Create(subplan_id_, output_attributes_);
+ return Create(subplan_id_, referenced_attributes_, output_attributes_);
}
} // namespace logical
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c5460f40/query_optimizer/logical/SharedSubplanReference.hpp
----------------------------------------------------------------------
diff --git a/query_optimizer/logical/SharedSubplanReference.hpp b/query_optimizer/logical/SharedSubplanReference.hpp
index 55035d1..8f0e37b 100644
--- a/query_optimizer/logical/SharedSubplanReference.hpp
+++ b/query_optimizer/logical/SharedSubplanReference.hpp
@@ -1,6 +1,8 @@
/**
* Copyright 2011-2015 Quickstep Technologies LLC.
* Copyright 2015 Pivotal Software, Inc.
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -28,6 +30,8 @@
#include "query_optimizer/logical/LogicalType.hpp"
#include "utility/Macros.hpp"
+#include "glog/logging.h"
+
namespace quickstep {
namespace optimizer {
namespace logical {
@@ -60,7 +64,14 @@ class SharedSubplanReference : public Logical {
}
/**
- * @return The output attributes of the shared subplan.
+ * @return The attributes from the referenced shared subplan.
+ */
+ const std::vector<expressions::AttributeReferencePtr>& referenced_attributes() const {
+ return referenced_attributes_;
+ }
+
+ /**
+ * @return The output attributes of this shared subplan reference.
*/
const std::vector<expressions::AttributeReferencePtr>& output_attributes() const {
return output_attributes_;
@@ -81,7 +92,7 @@ class SharedSubplanReference : public Logical {
LogicalPtr copyWithNewChildren(const std::vector<LogicalPtr> &new_children) const override;
std::vector<expressions::AttributeReferencePtr> getReferencedAttributes() const override {
- return output_attributes_;
+ return referenced_attributes_;
}
/**
@@ -89,21 +100,30 @@ class SharedSubplanReference : public Logical {
*
* @param subplan_id The ID of the shared subplan, which is the index of the referenced
* shared subplan in <shared_subplans_> of the TopLevelPlan.
- * @param output_attributes The output attributes of the shared subplan.
+ * @param referenced_attributes The attributes from the referenced shared subplan.
+ * @param output_attributes The output attributes of this shared subplan reference.
* @return An immutable SharedSubplanReference.
*/
- static SharedSubplanReferencePtr Create(int subplan_id,
- const std::vector<expressions::AttributeReferencePtr> &output_attributes) {
- return SharedSubplanReferencePtr(new SharedSubplanReference(subplan_id, output_attributes));
+ static SharedSubplanReferencePtr Create(
+ int subplan_id,
+ const std::vector<expressions::AttributeReferencePtr> &referenced_attributes,
+ const std::vector<expressions::AttributeReferencePtr> &output_attributes) {
+ return SharedSubplanReferencePtr(
+ new SharedSubplanReference(subplan_id, referenced_attributes, output_attributes));
}
private:
SharedSubplanReference(int subplan_id,
+ const std::vector<expressions::AttributeReferencePtr> &referenced_attributes,
const std::vector<expressions::AttributeReferencePtr> &output_attributes)
: subplan_id_(subplan_id),
- output_attributes_(output_attributes) {}
+ referenced_attributes_(referenced_attributes),
+ output_attributes_(output_attributes) {
+ DCHECK_EQ(output_attributes_.size(), referenced_attributes_.size());
+ }
int subplan_id_;
+ std::vector<expressions::AttributeReferencePtr> referenced_attributes_;
std::vector<expressions::AttributeReferencePtr> output_attributes_;
DISALLOW_COPY_AND_ASSIGN(SharedSubplanReference);
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c5460f40/query_optimizer/physical/SharedSubplanReference.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/physical/SharedSubplanReference.cpp b/query_optimizer/physical/SharedSubplanReference.cpp
index ce851ac..eb33d32 100644
--- a/query_optimizer/physical/SharedSubplanReference.cpp
+++ b/query_optimizer/physical/SharedSubplanReference.cpp
@@ -1,6 +1,8 @@
/**
* Copyright 2011-2015 Quickstep Technologies LLC.
* Copyright 2015 Pivotal Software, Inc.
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -40,6 +42,9 @@ void SharedSubplanReference::getFieldStringItems(
inline_field_names->push_back("subplan_id");
inline_field_values->push_back(std::to_string(subplan_id_));
+ container_child_field_names->push_back("referenced_attributes");
+ container_child_fields->push_back(CastSharedPtrVector<OptimizerTreeBase>(referenced_attributes_));
+
container_child_field_names->push_back("output_attributes");
container_child_fields->push_back(CastSharedPtrVector<OptimizerTreeBase>(output_attributes_));
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c5460f40/query_optimizer/physical/SharedSubplanReference.hpp
----------------------------------------------------------------------
diff --git a/query_optimizer/physical/SharedSubplanReference.hpp b/query_optimizer/physical/SharedSubplanReference.hpp
index 5005598..4439256 100644
--- a/query_optimizer/physical/SharedSubplanReference.hpp
+++ b/query_optimizer/physical/SharedSubplanReference.hpp
@@ -1,6 +1,8 @@
/**
* Copyright 2011-2015 Quickstep Technologies LLC.
* Copyright 2015 Pivotal Software, Inc.
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -63,7 +65,14 @@ class SharedSubplanReference : public physical::Physical {
}
/**
- * @return The output attributes of the shared subplan.
+ * @return The attributes from the referenced shared subplan.
+ */
+ const std::vector<expressions::AttributeReferencePtr>& referenced_attributes() const {
+ return referenced_attributes_;
+ }
+
+ /**
+ * @return The output attributes of this shared subplan reference.
*/
const std::vector<expressions::AttributeReferencePtr>& output_attributes() const {
return output_attributes_;
@@ -83,11 +92,11 @@ class SharedSubplanReference : public physical::Physical {
PhysicalPtr copyWithNewChildren(const std::vector<PhysicalPtr> &new_children) const override {
DCHECK(new_children.empty());
- return Create(subplan_id_, output_attributes_);
+ return Create(subplan_id_, referenced_attributes_, output_attributes_);
}
std::vector<expressions::AttributeReferencePtr> getReferencedAttributes() const override {
- return output_attributes_;
+ return referenced_attributes_;
}
bool maybeCopyWithPrunedExpressions(
@@ -101,21 +110,30 @@ class SharedSubplanReference : public physical::Physical {
*
* @param subplan_id The ID of the shared subplan, which is the index of
* the referenced shared subplan in <shared_subplans_> of the TopLevelPlan.
- * @param output_attributes The output attributes of the shared subplan.
+ * @param referenced_attributes The attributes from the referenced shared subplan.
+ * @param output_attributes The output attributes of this shared subplan reference.
* @return An immutable SharedSubplanReference.
*/
- static SharedSubplanReferencePtr Create(int subplan_id,
- const std::vector<expressions::AttributeReferencePtr> &output_attributes) {
- return SharedSubplanReferencePtr(new SharedSubplanReference(subplan_id, output_attributes));
+ static SharedSubplanReferencePtr Create(
+ int subplan_id,
+ const std::vector<expressions::AttributeReferencePtr> &referenced_attributes,
+ const std::vector<expressions::AttributeReferencePtr> &output_attributes) {
+ return SharedSubplanReferencePtr(
+ new SharedSubplanReference(subplan_id, referenced_attributes, output_attributes));
}
private:
SharedSubplanReference(int subplan_id,
+ const std::vector<expressions::AttributeReferencePtr> &referenced_attributes,
const std::vector<expressions::AttributeReferencePtr> &output_attributes)
: subplan_id_(subplan_id),
- output_attributes_(output_attributes) {}
+ referenced_attributes_(referenced_attributes),
+ output_attributes_(output_attributes) {
+ DCHECK_EQ(output_attributes_.size(), referenced_attributes_.size());
+ }
int subplan_id_;
+ std::vector<expressions::AttributeReferencePtr> referenced_attributes_;
std::vector<expressions::AttributeReferencePtr> output_attributes_;
DISALLOW_COPY_AND_ASSIGN(SharedSubplanReference);
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c5460f40/query_optimizer/resolver/Resolver.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/resolver/Resolver.cpp b/query_optimizer/resolver/Resolver.cpp
index 1cf5c28..45ecf33 100644
--- a/query_optimizer/resolver/Resolver.cpp
+++ b/query_optimizer/resolver/Resolver.cpp
@@ -1541,9 +1541,28 @@ L::LogicalPtr Resolver::resolveSimpleTableReference(
with_queries_info_.with_query_name_to_vector_position.find(lower_table_name);
if (subplan_it != with_queries_info_.with_query_name_to_vector_position.end()) {
with_queries_info_.unreferenced_query_indexes.erase(subplan_it->second);
- return L::SharedSubplanReference::Create(
- subplan_it->second,
- with_queries_info_.with_query_plans[subplan_it->second]->getOutputAttributes());
+
+ const std::vector<E::AttributeReferencePtr> with_query_attributes =
+ with_queries_info_.with_query_plans[subplan_it->second]->getOutputAttributes();
+
+ // Create a vector of new attributes to delegate the original output attributes
+ // from the WITH query, to avoid (ExprId -> CatalogAttribute) mapping collision
+ // later in ExecutionGenerator when there are multiple SharedSubplanReference's
+ // referencing a same shared subplan.
+ std::vector<E::AttributeReferencePtr> delegator_attributes;
+ for (const E::AttributeReferencePtr &attribute : with_query_attributes) {
+ delegator_attributes.emplace_back(
+ E::AttributeReference::Create(context_->nextExprId(),
+ attribute->attribute_name(),
+ attribute->attribute_alias(),
+ attribute->relation_name(),
+ attribute->getValueType(),
+ attribute->scope()));
+ }
+
+ return L::SharedSubplanReference::Create(subplan_it->second,
+ with_query_attributes,
+ delegator_attributes);
}
// Then look up the name in the database.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c5460f40/query_optimizer/strategy/OneToOne.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/strategy/OneToOne.cpp b/query_optimizer/strategy/OneToOne.cpp
index d4d79f9..7f59151 100644
--- a/query_optimizer/strategy/OneToOne.cpp
+++ b/query_optimizer/strategy/OneToOne.cpp
@@ -84,6 +84,7 @@ bool OneToOne::generatePlan(const L::LogicalPtr &logical_input,
const L::SharedSubplanReferencePtr shared_subplan_reference =
std::static_pointer_cast<const L::SharedSubplanReference>(logical_input);
*physical_output = P::SharedSubplanReference::Create(shared_subplan_reference->subplan_id(),
+ shared_subplan_reference->referenced_attributes(),
shared_subplan_reference->output_attributes());
return true;
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c5460f40/query_optimizer/tests/execution_generator/Select.test
----------------------------------------------------------------------
diff --git a/query_optimizer/tests/execution_generator/Select.test b/query_optimizer/tests/execution_generator/Select.test
index 3a64c9b..a08b012 100644
--- a/query_optimizer/tests/execution_generator/Select.test
+++ b/query_optimizer/tests/execution_generator/Select.test
@@ -891,6 +891,31 @@ FROM (
+-----------+--------------------+-----------+------------------------+
==
+# Same shared subplan referenced multiple times.
+WITH t(x, y) AS (
+ SELECT i % 5, i
+ FROM generate_series(1, 20) AS g(i)
+)
+SELECT *
+FROM t
+WHERE t.y = (
+ SELECT MAX(y)
+ FROM t t1
+ WHERE t.x = t1.x
+)
+ORDER BY x;
+--
++-----------+-----------+
+|x |y |
++-----------+-----------+
+| 0| 20|
+| 1| 16|
+| 2| 17|
+| 3| 18|
+| 4| 19|
++-----------+-----------+
+==
+
# TODO(team): Fix Issue #9 to enable COUNT(*).
SELECT COUNT(long_col)
FROM test,
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c5460f40/query_optimizer/tests/logical_generator/Select.test
----------------------------------------------------------------------
diff --git a/query_optimizer/tests/logical_generator/Select.test b/query_optimizer/tests/logical_generator/Select.test
index 6dff3e7..3c152e8 100644
--- a/query_optimizer/tests/logical_generator/Select.test
+++ b/query_optimizer/tests/logical_generator/Select.test
@@ -502,22 +502,29 @@ TopLevelPlan
| | | | type=VarChar(20) NULL]
| | | +-right=Filter
| | | | +-input=SharedSubplanReference[subplan_id=0]
+| | | | | +-referenced_attributes=
+| | | | | | +-AttributeReference[id=0,name=subquery_int0,relation=subquery,
+| | | | | | | type=Int NULL]
+| | | | | | +-AttributeReference[id=7,name=subquery_long,relation=subquery,
+| | | | | | | type=Long]
+| | | | | | +-AttributeReference[id=12,name=subquery_int1,relation=subquery,
+| | | | | | type=Int NULL]
| | | | | +-output_attributes=
-| | | | | +-AttributeReference[id=0,name=subquery_int0,relation=subquery,
+| | | | | +-AttributeReference[id=36,name=subquery_int0,relation=subquery,
| | | | | | type=Int NULL]
-| | | | | +-AttributeReference[id=7,name=subquery_long,relation=subquery,
+| | | | | +-AttributeReference[id=37,name=subquery_long,relation=subquery,
| | | | | | type=Long]
-| | | | | +-AttributeReference[id=12,name=subquery_int1,relation=subquery,
+| | | | | +-AttributeReference[id=38,name=subquery_int1,relation=subquery,
| | | | | type=Int NULL]
| | | | +-filter_predicate=Equal
-| | | | +-AttributeReference[id=12,name=subquery_int1,relation=subquery,
+| | | | +-AttributeReference[id=38,name=subquery_int1,relation=subquery,
| | | | | type=Int NULL]
-| | | | +-AttributeReference[id=7,name=subquery_long,relation=subquery,
+| | | | +-AttributeReference[id=37,name=subquery_long,relation=subquery,
| | | | type=Long]
| | | +-left_join_attributes=
| | | | +-AttributeReference[id=30,name=int_col,relation=c,type=Int NULL]
| | | +-right_join_attributes=
-| | | +-AttributeReference[id=0,name=subquery_int0,relation=subquery,
+| | | +-AttributeReference[id=36,name=subquery_int0,relation=subquery,
| | | type=Int NULL]
| | +-join_predicate=Literal[value=true]
| +-project_list=
@@ -572,11 +579,14 @@ select * from subquery
TopLevelPlan
+-plan=Project
| +-input=SharedSubplanReference[subplan_id=0]
+| | +-referenced_attributes=
+| | | +-AttributeReference[id=7,name=,alias=(int_col+2),relation=subquery,
+| | | type=Int NULL]
| | +-output_attributes=
-| | +-AttributeReference[id=7,name=,alias=(int_col+2),relation=subquery,
+| | +-AttributeReference[id=8,name=,alias=(int_col+2),relation=subquery,
| | type=Int NULL]
| +-project_list=
-| +-AttributeReference[id=7,name=,alias=(int_col+2),relation=subquery,
+| +-AttributeReference[id=8,name=,alias=(int_col+2),relation=subquery,
| type=Int NULL]
+-shared_subplans=
| +-Project
@@ -596,7 +606,7 @@ TopLevelPlan
| | +-Literal[value=1,type=Int]
| +-Literal[value=2,type=Int]
+-output_attributes=
- +-AttributeReference[id=7,name=,alias=(int_col+2),relation=subquery,
+ +-AttributeReference[id=8,name=,alias=(int_col+2),relation=subquery,
type=Int NULL]
==
@@ -1266,3 +1276,81 @@ SELECT x + (
FROM b;
--
ERROR: Nested queries can only reference attributes in the outer query one level above
+==
+
+# Same shared subplan referenced multiple times.
+WITH t(x, y) AS (
+ SELECT i % 5, i
+ FROM generate_series(1, 20) AS g(i)
+)
+SELECT *
+FROM t
+WHERE t.y = (
+ SELECT MAX(y)
+ FROM t t1
+ WHERE t.x = t1.x
+)
+ORDER BY x;
+--
+TopLevelPlan
++-plan=Project
+| +-input=Sort[is_ascending=[true],nulls_first=[false]]
+| | +-input=Filter
+| | | +-input=HashJoin
+| | | | +-left=SharedSubplanReference[subplan_id=0]
+| | | | | +-referenced_attributes=
+| | | | | | +-AttributeReference[id=3,name=x,relation=,type=Int]
+| | | | | | +-AttributeReference[id=4,name=y,relation=,type=Int]
+| | | | | +-output_attributes=
+| | | | | +-AttributeReference[id=5,name=x,relation=,type=Int]
+| | | | | +-AttributeReference[id=6,name=y,relation=,type=Int]
+| | | | +-right=Project
+| | | | | +-input=Aggregate
+| | | | | | +-input=SharedSubplanReference[subplan_id=0]
+| | | | | | | +-referenced_attributes=
+| | | | | | | | +-AttributeReference[id=3,name=x,relation=,type=Int]
+| | | | | | | | +-AttributeReference[id=4,name=y,relation=,type=Int]
+| | | | | | | +-output_attributes=
+| | | | | | | +-AttributeReference[id=7,name=x,relation=,type=Int]
+| | | | | | | +-AttributeReference[id=8,name=y,relation=,type=Int]
+| | | | | | +-grouping_expressions=
+| | | | | | | +-AttributeReference[id=7,name=x,relation=,type=Int]
+| | | | | | +-aggregate_expressions=
+| | | | | | +-Alias[id=9,name=,alias=$aggregate0,relation=$aggregate,
+| | | | | | type=Int NULL]
+| | | | | | +-AggregateFunction[function=MAX]
+| | | | | | +-AttributeReference[id=8,name=y,relation=,type=Int]
+| | | | | +-project_list=
+| | | | | +-AttributeReference[id=7,name=x,relation=,type=Int]
+| | | | | +-Alias[id=9,name=,alias=MAX(y),relation=,type=Int NULL]
+| | | | | +-AttributeReference[id=9,name=,alias=$aggregate0,
+| | | | | relation=$aggregate,type=Int NULL]
+| | | | +-left_join_attributes=
+| | | | | +-AttributeReference[id=5,name=x,relation=,type=Int]
+| | | | +-right_join_attributes=
+| | | | +-AttributeReference[id=7,name=x,relation=,type=Int]
+| | | +-filter_predicate=Equal
+| | | +-AttributeReference[id=6,name=y,relation=,type=Int]
+| | | +-AttributeReference[id=9,name=,alias=MAX(y),relation=,type=Int NULL]
+| | +-sort_expressions=
+| | +-AttributeReference[id=5,name=x,relation=,type=Int]
+| +-project_list=
+| +-AttributeReference[id=5,name=x,relation=,type=Int]
+| +-AttributeReference[id=6,name=y,relation=,type=Int]
++-shared_subplans=
+| +-Project
+| +-input=TableGenerator[function_name=generate_series,table_alias=g]
+| | +-AttributeReference[id=0,name=generate_series,alias=g,
+| | relation=generate_series,type=Int]
+| +-project_list=
+| +-Alias[id=3,name=x,relation=,type=Int]
+| | +-Modulo
+| | +-AttributeReference[id=0,name=generate_series,alias=g,
+| | | relation=generate_series,type=Int]
+| | +-Literal[value=5,type=Int]
+| +-Alias[id=4,name=y,relation=,type=Int]
+| +-AttributeReference[id=0,name=generate_series,alias=g,
+| relation=generate_series,type=Int]
++-output_attributes=
+ +-AttributeReference[id=5,name=x,relation=,type=Int]
+ +-AttributeReference[id=6,name=y,relation=,type=Int]
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c5460f40/query_optimizer/tests/physical_generator/Select.test
----------------------------------------------------------------------
diff --git a/query_optimizer/tests/physical_generator/Select.test b/query_optimizer/tests/physical_generator/Select.test
index 62d09f5..3365206 100644
--- a/query_optimizer/tests/physical_generator/Select.test
+++ b/query_optimizer/tests/physical_generator/Select.test
@@ -1542,12 +1542,16 @@ select int_col from subquery
TopLevelPlan
+-plan=Project
| +-input=SharedSubplanReference[subplan_id=0]
+| | +-referenced_attributes=
+| | | +-AttributeReference[id=0,name=int_col,relation=test,type=Int NULL]
+| | | +-AttributeReference[id=1,name=long_col,relation=test,type=Long]
+| | | +-AttributeReference[id=3,name=double_col,relation=test,type=Double NULL]
| | +-output_attributes=
-| | +-AttributeReference[id=0,name=int_col,relation=test,type=Int NULL]
-| | +-AttributeReference[id=1,name=long_col,relation=test,type=Long]
-| | +-AttributeReference[id=3,name=double_col,relation=test,type=Double NULL]
+| | +-AttributeReference[id=6,name=int_col,relation=test,type=Int NULL]
+| | +-AttributeReference[id=7,name=long_col,relation=test,type=Long]
+| | +-AttributeReference[id=8,name=double_col,relation=test,type=Double NULL]
| +-project_list=
-| +-AttributeReference[id=0,name=int_col,relation=test,type=Int NULL]
+| +-AttributeReference[id=6,name=int_col,relation=test,type=Int NULL]
+-shared_subplans=
| +-Project
| +-input=TableReference[relation_name=Test,relation_alias=test]
@@ -1563,17 +1567,21 @@ TopLevelPlan
| +-AttributeReference[id=1,name=long_col,relation=test,type=Long]
| +-AttributeReference[id=3,name=double_col,relation=test,type=Double NULL]
+-output_attributes=
- +-AttributeReference[id=0,name=int_col,relation=test,type=Int NULL]
+ +-AttributeReference[id=6,name=int_col,relation=test,type=Int NULL]
[Physical Plan]
TopLevelPlan
+-plan=Selection
| +-input=SharedSubplanReference[subplan_id=0]
+| | +-referenced_attributes=
+| | | +-AttributeReference[id=0,name=int_col,relation=test,type=Int NULL]
+| | | +-AttributeReference[id=1,name=long_col,relation=test,type=Long]
+| | | +-AttributeReference[id=3,name=double_col,relation=test,type=Double NULL]
| | +-output_attributes=
-| | +-AttributeReference[id=0,name=int_col,relation=test,type=Int NULL]
-| | +-AttributeReference[id=1,name=long_col,relation=test,type=Long]
-| | +-AttributeReference[id=3,name=double_col,relation=test,type=Double NULL]
+| | +-AttributeReference[id=6,name=int_col,relation=test,type=Int NULL]
+| | +-AttributeReference[id=7,name=long_col,relation=test,type=Long]
+| | +-AttributeReference[id=8,name=double_col,relation=test,type=Double NULL]
| +-project_expressions=
-| +-AttributeReference[id=0,name=int_col,relation=test,type=Int NULL]
+| +-AttributeReference[id=6,name=int_col,relation=test,type=Int NULL]
+-shared_subplans=
| +-Selection
| +-input=TableReference[relation=Test,alias=test]
@@ -1589,7 +1597,7 @@ TopLevelPlan
| +-AttributeReference[id=1,name=long_col,relation=test,type=Long]
| +-AttributeReference[id=3,name=double_col,relation=test,type=Double NULL]
+-output_attributes=
- +-AttributeReference[id=0,name=int_col,relation=test,type=Int NULL]
+ +-AttributeReference[id=6,name=int_col,relation=test,type=Int NULL]
==
SELECT COUNT(DISTINCT int_col), SUM(float_col)
@@ -2731,3 +2739,140 @@ TopLevelPlan
+-output_attributes=
+-AttributeReference[id=8,name=,alias=(x*SubqueryExpression),relation=,
type=Long NULL]
+==
+
+# Same shared subplan referenced multiple times.
+WITH t(x, y) AS (
+ SELECT i % 5, i
+ FROM generate_series(1, 20) AS g(i)
+)
+SELECT *
+FROM t
+WHERE t.y = (
+ SELECT MAX(y)
+ FROM t t1
+ WHERE t.x = t1.x
+)
+ORDER BY x;
+--
+[Optimized Logical Plan]
+TopLevelPlan
++-plan=Project
+| +-input=Sort[is_ascending=[true],nulls_first=[false]]
+| | +-input=Filter
+| | | +-input=HashJoin
+| | | | +-left=SharedSubplanReference[subplan_id=0]
+| | | | | +-referenced_attributes=
+| | | | | | +-AttributeReference[id=3,name=x,relation=,type=Int]
+| | | | | | +-AttributeReference[id=4,name=y,relation=,type=Int]
+| | | | | +-output_attributes=
+| | | | | +-AttributeReference[id=5,name=x,relation=,type=Int]
+| | | | | +-AttributeReference[id=6,name=y,relation=,type=Int]
+| | | | +-right=Project
+| | | | | +-input=Aggregate
+| | | | | | +-input=SharedSubplanReference[subplan_id=0]
+| | | | | | | +-referenced_attributes=
+| | | | | | | | +-AttributeReference[id=3,name=x,relation=,type=Int]
+| | | | | | | | +-AttributeReference[id=4,name=y,relation=,type=Int]
+| | | | | | | +-output_attributes=
+| | | | | | | +-AttributeReference[id=7,name=x,relation=,type=Int]
+| | | | | | | +-AttributeReference[id=8,name=y,relation=,type=Int]
+| | | | | | +-grouping_expressions=
+| | | | | | | +-AttributeReference[id=7,name=x,relation=,type=Int]
+| | | | | | +-aggregate_expressions=
+| | | | | | +-Alias[id=9,name=,alias=$aggregate0,relation=$aggregate,
+| | | | | | type=Int NULL]
+| | | | | | +-AggregateFunction[function=MAX]
+| | | | | | +-AttributeReference[id=8,name=y,relation=,type=Int]
+| | | | | +-project_list=
+| | | | | +-AttributeReference[id=7,name=x,relation=,type=Int]
+| | | | | +-Alias[id=9,name=,alias=MAX(y),relation=,type=Int NULL]
+| | | | | +-AttributeReference[id=9,name=,alias=$aggregate0,
+| | | | | relation=$aggregate,type=Int NULL]
+| | | | +-left_join_attributes=
+| | | | | +-AttributeReference[id=5,name=x,relation=,type=Int]
+| | | | +-right_join_attributes=
+| | | | +-AttributeReference[id=7,name=x,relation=,type=Int]
+| | | +-filter_predicate=Equal
+| | | +-AttributeReference[id=6,name=y,relation=,type=Int]
+| | | +-AttributeReference[id=9,name=,alias=MAX(y),relation=,type=Int NULL]
+| | +-sort_expressions=
+| | +-AttributeReference[id=5,name=x,relation=,type=Int]
+| +-project_list=
+| +-AttributeReference[id=5,name=x,relation=,type=Int]
+| +-AttributeReference[id=6,name=y,relation=,type=Int]
++-shared_subplans=
+| +-Project
+| +-input=TableGenerator[function_name=generate_series,table_alias=g]
+| | +-AttributeReference[id=0,name=generate_series,alias=g,
+| | relation=generate_series,type=Int]
+| +-project_list=
+| +-Alias[id=3,name=x,relation=,type=Int]
+| | +-Modulo
+| | +-AttributeReference[id=0,name=generate_series,alias=g,
+| | | relation=generate_series,type=Int]
+| | +-Literal[value=5,type=Int]
+| +-Alias[id=4,name=y,relation=,type=Int]
+| +-AttributeReference[id=0,name=generate_series,alias=g,
+| relation=generate_series,type=Int]
++-output_attributes=
+ +-AttributeReference[id=5,name=x,relation=,type=Int]
+ +-AttributeReference[id=6,name=y,relation=,type=Int]
+[Physical Plan]
+TopLevelPlan
++-plan=Selection
+| +-input=Sort[is_ascending=[true],nulls_first=[false]]
+| | +-input=HashJoin
+| | | +-left=SharedSubplanReference[subplan_id=0]
+| | | | +-referenced_attributes=
+| | | | | +-AttributeReference[id=3,name=x,relation=,type=Int]
+| | | | | +-AttributeReference[id=4,name=y,relation=,type=Int]
+| | | | +-output_attributes=
+| | | | +-AttributeReference[id=5,name=x,relation=,type=Int]
+| | | | +-AttributeReference[id=6,name=y,relation=,type=Int]
+| | | +-right=Aggregate
+| | | | +-input=SharedSubplanReference[subplan_id=0]
+| | | | | +-referenced_attributes=
+| | | | | | +-AttributeReference[id=3,name=x,relation=,type=Int]
+| | | | | | +-AttributeReference[id=4,name=y,relation=,type=Int]
+| | | | | +-output_attributes=
+| | | | | +-AttributeReference[id=7,name=x,relation=,type=Int]
+| | | | | +-AttributeReference[id=8,name=y,relation=,type=Int]
+| | | | +-grouping_expressions=
+| | | | | +-AttributeReference[id=7,name=x,relation=,type=Int]
+| | | | +-aggregate_expressions=
+| | | | +-Alias[id=9,name=,alias=$aggregate0,relation=$aggregate,type=Int NULL]
+| | | | +-AggregateFunction[function=MAX]
+| | | | +-AttributeReference[id=8,name=y,relation=,type=Int]
+| | | +-project_expressions=
+| | | | +-AttributeReference[id=5,name=x,relation=,type=Int]
+| | | | +-AttributeReference[id=6,name=y,relation=,type=Int]
+| | | +-left_join_attributes=
+| | | | +-AttributeReference[id=5,name=x,relation=,type=Int]
+| | | | +-AttributeReference[id=6,name=y,relation=,type=Int]
+| | | +-right_join_attributes=
+| | | +-AttributeReference[id=7,name=x,relation=,type=Int]
+| | | +-AttributeReference[id=9,name=,alias=$aggregate0,relation=$aggregate,
+| | | type=Int NULL]
+| | +-sort_attributes=
+| | +-AttributeReference[id=5,name=x,relation=,type=Int]
+| +-project_expressions=
+| +-AttributeReference[id=5,name=x,relation=,type=Int]
+| +-AttributeReference[id=6,name=y,relation=,type=Int]
++-shared_subplans=
+| +-Selection
+| +-input=TableGenerator[function_name=generate_series,table_alias=g]
+| | +-AttributeReference[id=0,name=generate_series,alias=g,
+| | relation=generate_series,type=Int]
+| +-project_expressions=
+| +-Alias[id=3,name=x,relation=,type=Int]
+| | +-Modulo
+| | +-AttributeReference[id=0,name=generate_series,alias=g,
+| | | relation=generate_series,type=Int]
+| | +-Literal[value=5,type=Int]
+| +-Alias[id=4,name=y,relation=,type=Int]
+| +-AttributeReference[id=0,name=generate_series,alias=g,
+| relation=generate_series,type=Int]
++-output_attributes=
+ +-AttributeReference[id=5,name=x,relation=,type=Int]
+ +-AttributeReference[id=6,name=y,relation=,type=Int]
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c5460f40/query_optimizer/tests/resolver/Select.test
----------------------------------------------------------------------
diff --git a/query_optimizer/tests/resolver/Select.test b/query_optimizer/tests/resolver/Select.test
index 00ff18a..9897934 100644
--- a/query_optimizer/tests/resolver/Select.test
+++ b/query_optimizer/tests/resolver/Select.test
@@ -1723,11 +1723,14 @@ select subquery_col from subquery
TopLevelPlan
+-plan=Project
| +-input=SharedSubplanReference[subplan_id=0]
+| | +-referenced_attributes=
+| | | +-AttributeReference[id=0,name=subquery_col,relation=subquery,
+| | | type=Int NULL]
| | +-output_attributes=
-| | +-AttributeReference[id=0,name=subquery_col,relation=subquery,
+| | +-AttributeReference[id=6,name=subquery_col,relation=subquery,
| | type=Int NULL]
| +-project_list=
-| +-AttributeReference[id=0,name=subquery_col,relation=subquery,type=Int NULL]
+| +-AttributeReference[id=6,name=subquery_col,relation=subquery,type=Int NULL]
+-shared_subplans=
| +-Project
| +-input=TableReference[relation_name=Test,relation_alias=test]
@@ -1742,7 +1745,7 @@ TopLevelPlan
| +-Alias[id=0,name=subquery_col,relation=subquery,type=Int NULL]
| +-AttributeReference[id=0,name=int_col,relation=test,type=Int NULL]
+-output_attributes=
- +-AttributeReference[id=0,name=subquery_col,relation=subquery,type=Int NULL]
+ +-AttributeReference[id=6,name=subquery_col,relation=subquery,type=Int NULL]
==
# Multiple WITH queries.
@@ -1755,14 +1758,18 @@ TopLevelPlan
+-plan=Project
| +-input=MultiwayCartesianJoin
| | +-SharedSubplanReference[subplan_id=2]
+| | | +-referenced_attributes=
+| | | | +-AttributeReference[id=10,name=int_col,relation=subquery3,type=Int NULL]
| | | +-output_attributes=
-| | | +-AttributeReference[id=7,name=int_col,relation=subquery3,type=Int NULL]
+| | | +-AttributeReference[id=11,name=int_col,relation=subquery3,type=Int NULL]
| | +-SharedSubplanReference[subplan_id=1]
+| | +-referenced_attributes=
+| | | +-AttributeReference[id=7,name=int_col,relation=subquery2,type=Int NULL]
| | +-output_attributes=
-| | +-AttributeReference[id=6,name=int_col,relation=subquery2,type=Int NULL]
+| | +-AttributeReference[id=12,name=int_col,relation=subquery2,type=Int NULL]
| +-project_list=
-| +-AttributeReference[id=7,name=int_col,relation=subquery3,type=Int NULL]
-| +-AttributeReference[id=6,name=int_col,relation=subquery2,type=Int NULL]
+| +-AttributeReference[id=11,name=int_col,relation=subquery3,type=Int NULL]
+| +-AttributeReference[id=12,name=int_col,relation=subquery2,type=Int NULL]
+-shared_subplans=
| +-Project
| | +-input=TableReference[relation_name=Test,relation_alias=test]
@@ -1777,31 +1784,38 @@ TopLevelPlan
| | +-AttributeReference[id=0,name=int_col,relation=test,type=Int NULL]
| +-Project
| | +-input=SharedSubplanReference[subplan_id=0]
+| | | +-referenced_attributes=
+| | | | +-AttributeReference[id=0,name=int_col,relation=test,type=Int NULL]
| | | +-output_attributes=
-| | | +-AttributeReference[id=0,name=int_col,relation=test,type=Int NULL]
+| | | +-AttributeReference[id=6,name=int_col,relation=test,type=Int NULL]
| | +-project_list=
-| | +-Alias[id=6,name=int_col,relation=subquery2,type=Int NULL]
+| | +-Alias[id=7,name=int_col,relation=subquery2,type=Int NULL]
| | +-Add
-| | +-AttributeReference[id=0,name=int_col,relation=test,type=Int NULL]
+| | +-AttributeReference[id=6,name=int_col,relation=test,type=Int NULL]
| | +-Literal[value=5,type=Int]
| +-Project
| +-input=MultiwayCartesianJoin
| | +-SharedSubplanReference[subplan_id=1]
+| | | +-referenced_attributes=
+| | | | +-AttributeReference[id=7,name=int_col,relation=subquery2,
+| | | | type=Int NULL]
| | | +-output_attributes=
-| | | +-AttributeReference[id=6,name=int_col,relation=subquery2,
+| | | +-AttributeReference[id=8,name=int_col,relation=subquery2,
| | | type=Int NULL]
| | +-SharedSubplanReference[subplan_id=0]
+| | +-referenced_attributes=
+| | | +-AttributeReference[id=0,name=int_col,relation=test,type=Int NULL]
| | +-output_attributes=
-| | +-AttributeReference[id=0,name=int_col,relation=test,type=Int NULL]
+| | +-AttributeReference[id=9,name=int_col,relation=test,type=Int NULL]
| +-project_list=
-| +-Alias[id=7,name=int_col,relation=subquery3,type=Int NULL]
+| +-Alias[id=10,name=int_col,relation=subquery3,type=Int NULL]
| +-Add
-| +-AttributeReference[id=6,name=int_col,relation=subquery2,
+| +-AttributeReference[id=8,name=int_col,relation=subquery2,
| | type=Int NULL]
| +-Literal[value=6,type=Int]
+-output_attributes=
- +-AttributeReference[id=7,name=int_col,relation=subquery3,type=Int NULL]
- +-AttributeReference[id=6,name=int_col,relation=subquery2,type=Int NULL]
+ +-AttributeReference[id=11,name=int_col,relation=subquery3,type=Int NULL]
+ +-AttributeReference[id=12,name=int_col,relation=subquery2,type=Int NULL]
==
with subquery(int_col_alias, long_col_alias) as (select int_col, long_col from test)
@@ -1810,11 +1824,14 @@ select int_col_alias from subquery
TopLevelPlan
+-plan=Project
| +-input=SharedSubplanReference[subplan_id=0]
+| | +-referenced_attributes=
+| | | +-AttributeReference[id=6,name=int_col_alias,relation=,type=Int NULL]
+| | | +-AttributeReference[id=7,name=long_col_alias,relation=,type=Long]
| | +-output_attributes=
-| | +-AttributeReference[id=6,name=int_col_alias,relation=,type=Int NULL]
-| | +-AttributeReference[id=7,name=long_col_alias,relation=,type=Long]
+| | +-AttributeReference[id=8,name=int_col_alias,relation=,type=Int NULL]
+| | +-AttributeReference[id=9,name=long_col_alias,relation=,type=Long]
| +-project_list=
-| +-AttributeReference[id=6,name=int_col_alias,relation=,type=Int NULL]
+| +-AttributeReference[id=8,name=int_col_alias,relation=,type=Int NULL]
+-shared_subplans=
| +-Project
| +-input=Project
@@ -1835,7 +1852,7 @@ TopLevelPlan
| +-Alias[id=7,name=long_col_alias,relation=,type=Long]
| +-AttributeReference[id=1,name=long_col,relation=test,type=Long]
+-output_attributes=
- +-AttributeReference[id=6,name=int_col_alias,relation=,type=Int NULL]
+ +-AttributeReference[id=8,name=int_col_alias,relation=,type=Int NULL]
==
with subquery as (select int_col, long_col from test)
@@ -1845,16 +1862,19 @@ TopLevelPlan
+-plan=Project
| +-input=Project
| | +-input=SharedSubplanReference[subplan_id=0]
+| | | +-referenced_attributes=
+| | | | +-AttributeReference[id=0,name=int_col,relation=test,type=Int NULL]
+| | | | +-AttributeReference[id=1,name=long_col,relation=test,type=Long]
| | | +-output_attributes=
-| | | +-AttributeReference[id=0,name=int_col,relation=test,type=Int NULL]
-| | | +-AttributeReference[id=1,name=long_col,relation=test,type=Long]
+| | | +-AttributeReference[id=6,name=int_col,relation=test,type=Int NULL]
+| | | +-AttributeReference[id=7,name=long_col,relation=test,type=Long]
| | +-project_list=
-| | +-Alias[id=6,name=int_col_alias,relation=,type=Int NULL]
-| | | +-AttributeReference[id=0,name=int_col,relation=test,type=Int NULL]
-| | +-Alias[id=7,name=long_col_alias,relation=,type=Long]
-| | +-AttributeReference[id=1,name=long_col,relation=test,type=Long]
+| | +-Alias[id=8,name=int_col_alias,relation=,type=Int NULL]
+| | | +-AttributeReference[id=6,name=int_col,relation=test,type=Int NULL]
+| | +-Alias[id=9,name=long_col_alias,relation=,type=Long]
+| | +-AttributeReference[id=7,name=long_col,relation=test,type=Long]
| +-project_list=
-| +-AttributeReference[id=6,name=int_col_alias,relation=,type=Int NULL]
+| +-AttributeReference[id=8,name=int_col_alias,relation=,type=Int NULL]
+-shared_subplans=
| +-Project
| +-input=TableReference[relation_name=Test,relation_alias=test]
@@ -1869,7 +1889,7 @@ TopLevelPlan
| +-AttributeReference[id=0,name=int_col,relation=test,type=Int NULL]
| +-AttributeReference[id=1,name=long_col,relation=test,type=Long]
+-output_attributes=
- +-AttributeReference[id=6,name=int_col_alias,relation=,type=Int NULL]
+ +-AttributeReference[id=8,name=int_col_alias,relation=,type=Int NULL]
==
with subquery as (select int_col int_col_alias, long_col from test)
@@ -1878,12 +1898,16 @@ select int_col_alias from subquery
TopLevelPlan
+-plan=Project
| +-input=SharedSubplanReference[subplan_id=0]
+| | +-referenced_attributes=
+| | | +-AttributeReference[id=0,name=int_col_alias,relation=subquery,
+| | | | type=Int NULL]
+| | | +-AttributeReference[id=1,name=long_col,relation=test,type=Long]
| | +-output_attributes=
-| | +-AttributeReference[id=0,name=int_col_alias,relation=subquery,
+| | +-AttributeReference[id=6,name=int_col_alias,relation=subquery,
| | | type=Int NULL]
-| | +-AttributeReference[id=1,name=long_col,relation=test,type=Long]
+| | +-AttributeReference[id=7,name=long_col,relation=test,type=Long]
| +-project_list=
-| +-AttributeReference[id=0,name=int_col_alias,relation=subquery,type=Int NULL]
+| +-AttributeReference[id=6,name=int_col_alias,relation=subquery,type=Int NULL]
+-shared_subplans=
| +-Project
| +-input=TableReference[relation_name=Test,relation_alias=test]
@@ -1899,7 +1923,7 @@ TopLevelPlan
| | +-AttributeReference[id=0,name=int_col,relation=test,type=Int NULL]
| +-AttributeReference[id=1,name=long_col,relation=test,type=Long]
+-output_attributes=
- +-AttributeReference[id=0,name=int_col_alias,relation=subquery,type=Int NULL]
+ +-AttributeReference[id=6,name=int_col_alias,relation=subquery,type=Int NULL]
==
# A WITH query cannot reference a subsequent WITH query.
@@ -1936,10 +1960,12 @@ select 1 from test
TopLevelPlan
+-plan=Project
| +-input=SharedSubplanReference[subplan_id=0]
+| | +-referenced_attributes=
+| | | +-AttributeReference[id=6,name=,alias=1,relation=test,type=Int]
| | +-output_attributes=
-| | +-AttributeReference[id=6,name=,alias=1,relation=test,type=Int]
+| | +-AttributeReference[id=7,name=,alias=1,relation=test,type=Int]
| +-project_list=
-| +-Alias[id=7,name=,alias=1,relation=,type=Int]
+| +-Alias[id=8,name=,alias=1,relation=,type=Int]
| +-Literal[value=1,type=Int]
+-shared_subplans=
| +-Project
@@ -1955,7 +1981,7 @@ TopLevelPlan
| +-Alias[id=6,name=,alias=1,relation=test,type=Int]
| +-Literal[value=1,type=Int]
+-output_attributes=
- +-AttributeReference[id=7,name=,alias=1,relation=,type=Int]
+ +-AttributeReference[id=8,name=,alias=1,relation=,type=Int]
==
# Identifies with special characters.
@@ -2937,3 +2963,88 @@ TopLevelPlan
+-output_attributes=
+-AttributeReference[id=11,name=,alias=(x+SubqueryExpression),relation=,
type=Long NULL]
+==
+
+# Same shared subplan referenced multiple times.
+WITH t(x, y) AS (
+ SELECT i % 5, i
+ FROM generate_series(1, 20) AS g(i)
+)
+SELECT *
+FROM t
+WHERE t.y = (
+ SELECT MAX(y)
+ FROM t t1
+ WHERE t.x = t1.x
+)
+ORDER BY x;
+--
+TopLevelPlan
++-plan=Project
+| +-input=Sort[is_ascending=[true],nulls_first=[false]]
+| | +-input=Filter
+| | | +-input=SharedSubplanReference[subplan_id=0]
+| | | | +-referenced_attributes=
+| | | | | +-AttributeReference[id=3,name=x,relation=,type=Int]
+| | | | | +-AttributeReference[id=4,name=y,relation=,type=Int]
+| | | | +-output_attributes=
+| | | | +-AttributeReference[id=5,name=x,relation=,type=Int]
+| | | | +-AttributeReference[id=6,name=y,relation=,type=Int]
+| | | +-filter_predicate=Equal
+| | | +-AttributeReference[id=6,name=y,relation=,type=Int]
+| | | +-SubqueryExpression
+| | | +-subquery=Project
+| | | +-input=Aggregate
+| | | | +-input=Filter
+| | | | | +-input=SharedSubplanReference[subplan_id=0]
+| | | | | | +-referenced_attributes=
+| | | | | | | +-AttributeReference[id=3,name=x,relation=,type=Int]
+| | | | | | | +-AttributeReference[id=4,name=y,relation=,type=Int]
+| | | | | | +-output_attributes=
+| | | | | | +-AttributeReference[id=7,name=x,relation=,type=Int]
+| | | | | | +-AttributeReference[id=8,name=y,relation=,type=Int]
+| | | | | +-filter_predicate=Equal
+| | | | | +-AttributeReference[id=5,name=x,relation=,type=Int,
+| | | | | | is_outer_reference=true]
+| | | | | +-AttributeReference[id=7,name=x,relation=,type=Int]
+| | | | +-grouping_expressions=
+| | | | | +-[]
+| | | | +-aggregate_expressions=
+| | | | +-Alias[id=9,name=,alias=$aggregate0,relation=$aggregate,
+| | | | type=Int NULL]
+| | | | +-AggregateFunction[function=MAX]
+| | | | +-AttributeReference[id=8,name=y,relation=,type=Int]
+| | | +-project_list=
+| | | +-Alias[id=9,name=,alias=MAX(y),relation=,type=Int NULL]
+| | | +-AttributeReference[id=9,name=,alias=$aggregate0,
+| | | relation=$aggregate,type=Int NULL]
+| | +-sort_expressions=
+| | +-AttributeReference[id=5,name=x,relation=,type=Int]
+| +-project_list=
+| +-AttributeReference[id=5,name=x,relation=,type=Int]
+| +-AttributeReference[id=6,name=y,relation=,type=Int]
++-shared_subplans=
+| +-Project
+| +-input=Project
+| | +-input=Project
+| | | +-input=TableGenerator[function_name=generate_series,table_alias=g]
+| | | | +-AttributeReference[id=0,name=generate_series,alias=g,
+| | | | relation=generate_series,type=Int]
+| | | +-project_list=
+| | | +-Alias[id=1,name=i,relation=,type=Int]
+| | | +-AttributeReference[id=0,name=generate_series,alias=g,
+| | | relation=generate_series,type=Int]
+| | +-project_list=
+| | +-Alias[id=2,name=,alias=(i%5),relation=t,type=Int]
+| | | +-Modulo
+| | | +-AttributeReference[id=1,name=i,relation=,type=Int]
+| | | +-Literal[value=5,type=Int]
+| | +-AttributeReference[id=1,name=i,relation=,type=Int]
+| +-project_list=
+| +-Alias[id=3,name=x,relation=,type=Int]
+| | +-AttributeReference[id=2,name=,alias=(i%5),relation=t,type=Int]
+| +-Alias[id=4,name=y,relation=,type=Int]
+| +-AttributeReference[id=1,name=i,relation=,type=Int]
++-output_attributes=
+ +-AttributeReference[id=5,name=x,relation=,type=Int]
+ +-AttributeReference[id=6,name=y,relation=,type=Int]
[40/50] [abbrv] incubator-quickstep git commit: Visualize optimized
physical plan in DOT format (#232)
Posted by zu...@apache.org.
Visualize optimized physical plan in DOT format (#232)
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/1605fd84
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/1605fd84
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/1605fd84
Branch: refs/heads/work-order-serialization
Commit: 1605fd8445a59a7cc38bf23088d9af434a2b75eb
Parents: d136e1d
Author: Jianqiao Zhu <ji...@cs.wisc.edu>
Authored: Fri May 20 16:47:31 2016 -0500
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Mon May 30 15:47:52 2016 -0700
----------------------------------------------------------------------
query_optimizer/CMakeLists.txt | 3 +-
query_optimizer/PhysicalGenerator.cpp | 13 ++-
utility/CMakeLists.txt | 13 +++
utility/PlanVisualizer.cpp | 161 +++++++++++++++++++++++++++++
utility/PlanVisualizer.hpp | 94 +++++++++++++++++
utility/StringUtil.hpp | 30 +++++-
6 files changed, 310 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1605fd84/query_optimizer/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_optimizer/CMakeLists.txt b/query_optimizer/CMakeLists.txt
index aa2873e..5c9438d 100644
--- a/query_optimizer/CMakeLists.txt
+++ b/query_optimizer/CMakeLists.txt
@@ -194,7 +194,8 @@ target_link_libraries(quickstep_queryoptimizer_PhysicalGenerator
quickstep_queryoptimizer_strategy_Selection
quickstep_queryoptimizer_strategy_Strategy
quickstep_queryoptimizer_Validator
- quickstep_utility_Macros)
+ quickstep_utility_Macros
+ quickstep_utility_PlanVisualizer)
target_link_libraries(quickstep_queryoptimizer_QueryHandle
quickstep_catalog_Catalog_proto
quickstep_queryexecution_QueryContext_proto
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1605fd84/query_optimizer/PhysicalGenerator.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/PhysicalGenerator.cpp b/query_optimizer/PhysicalGenerator.cpp
index 662236f..75a7bc9 100644
--- a/query_optimizer/PhysicalGenerator.cpp
+++ b/query_optimizer/PhysicalGenerator.cpp
@@ -33,6 +33,7 @@
#include "query_optimizer/strategy/OneToOne.hpp"
#include "query_optimizer/strategy/Selection.hpp"
#include "query_optimizer/strategy/Strategy.hpp"
+#include "utility/PlanVisualizer.hpp"
#include "gflags/gflags.h"
@@ -45,7 +46,12 @@ DEFINE_bool(reorder_hash_joins, true,
"If true, apply hash join order optimization to each group of hash "
"joins. The optimization applies a greedy algorithm to favor smaller "
"cardinality and selective tables to be joined first, which is suitable "
- "for queries on star-schema tables");
+ "for queries on star-schema tables.");
+
+DEFINE_bool(visualize_plan, false,
+ "If true, visualize the final physical plan into a graph in DOT format "
+ "(DOT is a plain text graph description language). Then print the "
+ "generated graph through stderr.");
namespace L = ::quickstep::optimizer::logical;
namespace P = ::quickstep::optimizer::physical;
@@ -101,6 +107,11 @@ P::PhysicalPtr PhysicalGenerator::optimizePlan() {
DVLOG(4) << "Optimized physical plan:\n" << physical_plan_->toString();
+ if (FLAGS_visualize_plan) {
+ quickstep::PlanVisualizer plan_visualizer;
+ std::cerr << "\n" << plan_visualizer.visualize(physical_plan_) << "\n";
+ }
+
#ifdef QUICKSTEP_DEBUG
Validate(physical_plan_);
#endif
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1605fd84/utility/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/utility/CMakeLists.txt b/utility/CMakeLists.txt
index 6d1eeab..2d3db8f 100644
--- a/utility/CMakeLists.txt
+++ b/utility/CMakeLists.txt
@@ -171,6 +171,7 @@ add_library(quickstep_utility_Glob Glob.cpp Glob.hpp)
add_library(quickstep_utility_HashPair ../empty_src.cpp HashPair.hpp)
add_library(quickstep_utility_Macros ../empty_src.cpp Macros.hpp)
add_library(quickstep_utility_MemStream ../empty_src.cpp MemStream.hpp)
+add_library(quickstep_utility_PlanVisualizer PlanVisualizer.cpp PlanVisualizer.hpp)
add_library(quickstep_utility_PrimeNumber PrimeNumber.cpp PrimeNumber.hpp)
add_library(quickstep_utility_PtrList ../empty_src.cpp PtrList.hpp)
add_library(quickstep_utility_PtrMap ../empty_src.cpp PtrMap.hpp)
@@ -231,6 +232,17 @@ target_link_libraries(quickstep_utility_MemStream
quickstep_utility_Macros)
target_link_libraries(quickstep_utility_PrimeNumber
glog)
+target_link_libraries(quickstep_utility_PlanVisualizer
+ quickstep_catalog_CatalogRelation
+ quickstep_queryoptimizer_costmodel_StarSchemaSimpleCostModel
+ quickstep_queryoptimizer_expressions_AttributeReference
+ quickstep_queryoptimizer_physical_HashJoin
+ quickstep_queryoptimizer_physical_Physical
+ quickstep_queryoptimizer_physical_PhysicalType
+ quickstep_queryoptimizer_physical_TableReference
+ quickstep_queryoptimizer_physical_TopLevelPlan
+ quickstep_utility_Macros
+ quickstep_utility_StringUtil)
target_link_libraries(quickstep_utility_PtrList
quickstep_utility_Macros)
target_link_libraries(quickstep_utility_PtrMap
@@ -295,6 +307,7 @@ target_link_libraries(quickstep_utility
quickstep_utility_HashPair
quickstep_utility_Macros
quickstep_utility_MemStream
+ quickstep_utility_PlanVisualizer
quickstep_utility_PrimeNumber
quickstep_utility_PtrList
quickstep_utility_PtrMap
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1605fd84/utility/PlanVisualizer.cpp
----------------------------------------------------------------------
diff --git a/utility/PlanVisualizer.cpp b/utility/PlanVisualizer.cpp
new file mode 100644
index 0000000..962d577
--- /dev/null
+++ b/utility/PlanVisualizer.cpp
@@ -0,0 +1,161 @@
+/**
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#include "utility/PlanVisualizer.hpp"
+
+#include <cstddef>
+#include <memory>
+#include <sstream>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include "catalog/CatalogRelation.hpp"
+
+#include "query_optimizer/cost_model/StarSchemaSimpleCostModel.hpp"
+#include "query_optimizer/expressions/AttributeReference.hpp"
+#include "query_optimizer/physical/HashJoin.hpp"
+#include "query_optimizer/physical/Physical.hpp"
+#include "query_optimizer/physical/PhysicalType.hpp"
+#include "query_optimizer/physical/TableReference.hpp"
+#include "query_optimizer/physical/TopLevelPlan.hpp"
+#include "utility/StringUtil.hpp"
+
+#include "glog/logging.h"
+
+namespace quickstep {
+
+namespace E = ::quickstep::optimizer::expressions;
+namespace P = ::quickstep::optimizer::physical;
+namespace C = ::quickstep::optimizer::cost;
+
+std::string PlanVisualizer::visualize(const P::PhysicalPtr &input) {
+ DCHECK(input->getPhysicalType() == P::PhysicalType::kTopLevelPlan);
+ cost_model_.reset(
+ new C::StarSchemaSimpleCostModel(
+ std::static_pointer_cast<const P::TopLevelPlan>(input)->shared_subplans()));
+
+ color_map_["TableReference"] = "skyblue";
+ color_map_["Selection"] = "#90EE90";
+ color_map_["HashJoin"] = "red";
+ color_map_["HashLeftOuterJoin"] = "orange";
+ color_map_["HashLeftSemiJoin"] = "orange";
+ color_map_["HashLeftAntiJoin"] = "orange";
+
+ visit(input);
+
+ // Format output graph
+ std::ostringstream graph_oss;
+ graph_oss << "digraph g {\n";
+ graph_oss << " rankdir=BT\n";
+ graph_oss << " node [penwidth=2]\n";
+ graph_oss << " edge [fontsize=16 fontcolor=gray penwidth=2]\n\n";
+
+ // Format nodes
+ for (const NodeInfo &node_info : nodes_) {
+ graph_oss << " " << node_info.id << " [";
+ if (!node_info.labels.empty()) {
+ graph_oss << "label=\""
+ << EscapeSpecialChars(JoinToString(node_info.labels, " "))
+ << "\"";
+ }
+ if (!node_info.color.empty()) {
+ graph_oss << " style=filled fillcolor=\"" << node_info.color << "\"";
+ }
+ graph_oss << "]\n";
+ }
+ graph_oss << "\n";
+
+ // Format edges
+ for (const EdgeInfo &edge_info : edges_) {
+ graph_oss << " " << edge_info.src_node_id << " -> "
+ << edge_info.dst_node_id << " [";
+ if (!edge_info.labels.empty()) {
+ graph_oss << "label=\""
+ << EscapeSpecialChars(JoinToString(edge_info.labels, " "))
+ << "\"";
+ }
+ graph_oss << "]\n";
+ }
+
+ graph_oss << "}\n";
+
+ return graph_oss.str();
+}
+
+void PlanVisualizer::visit(const P::PhysicalPtr &input) {
+ int node_id = ++id_counter_;
+ node_id_map_.emplace(input, node_id);
+
+ for (const auto &child : input->children()) {
+ visit(child);
+
+ int child_id = node_id_map_[child];
+
+ edges_.emplace_back(EdgeInfo());
+ EdgeInfo &edge_info = edges_.back();
+ edge_info.src_node_id = child_id;
+ edge_info.dst_node_id = node_id;
+
+ // Print output attributes except for TableReference -- there are just too many
+ // attributes out of TableReference.
+ if (child->getPhysicalType() != P::PhysicalType::kTableReference) {
+ for (const auto &attr : child->getOutputAttributes()) {
+ edge_info.labels.emplace_back(attr->attribute_alias());
+ }
+ }
+ }
+
+ nodes_.emplace_back(NodeInfo());
+ NodeInfo &node_info = nodes_.back();
+ node_info.id = node_id;
+ if (color_map_.find(input->getName()) != color_map_.end()) {
+ node_info.color = color_map_[input->getName()];
+ }
+
+ switch (input->getPhysicalType()) {
+ case P::PhysicalType::kTableReference: {
+ const P::TableReferencePtr table_reference =
+ std::static_pointer_cast<const P::TableReference>(input);
+ node_info.labels.emplace_back(table_reference->relation()->getName());
+ break;
+ }
+ case P::PhysicalType::kHashJoin: {
+ const P::HashJoinPtr hash_join =
+ std::static_pointer_cast<const P::HashJoin>(input);
+ node_info.labels.emplace_back(input->getName());
+
+ const auto &left_attributes = hash_join->left_join_attributes();
+ const auto &right_attributes = hash_join->right_join_attributes();
+ for (std::size_t i = 0; i < left_attributes.size(); ++i) {
+ node_info.labels.emplace_back(
+ left_attributes[i]->attribute_alias() + " = " + right_attributes[i]->attribute_alias());
+ }
+ break;
+ }
+ default: {
+ node_info.labels.emplace_back(input->getName());
+ break;
+ }
+ }
+ node_info.labels.emplace_back(
+ "est. # = " + std::to_string(cost_model_->estimateCardinality(input)));
+ node_info.labels.emplace_back(
+ "est. Selectivity = " + std::to_string(cost_model_->estimateSelectivity(input)));
+}
+
+} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1605fd84/utility/PlanVisualizer.hpp
----------------------------------------------------------------------
diff --git a/utility/PlanVisualizer.hpp b/utility/PlanVisualizer.hpp
new file mode 100644
index 0000000..080b7de
--- /dev/null
+++ b/utility/PlanVisualizer.hpp
@@ -0,0 +1,94 @@
+/**
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#ifndef QUICKSTEP_UTILITY_PLAN_VISUALIZER_HPP_
+#define QUICKSTEP_UTILITY_PLAN_VISUALIZER_HPP_
+
+#include <memory>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include "query_optimizer/cost_model/StarSchemaSimpleCostModel.hpp"
+#include "query_optimizer/physical/Physical.hpp"
+#include "utility/Macros.hpp"
+
+namespace quickstep {
+
+/** \addtogroup Utility
+ * @{
+ */
+
+/**
+ * @brief A query plan visualizer that converts a physical plan into a graph in
+ * DOT format. Note that DOT is a plain text graph description language.
+ *
+ * @note This utility tool can be further extended to be more generic.
+ */
+class PlanVisualizer {
+ public:
+ PlanVisualizer()
+ : id_counter_(0) {}
+
+ ~PlanVisualizer() {}
+
+ /**
+ * @brief Visualize the query plan into a graph in DOT format (DOT is a plain
+ * text graph description language).
+ *
+ * @return The visualized query plan graph in DOT format.
+ */
+ std::string visualize(const optimizer::physical::PhysicalPtr &input);
+
+ private:
+ /**
+ * @brief Information of a graph node.
+ */
+ struct NodeInfo {
+ int id;
+ std::vector<std::string> labels;
+ std::string color;
+ };
+
+ /**
+ * @brief Information of a graph edge.
+ */
+ struct EdgeInfo {
+ int src_node_id;
+ int dst_node_id;
+ std::vector<std::string> labels;
+ };
+
+ void visit(const optimizer::physical::PhysicalPtr &input);
+
+ int id_counter_;
+ std::unordered_map<optimizer::physical::PhysicalPtr, int> node_id_map_;
+ std::unordered_map<std::string, std::string> color_map_;
+
+ std::vector<NodeInfo> nodes_;
+ std::vector<EdgeInfo> edges_;
+
+ std::unique_ptr<optimizer::cost::StarSchemaSimpleCostModel> cost_model_;
+
+ DISALLOW_COPY_AND_ASSIGN(PlanVisualizer);
+};
+
+/** @} */
+
+} // namespace quickstep
+
+#endif /* QUICKSTEP_UTILITY_PLAN_VISUALIZER_HPP_ */
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1605fd84/utility/StringUtil.hpp
----------------------------------------------------------------------
diff --git a/utility/StringUtil.hpp b/utility/StringUtil.hpp
index bd138bb..6477ded 100644
--- a/utility/StringUtil.hpp
+++ b/utility/StringUtil.hpp
@@ -1,6 +1,8 @@
/**
* Copyright 2011-2015 Quickstep Technologies LLC.
* Copyright 2015 Pivotal Software, Inc.
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -19,6 +21,7 @@
#define QUICKSTEP_UTILITY_STRING_UTIL_HPP_
#include <cstdint>
+#include <sstream>
#include <string>
#include <vector>
@@ -34,7 +37,7 @@ namespace quickstep {
* @param str The string to be converted.
* @return The converted string with all lower case characters bing converted to upper case characters.
*/
-extern std::string ToLower(const std::string& str);
+extern std::string ToLower(const std::string &str);
/**
* @brief Converts special characters to escape characters.
@@ -42,7 +45,30 @@ extern std::string ToLower(const std::string& str);
* @param text The string to be unescaped.
* @return Unescaped string.
*/
-extern std::string EscapeSpecialChars(const std::string& text);
+extern std::string EscapeSpecialChars(const std::string &text);
+
+/**
+ * @brief Join all objects in a iterable container into a single string. The object
+ * must have implemented the operator<< overloading with std::stringstream.
+ *
+ * @param container The iterable container of objects.
+ * @param separator A string to separate each object.
+ */
+template <typename ContainerType>
+std::string JoinToString(const ContainerType &container,
+ const std::string &separator) {
+ std::ostringstream oss;
+ bool is_first = true;
+ for (const auto &item : container) {
+ if (is_first) {
+ is_first = false;
+ } else {
+ oss << separator;
+ }
+ oss << item;
+ }
+ return oss.str();
+}
/**
* @brief Parse a string of base-10 integers separated by delimiter characters
[47/50] [abbrv] incubator-quickstep git commit: Added BlockLocator.
Posted by zu...@apache.org.
Added BlockLocator.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/2221b7e4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/2221b7e4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/2221b7e4
Branch: refs/heads/work-order-serialization
Commit: 2221b7e45bf5b01897e7ccee3016107d78fcea3e
Parents: 8f8a03a
Author: Zuyu Zhang <zz...@pivotal.io>
Authored: Sat May 28 14:24:13 2016 -0700
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Mon May 30 15:47:54 2016 -0700
----------------------------------------------------------------------
query_execution/BlockLocator.cpp | 223 +++++++++++++++
query_execution/BlockLocator.hpp | 125 +++++++++
query_execution/CMakeLists.txt | 48 ++++
query_execution/QueryExecutionMessages.proto | 34 +++
query_execution/QueryExecutionTypedefs.hpp | 16 ++
query_execution/tests/BlockLocator_unittest.cpp | 270 +++++++++++++++++++
storage/CMakeLists.txt | 9 +-
storage/StorageManager.cpp | 190 ++++++++++++-
storage/StorageManager.hpp | 76 +++++-
9 files changed, 984 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2221b7e4/query_execution/BlockLocator.cpp
----------------------------------------------------------------------
diff --git a/query_execution/BlockLocator.cpp b/query_execution/BlockLocator.cpp
new file mode 100644
index 0000000..6cf5249
--- /dev/null
+++ b/query_execution/BlockLocator.cpp
@@ -0,0 +1,223 @@
+/**
+ * Copyright 2016 Pivotal Software, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#include "query_execution/BlockLocator.hpp"
+
+#include <cstdlib>
+#include <string>
+#include <utility>
+
+#include "query_execution/QueryExecutionMessages.pb.h"
+#include "query_execution/QueryExecutionTypedefs.hpp"
+#include "query_execution/QueryExecutionUtil.hpp"
+#include "storage/StorageBlockInfo.hpp"
+#include "threading/ThreadUtil.hpp"
+
+#include "glog/logging.h"
+
+#include "tmb/id_typedefs.h"
+#include "tmb/message_bus.h"
+#include "tmb/tagged_message.h"
+
+using std::free;
+using std::malloc;
+using std::move;
+
+using tmb::TaggedMessage;
+using tmb::client_id;
+
+namespace quickstep {
+
+void BlockLocator::run() {
+ if (cpu_id_ >= 0) {
+ ThreadUtil::BindToCPU(cpu_id_);
+ }
+
+ for (;;) {
+ // Receive() is a blocking call, causing this thread to sleep until next
+ // message is received.
+ const tmb::AnnotatedMessage annotated_message = bus_->Receive(locator_client_id_, 0, true);
+ const TaggedMessage &tagged_message = annotated_message.tagged_message;
+ const client_id sender = annotated_message.sender;
+ LOG(INFO) << "BlockLocator received the typed '" << tagged_message.message_type()
+ << "' message from TMB Client " << sender;
+ switch (tagged_message.message_type()) {
+ case kBlockDomainRegistrationMessage: {
+ serialization::BlockDomainRegistrationMessage proto;
+ CHECK(proto.ParseFromArray(tagged_message.message(), tagged_message.message_bytes()));
+
+ processBlockDomainRegistrationMessage(sender, proto.domain_network_address());
+ break;
+ }
+ case kAddBlockLocationMessage: {
+ serialization::BlockLocationMessage proto;
+ CHECK(proto.ParseFromArray(tagged_message.message(), tagged_message.message_bytes()));
+
+ const block_id block = proto.block_id();
+ const block_id_domain domain = proto.block_domain();
+
+ const auto result_block_locations = block_locations_[block].insert(domain);
+ const auto result_domain_blocks = domain_blocks_[domain].insert(block);
+ DCHECK_EQ(result_block_locations.second, result_domain_blocks.second);
+
+ if (result_domain_blocks.second) {
+ LOG(INFO) << "Block " << BlockIdUtil::ToString(block) << " loaded in Domain " << domain;
+ } else {
+ LOG(INFO) << "Block " << BlockIdUtil::ToString(block) << " existed in Domain " << domain;
+ }
+ break;
+ }
+ case kDeleteBlockLocationMessage: {
+ serialization::BlockLocationMessage proto;
+ CHECK(proto.ParseFromArray(tagged_message.message(), tagged_message.message_bytes()));
+
+ const block_id block = proto.block_id();
+ const block_id_domain domain = proto.block_domain();
+
+ const auto cit = block_locations_[block].find(domain);
+ if (cit != block_locations_[block].end()) {
+ block_locations_[block].erase(domain);
+ domain_blocks_[domain].erase(block);
+
+ LOG(INFO) << "Block " << BlockIdUtil::ToString(block) << " evicted in Domain " << domain;
+ } else {
+ LOG(INFO) << "Block " << BlockIdUtil::ToString(block) << " not found in Domain " << domain;
+ }
+ break;
+ }
+ case kLocateBlockMessage: {
+ serialization::BlockMessage proto;
+ CHECK(proto.ParseFromArray(tagged_message.message(), tagged_message.message_bytes()));
+
+ processLocateBlockMessage(sender, proto.block_id());
+ break;
+ }
+ case kGetPeerDomainNetworkAddressesMessage: {
+ serialization::BlockMessage proto;
+ CHECK(proto.ParseFromArray(tagged_message.message(), tagged_message.message_bytes()));
+
+ processGetPeerDomainNetworkAddressesMessage(sender, proto.block_id());
+ break;
+ }
+ case kBlockDomainUnregistrationMessage: {
+ serialization::BlockDomainMessage proto;
+ CHECK(proto.ParseFromArray(tagged_message.message(), tagged_message.message_bytes()));
+
+ const block_id_domain domain = proto.block_domain();
+
+ domain_network_addresses_.erase(domain);
+
+ for (const block_id block : domain_blocks_[domain]) {
+ block_locations_[block].erase(domain);
+ }
+ domain_blocks_.erase(domain);
+
+ LOG(INFO) << "Unregistered Domain " << domain;
+ break;
+ }
+ case kPoisonMessage: {
+ return;
+ }
+ }
+ }
+}
+
+void BlockLocator::processBlockDomainRegistrationMessage(const client_id receiver,
+ const std::string &network_address) {
+ DCHECK_LT(block_domain_, kMaxDomain);
+
+ domain_network_addresses_.emplace(++block_domain_, network_address);
+ domain_blocks_[block_domain_];
+
+ serialization::BlockDomainMessage proto;
+ proto.set_block_domain(block_domain_);
+
+ const int proto_length = proto.ByteSize();
+ char *proto_bytes = static_cast<char*>(malloc(proto_length));
+ CHECK(proto.SerializeToArray(proto_bytes, proto_length));
+
+ TaggedMessage message(static_cast<const void*>(proto_bytes),
+ proto_length,
+ kBlockDomainRegistrationResponseMessage);
+ free(proto_bytes);
+
+ LOG(INFO) << "BlockLocator (id '" << locator_client_id_
+ << "') sent BlockDomainRegistrationResponseMessage (typed '"
+ << kBlockDomainRegistrationResponseMessage
+ << "') to Worker (id '" << receiver << "')";
+ CHECK(tmb::MessageBus::SendStatus::kOK ==
+ QueryExecutionUtil::SendTMBMessage(bus_,
+ locator_client_id_,
+ receiver,
+ move(message)));
+}
+
+void BlockLocator::processLocateBlockMessage(const client_id receiver,
+ const block_id block) {
+ serialization::LocateBlockResponseMessage proto;
+
+ for (const block_id_domain domain : block_locations_[block]) {
+ proto.add_block_domains(domain);
+ }
+
+ const int proto_length = proto.ByteSize();
+ char *proto_bytes = static_cast<char*>(malloc(proto_length));
+ CHECK(proto.SerializeToArray(proto_bytes, proto_length));
+
+ TaggedMessage message(static_cast<const void*>(proto_bytes),
+ proto_length,
+ kLocateBlockResponseMessage);
+ free(proto_bytes);
+
+ LOG(INFO) << "BlockLocator (id '" << locator_client_id_
+ << "') sent LocateBlockResponseMessage (typed '" << kLocateBlockResponseMessage
+ << "') to StorageManager (id '" << receiver << "')";
+ CHECK(tmb::MessageBus::SendStatus::kOK ==
+ QueryExecutionUtil::SendTMBMessage(bus_,
+ locator_client_id_,
+ receiver,
+ move(message)));
+}
+
+void BlockLocator::processGetPeerDomainNetworkAddressesMessage(const client_id receiver,
+ const block_id block) {
+ serialization::GetPeerDomainNetworkAddressesResponseMessage proto;
+
+ for (const block_id_domain domain : block_locations_[block]) {
+ proto.add_domain_network_addresses(domain_network_addresses_[domain]);
+ }
+
+ const int proto_length = proto.ByteSize();
+ char *proto_bytes = static_cast<char*>(malloc(proto_length));
+ CHECK(proto.SerializeToArray(proto_bytes, proto_length));
+
+ TaggedMessage message(static_cast<const void*>(proto_bytes),
+ proto_length,
+ kGetPeerDomainNetworkAddressesResponseMessage);
+ free(proto_bytes);
+
+ LOG(INFO) << "BlockLocator (id '" << locator_client_id_
+ << "') sent GetPeerDomainNetworkAddressesResponseMessage (typed '"
+ << kGetPeerDomainNetworkAddressesResponseMessage
+ << "') to StorageManager (id '" << receiver << "')";
+ CHECK(tmb::MessageBus::SendStatus::kOK ==
+ QueryExecutionUtil::SendTMBMessage(bus_,
+ locator_client_id_,
+ receiver,
+ move(message)));
+}
+
+} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2221b7e4/query_execution/BlockLocator.hpp
----------------------------------------------------------------------
diff --git a/query_execution/BlockLocator.hpp b/query_execution/BlockLocator.hpp
new file mode 100644
index 0000000..bbd9b8f
--- /dev/null
+++ b/query_execution/BlockLocator.hpp
@@ -0,0 +1,125 @@
+/**
+ * Copyright 2016 Pivotal Software, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#ifndef QUICKSTEP_QUERY_EXECUTION_BLOCK_LOCATOR_HPP_
+#define QUICKSTEP_QUERY_EXECUTION_BLOCK_LOCATOR_HPP_
+
+#include <atomic>
+#include <string>
+#include <unordered_map>
+#include <unordered_set>
+
+#include "query_execution/QueryExecutionTypedefs.hpp"
+#include "storage/StorageBlockInfo.hpp"
+#include "storage/StorageConstants.hpp"
+#include "threading/Thread.hpp"
+#include "utility/Macros.hpp"
+
+#include "glog/logging.h"
+
+#include "tmb/id_typedefs.h"
+#include "tmb/message_bus.h"
+
+namespace quickstep {
+
+/** \addtogroup QueryExecution
+ * @{
+ */
+
+/**
+ * @brief A class for keeping trace of blocks loaded in a Worker's buffer pool
+ * in the distributed version.
+ **/
+class BlockLocator : public Thread {
+ public:
+ /**
+ * @brief Constructor.
+ *
+ * @param bus A pointer to the TMB.
+ * @param cpu_id The ID of the CPU to which the BlockLocator thread can be pinned.
+ *
+ * @note If cpu_id is not specified, BlockLocator thread can be possibly moved
+ * around on different CPUs by the OS.
+ **/
+ BlockLocator(tmb::MessageBus *bus,
+ const int cpu_id = -1)
+ : bus_(DCHECK_NOTNULL(bus)),
+ cpu_id_(cpu_id),
+ block_domain_(0) {
+ locator_client_id_ = bus_->Connect();
+
+ bus_->RegisterClientAsReceiver(locator_client_id_, kBlockDomainRegistrationMessage);
+ bus_->RegisterClientAsSender(locator_client_id_, kBlockDomainRegistrationResponseMessage);
+
+ bus_->RegisterClientAsReceiver(locator_client_id_, kAddBlockLocationMessage);
+ bus_->RegisterClientAsReceiver(locator_client_id_, kDeleteBlockLocationMessage);
+
+ bus_->RegisterClientAsReceiver(locator_client_id_, kLocateBlockMessage);
+ bus_->RegisterClientAsSender(locator_client_id_, kLocateBlockResponseMessage);
+
+ bus_->RegisterClientAsReceiver(locator_client_id_, kGetPeerDomainNetworkAddressesMessage);
+ bus_->RegisterClientAsSender(locator_client_id_, kGetPeerDomainNetworkAddressesResponseMessage);
+
+ bus_->RegisterClientAsReceiver(locator_client_id_, kBlockDomainUnregistrationMessage);
+ bus_->RegisterClientAsReceiver(locator_client_id_, kPoisonMessage);
+ }
+
+ ~BlockLocator() override {}
+
+ /**
+ * @brief Get the TMB client ID of BlockLocator thread.
+ *
+ * @return TMB client ID of BlockLocator thread.
+ **/
+ tmb::client_id getBusClientID() const {
+ return locator_client_id_;
+ }
+
+ protected:
+ void run() override;
+
+ private:
+ void processBlockDomainRegistrationMessage(const tmb::client_id receiver, const std::string &network_address);
+ void processLocateBlockMessage(const tmb::client_id receiver, const block_id block);
+ void processGetPeerDomainNetworkAddressesMessage(const tmb::client_id receiver, const block_id block);
+
+ tmb::MessageBus *bus_;
+
+ // The ID of the CPU that the BlockLocator thread can optionally be pinned to.
+ const int cpu_id_;
+
+ alignas(kCacheLineBytes) std::atomic<block_id_domain> block_domain_;
+
+ // From a block domain to its network info in the ip:port format, i.e.,
+ // "0.0.0.0:0".
+ std::unordered_map<block_id_domain, const std::string> domain_network_addresses_;
+
+ // From a block to its domains.
+ std::unordered_map<block_id, std::unordered_set<block_id_domain>> block_locations_;
+
+ // From a block domain to all blocks loaded in its buffer pool.
+ std::unordered_map<block_id_domain, std::unordered_set<block_id>> domain_blocks_;
+
+ tmb::client_id locator_client_id_;
+
+ DISALLOW_COPY_AND_ASSIGN(BlockLocator);
+};
+
+/** @} */
+
+} // namespace quickstep
+
+#endif // QUICKSTEP_QUERY_EXECUTION_BLOCK_LOCATOR_HPP_
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2221b7e4/query_execution/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_execution/CMakeLists.txt b/query_execution/CMakeLists.txt
index 04a0348..7d9d601 100644
--- a/query_execution/CMakeLists.txt
+++ b/query_execution/CMakeLists.txt
@@ -20,6 +20,9 @@ QS_PROTOBUF_GENERATE_CPP(queryexecution_QueryExecutionMessages_proto_srcs
QueryExecutionMessages.proto)
# Declare micro-libs:
+if (ENABLE_DISTRIBUTED)
+ add_library(quickstep_queryexecution_BlockLocator BlockLocator.cpp BlockLocator.hpp)
+endif()
add_library(quickstep_queryexecution_Foreman Foreman.cpp Foreman.hpp)
add_library(quickstep_queryexecution_ForemanLite ../empty_src.cpp ForemanLite.hpp)
add_library(quickstep_queryexecution_QueryContext QueryContext.cpp QueryContext.hpp)
@@ -40,6 +43,19 @@ add_library(quickstep_queryexecution_WorkerMessage ../empty_src.cpp WorkerMessag
add_library(quickstep_queryexecution_WorkerSelectionPolicy ../empty_src.cpp WorkerSelectionPolicy.hpp)
# Link dependencies:
+if (ENABLE_DISTRIBUTED)
+ target_link_libraries(quickstep_queryexecution_BlockLocator
+ glog
+ quickstep_queryexecution_QueryExecutionMessages_proto
+ quickstep_queryexecution_QueryExecutionTypedefs
+ quickstep_queryexecution_QueryExecutionUtil
+ quickstep_storage_StorageBlockInfo
+ quickstep_storage_StorageConstants
+ quickstep_threading_Thread
+ quickstep_threading_ThreadUtil
+ quickstep_utility_Macros
+ tmb)
+endif()
target_link_libraries(quickstep_queryexecution_Foreman
glog
gtest
@@ -176,7 +192,37 @@ target_link_libraries(quickstep_queryexecution
quickstep_queryexecution_WorkerDirectory
quickstep_queryexecution_WorkerMessage
quickstep_queryexecution_WorkerSelectionPolicy)
+if (ENABLE_DISTRIBUTED)
+ target_link_libraries(quickstep_queryexecution
+ quickstep_queryexecution_BlockLocator)
+endif()
+
# Tests:
+if (ENABLE_DISTRIBUTED)
+ add_executable(BlockLocator_unittest
+ "${CMAKE_CURRENT_SOURCE_DIR}/tests/BlockLocator_unittest.cpp")
+ target_link_libraries(BlockLocator_unittest
+ gflags_nothreads-static
+ glog
+ gtest
+ quickstep_catalog_CatalogAttribute
+ quickstep_catalog_CatalogRelation
+ quickstep_queryexecution_BlockLocator
+ quickstep_queryexecution_QueryExecutionMessages_proto
+ quickstep_queryexecution_QueryExecutionTypedefs
+ quickstep_queryexecution_QueryExecutionUtil
+ quickstep_storage_StorageBlob
+ quickstep_storage_StorageBlock
+ quickstep_storage_StorageBlockInfo
+ quickstep_storage_StorageConstants
+ quickstep_storage_StorageManager
+ quickstep_types_TypeFactory
+ quickstep_types_TypeID
+ tmb
+ ${LIBS})
+ add_test(BlockLocator_unittest BlockLocator_unittest)
+endif()
+
add_executable(Foreman_unittest
"${CMAKE_CURRENT_SOURCE_DIR}/tests/Foreman_unittest.cpp")
target_link_libraries(Foreman_unittest
@@ -269,3 +315,5 @@ target_link_libraries(WorkerSelectionPolicy_unittest
quickstep_queryexecution_WorkerDirectory
quickstep_queryexecution_WorkerSelectionPolicy)
add_test(WorkerSelectionPolicy_unittest WorkerSelectionPolicy_unittest)
+
+file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/block_locator_test_data/)
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2221b7e4/query_execution/QueryExecutionMessages.proto
----------------------------------------------------------------------
diff --git a/query_execution/QueryExecutionMessages.proto b/query_execution/QueryExecutionMessages.proto
index 8d2efd0..15803cf 100644
--- a/query_execution/QueryExecutionMessages.proto
+++ b/query_execution/QueryExecutionMessages.proto
@@ -16,6 +16,10 @@ syntax = "proto2";
package quickstep.serialization;
+// Used for any messages that do not carry payloads.
+message EmptyMessage {
+}
+
// Used for both Normal WorkOrders and RebuildWorkOrders.
// NOTE(zuyu): we might need to seperate the completion messages to contain
// run-time information for Foreman to make better decisions on scheduling
@@ -42,3 +46,33 @@ message DataPipelineMessage {
message WorkOrdersAvailableMessage {
required uint64 operator_index = 1;
}
+
+// BlockLocator related messages.
+message BlockDomainRegistrationMessage {
+ // Format IP:Port, i.e., "0.0.0.0:0".
+ required string domain_network_address = 1;
+}
+
+// Used for RegistrationResponse, Unregistration, and FailureReport.
+message BlockDomainMessage {
+ required uint32 block_domain = 1;
+}
+
+// Used when StorageManager loads or evicts a block or a blob from its buffer
+// pool.
+message BlockLocationMessage {
+ required fixed64 block_id = 1;
+ required uint32 block_domain = 2;
+}
+
+message BlockMessage {
+ required fixed64 block_id = 1;
+}
+
+message LocateBlockResponseMessage {
+ repeated uint32 block_domains = 1;
+}
+
+message GetPeerDomainNetworkAddressesResponseMessage {
+ repeated string domain_network_addresses = 1;
+}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2221b7e4/query_execution/QueryExecutionTypedefs.hpp
----------------------------------------------------------------------
diff --git a/query_execution/QueryExecutionTypedefs.hpp b/query_execution/QueryExecutionTypedefs.hpp
index 36cfd82..fc253bc 100644
--- a/query_execution/QueryExecutionTypedefs.hpp
+++ b/query_execution/QueryExecutionTypedefs.hpp
@@ -18,6 +18,7 @@
#ifndef QUICKSTEP_QUERY_EXECUTION_QUERY_EXECUTION_TYPEDEFS_HPP_
#define QUICKSTEP_QUERY_EXECUTION_QUERY_EXECUTION_TYPEDEFS_HPP_
+#include "query_optimizer/QueryOptimizerConfig.h" // For QUICKSTEP_DISTRIBUTED
#include "threading/ThreadIDBasedMap.hpp"
#include "tmb/address.h"
@@ -55,6 +56,7 @@ using ClientIDMap = ThreadIDBasedMap<client_id,
'a',
'p'>;
+// We sort the following message types in the order of a life cycle of a query.
enum QueryExecutionMessageType : message_type_id {
kWorkOrderMessage, // From Foreman to Worker.
kWorkOrderCompleteMessage, // From Worker to Foreman.
@@ -66,6 +68,20 @@ enum QueryExecutionMessageType : message_type_id {
kRebuildWorkOrderMessage, // From Foreman to Worker.
kRebuildWorkOrderCompleteMessage, // From Worker to Foreman.
kPoisonMessage, // From the CLI shell to Foreman, then from Foreman to Workers.
+
+#ifdef QUICKSTEP_DISTRIBUTED
+ // BlockLocator related messages, sorted in a life cycle of StorageManager
+ // with a unique block domain.
+ kBlockDomainRegistrationMessage, // From Worker to BlockLocator.
+ kBlockDomainRegistrationResponseMessage, // From BlockLocator to Worker.
+ kAddBlockLocationMessage, // From StorageManager to BlockLocator.
+ kDeleteBlockLocationMessage, // From StorageManager to BlockLocator.
+ kLocateBlockMessage, // From StorageManager to BlockLocator.
+ kLocateBlockResponseMessage, // From BlockLocator to StorageManager.
+ kGetPeerDomainNetworkAddressesMessage, // From StorageManager to BlockLocator.
+ kGetPeerDomainNetworkAddressesResponseMessage, // From BlockLocator to StorageManager.
+ kBlockDomainUnregistrationMessage, // From StorageManager to BlockLocator.
+#endif
};
/** @} */
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2221b7e4/query_execution/tests/BlockLocator_unittest.cpp
----------------------------------------------------------------------
diff --git a/query_execution/tests/BlockLocator_unittest.cpp b/query_execution/tests/BlockLocator_unittest.cpp
new file mode 100644
index 0000000..fe7b86b
--- /dev/null
+++ b/query_execution/tests/BlockLocator_unittest.cpp
@@ -0,0 +1,270 @@
+/**
+ * Copyright 2016 Pivotal Software, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#include <cstdlib>
+#include <memory>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "catalog/CatalogAttribute.hpp"
+#include "catalog/CatalogRelation.hpp"
+#include "query_execution/BlockLocator.hpp"
+#include "query_execution/QueryExecutionMessages.pb.h"
+#include "query_execution/QueryExecutionTypedefs.hpp"
+#include "query_execution/QueryExecutionUtil.hpp"
+#include "storage/StorageBlob.hpp"
+#include "storage/StorageBlock.hpp"
+#include "storage/StorageBlockInfo.hpp"
+#include "storage/StorageConstants.hpp"
+#include "storage/StorageManager.hpp"
+#include "types/TypeFactory.hpp"
+#include "types/TypeID.hpp"
+
+#include "gflags/gflags.h"
+#include "glog/logging.h"
+#include "gtest/gtest.h"
+
+#include "tmb/id_typedefs.h"
+#include "tmb/message_bus.h"
+#include "tmb/tagged_message.h"
+
+using std::free;
+using std::malloc;
+using std::move;
+using std::string;
+using std::unique_ptr;
+using std::vector;
+
+using tmb::AnnotatedMessage;
+using tmb::MessageBus;
+using tmb::TaggedMessage;
+
+namespace quickstep {
+
+class BlockLocatorTest : public ::testing::Test {
+ protected:
+ static const char kStoragePath[];
+ static const char kDomainNetworkAddress[];
+
+ ~BlockLocatorTest() {
+ locator_->join();
+ }
+
+ virtual void SetUp() {
+ bus_.Initialize();
+
+ locator_.reset(new BlockLocator(&bus_));
+ locator_client_id_ = locator_->getBusClientID();
+ locator_->start();
+
+ worker_client_id_ = bus_.Connect();
+
+ bus_.RegisterClientAsSender(worker_client_id_, kBlockDomainRegistrationMessage);
+ bus_.RegisterClientAsReceiver(worker_client_id_, kBlockDomainRegistrationResponseMessage);
+
+ bus_.RegisterClientAsSender(worker_client_id_, kLocateBlockMessage);
+ bus_.RegisterClientAsReceiver(worker_client_id_, kLocateBlockResponseMessage);
+
+ bus_.RegisterClientAsSender(worker_client_id_, kPoisonMessage);
+
+ block_domain_ = getBlockDomain(kDomainNetworkAddress);
+
+ storage_manager_.reset(
+ new StorageManager(kStoragePath, block_domain_, locator_client_id_, &bus_));
+ }
+
+ virtual void TearDown() {
+ storage_manager_.reset();
+
+ serialization::EmptyMessage proto;
+
+ const int proto_length = proto.ByteSize();
+ char *proto_bytes = static_cast<char*>(malloc(proto_length));
+ CHECK(proto.SerializeToArray(proto_bytes, proto_length));
+
+ TaggedMessage message(static_cast<const void*>(proto_bytes),
+ proto_length,
+ kPoisonMessage);
+ free(proto_bytes);
+
+ LOG(INFO) << "Worker (id '" << worker_client_id_
+ << "') sent PoisonMessage (typed '" << kPoisonMessage
+ << "') to BlockLocator (id '" << locator_client_id_ << "')";
+ CHECK(MessageBus::SendStatus::kOK ==
+ QueryExecutionUtil::SendTMBMessage(&bus_,
+ worker_client_id_,
+ locator_client_id_,
+ move(message)));
+ }
+
+ vector<block_id_domain> getPeerDomains(const block_id block) {
+ serialization::BlockMessage proto;
+ proto.set_block_id(block);
+
+ const int proto_length = proto.ByteSize();
+ char *proto_bytes = static_cast<char*>(malloc(proto_length));
+ CHECK(proto.SerializeToArray(proto_bytes, proto_length));
+
+ TaggedMessage message(static_cast<const void*>(proto_bytes),
+ proto_length,
+ kLocateBlockMessage);
+ free(proto_bytes);
+
+ LOG(INFO) << "Worker (id '" << worker_client_id_
+ << "') sent LocateBlockMessage (typed '" << kLocateBlockMessage
+ << "') to BlockLocator";
+ CHECK(MessageBus::SendStatus::kOK ==
+ QueryExecutionUtil::SendTMBMessage(&bus_,
+ worker_client_id_,
+ locator_client_id_,
+ move(message)));
+
+ const AnnotatedMessage annotated_message(bus_.Receive(worker_client_id_, 0, true));
+ const TaggedMessage &tagged_message = annotated_message.tagged_message;
+ CHECK_EQ(kLocateBlockResponseMessage, tagged_message.message_type());
+ LOG(INFO) << "Worker (id '" << worker_client_id_
+ << "') received LocateBlockResponseMessage from BlockLocator";
+
+ serialization::LocateBlockResponseMessage response_proto;
+ CHECK(response_proto.ParseFromArray(tagged_message.message(), tagged_message.message_bytes()));
+
+ vector<block_id_domain> domains;
+ for (int i = 0; i < response_proto.block_domains_size(); ++i) {
+ domains.push_back(response_proto.block_domains(i));
+ }
+
+ return domains;
+ }
+
+ void checkLoaded(const block_id block) {
+ const vector<string> peer_domain_network_addresses = storage_manager_->getPeerDomainNetworkAddresses(block);
+ EXPECT_EQ(1u, peer_domain_network_addresses.size());
+ EXPECT_STREQ(kDomainNetworkAddress, peer_domain_network_addresses[0].data());
+
+ const vector<block_id_domain> domains = getPeerDomains(block);
+ EXPECT_EQ(1u, domains.size());
+ EXPECT_EQ(block_domain_, domains[0]);
+ }
+
+ void checkEvicted(const block_id block) {
+ const vector<string> peer_domain_network_addresses = storage_manager_->getPeerDomainNetworkAddresses(block);
+ EXPECT_TRUE(peer_domain_network_addresses.empty());
+
+ const vector<block_id_domain> domains = getPeerDomains(block);
+ EXPECT_TRUE(domains.empty());
+ }
+
+ tmb::client_id worker_client_id_;
+
+ block_id_domain block_domain_;
+ unique_ptr<StorageManager> storage_manager_;
+
+ private:
+ block_id_domain getBlockDomain(const string &network_address) {
+ serialization::BlockDomainRegistrationMessage proto;
+ proto.set_domain_network_address(network_address);
+
+ const int proto_length = proto.ByteSize();
+ char *proto_bytes = static_cast<char*>(malloc(proto_length));
+ CHECK(proto.SerializeToArray(proto_bytes, proto_length));
+
+ TaggedMessage message(static_cast<const void*>(proto_bytes),
+ proto_length,
+ kBlockDomainRegistrationMessage);
+ free(proto_bytes);
+
+ LOG(INFO) << "Worker (id '" << worker_client_id_
+ << "') sent BlockDomainRegistrationMessage (typed '" << kBlockDomainRegistrationMessage
+ << "') to BlockLocator (id '" << locator_client_id_ << "')";
+
+ CHECK(MessageBus::SendStatus::kOK ==
+ QueryExecutionUtil::SendTMBMessage(&bus_,
+ worker_client_id_,
+ locator_client_id_,
+ move(message)));
+
+ const AnnotatedMessage annotated_message(bus_.Receive(worker_client_id_, 0, true));
+ const TaggedMessage &tagged_message = annotated_message.tagged_message;
+ EXPECT_EQ(locator_client_id_, annotated_message.sender);
+ EXPECT_EQ(kBlockDomainRegistrationResponseMessage, tagged_message.message_type());
+ LOG(INFO) << "Worker (id '" << worker_client_id_
+ << "') received BlockDomainRegistrationResponseMessage from BlockLocator";
+
+ serialization::BlockDomainMessage response_proto;
+ CHECK(response_proto.ParseFromArray(tagged_message.message(), tagged_message.message_bytes()));
+
+ return static_cast<block_id_domain>(response_proto.block_domain());
+ }
+
+ MessageBusImpl bus_;
+
+ unique_ptr<BlockLocator> locator_;
+ tmb::client_id locator_client_id_;
+};
+
+const char BlockLocatorTest::kStoragePath[] = "./block_locator_test_data/";
+const char BlockLocatorTest::kDomainNetworkAddress[] = "ip:port";
+
+TEST_F(BlockLocatorTest, BlockTest) {
+ CatalogRelation relation(nullptr, "rel");
+ relation.addAttribute(new CatalogAttribute(nullptr, "attr_int", TypeFactory::GetType(kInt)));
+
+ const block_id block =
+ storage_manager_->createBlock(relation, relation.getDefaultStorageBlockLayout());
+ checkLoaded(block);
+
+ ASSERT_TRUE(storage_manager_->saveBlockOrBlob(block));
+ storage_manager_->evictBlockOrBlob(block);
+ checkEvicted(block);
+
+ {
+ const BlockReference block_ref = storage_manager_->getBlock(block, relation);
+ }
+ checkLoaded(block);
+
+ storage_manager_->deleteBlockOrBlobFile(block);
+ checkEvicted(block);
+}
+
+TEST_F(BlockLocatorTest, BlobTest) {
+ const block_id blob = storage_manager_->createBlob(kDefaultBlockSizeInSlots);
+ checkLoaded(blob);
+
+ ASSERT_TRUE(storage_manager_->saveBlockOrBlob(blob));
+ storage_manager_->evictBlockOrBlob(blob);
+ checkEvicted(blob);
+
+ {
+ const BlobReference blob_ref = storage_manager_->getBlob(blob);
+ }
+ checkLoaded(blob);
+
+ storage_manager_->deleteBlockOrBlobFile(blob);
+ checkEvicted(blob);
+}
+
+} // namespace quickstep
+
+int main(int argc, char **argv) {
+ google::InitGoogleLogging(argv[0]);
+ // Honor FLAGS_buffer_pool_slots in StorageManager.
+ gflags::ParseCommandLineFlags(&argc, &argv, true);
+
+ ::testing::InitGoogleTest(&argc, argv);
+
+ return RUN_ALL_TESTS();
+}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2221b7e4/storage/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/storage/CMakeLists.txt b/storage/CMakeLists.txt
index 87a5e54..4da16ea 100644
--- a/storage/CMakeLists.txt
+++ b/storage/CMakeLists.txt
@@ -941,7 +941,8 @@ target_link_libraries(quickstep_storage_StorageManager
quickstep_utility_Alignment
quickstep_utility_CalculateInstalledMemory
quickstep_utility_Macros
- quickstep_utility_ShardedLockManager)
+ quickstep_utility_ShardedLockManager
+ tmb)
if (QUICKSTEP_HAVE_FILE_MANAGER_HDFS)
target_link_libraries(quickstep_storage_StorageManager
quickstep_storage_FileManagerHdfs)
@@ -950,6 +951,12 @@ if (QUICKSTEP_HAVE_LIBNUMA)
target_link_libraries(quickstep_storage_StorageManager
${LIBNUMA_LIBRARY})
endif()
+if (ENABLE_DISTRIBUTED)
+ target_link_libraries(quickstep_storage_StorageManager
+ quickstep_queryexecution_QueryExecutionMessages_proto
+ quickstep_queryexecution_QueryExecutionTypedefs
+ quickstep_queryexecution_QueryExecutionUtil)
+endif(ENABLE_DISTRIBUTED)
target_link_libraries(quickstep_storage_SubBlockTypeRegistry
glog
quickstep_storage_StorageBlockLayout_proto
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2221b7e4/storage/StorageManager.cpp
----------------------------------------------------------------------
diff --git a/storage/StorageManager.cpp b/storage/StorageManager.cpp
index 5d91052..15e2503 100644
--- a/storage/StorageManager.cpp
+++ b/storage/StorageManager.cpp
@@ -18,6 +18,7 @@
// This is included before other files so that we can conditionally determine
// what else to include.
#include "catalog/CatalogConfig.h"
+#include "query_optimizer/QueryOptimizerConfig.h" // For QUICKSTEP_DISTRIBUTED
#include "storage/StorageConfig.h"
// Define feature test macros to enable large page support for mmap.
@@ -52,6 +53,12 @@
#include <unordered_map>
#include <vector>
+#ifdef QUICKSTEP_DISTRIBUTED
+#include "query_execution/QueryExecutionMessages.pb.h"
+#include "query_execution/QueryExecutionTypedefs.hpp"
+#include "query_execution/QueryExecutionUtil.hpp"
+#endif
+
#include "storage/CountedReference.hpp"
#include "storage/EvictionPolicy.hpp"
#include "storage/FileManagerLocal.hpp"
@@ -74,6 +81,13 @@
#include "gflags/gflags.h"
#include "glog/logging.h"
+#include "tmb/id_typedefs.h"
+
+#ifdef QUICKSTEP_DISTRIBUTED
+#include "tmb/message_bus.h"
+#include "tmb/tagged_message.h"
+#endif
+
using std::free;
using std::int32_t;
using std::memset;
@@ -81,6 +95,15 @@ using std::size_t;
using std::string;
using std::vector;
+#ifdef QUICKSTEP_DISTRIBUTED
+using std::malloc;
+using std::move;
+using std::unique_ptr;
+
+using tmb::MessageBus;
+using tmb::TaggedMessage;
+#endif
+
namespace quickstep {
static bool ValidateBlockDomain(const char *flagname,
@@ -157,14 +180,21 @@ DEFINE_bool(use_hdfs, false, "Use HDFS as the persistent storage, instead of the
#endif
StorageManager::StorageManager(
- const std::string &storage_path,
- const block_id_domain block_domain,
- const size_t max_memory_usage,
- EvictionPolicy *eviction_policy)
+ const std::string &storage_path,
+ const block_id_domain block_domain,
+ const size_t max_memory_usage,
+ EvictionPolicy *eviction_policy,
+ const tmb::client_id block_locator_client_id,
+ tmb::MessageBus *bus)
: storage_path_(storage_path),
total_memory_usage_(0),
max_memory_usage_(max_memory_usage),
- eviction_policy_(eviction_policy) {
+ eviction_policy_(eviction_policy),
+#ifdef QUICKSTEP_DISTRIBUTED
+ block_domain_(block_domain),
+#endif
+ block_locator_client_id_(block_locator_client_id),
+ bus_(bus) {
#ifdef QUICKSTEP_HAVE_FILE_MANAGER_HDFS
if (FLAGS_use_hdfs) {
file_manager_.reset(new FileManagerHdfs(storage_path));
@@ -175,10 +205,55 @@ StorageManager::StorageManager(
file_manager_.reset(new FileManagerLocal(storage_path));
#endif
+#ifdef QUICKSTEP_DISTRIBUTED
+ // NOTE(zuyu): The following if-condition is a workaround to bypass code for
+ // the distributed version in some unittests that does not use TMB. The
+ // end-to-end functional tests for the distributed version, however, would not
+ // be affected.
+ if (bus_) {
+ storage_manager_client_id_ = bus_->Connect();
+
+ bus_->RegisterClientAsSender(storage_manager_client_id_, kGetPeerDomainNetworkAddressesMessage);
+ bus_->RegisterClientAsReceiver(storage_manager_client_id_, kGetPeerDomainNetworkAddressesResponseMessage);
+
+ bus_->RegisterClientAsSender(storage_manager_client_id_, kAddBlockLocationMessage);
+ bus_->RegisterClientAsSender(storage_manager_client_id_, kDeleteBlockLocationMessage);
+ bus_->RegisterClientAsSender(storage_manager_client_id_, kBlockDomainUnregistrationMessage);
+
+ LOG(INFO) << "StorageManager (id '" << storage_manager_client_id_
+ << "') starts with Domain " << block_domain;
+ }
+#endif
+
block_index_ = BlockIdUtil::GetBlockId(block_domain, file_manager_->getMaxUsedBlockCounter(block_domain));
}
StorageManager::~StorageManager() {
+#ifdef QUICKSTEP_DISTRIBUTED
+ if (bus_) {
+ serialization::BlockDomainMessage proto;
+ proto.set_block_domain(block_domain_);
+
+ const int proto_length = proto.ByteSize();
+ char *proto_bytes = static_cast<char*>(malloc(proto_length));
+ CHECK(proto.SerializeToArray(proto_bytes, proto_length));
+
+ TaggedMessage message(static_cast<const void*>(proto_bytes),
+ proto_length,
+ kBlockDomainUnregistrationMessage);
+ free(proto_bytes);
+
+ LOG(INFO) << "StorageManager (id '" << storage_manager_client_id_
+ << "') sent BlockDomainUnregistrationMessage (typed '" << kBlockDomainUnregistrationMessage
+ << "') to BlockLocator";
+ CHECK(MessageBus::SendStatus::kOK ==
+ QueryExecutionUtil::SendTMBMessage(bus_,
+ storage_manager_client_id_,
+ block_locator_client_id_,
+ move(message)));
+ }
+#endif
+
for (std::unordered_map<block_id, BlockHandle>::iterator it = blocks_.begin();
it != blocks_.end();
++it) {
@@ -222,6 +297,12 @@ block_id StorageManager::createBlock(const CatalogRelationSchema &relation,
// Make '*eviction_policy_' aware of the new block's existence.
eviction_policy_->blockCreated(new_block_id);
+#ifdef QUICKSTEP_DISTRIBUTED
+ if (bus_) {
+ sendBlockLocationMessage(new_block_id, kAddBlockLocationMessage);
+ }
+#endif
+
return new_block_id;
}
@@ -249,6 +330,12 @@ block_id StorageManager::createBlob(const std::size_t num_slots,
// Make '*eviction_policy_' aware of the new blob's existence.
eviction_policy_->blockCreated(new_block_id);
+#ifdef QUICKSTEP_DISTRIBUTED
+ if (bus_) {
+ sendBlockLocationMessage(new_block_id, kAddBlockLocationMessage);
+ }
+#endif
+
return new_block_id;
}
@@ -315,6 +402,12 @@ bool StorageManager::saveBlockOrBlob(const block_id block, const bool force) {
}
void StorageManager::evictBlockOrBlob(const block_id block) {
+#ifdef QUICKSTEP_DISTRIBUTED
+ if (bus_) {
+ sendBlockLocationMessage(block, kDeleteBlockLocationMessage);
+ }
+#endif
+
BlockHandle handle;
{
SpinSharedMutexExclusiveLock<false> write_lock(blocks_shared_mutex_);
@@ -362,6 +455,87 @@ block_id StorageManager::allocateNewBlockOrBlob(const std::size_t num_slots,
return ++block_index_;
}
+#ifdef QUICKSTEP_DISTRIBUTED
+vector<string> StorageManager::getPeerDomainNetworkAddresses(const block_id block) {
+ serialization::BlockMessage proto;
+ proto.set_block_id(block);
+
+ const int proto_length = proto.ByteSize();
+ char *proto_bytes = static_cast<char*>(malloc(proto_length));
+ CHECK(proto.SerializeToArray(proto_bytes, proto_length));
+
+ TaggedMessage message(static_cast<const void*>(proto_bytes),
+ proto_length,
+ kGetPeerDomainNetworkAddressesMessage);
+ free(proto_bytes);
+
+ LOG(INFO) << "StorageManager (id '" << storage_manager_client_id_
+ << "') sent GetPeerDomainNetworkAddressesMessage (typed '" << kGetPeerDomainNetworkAddressesMessage
+ << "') to BlockLocator";
+
+ DCHECK_NE(block_locator_client_id_, tmb::kClientIdNone);
+ DCHECK(bus_ != nullptr);
+ CHECK(MessageBus::SendStatus::kOK ==
+ QueryExecutionUtil::SendTMBMessage(bus_,
+ storage_manager_client_id_,
+ block_locator_client_id_,
+ move(message)));
+
+ const tmb::AnnotatedMessage annotated_message(bus_->Receive(storage_manager_client_id_, 0, true));
+ const TaggedMessage &tagged_message = annotated_message.tagged_message;
+ CHECK_EQ(block_locator_client_id_, annotated_message.sender);
+ CHECK_EQ(kGetPeerDomainNetworkAddressesResponseMessage, tagged_message.message_type());
+ LOG(INFO) << "StorageManager (id '" << storage_manager_client_id_
+ << "') received GetPeerDomainNetworkAddressesResponseMessage from BlockLocator";
+
+ serialization::GetPeerDomainNetworkAddressesResponseMessage response_proto;
+ CHECK(response_proto.ParseFromArray(tagged_message.message(), tagged_message.message_bytes()));
+
+ vector<string> domain_network_addresses;
+ for (int i = 0; i < response_proto.domain_network_addresses_size(); ++i) {
+ domain_network_addresses.push_back(response_proto.domain_network_addresses(i));
+ }
+
+ return domain_network_addresses;
+}
+
+void StorageManager::sendBlockLocationMessage(const block_id block,
+ const tmb::message_type_id message_type) {
+ switch (message_type) {
+ case kAddBlockLocationMessage:
+ LOG(INFO) << "Loaded Block " << BlockIdUtil::ToString(block) << " in Domain " << block_domain_;
+ break;
+ case kDeleteBlockLocationMessage:
+ LOG(INFO) << "Evicted Block " << BlockIdUtil::ToString(block) << " in Domain " << block_domain_;
+ break;
+ default:
+ LOG(FATAL) << "Unknown message type " << message_type;
+ }
+
+ serialization::BlockLocationMessage proto;
+ proto.set_block_id(block);
+ proto.set_block_domain(block_domain_);
+
+ const int proto_length = proto.ByteSize();
+ char *proto_bytes = static_cast<char*>(malloc(proto_length));
+ CHECK(proto.SerializeToArray(proto_bytes, proto_length));
+
+ TaggedMessage message(static_cast<const void*>(proto_bytes),
+ proto_length,
+ message_type);
+ free(proto_bytes);
+
+ LOG(INFO) << "StorageManager (id '" << storage_manager_client_id_
+ << "') sent BlockLocationMessage (typed '" << message_type
+ << "') to BlockLocator";
+ CHECK(MessageBus::SendStatus::kOK ==
+ QueryExecutionUtil::SendTMBMessage(bus_,
+ storage_manager_client_id_,
+ block_locator_client_id_,
+ move(message)));
+}
+#endif
+
StorageManager::BlockHandle StorageManager::loadBlockOrBlob(
const block_id block, const int numa_node) {
// The caller of this function holds an exclusive lock on this block/blob's
@@ -378,6 +552,12 @@ StorageManager::BlockHandle StorageManager::loadBlockOrBlob(
loaded_handle.block_memory = block_buffer;
loaded_handle.block_memory_size = num_slots;
+#ifdef QUICKSTEP_DISTRIBUTED
+ if (bus_) {
+ sendBlockLocationMessage(block, kAddBlockLocationMessage);
+ }
+#endif
+
return loaded_handle;
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/2221b7e4/storage/StorageManager.hpp
----------------------------------------------------------------------
diff --git a/storage/StorageManager.hpp b/storage/StorageManager.hpp
index 52326c2..55a011e 100644
--- a/storage/StorageManager.hpp
+++ b/storage/StorageManager.hpp
@@ -20,11 +20,14 @@
#include <atomic>
#include <chrono>
+#include <cstddef>
#include <memory>
#include <string>
#include <unordered_map>
#include <vector>
+#include "query_optimizer/QueryOptimizerConfig.h" // For QUICKSTEP_DISTRIBUTED
+
#include "storage/CountedReference.hpp"
#include "storage/EvictionPolicy.hpp"
#include "storage/FileManager.hpp"
@@ -40,6 +43,10 @@
#include "gflags/gflags.h"
#include "gtest/gtest_prod.h"
+#include "tmb/id_typedefs.h"
+
+namespace tmb { class MessageBus; }
+
namespace quickstep {
DECLARE_int32(block_domain);
@@ -50,6 +57,7 @@ DECLARE_bool(use_hdfs);
#endif
class CatalogRelationSchema;
+
class StorageBlockLayout;
/** \addtogroup Storage
@@ -104,6 +112,33 @@ class StorageManager {
std::chrono::milliseconds(200))) {
}
+#ifdef QUICKSTEP_DISTRIBUTED
+ /**
+ * @brief Constructor.
+ * @param storage_path The filesystem directory where blocks have persistent
+ * storage.
+ * @param block_domain The unique block domain.
+ * @param block_locator_client_id The TMB client ID of the block locator.
+ * @param bus A pointer to the TMB.
+ *
+ * @exception CorruptPersistentStorage The storage directory layout is not
+ * in the expected format.
+ **/
+ StorageManager(const std::string &storage_path,
+ const block_id_domain block_domain,
+ const tmb::client_id block_locator_client_id,
+ tmb::MessageBus *bus)
+ : StorageManager(storage_path,
+ block_domain,
+ FLAGS_buffer_pool_slots,
+ LRUKEvictionPolicyFactory::ConstructLRUKEvictionPolicy(
+ 2,
+ std::chrono::milliseconds(200)),
+ block_locator_client_id,
+ bus) {
+ }
+#endif
+
/**
* @brief Constructor.
* @param storage_path The filesystem directory where blocks have persistent
@@ -121,13 +156,18 @@ class StorageManager {
* @param eviction_policy The eviction policy that the storage manager should
* use to manage the cache. The storage manager takes
* ownership of *eviction_policy.
+ * @param block_locator_client_id The TMB client ID of the block locator.
+ * @param bus A pointer to the TMB.
+ *
* @exception CorruptPersistentStorage The storage directory layout is not
* in the expected format.
**/
StorageManager(const std::string &storage_path,
const block_id_domain block_domain,
const size_t max_memory_usage,
- EvictionPolicy *eviction_policy);
+ EvictionPolicy *eviction_policy,
+ const tmb::client_id block_locator_client_id = tmb::kClientIdNone,
+ tmb::MessageBus *bus = nullptr);
/**
* @brief Destructor which also destroys all managed blocks.
@@ -332,6 +372,27 @@ class StorageManager {
StorageBlockBase *block;
};
+#ifdef QUICKSTEP_DISTRIBUTED
+ /**
+ * @brief Get the network info of all the remote StorageManagers which may
+ * load the given block in the buffer pool.
+ *
+ * @param block The block or blob to pull.
+ *
+ * @return The network info of all the possible peers to pull.
+ **/
+ std::vector<std::string> getPeerDomainNetworkAddresses(const block_id block);
+
+ /**
+ * @brief Update the block location info in BlockLocator.
+ *
+ * @param block The given block or blob.
+ * @param message_type Indicate whether to add or delete a block location.
+ **/
+ void sendBlockLocationMessage(const block_id block,
+ const tmb::message_type_id message_type);
+#endif
+
// Helper for createBlock() and createBlob(). Allocates a block ID and memory
// slots for a new StorageBlock or StorageBlob. Returns the allocated ID and
// writes the allocated slot range into 'handle->slot_index_low' and
@@ -459,6 +520,15 @@ class StorageManager {
std::unique_ptr<EvictionPolicy> eviction_policy_;
+#ifdef QUICKSTEP_DISTRIBUTED
+ const block_id_domain block_domain_;
+
+ tmb::client_id storage_manager_client_id_;
+#endif
+
+ const tmb::client_id block_locator_client_id_;
+ tmb::MessageBus *bus_;
+
std::unique_ptr<FileManager> file_manager_;
// Used to generate unique IDs in allocateNewBlockOrBlob().
@@ -486,6 +556,10 @@ class StorageManager {
static constexpr std::size_t kLockManagerNumShards = 0x2000-1;
ShardedLockManager<block_id, kLockManagerNumShards, SpinSharedMutex<false>> lock_manager_;
+ friend class BlockLocatorTest;
+ FRIEND_TEST(BlockLocatorTest, BlockTest);
+ FRIEND_TEST(BlockLocatorTest, BlobTest);
+
FRIEND_TEST(StorageManagerTest, DifferentNUMANodeBlobTestWithEviction);
FRIEND_TEST(StorageManagerTest, EvictFromSameShardTest);
[18/50] [abbrv] incubator-quickstep git commit: Cleanup in Preloader
class (#209)
Posted by zu...@apache.org.
Cleanup in Preloader class (#209)
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/32e7c1bd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/32e7c1bd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/32e7c1bd
Branch: refs/heads/work-order-serialization
Commit: 32e7c1bd8159b7c3883e0cb8213d330276b4cf41
Parents: 767b2ef
Author: Harshad Deshmukh <d....@gmail.com>
Authored: Fri May 6 22:29:41 2016 -0500
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Mon May 30 15:46:31 2016 -0700
----------------------------------------------------------------------
storage/PreloaderThread.cpp | 44 ++++++++++++++++++++++++----------------
storage/PreloaderThread.hpp | 7 +++++++
2 files changed, 34 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/32e7c1bd/storage/PreloaderThread.cpp
----------------------------------------------------------------------
diff --git a/storage/PreloaderThread.cpp b/storage/PreloaderThread.cpp
index 8f600b8..4bfa9c5 100644
--- a/storage/PreloaderThread.cpp
+++ b/storage/PreloaderThread.cpp
@@ -49,37 +49,46 @@ void PreloaderThread::run() {
std::size_t blocks_loaded = 0;
for (const CatalogRelation &relation : database_) {
- if (relation.hasPartitionScheme()) {
+ if (relation.hasPartitionScheme() && relation.hasNUMAPlacementScheme()) {
+#ifdef QUICKSTEP_HAVE_LIBNUMA
blocks_loaded += preloadNUMAAware(relation, blocks_loaded, num_slots);
+#endif
} else {
+ // NUMA agnostic preloading of relation.
std::vector<block_id> blocks = relation.getBlocksSnapshot();
for (block_id current_block_id : blocks) {
try {
- BlockReference current_block = storage_manager_->getBlock(current_block_id, relation);
+ BlockReference current_block =
+ storage_manager_->getBlock(current_block_id, relation);
} catch (...) {
- LOG(ERROR) << "Error after loading " << blocks_loaded << "blocks\n";
+ LOG(ERROR) << "Error after loading " << blocks_loaded << "blocks";
throw;
}
++blocks_loaded;
if (blocks_loaded == num_slots) {
- // The buffer pool has filled up. But, some database blocks are not loaded.
- printf(" The database is larger than the buffer pool. Only %lu blocks were loaded ",
- blocks_loaded);
+ // The buffer pool has filled up. But, some database blocks are not
+ // loaded.
+ printf(
+ " The database is larger than the buffer pool. Only %lu blocks "
+ "were loaded ", blocks_loaded);
return;
}
}
+ LOG(INFO) << "Relation " << relation.getName()
+ << " completely preloaded in buffer pool";
}
}
printf(" Loaded %lu blocks ", blocks_loaded);
}
+#ifdef QUICKSTEP_HAVE_LIBNUMA
std::size_t PreloaderThread::preloadNUMAAware(
const CatalogRelation &relation,
const std::size_t num_previously_loaded_blocks,
const std::size_t num_slots) {
-#ifdef QUICKSTEP_HAVE_LIBNUMA
std::size_t blocks_loaded = 0;
- const NUMAPlacementScheme *placement_scheme = relation.getNUMAPlacementSchemePtr();
+ const NUMAPlacementScheme *placement_scheme =
+ relation.getNUMAPlacementSchemePtr();
DCHECK(placement_scheme != nullptr);
DCHECK(relation.hasPartitionScheme());
const PartitionScheme &part_scheme = relation.getPartitionScheme();
@@ -96,15 +105,17 @@ std::size_t PreloaderThread::preloadNUMAAware(
BlockReference current_block = storage_manager_->getBlock(
curr_block_id, relation, partition_numa_node_id);
} catch (...) {
- LOG(ERROR) << "Error after loading "
+ LOG(ERROR) << "Error while preloading: After loading total "
<< blocks_loaded + num_previously_loaded_blocks
- << " blocks\n";
+ << " blocks and " << blocks_loaded
+ << " blocks of relation " << relation.getName();
throw;
}
++blocks_loaded;
num_blocks_loaded[partition_numa_node_id]++;
if ((blocks_loaded + num_previously_loaded_blocks) == num_slots) {
- // The buffer pool has filled up. But, some database blocks are not loaded.
+ // The buffer pool has filled up. But, some database blocks are not
+ // loaded.
printf(
" The database is larger than the buffer pool. Only %lu blocks "
"were loaded ",
@@ -116,14 +127,13 @@ std::size_t PreloaderThread::preloadNUMAAware(
LOG(INFO) << "For relation: " << relation.getName();
for (auto numa_block_loaded_info : num_blocks_loaded) {
LOG(INFO) << "NUMA node: " << numa_block_loaded_info.first
- << " Number of loaded blocks: " << numa_block_loaded_info.second;
+ << " Number of loaded blocks: "
+ << numa_block_loaded_info.second;
}
+ LOG(INFO) << "Relation " << relation.getName()
+ << " completely preloaded in buffer pool in a NUMA aware fashion";
return blocks_loaded;
-#else
- LOG(INFO) << "Relation: " << relation.getName()
- << " has partition scheme but the system doesn't support NUMA";
- return 0;
-#endif
}
+#endif
} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/32e7c1bd/storage/PreloaderThread.hpp
----------------------------------------------------------------------
diff --git a/storage/PreloaderThread.hpp b/storage/PreloaderThread.hpp
index f16fd50..16df34a 100644
--- a/storage/PreloaderThread.hpp
+++ b/storage/PreloaderThread.hpp
@@ -20,6 +20,7 @@
#include <cstddef>
+#include "storage/StorageConfig.h"
#include "threading/Thread.hpp"
#include "utility/Macros.hpp"
@@ -68,6 +69,7 @@ class PreloaderThread : public Thread {
void run() override;
private:
+#ifdef QUICKSTEP_HAVE_LIBNUMA
/**
* @brief Preload a relation which has a partition and a NUMA placement scheme.
*
@@ -82,11 +84,16 @@ class PreloaderThread : public Thread {
* allocated sufficient amount of memory so as not to exceed that
* socket's memory limit.
*
+ * TODO(harshad) - Allow multiple preloader threads, each pinned to a NUMA
+ * socket in the system. Each thread should preload only its share of
+ * storage blocks.
+ *
* @return The number of blocks loaded during this function call.
**/
std::size_t preloadNUMAAware(const CatalogRelation &relation,
const std::size_t num_previously_loaded_blocks,
const std::size_t num_slots);
+#endif
const CatalogDatabase &database_;
StorageManager *storage_manager_;
[24/50] [abbrv] incubator-quickstep git commit: Leave 10% more memory
for the OS. (#218)
Posted by zu...@apache.org.
Leave 10% more memory for the OS. (#218)
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/6f184957
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/6f184957
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/6f184957
Branch: refs/heads/work-order-serialization
Commit: 6f1849575e25a1ca4cc6d38898918cff7207ddf7
Parents: 4937e68
Author: Jignesh Patel <pa...@users.noreply.github.com>
Authored: Sun May 8 22:41:17 2016 -0500
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Mon May 30 15:46:32 2016 -0700
----------------------------------------------------------------------
storage/StorageConstants.hpp | 4 ++--
storage/StorageManager.cpp | 6 +++---
2 files changed, 5 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/6f184957/storage/StorageConstants.hpp
----------------------------------------------------------------------
diff --git a/storage/StorageConstants.hpp b/storage/StorageConstants.hpp
index 154d2f7..0609cba 100644
--- a/storage/StorageConstants.hpp
+++ b/storage/StorageConstants.hpp
@@ -58,8 +58,8 @@ const std::uint64_t kLargeMemorySystemThresholdInGB = 32;
// we grab a bigger fraction of the installed memory than for a "small" system.
// The two constants below define the percentages to grab in each case.
// TODO(jmp): May need to generalize this to more than two levels in the future.
-const std::uint64_t kPercentageToGrabForSmallSystems = 80;
-const std::uint64_t kPercentageToGrabForLargeSystems = 90;
+const std::uint64_t kPercentageToGrabForSmallSystems = 70;
+const std::uint64_t kPercentageToGrabForLargeSystems = 80;
// The default size of the buffer pool (in terms of the number of slots).
const std::uint64_t kDefaultBufferPoolSizeInSlots = 1024;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/6f184957/storage/StorageManager.cpp
----------------------------------------------------------------------
diff --git a/storage/StorageManager.cpp b/storage/StorageManager.cpp
index b990a22..dfc95b8 100644
--- a/storage/StorageManager.cpp
+++ b/storage/StorageManager.cpp
@@ -100,8 +100,8 @@ static const volatile bool block_domain_dummy
/**
* @brief Set or validate the buffer pool slots. When automatically picking a
* default value, check if the system is "small" or "large." Set the
- * buffer pool space to 80% of the installed main memory for small
- * and 90% otherwise.
+ * buffer pool space to 70% of the installed main memory for small
+ * and 80% otherwise.
* This method follows the signature that is set by the gflags module.
* @param flagname The name of the buffer pool flag.
* @param value The value of this flag from the command line, or default (0)
@@ -142,7 +142,7 @@ static bool SetOrValidateBufferPoolSlots(const char *flagname,
DEFINE_uint64(buffer_pool_slots, 0,
"By default the value is 0 and the system automatically sets the "
- "buffer pool size/slots at 80-90% of the total installed memory. "
+ "buffer pool size/slots at 70-80% of the total installed memory. "
"The user can also explicity define the number of slots. "
"The units for this variable is the number of 2-megabyte slots "
"that is allocated in the buffer pool. This is a \"soft\" limit: "
[20/50] [abbrv] incubator-quickstep git commit: Added support for the
substring function. (#211)
Posted by zu...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/767b2ef1/parser/preprocessed/SqlParser_gen.cpp
----------------------------------------------------------------------
diff --git a/parser/preprocessed/SqlParser_gen.cpp b/parser/preprocessed/SqlParser_gen.cpp
index f2cb8ca..038db14 100644
--- a/parser/preprocessed/SqlParser_gen.cpp
+++ b/parser/preprocessed/SqlParser_gen.cpp
@@ -244,67 +244,69 @@ extern int quickstep_yydebug;
TOKEN_FALSE = 314,
TOKEN_FIRST = 315,
TOKEN_FLOAT = 316,
- TOKEN_FOREIGN = 317,
- TOKEN_FROM = 318,
- TOKEN_FULL = 319,
- TOKEN_GROUP = 320,
- TOKEN_HASH = 321,
- TOKEN_HAVING = 322,
- TOKEN_HOUR = 323,
- TOKEN_IN = 324,
- TOKEN_INDEX = 325,
- TOKEN_INNER = 326,
- TOKEN_INSERT = 327,
- TOKEN_INTEGER = 328,
- TOKEN_INTERVAL = 329,
- TOKEN_INTO = 330,
- TOKEN_JOIN = 331,
- TOKEN_KEY = 332,
- TOKEN_LAST = 333,
- TOKEN_LEFT = 334,
- TOKEN_LIMIT = 335,
- TOKEN_LONG = 336,
- TOKEN_MINUTE = 337,
- TOKEN_MONTH = 338,
- TOKEN_NULL = 339,
- TOKEN_NULLS = 340,
- TOKEN_OFF = 341,
- TOKEN_ON = 342,
- TOKEN_ORDER = 343,
- TOKEN_OUTER = 344,
- TOKEN_PARTITION = 345,
- TOKEN_PARTITIONS = 346,
- TOKEN_PERCENT = 347,
- TOKEN_PRIMARY = 348,
- TOKEN_QUIT = 349,
- TOKEN_RANGE = 350,
- TOKEN_REAL = 351,
- TOKEN_REFERENCES = 352,
- TOKEN_RIGHT = 353,
- TOKEN_ROW_DELIMITER = 354,
- TOKEN_SECOND = 355,
- TOKEN_SELECT = 356,
- TOKEN_SET = 357,
- TOKEN_SMA = 358,
- TOKEN_SMALLINT = 359,
- TOKEN_TABLE = 360,
- TOKEN_THEN = 361,
- TOKEN_TIME = 362,
- TOKEN_TIMESTAMP = 363,
- TOKEN_TRUE = 364,
- TOKEN_TUPLESAMPLE = 365,
- TOKEN_UNIQUE = 366,
- TOKEN_UPDATE = 367,
- TOKEN_USING = 368,
- TOKEN_VALUES = 369,
- TOKEN_VARCHAR = 370,
- TOKEN_WHEN = 371,
- TOKEN_WHERE = 372,
- TOKEN_WITH = 373,
- TOKEN_YEAR = 374,
- TOKEN_YEARMONTH = 375,
- TOKEN_EOF = 376,
- TOKEN_LEX_ERROR = 377
+ TOKEN_FOR = 317,
+ TOKEN_FOREIGN = 318,
+ TOKEN_FROM = 319,
+ TOKEN_FULL = 320,
+ TOKEN_GROUP = 321,
+ TOKEN_HASH = 322,
+ TOKEN_HAVING = 323,
+ TOKEN_HOUR = 324,
+ TOKEN_IN = 325,
+ TOKEN_INDEX = 326,
+ TOKEN_INNER = 327,
+ TOKEN_INSERT = 328,
+ TOKEN_INTEGER = 329,
+ TOKEN_INTERVAL = 330,
+ TOKEN_INTO = 331,
+ TOKEN_JOIN = 332,
+ TOKEN_KEY = 333,
+ TOKEN_LAST = 334,
+ TOKEN_LEFT = 335,
+ TOKEN_LIMIT = 336,
+ TOKEN_LONG = 337,
+ TOKEN_MINUTE = 338,
+ TOKEN_MONTH = 339,
+ TOKEN_NULL = 340,
+ TOKEN_NULLS = 341,
+ TOKEN_OFF = 342,
+ TOKEN_ON = 343,
+ TOKEN_ORDER = 344,
+ TOKEN_OUTER = 345,
+ TOKEN_PARTITION = 346,
+ TOKEN_PARTITIONS = 347,
+ TOKEN_PERCENT = 348,
+ TOKEN_PRIMARY = 349,
+ TOKEN_QUIT = 350,
+ TOKEN_RANGE = 351,
+ TOKEN_REAL = 352,
+ TOKEN_REFERENCES = 353,
+ TOKEN_RIGHT = 354,
+ TOKEN_ROW_DELIMITER = 355,
+ TOKEN_SECOND = 356,
+ TOKEN_SELECT = 357,
+ TOKEN_SET = 358,
+ TOKEN_SMA = 359,
+ TOKEN_SMALLINT = 360,
+ TOKEN_SUBSTRING = 361,
+ TOKEN_TABLE = 362,
+ TOKEN_THEN = 363,
+ TOKEN_TIME = 364,
+ TOKEN_TIMESTAMP = 365,
+ TOKEN_TRUE = 366,
+ TOKEN_TUPLESAMPLE = 367,
+ TOKEN_UNIQUE = 368,
+ TOKEN_UPDATE = 369,
+ TOKEN_USING = 370,
+ TOKEN_VALUES = 371,
+ TOKEN_VARCHAR = 372,
+ TOKEN_WHEN = 373,
+ TOKEN_WHERE = 374,
+ TOKEN_WITH = 375,
+ TOKEN_YEAR = 376,
+ TOKEN_YEARMONTH = 377,
+ TOKEN_EOF = 378,
+ TOKEN_LEX_ERROR = 379
};
#endif
@@ -405,7 +407,7 @@ union YYSTYPE
quickstep::PtrVector<quickstep::ParseSubqueryTableReference> *with_list_;
quickstep::ParseSubqueryTableReference *with_list_element_;
-#line 409 "SqlParser_gen.cpp" /* yacc.c:355 */
+#line 411 "SqlParser_gen.cpp" /* yacc.c:355 */
};
typedef union YYSTYPE YYSTYPE;
@@ -440,7 +442,7 @@ int quickstep_yyparse (yyscan_t yyscanner, quickstep::ParseStatement **parsedSta
#include "SqlLexer_gen.hpp"
void NotSupported(const YYLTYPE *location, yyscan_t yyscanner, const std::string &feature);
-#line 444 "SqlParser_gen.cpp" /* yacc.c:358 */
+#line 446 "SqlParser_gen.cpp" /* yacc.c:358 */
#ifdef short
# undef short
@@ -684,21 +686,21 @@ union yyalloc
/* YYFINAL -- State number of the termination state. */
#define YYFINAL 47
/* YYLAST -- Last index in YYTABLE. */
-#define YYLAST 1191
+#define YYLAST 1215
/* YYNTOKENS -- Number of terminals. */
-#define YYNTOKENS 134
+#define YYNTOKENS 136
/* YYNNTS -- Number of nonterminals. */
-#define YYNNTS 95
+#define YYNNTS 96
/* YYNRULES -- Number of rules. */
-#define YYNRULES 262
+#define YYNRULES 265
/* YYNSTATES -- Number of states. */
-#define YYNSTATES 478
+#define YYNSTATES 488
/* YYTRANSLATE[YYX] -- Symbol number corresponding to YYX as returned
by yylex, with out-of-bounds checking. */
#define YYUNDEFTOK 2
-#define YYMAXUTOK 377
+#define YYMAXUTOK 379
#define YYTRANSLATE(YYX) \
((unsigned int) (YYX) <= YYMAXUTOK ? yytranslate[YYX] : YYUNDEFTOK)
@@ -708,11 +710,11 @@ union yyalloc
static const yytype_uint8 yytranslate[] =
{
0, 2, 2, 2, 2, 2, 2, 2, 2, 2,
- 129, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+ 131, 2, 2, 2, 2, 2, 2, 2, 2, 2,
2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
- 2, 2, 2, 2, 2, 2, 2, 133, 2, 2,
- 130, 131, 23, 21, 132, 22, 27, 24, 2, 2,
- 2, 2, 2, 2, 2, 2, 2, 2, 2, 128,
+ 2, 2, 2, 2, 2, 2, 2, 135, 2, 2,
+ 132, 133, 23, 21, 134, 22, 27, 24, 2, 2,
+ 2, 2, 2, 2, 2, 2, 2, 2, 2, 130,
2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
@@ -744,40 +746,40 @@ static const yytype_uint8 yytranslate[] =
90, 91, 92, 93, 94, 95, 96, 97, 98, 99,
100, 101, 102, 103, 104, 105, 106, 107, 108, 109,
110, 111, 112, 113, 114, 115, 116, 117, 118, 119,
- 120, 121, 122, 123, 124, 125, 126, 127
+ 120, 121, 122, 123, 124, 125, 126, 127, 128, 129
};
#if YYDEBUG
/* YYRLINE[YYN] -- Source line where rule number YYN was defined. */
static const yytype_uint16 yyrline[] =
{
- 0, 582, 582, 586, 590, 594, 598, 601, 608, 611,
- 614, 617, 620, 623, 626, 629, 632, 635, 641, 647,
- 654, 660, 667, 676, 681, 690, 695, 700, 704, 710,
- 715, 718, 721, 726, 729, 732, 735, 738, 741, 744,
- 747, 750, 753, 765, 768, 771, 789, 809, 812, 815,
- 820, 825, 831, 837, 846, 850, 856, 859, 864, 869,
- 874, 881, 888, 892, 898, 901, 906, 909, 914, 917,
- 922, 925, 944, 948, 954, 958, 964, 967, 970, 975,
- 978, 985, 990, 1001, 1006, 1010, 1014, 1020, 1023, 1029,
- 1037, 1040, 1043, 1049, 1054, 1057, 1062, 1066, 1070, 1074,
- 1080, 1085, 1090, 1094, 1100, 1106, 1109, 1114, 1119, 1123,
- 1129, 1135, 1141, 1144, 1148, 1154, 1157, 1162, 1166, 1172,
- 1175, 1178, 1183, 1188, 1193, 1196, 1199, 1204, 1207, 1210,
- 1213, 1216, 1219, 1222, 1225, 1230, 1233, 1238, 1242, 1246,
- 1249, 1253, 1256, 1261, 1264, 1269, 1272, 1277, 1281, 1287,
- 1290, 1295, 1298, 1303, 1306, 1311, 1314, 1333, 1337, 1343,
- 1350, 1353, 1356, 1361, 1364, 1367, 1373, 1376, 1381, 1386,
- 1395, 1400, 1409, 1414, 1417, 1422, 1425, 1430, 1436, 1442,
- 1445, 1448, 1451, 1454, 1457, 1463, 1472, 1475, 1480, 1483,
- 1488, 1491, 1496, 1499, 1502, 1505, 1508, 1511, 1514, 1519,
- 1523, 1527, 1530, 1535, 1540, 1543, 1548, 1552, 1558, 1563,
- 1567, 1573, 1578, 1581, 1586, 1590, 1596, 1599, 1602, 1605,
- 1617, 1621, 1640, 1653, 1668, 1671, 1674, 1677, 1680, 1683,
- 1688, 1692, 1698, 1701, 1706, 1710, 1717, 1720, 1723, 1726,
- 1729, 1732, 1735, 1738, 1741, 1744, 1749, 1760, 1763, 1768,
- 1771, 1774, 1780, 1784, 1790, 1793, 1801, 1804, 1807, 1810,
- 1816, 1821, 1826
+ 0, 585, 585, 589, 593, 597, 601, 604, 611, 614,
+ 617, 620, 623, 626, 629, 632, 635, 638, 644, 650,
+ 657, 663, 670, 679, 684, 693, 698, 703, 707, 713,
+ 718, 721, 724, 729, 732, 735, 738, 741, 744, 747,
+ 750, 753, 756, 768, 771, 774, 792, 812, 815, 818,
+ 823, 828, 834, 840, 849, 853, 859, 862, 867, 872,
+ 877, 884, 891, 895, 901, 904, 909, 912, 917, 920,
+ 925, 928, 947, 951, 957, 961, 967, 970, 973, 978,
+ 981, 988, 993, 1004, 1009, 1013, 1017, 1023, 1026, 1032,
+ 1040, 1043, 1046, 1052, 1057, 1060, 1065, 1069, 1073, 1077,
+ 1083, 1088, 1093, 1097, 1103, 1109, 1112, 1117, 1122, 1126,
+ 1132, 1138, 1144, 1147, 1151, 1157, 1160, 1165, 1169, 1175,
+ 1178, 1181, 1186, 1191, 1196, 1199, 1202, 1207, 1210, 1213,
+ 1216, 1219, 1222, 1225, 1228, 1233, 1236, 1241, 1245, 1249,
+ 1252, 1256, 1259, 1264, 1267, 1272, 1275, 1280, 1284, 1290,
+ 1293, 1298, 1301, 1306, 1309, 1314, 1317, 1336, 1340, 1346,
+ 1353, 1356, 1359, 1364, 1367, 1370, 1376, 1379, 1384, 1389,
+ 1398, 1403, 1412, 1417, 1420, 1425, 1428, 1433, 1439, 1445,
+ 1448, 1451, 1454, 1457, 1460, 1466, 1475, 1478, 1483, 1486,
+ 1491, 1494, 1499, 1502, 1505, 1508, 1511, 1514, 1517, 1520,
+ 1525, 1529, 1533, 1536, 1541, 1546, 1550, 1556, 1559, 1564,
+ 1568, 1574, 1579, 1583, 1589, 1594, 1597, 1602, 1606, 1612,
+ 1615, 1618, 1621, 1633, 1637, 1656, 1669, 1684, 1687, 1690,
+ 1693, 1696, 1699, 1704, 1708, 1714, 1717, 1722, 1726, 1733,
+ 1736, 1739, 1742, 1745, 1748, 1751, 1754, 1757, 1760, 1765,
+ 1776, 1779, 1784, 1787, 1790, 1796, 1800, 1806, 1809, 1817,
+ 1820, 1823, 1826, 1832, 1837, 1842
};
#endif
@@ -801,7 +803,7 @@ static const char *const yytname[] =
"TOKEN_DELETE", "TOKEN_DELIMITER", "TOKEN_DESC", "TOKEN_DISTINCT",
"TOKEN_DOUBLE", "TOKEN_DROP", "TOKEN_ELSE", "TOKEN_END",
"TOKEN_ESCAPE_STRINGS", "TOKEN_EXISTS", "TOKEN_EXTRACT", "TOKEN_FALSE",
- "TOKEN_FIRST", "TOKEN_FLOAT", "TOKEN_FOREIGN", "TOKEN_FROM",
+ "TOKEN_FIRST", "TOKEN_FLOAT", "TOKEN_FOR", "TOKEN_FOREIGN", "TOKEN_FROM",
"TOKEN_FULL", "TOKEN_GROUP", "TOKEN_HASH", "TOKEN_HAVING", "TOKEN_HOUR",
"TOKEN_IN", "TOKEN_INDEX", "TOKEN_INNER", "TOKEN_INSERT",
"TOKEN_INTEGER", "TOKEN_INTERVAL", "TOKEN_INTO", "TOKEN_JOIN",
@@ -811,26 +813,27 @@ static const char *const yytname[] =
"TOKEN_PARTITIONS", "TOKEN_PERCENT", "TOKEN_PRIMARY", "TOKEN_QUIT",
"TOKEN_RANGE", "TOKEN_REAL", "TOKEN_REFERENCES", "TOKEN_RIGHT",
"TOKEN_ROW_DELIMITER", "TOKEN_SECOND", "TOKEN_SELECT", "TOKEN_SET",
- "TOKEN_SMA", "TOKEN_SMALLINT", "TOKEN_TABLE", "TOKEN_THEN", "TOKEN_TIME",
- "TOKEN_TIMESTAMP", "TOKEN_TRUE", "TOKEN_TUPLESAMPLE", "TOKEN_UNIQUE",
- "TOKEN_UPDATE", "TOKEN_USING", "TOKEN_VALUES", "TOKEN_VARCHAR",
- "TOKEN_WHEN", "TOKEN_WHERE", "TOKEN_WITH", "TOKEN_YEAR",
- "TOKEN_YEARMONTH", "TOKEN_EOF", "TOKEN_LEX_ERROR", "';'", "'\\n'", "'('",
- "')'", "','", "'%'", "$accept", "start", "sql_statement",
- "quit_statement", "alter_table_statement", "create_table_statement",
- "create_index_statement", "drop_table_statement", "column_def",
- "column_def_commalist", "data_type", "column_constraint_def",
- "column_constraint_def_list", "opt_column_constraint_def_list",
- "table_constraint_def", "table_constraint_def_commalist",
- "opt_table_constraint_def_commalist", "opt_column_list",
- "opt_block_properties", "opt_partition_clause", "partition_type",
- "key_value_list", "key_value", "key_string_value", "key_string_list",
- "key_integer_value", "index_type", "opt_index_properties",
- "insert_statement", "copy_from_statement", "opt_copy_from_params",
- "copy_from_params", "update_statement", "delete_statement",
- "assignment_list", "assignment_item", "select_statement", "with_clause",
- "with_list", "with_list_element", "select_query", "opt_all_distinct",
- "selection", "selection_item_commalist", "selection_item", "from_clause",
+ "TOKEN_SMA", "TOKEN_SMALLINT", "TOKEN_SUBSTRING", "TOKEN_TABLE",
+ "TOKEN_THEN", "TOKEN_TIME", "TOKEN_TIMESTAMP", "TOKEN_TRUE",
+ "TOKEN_TUPLESAMPLE", "TOKEN_UNIQUE", "TOKEN_UPDATE", "TOKEN_USING",
+ "TOKEN_VALUES", "TOKEN_VARCHAR", "TOKEN_WHEN", "TOKEN_WHERE",
+ "TOKEN_WITH", "TOKEN_YEAR", "TOKEN_YEARMONTH", "TOKEN_EOF",
+ "TOKEN_LEX_ERROR", "';'", "'\\n'", "'('", "')'", "','", "'%'", "$accept",
+ "start", "sql_statement", "quit_statement", "alter_table_statement",
+ "create_table_statement", "create_index_statement",
+ "drop_table_statement", "column_def", "column_def_commalist",
+ "data_type", "column_constraint_def", "column_constraint_def_list",
+ "opt_column_constraint_def_list", "table_constraint_def",
+ "table_constraint_def_commalist", "opt_table_constraint_def_commalist",
+ "opt_column_list", "opt_block_properties", "opt_partition_clause",
+ "partition_type", "key_value_list", "key_value", "key_string_value",
+ "key_string_list", "key_integer_value", "index_type",
+ "opt_index_properties", "insert_statement", "copy_from_statement",
+ "opt_copy_from_params", "copy_from_params", "update_statement",
+ "delete_statement", "assignment_list", "assignment_item",
+ "select_statement", "with_clause", "with_list", "with_list_element",
+ "select_query", "opt_all_distinct", "selection",
+ "selection_item_commalist", "selection_item", "from_clause",
"subquery_expression", "opt_sample_clause", "join_type",
"joined_table_reference", "table_reference", "table_reference_signature",
"table_reference_signature_primary", "joined_table_reference_commalist",
@@ -840,14 +843,14 @@ static const char *const yytname[] =
"where_clause", "or_expression", "and_expression", "not_expression",
"predicate_expression_base", "add_expression", "multiply_expression",
"unary_expression", "expression_base", "function_call",
- "extract_function", "case_expression", "simple_when_clause_list",
- "simple_when_clause", "searched_when_clause_list",
- "searched_when_clause", "opt_else_clause", "expression_list",
- "literal_value", "datetime_unit", "literal_value_commalist",
- "attribute_ref", "attribute_ref_list", "comparison_operation",
- "unary_operation", "add_operation", "multiply_operation",
- "name_commalist", "any_name", "boolean_value", "command",
- "command_argument_list", YY_NULLPTR
+ "extract_function", "substr_function", "case_expression",
+ "simple_when_clause_list", "simple_when_clause",
+ "searched_when_clause_list", "searched_when_clause", "opt_else_clause",
+ "expression_list", "literal_value", "datetime_unit",
+ "literal_value_commalist", "attribute_ref", "attribute_ref_list",
+ "comparison_operation", "unary_operation", "add_operation",
+ "multiply_operation", "name_commalist", "any_name", "boolean_value",
+ "command", "command_argument_list", YY_NULLPTR
};
#endif
@@ -868,15 +871,15 @@ static const yytype_uint16 yytoknum[] =
340, 341, 342, 343, 344, 345, 346, 347, 348, 349,
350, 351, 352, 353, 354, 355, 356, 357, 358, 359,
360, 361, 362, 363, 364, 365, 366, 367, 368, 369,
- 370, 371, 372, 373, 374, 375, 376, 377, 59, 10,
- 40, 41, 44, 37
+ 370, 371, 372, 373, 374, 375, 376, 377, 378, 379,
+ 59, 10, 40, 41, 44, 37
};
# endif
-#define YYPACT_NINF -223
+#define YYPACT_NINF -230
#define yypact_value_is_default(Yystate) \
- (!!((Yystate) == (-223)))
+ (!!((Yystate) == (-230)))
#define YYTABLE_NINF -128
@@ -887,54 +890,55 @@ static const yytype_uint16 yytoknum[] =
STATE-NUM. */
static const yytype_int16 yypact[] =
{
- 921, -223, -223, -82, 231, -14, 56, 22, 74, -223,
- 70, 231, 231, -223, 135, 120, -223, -223, -223, -223,
- -223, -223, -223, -223, -223, -223, -35, -223, -73, 177,
- 231, -223, -223, 121, 231, 231, 231, 231, 231, -223,
- -223, 576, 85, 63, -223, 173, 77, -223, -223, -223,
- 140, -223, -223, -223, -223, 18, 218, 144, 97, 119,
- -223, 4, -223, -223, 240, 245, -223, -223, -223, 642,
- 134, -223, 187, -223, -223, 146, -223, -223, 265, -223,
- -223, -223, -223, -223, -223, 164, 203, 708, 290, 230,
- 176, -223, -223, 256, 20, -223, -223, -223, -223, -223,
- -223, -223, 840, -11, 231, 231, 182, 231, 231, 167,
- 206, 204, 231, 231, 483, -223, -223, 205, 231, -223,
- -223, -223, 483, 47, -10, -223, 329, -223, 129, 129,
- 330, -223, 207, 26, -223, 32, 119, 906, -223, -223,
- 231, 906, -223, -223, -223, -223, 906, 245, -223, 231,
- 324, -70, -223, 331, -223, 234, -223, 125, -223, 234,
- 231, 49, 231, 231, 211, -223, 213, -223, 137, 991,
- 774, 182, 390, 336, 339, -223, -223, 1117, 335, 1001,
- 142, 10, 906, -9, -223, 906, -223, 296, 228, -223,
- -223, -223, -223, -223, -223, 292, -223, 232, -223, -223,
- 21, 163, 122, -223, 229, 163, -13, 294, -223, -223,
- 20, -223, -223, 236, 906, -223, 259, 152, 231, -223,
- 906, -223, 231, -223, -223, 238, 287, 288, 241, -223,
- -223, -223, 61, 231, 258, 49, 231, -223, 156, -223,
- -223, -6, 69, 483, 483, 55, -223, -223, -223, -223,
- -223, -223, -223, -223, 906, 244, 906, 5, -223, 154,
- 260, 906, 37, -223, 317, 259, -223, -223, 906, -223,
- 102, 231, -223, -223, 284, -223, 291, 297, 301, 32,
- -223, 377, 379, 163, 348, 320, -223, 157, -223, 906,
- -223, 259, -223, 483, 263, 268, 231, 394, 151, 159,
- -223, 168, 378, 25, -223, 272, 283, -223, 318, 279,
- 1001, -223, 332, 231, -223, -223, 156, -223, -223, 339,
- -223, -223, -223, 906, 285, 161, 708, -223, 259, 327,
- -223, -223, 1001, 289, 259, 906, -223, 33, -223, -223,
- -223, -223, -223, 32, 122, 321, 323, -223, 906, 483,
- 328, -223, 259, 13, 231, 231, 170, -223, -223, -223,
- -223, -223, -223, -223, 76, -223, 231, -223, -223, -223,
- -223, 299, 49, 381, 340, -223, 483, -223, -223, 300,
- -223, 200, 708, -223, 906, 172, -223, -223, 1001, 259,
- -223, 342, -223, -223, 311, 336, 382, 361, -223, 175,
- 179, -223, 443, 151, -223, 231, -223, -223, 333, 410,
- -223, 29, 231, 906, 183, 259, -223, 185, 483, 906,
- 444, -223, 355, -223, -223, -223, 188, -223, -223, -223,
- -223, 16, 231, -5, -223, 334, 259, -223, -223, 336,
- 326, -223, 155, -223, 231, -223, 231, -223, -223, 231,
- -223, 190, -223, -223, 337, -223, 906, -223, -223, 369,
- 341, -223, 192, -223, 231, -223, 110, -223, 231, -223,
- 194, -223, -223, 201, 365, -223, 455, -223
+ 162, -230, -230, -75, 181, 7, 17, 23, 19, -230,
+ 41, 181, 181, -230, 151, 129, -230, -230, -230, -230,
+ -230, -230, -230, -230, -230, -230, -38, -230, -36, 166,
+ 181, -230, -230, 138, 181, 181, 181, 181, 181, -230,
+ -230, 644, 111, 93, -230, 220, 128, -230, -230, -230,
+ 202, -230, -230, -230, -230, 65, 273, 192, 163, 172,
+ -230, 11, -230, -230, 295, 300, -230, -230, -230, 699,
+ 207, -230, 233, -230, -230, 218, -230, -230, 310, -230,
+ -230, -230, -230, 219, -230, -230, 229, 245, 786, 322,
+ 261, 224, -230, -230, 182, 35, -230, -230, -230, -230,
+ -230, -230, -230, -230, 928, -12, 181, 181, 235, 181,
+ 181, 194, 226, 238, 181, 181, 542, -230, -230, 237,
+ 181, -230, -230, -230, 542, 53, -34, -230, 359, -230,
+ 105, 105, 983, 363, -230, 241, 40, -230, 18, 172,
+ 983, -230, -230, 181, 983, -230, -230, -230, -230, 983,
+ 300, -230, 181, 331, 82, -230, 360, -230, 268, -230,
+ -7, -230, 268, 181, 146, 181, 181, 244, -230, 246,
+ -230, 143, 1088, 841, 235, 455, 369, 372, -230, -230,
+ 624, 364, 1069, 155, 15, 983, 8, -230, 983, -230,
+ 323, 252, -230, -230, -230, -230, -230, -230, 317, -230,
+ 60, 256, -230, -230, 10, 90, 197, -230, 257, 90,
+ -10, 319, -230, -230, 35, -230, -230, 259, 983, -230,
+ 270, 160, 181, -230, 983, -230, 181, -230, -230, 263,
+ 313, 315, 267, -230, -230, -230, 174, 181, 281, 146,
+ 181, -230, 81, -230, -230, 20, 34, 542, 542, 61,
+ -230, -230, -230, -230, -230, -230, -230, -230, 983, 271,
+ 983, 47, -230, 165, 283, 983, 55, -230, 342, 270,
+ -230, -230, 983, 398, -230, 125, 181, -230, -230, 311,
+ -230, 312, 318, 326, 18, -230, 405, 407, 90, 375,
+ 344, -230, 171, -230, 983, -230, 270, -230, 542, 287,
+ 288, 181, 417, 157, 175, -230, 177, 396, 49, -230,
+ 290, 301, -230, 335, 296, 1069, -230, 346, 181, -230,
+ -230, 81, -230, -230, 372, -230, -230, -230, 983, 298,
+ 203, 786, -230, 270, 337, -230, -230, 1069, 299, 270,
+ 983, -230, 42, -25, -230, -230, -230, -230, -230, 18,
+ 197, 334, 336, -230, 983, 542, 341, -230, 270, 21,
+ 181, 181, 183, -230, -230, -230, -230, -230, -230, -230,
+ 193, -230, 181, -230, -230, -230, -230, 307, 146, 404,
+ 347, -230, 542, -230, -230, 316, -230, 234, 786, -230,
+ 983, 185, -230, -230, 1069, 270, -230, 437, -230, 354,
+ -230, -230, 320, 369, 409, 365, -230, 188, 190, -230,
+ 451, 157, -230, 181, -230, -230, 325, 426, -230, 31,
+ 181, 983, 195, 270, -230, 198, 338, 542, 983, 460,
+ -230, 366, -230, -230, -230, 200, -230, -230, -230, -230,
+ 14, 181, 13, -230, 339, 270, -230, -230, -230, 369,
+ 345, -230, 231, -230, 181, -230, 181, -230, -230, 181,
+ -230, 208, -230, -230, 343, -230, 983, -230, -230, 377,
+ 348, -230, 210, -230, 181, -230, 113, -230, 181, -230,
+ 213, -230, -230, 215, 376, -230, 471, -230
};
/* YYDEFACT[STATE-NUM] -- Default reduction number in state STATE-NUM.
@@ -942,50 +946,51 @@ static const yytype_int16 yypact[] =
means the default is an error. */
static const yytype_uint16 yydefact[] =
{
- 0, 6, 262, 0, 0, 0, 0, 0, 0, 18,
+ 0, 6, 265, 0, 0, 0, 0, 0, 0, 18,
112, 0, 0, 7, 0, 0, 15, 8, 10, 11,
- 13, 14, 9, 17, 12, 16, 0, 105, 0, 260,
- 0, 254, 255, 0, 0, 0, 0, 0, 0, 113,
+ 13, 14, 9, 17, 12, 16, 0, 105, 0, 263,
+ 0, 257, 258, 0, 0, 0, 0, 0, 0, 113,
114, 0, 0, 107, 108, 0, 145, 1, 3, 2,
- 0, 106, 5, 4, 261, 0, 0, 0, 0, 166,
- 25, 0, 220, 217, 0, 246, 115, 40, 29, 0,
+ 0, 106, 5, 4, 264, 0, 0, 0, 0, 166,
+ 25, 0, 223, 220, 0, 249, 115, 40, 29, 0,
0, 30, 31, 34, 36, 0, 37, 39, 0, 41,
- 216, 35, 38, 32, 33, 0, 0, 0, 0, 0,
- 116, 117, 198, 121, 187, 189, 191, 194, 195, 196,
- 193, 192, 0, 232, 0, 0, 0, 0, 0, 0,
- 0, 94, 0, 0, 0, 101, 167, 0, 0, 91,
- 218, 219, 0, 0, 212, 209, 0, 43, 0, 221,
- 0, 44, 0, 0, 223, 0, 166, 0, 247, 248,
- 0, 0, 120, 250, 251, 249, 0, 0, 190, 0,
- 0, 166, 103, 0, 109, 0, 110, 0, 252, 0,
- 0, 0, 0, 0, 0, 93, 66, 27, 0, 0,
- 0, 0, 0, 168, 170, 172, 174, 0, 192, 0,
- 0, 0, 0, 212, 206, 0, 210, 0, 0, 226,
- 227, 228, 225, 229, 224, 0, 222, 0, 123, 197,
- 0, 0, 147, 136, 122, 141, 124, 149, 118, 119,
- 186, 188, 233, 0, 0, 199, 214, 0, 0, 100,
- 0, 146, 0, 92, 19, 0, 0, 0, 0, 20,
- 21, 22, 0, 0, 0, 64, 0, 42, 56, 173,
- 181, 0, 0, 0, 0, 0, 236, 238, 239, 240,
- 241, 237, 242, 244, 0, 0, 0, 0, 230, 0,
- 0, 0, 0, 207, 0, 213, 205, 45, 0, 46,
- 127, 0, 137, 143, 133, 128, 129, 131, 0, 0,
- 140, 0, 0, 139, 0, 151, 200, 0, 201, 0,
- 102, 104, 253, 0, 0, 0, 0, 0, 0, 0,
- 234, 0, 232, 0, 63, 65, 68, 28, 0, 0,
- 0, 47, 0, 0, 49, 55, 57, 26, 180, 169,
- 171, 243, 245, 0, 0, 0, 0, 182, 179, 0,
- 178, 90, 0, 0, 211, 0, 204, 0, 142, 144,
- 134, 130, 132, 0, 148, 0, 0, 138, 0, 0,
- 153, 202, 215, 0, 0, 0, 0, 96, 258, 259,
- 257, 256, 97, 95, 0, 67, 0, 83, 84, 85,
- 86, 87, 0, 0, 70, 48, 0, 51, 50, 0,
- 54, 0, 0, 184, 0, 0, 177, 231, 0, 208,
- 203, 0, 125, 126, 150, 152, 0, 155, 61, 0,
- 0, 58, 0, 0, 235, 0, 24, 62, 0, 0,
- 23, 0, 0, 0, 0, 175, 183, 0, 0, 0,
- 0, 111, 0, 59, 98, 99, 0, 74, 76, 77,
- 78, 0, 0, 0, 52, 0, 176, 185, 89, 135,
+ 219, 35, 38, 0, 32, 33, 0, 0, 0, 0,
+ 0, 116, 117, 199, 121, 187, 189, 191, 194, 195,
+ 196, 197, 193, 192, 0, 235, 0, 0, 0, 0,
+ 0, 0, 0, 94, 0, 0, 0, 101, 167, 0,
+ 0, 91, 221, 222, 0, 0, 215, 212, 0, 43,
+ 0, 224, 0, 0, 44, 0, 0, 226, 0, 166,
+ 0, 250, 251, 0, 0, 120, 253, 254, 252, 0,
+ 0, 190, 0, 0, 166, 103, 0, 109, 0, 110,
+ 0, 255, 0, 0, 0, 0, 0, 0, 93, 66,
+ 27, 0, 0, 0, 0, 0, 168, 170, 172, 174,
+ 0, 192, 0, 0, 0, 0, 215, 209, 0, 213,
+ 0, 0, 229, 230, 231, 228, 232, 227, 0, 225,
+ 0, 0, 123, 198, 0, 0, 147, 136, 122, 141,
+ 124, 149, 118, 119, 186, 188, 236, 0, 0, 200,
+ 217, 0, 0, 100, 0, 146, 0, 92, 19, 0,
+ 0, 0, 0, 20, 21, 22, 0, 0, 0, 64,
+ 0, 42, 56, 173, 181, 0, 0, 0, 0, 0,
+ 239, 241, 242, 243, 244, 240, 245, 247, 0, 0,
+ 0, 0, 233, 0, 0, 0, 0, 210, 0, 216,
+ 208, 45, 0, 0, 46, 127, 0, 137, 143, 133,
+ 128, 129, 131, 0, 0, 140, 0, 0, 139, 0,
+ 151, 201, 0, 202, 0, 102, 104, 256, 0, 0,
+ 0, 0, 0, 0, 0, 237, 0, 235, 0, 63,
+ 65, 68, 28, 0, 0, 0, 47, 0, 0, 49,
+ 55, 57, 26, 180, 169, 171, 246, 248, 0, 0,
+ 0, 0, 182, 179, 0, 178, 90, 0, 0, 214,
+ 0, 207, 0, 0, 142, 144, 134, 130, 132, 0,
+ 148, 0, 0, 138, 0, 0, 153, 203, 218, 0,
+ 0, 0, 0, 96, 261, 262, 260, 259, 97, 95,
+ 0, 67, 0, 83, 84, 85, 86, 87, 0, 0,
+ 70, 48, 0, 51, 50, 0, 54, 0, 0, 184,
+ 0, 0, 177, 234, 0, 211, 204, 0, 205, 0,
+ 125, 126, 150, 152, 0, 155, 61, 0, 0, 58,
+ 0, 0, 238, 0, 24, 62, 0, 0, 23, 0,
+ 0, 0, 0, 175, 183, 0, 0, 0, 0, 0,
+ 111, 0, 59, 98, 99, 0, 74, 76, 77, 78,
+ 0, 0, 0, 52, 0, 176, 185, 89, 206, 135,
154, 157, 160, 156, 0, 88, 0, 82, 80, 0,
79, 0, 72, 73, 0, 53, 0, 161, 162, 163,
0, 75, 0, 69, 0, 158, 0, 159, 0, 81,
@@ -995,31 +1000,31 @@ static const yytype_uint16 yydefact[] =
/* YYPGOTO[NTERM-NUM]. */
static const yytype_int16 yypgoto[] =
{
- -223, -223, -223, -223, -223, -223, -223, -223, -131, -223,
- 303, 150, -223, -223, -222, -223, -223, -223, -223, -223,
- -223, 38, 27, -223, -223, -223, -223, -223, -223, -223,
- -223, -223, -223, -223, -223, 257, -223, -223, -223, 372,
- 14, -223, -223, -223, 343, -223, -94, -223, -223, -181,
- 131, -170, -8, -223, -223, -223, -223, -223, -223, 28,
- -223, -223, -58, -223, -121, 235, 237, 312, -30, 344,
- 346, 384, -130, -223, -223, -223, 314, -223, 359, 315,
- -208, -169, 366, 106, -105, -223, -223, -223, -223, -223,
- -115, -4, 98, -223, -223
+ -230, -230, -230, -230, -230, -230, -230, -230, -127, -230,
+ 309, 161, -230, -230, -229, -230, -230, -230, -230, -230,
+ -230, 43, 27, -230, -230, -230, -230, -230, -230, -230,
+ -230, -230, -230, -230, -230, 264, -230, -230, -230, 378,
+ 12, -230, -230, -230, 350, -230, -103, -230, -230, -139,
+ 142, -190, -8, -230, -230, -230, -230, -230, -230, 26,
+ -230, -230, 44, -230, -121, 240, 247, 321, -28, 349,
+ 351, 394, -132, -230, -230, -230, -230, 324, -230, 373,
+ 327, -216, -171, 370, 108, -107, -230, -230, -230, -230,
+ -230, -119, -4, 94, -230, -230
};
/* YYDEFGOTO[NTERM-NUM]. */
static const yytype_int16 yydefgoto[] =
{
- -1, 14, 15, 16, 17, 18, 19, 20, 167, 168,
- 88, 315, 316, 317, 229, 305, 306, 234, 374, 410,
- 454, 426, 427, 428, 429, 430, 371, 406, 21, 22,
- 165, 299, 23, 24, 151, 152, 25, 26, 43, 44,
- 132, 41, 89, 90, 91, 136, 92, 283, 278, 202,
- 203, 272, 273, 204, 285, 350, 397, 421, 440, 441,
- 459, 467, 115, 116, 173, 174, 175, 176, 177, 94,
- 95, 96, 97, 98, 99, 183, 184, 124, 125, 187,
- 217, 100, 195, 259, 101, 301, 256, 102, 141, 146,
- 157, 103, 362, 28, 29
+ -1, 14, 15, 16, 17, 18, 19, 20, 170, 171,
+ 89, 320, 321, 322, 233, 310, 311, 238, 380, 418,
+ 464, 435, 436, 437, 438, 439, 377, 414, 21, 22,
+ 168, 304, 23, 24, 154, 155, 25, 26, 43, 44,
+ 135, 41, 90, 91, 92, 139, 93, 288, 283, 206,
+ 207, 277, 278, 208, 290, 356, 405, 430, 450, 451,
+ 469, 477, 117, 118, 176, 177, 178, 179, 180, 95,
+ 96, 97, 98, 99, 100, 101, 186, 187, 126, 127,
+ 190, 221, 102, 198, 263, 103, 306, 260, 104, 144,
+ 149, 160, 105, 368, 28, 29
};
/* YYTABLE[YYPACT[STATE-NUM]] -- What to do in state STATE-NUM. If
@@ -1027,336 +1032,341 @@ static const yytype_int16 yydefgoto[] =
number is the opposite. If YYTABLE_NINF, syntax error. */
static const yytype_int16 yytable[] =
{
- 33, 181, 243, 180, 45, 205, 287, 42, 46, 178,
- 258, 93, 156, 304, 27, 329, 149, 178, 243, 270,
- 31, 243, 32, 447, 281, 31, 55, 32, 30, 224,
- 57, 58, 59, 60, 61, 280, 31, 243, 32, 123,
- 51, 201, 50, 143, 144, 448, 109, 138, 139, 185,
- 185, 241, 114, 52, 138, 139, 53, 133, 138, 139,
- 367, 34, 218, 368, 369, 178, 452, 178, 138, 139,
- 205, 10, 321, 322, 323, 119, 110, 240, 207, 245,
- 246, 247, 248, 249, 250, 251, 252, 253, 254, 142,
- 138, 139, 225, 219, 330, 453, 35, 45, 344, 39,
- 153, 46, 282, 158, 159, 307, 201, 93, 166, 169,
- 10, 122, 182, 347, 158, 297, 226, 150, 385, 150,
- 216, 261, 298, 117, 36, 318, 40, 10, 300, 324,
- 402, 206, 37, 370, 118, 47, 209, 403, 178, 178,
- 394, 377, 242, 255, 398, 212, 449, 227, 335, 205,
- 407, 200, 262, 145, 38, 265, 169, 199, 230, 231,
- 434, 327, 200, 387, 390, 228, 308, 31, 182, 32,
- 384, 274, 353, 223, 414, 471, 138, 139, 275, 189,
- 54, 356, 138, 139, 216, 201, 276, 457, 178, 56,
- 291, 274, 104, 472, 271, 105, 206, 46, 275, 309,
- 199, 46, 190, -127, 106, 277, 276, 107, 310, 413,
- 458, 160, 161, 205, 153, 358, 191, 192, 292, 258,
- 108, 138, 139, 111, 325, 277, 328, 113, 395, 302,
- 383, 334, 169, 338, 193, 31, 112, 32, 337, 399,
- 400, 114, 359, 360, 178, 311, 48, 120, 49, 201,
- 162, 163, 121, 194, 312, 411, 221, 222, 313, 352,
- 31, 404, 32, 339, 126, 361, 127, 46, 235, 236,
- 129, 178, 314, 260, 222, 206, 128, 138, 139, 46,
- 138, 139, 131, 288, 289, 331, 332, 140, 351, 289,
- 363, 364, 158, 381, 130, 134, 216, 439, 135, 365,
- 366, 401, 222, 416, 289, 389, 422, 222, 137, 379,
- 423, 222, 155, 178, 437, 289, 438, 332, 216, 445,
- 446, 463, 446, 469, 222, 474, 222, 164, 31, 62,
- 32, 63, 475, 222, 462, 179, 188, 197, 198, 206,
- 10, 232, 220, 233, 243, 64, 65, 213, 244, 470,
- 158, 158, 216, 473, 415, 257, 266, 67, 68, 267,
- 268, 279, 302, 269, 284, 69, 70, 286, 293, 294,
- 295, 296, 71, 72, 326, 73, 303, 336, 340, 333,
- 214, 74, 343, 436, 345, 341, 346, 75, 348, 442,
- 76, 342, 349, 354, 31, 62, 32, 63, 355, 357,
- 170, 431, 77, 78, 372, 149, 373, 375, 435, 376,
- 79, 64, 65, 80, 378, 382, 386, 408, 392, 388,
- 393, 396, 419, 67, 68, 81, 442, 450, 431, 405,
- 412, 69, 70, 82, 418, 409, 83, 84, 71, 72,
- 460, 73, 431, 289, 85, 158, 420, 74, 424, 86,
- 433, 443, 171, 75, 87, 215, 76, 444, 456, 466,
- 158, 476, 477, 432, 158, 455, 380, 464, 77, 78,
- 451, 468, 238, 461, 391, 290, 79, 154, 319, 80,
- 208, 320, 239, 186, 465, 210, 148, 31, 62, 32,
- 63, 81, 211, 170, 417, 196, 10, 263, 264, 82,
- 0, 425, 83, 84, 64, 65, 0, 0, 0, 0,
- 85, 0, 0, 0, 0, 86, 67, 68, 0, 0,
- 172, 0, 0, 0, 69, 70, 0, 0, 0, 0,
- 0, 71, 72, 0, 73, 0, 0, 0, 0, 0,
- 74, 0, 0, 0, 0, 171, 75, 0, 0, 76,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 77, 78, 0, 0, 0, 0, 0, 0, 79,
- 0, 0, 80, 0, 0, 0, 0, 0, 0, 0,
- 31, 62, 32, 63, 81, 0, 0, 0, 0, 0,
- 0, 0, 82, 0, 0, 83, 84, 64, 65, 66,
- 0, 0, 0, 85, 0, 0, 0, 0, 86, 67,
- 68, 0, 0, 172, 0, 0, 0, 69, 70, 0,
- 0, 0, 0, 0, 71, 72, 0, 73, 0, 0,
- 0, 0, 0, 74, 0, 0, 0, 0, 0, 75,
- 0, 0, 76, 0, 0, 0, 31, 62, 32, 63,
+ 33, 183, 292, 184, 45, 159, 209, 42, 46, 181,
+ 309, 262, 27, 94, 31, 152, 32, 181, 31, 285,
+ 32, 457, 31, 247, 32, 188, 55, 286, 247, 247,
+ 57, 58, 59, 60, 61, 205, 228, 30, 51, 247,
+ 50, 125, 397, 458, 249, 250, 251, 252, 253, 254,
+ 255, 256, 257, 258, 245, 141, 142, 334, 146, 147,
+ 136, 141, 142, 141, 142, 275, 181, 188, 181, 10,
+ 39, 244, 209, 121, 141, 142, 141, 142, 326, 327,
+ 328, 141, 142, 34, 373, 462, 36, 374, 375, 124,
+ 145, 313, 52, 111, 31, 53, 32, 40, 353, 45,
+ 38, 205, 156, 46, 200, 161, 162, 287, 398, 259,
+ 169, 172, 94, 312, 463, 391, 161, 10, 10, 35,
+ 153, 276, 153, 112, 314, 220, 225, 226, 265, 273,
+ 305, 185, 119, 315, 210, 37, 329, 335, 402, 213,
+ 181, 181, 204, 120, 383, 350, 459, 246, 216, 415,
+ 204, 47, 209, 323, 406, 192, 332, 266, 376, 172,
+ 269, 234, 235, 1, 443, 2, 393, 203, 340, 54,
+ 148, 316, 422, 203, 227, 396, 185, 359, 481, 193,
+ 317, 205, 362, 211, 318, 31, 31, 32, 32, 229,
+ 220, 181, 3, 194, 195, 279, 296, 482, 223, 319,
+ 210, 46, 280, 141, 142, 46, 116, 56, 4, 5,
+ 281, 196, 390, 143, 230, 6, 222, 209, 156, 106,
+ 7, 364, 297, 262, 141, 142, 389, 107, 302, 282,
+ 330, 197, 333, 307, 403, 303, 172, 339, 163, 164,
+ 8, 407, 408, 421, 342, 231, 205, 410, 181, 365,
+ 366, 108, 141, 142, 411, 141, 142, 48, 344, 49,
+ 109, 419, 9, 467, 232, 412, 358, 279, 345, 10,
+ 165, 166, 46, 367, 280, 181, 239, 240, 113, -127,
+ 210, 11, 281, 110, 46, 114, 468, 12, 264, 226,
+ 13, 141, 142, 293, 294, 115, 116, 161, 336, 337,
+ 387, 282, 122, 220, 357, 294, 449, 123, 369, 370,
+ 371, 372, 395, 129, 385, 131, 409, 226, 424, 294,
+ 181, 431, 226, 432, 226, 134, 220, 137, 446, 294,
+ 138, 447, 337, 455, 456, 31, 62, 32, 63, 128,
+ 472, 473, 456, 479, 226, 210, 484, 226, 485, 226,
+ 130, 132, 64, 65, 217, 480, 161, 161, 140, 483,
+ 220, 133, 423, 167, 67, 68, 191, 158, 307, 182,
+ 201, 224, 69, 70, 202, 10, 236, 247, 237, 71,
+ 72, 248, 73, 270, 261, 271, 272, 218, 74, 274,
+ 289, 284, 291, 445, 75, 298, 299, 76, 300, 301,
+ 452, 308, 341, 331, 338, 343, 346, 347, 349, 440,
+ 77, 78, 351, 348, 352, 354, 444, 355, 79, 360,
+ 361, 80, 363, 152, 378, 381, 379, 392, 382, 384,
+ 388, 394, 400, 81, 401, 404, 460, 440, 452, 413,
+ 416, 82, 83, 417, 426, 84, 85, 427, 420, 428,
+ 470, 429, 440, 86, 294, 161, 433, 441, 87, 31,
+ 62, 32, 63, 88, 219, 173, 442, 453, 476, 454,
+ 161, 448, 465, 486, 161, 474, 64, 65, 487, 466,
+ 478, 242, 386, 471, 461, 157, 295, 324, 67, 68,
+ 212, 399, 475, 214, 243, 325, 69, 70, 151, 189,
+ 215, 199, 425, 71, 72, 434, 73, 0, 0, 0,
+ 267, 0, 74, 268, 0, 0, 0, 174, 75, 0,
+ 0, 76, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 77, 78, 0, 0, 0, 0,
- 0, 0, 79, 64, 65, 80, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 67, 68, 81, 0, 0,
- 0, 0, 0, 69, 70, 82, 0, 0, 83, 84,
- 71, 72, 0, 73, 0, 0, 85, 0, 0, 74,
- 0, 86, 0, 0, 0, 75, 87, 0, 76, 0,
- 0, 0, 31, 62, 32, 63, 0, 0, 0, 0,
- 77, 78, 0, 0, 0, 0, 0, 0, 79, 64,
- 65, 80, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 67, 68, 81, 0, 0, 0, 0, 0, 69,
- 70, 82, 0, 0, 83, 84, 71, 72, 0, 73,
- 0, 0, 85, 122, 0, 74, 0, 86, 0, 0,
- 0, 75, 87, 0, 76, 0, 0, 0, 31, 62,
- 32, 63, 0, 0, 0, 0, 77, 78, 0, 0,
- 0, 0, 0, 0, 79, 64, 65, 80, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 67, 68, 81,
- 0, 0, 0, 0, 10, 69, 70, 82, 0, 0,
- 83, 84, 71, 72, 0, 73, 0, 0, 85, 0,
- 0, 74, 0, 86, 0, 0, 171, 75, 87, 0,
- 76, 0, 0, 0, 31, 62, 32, 63, 0, 0,
- 0, 0, 77, 78, 0, 0, 0, 0, 0, 0,
- 79, 64, 147, 80, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 67, 68, 81, 0, 0, 0, 0,
- 0, 69, 70, 82, 0, 0, 83, 84, 71, 72,
- 0, 73, 0, 0, 85, 0, 0, 74, 0, 86,
- 0, 0, 0, 75, 172, 0, 76, 0, 0, 0,
- 31, 62, 32, 63, 0, 0, 0, 0, 77, 78,
- 0, 0, 1, 0, 2, 0, 79, 64, 65, 80,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 67,
- 68, 81, 0, 0, 0, 0, 0, 69, 70, 82,
- 0, 3, 83, 84, 71, 72, 0, 73, 0, 0,
- 85, 0, 0, 74, 0, 86, 0, 4, 5, 75,
- 87, 0, 76, 0, 6, 0, 0, 0, 0, 7,
- 0, 0, 0, 0, 77, 78, 0, 0, 0, 0,
- 0, 0, 79, 0, 0, 80, 0, 0, 8, 0,
- 0, 0, 0, 0, 0, 0, 62, 81, 63, 0,
- 0, 0, 0, 0, 0, 82, 0, 0, 83, 84,
- 9, 0, 64, 147, 67, 68, 85, 10, 0, 0,
- 0, 86, 0, 70, 67, 68, 87, 0, 11, 71,
- 72, 0, 73, 70, 12, 0, 0, 13, 74, 71,
- 72, 0, 73, 0, 0, 0, 0, 76, 74, 0,
- 0, 0, 0, 0, 0, 0, 0, 76, 0, 77,
- 237, 0, 0, 0, 0, 0, 0, 79, 0, 77,
- 78, 0, 0, 0, 0, 0, 0, 79, 0, 0,
- 80, 0, 81, 0, 0, 0, 0, 0, 0, 0,
- 82, 0, 81, 83, 84, 0, 0, 0, 0, 0,
- 82, 85, 0, 83, 84, 0, 86, 0, 0, 0,
- 0, 85, 0, 0, 0, 0, 86, 245, 246, 247,
- 248, 249, 250, 251, 252, 253, 254, 0, 138, 139,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 79, 0, 0, 80, 31, 62, 32, 63,
+ 0, 0, 173, 0, 0, 0, 0, 81, 0, 0,
+ 0, 0, 10, 64, 65, 82, 83, 0, 0, 84,
+ 85, 0, 0, 0, 0, 67, 68, 86, 0, 0,
+ 0, 0, 87, 69, 70, 0, 0, 175, 0, 0,
+ 71, 72, 0, 73, 0, 0, 0, 0, 0, 74,
+ 0, 0, 0, 0, 174, 75, 0, 0, 76, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 255
+ 0, 77, 78, 0, 0, 0, 0, 0, 0, 79,
+ 0, 0, 80, 0, 249, 250, 251, 252, 253, 254,
+ 255, 256, 257, 258, 81, 141, 142, 0, 31, 62,
+ 32, 63, 82, 83, 0, 0, 84, 85, 0, 0,
+ 0, 0, 0, 0, 86, 64, 65, 66, 0, 87,
+ 0, 0, 0, 0, 175, 0, 0, 67, 68, 0,
+ 0, 0, 0, 0, 0, 69, 70, 0, 0, 0,
+ 0, 0, 71, 72, 0, 73, 0, 0, 0, 259,
+ 0, 74, 0, 31, 62, 32, 63, 75, 0, 0,
+ 76, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 64, 65, 0, 77, 78, 0, 0, 0, 0, 0,
+ 0, 79, 67, 68, 80, 0, 0, 0, 0, 0,
+ 69, 70, 0, 0, 0, 0, 81, 71, 72, 0,
+ 73, 0, 0, 0, 82, 83, 74, 0, 84, 85,
+ 0, 0, 75, 0, 0, 76, 86, 0, 0, 0,
+ 0, 87, 0, 0, 0, 0, 88, 0, 77, 78,
+ 0, 0, 0, 0, 0, 0, 79, 0, 0, 80,
+ 31, 62, 32, 63, 0, 0, 0, 0, 0, 0,
+ 0, 81, 0, 0, 0, 0, 0, 64, 65, 82,
+ 83, 0, 0, 84, 85, 0, 0, 0, 0, 67,
+ 68, 86, 124, 0, 0, 0, 87, 69, 70, 0,
+ 0, 88, 0, 0, 71, 72, 0, 73, 0, 0,
+ 0, 0, 0, 74, 0, 31, 62, 32, 63, 75,
+ 0, 0, 76, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 64, 65, 0, 77, 78, 0, 0, 0,
+ 0, 0, 0, 79, 67, 68, 80, 0, 0, 0,
+ 0, 0, 69, 70, 0, 0, 0, 0, 81, 71,
+ 72, 0, 73, 10, 0, 0, 82, 83, 74, 0,
+ 84, 85, 0, 174, 75, 0, 0, 76, 86, 0,
+ 0, 0, 0, 87, 0, 0, 0, 0, 88, 0,
+ 77, 78, 0, 0, 0, 0, 0, 0, 79, 0,
+ 0, 80, 31, 62, 32, 63, 0, 0, 0, 0,
+ 0, 0, 0, 81, 0, 0, 0, 0, 0, 64,
+ 150, 82, 83, 0, 0, 84, 85, 0, 0, 0,
+ 0, 67, 68, 86, 0, 0, 0, 0, 87, 69,
+ 70, 0, 0, 175, 0, 0, 71, 72, 0, 73,
+ 0, 0, 0, 0, 0, 74, 0, 31, 62, 32,
+ 63, 75, 0, 0, 76, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 64, 65, 0, 77, 78, 0,
+ 0, 0, 0, 0, 0, 79, 67, 68, 80, 0,
+ 0, 0, 0, 0, 69, 70, 0, 0, 0, 0,
+ 81, 71, 72, 0, 73, 0, 0, 0, 82, 83,
+ 74, 0, 84, 85, 0, 0, 75, 0, 0, 76,
+ 86, 0, 0, 0, 0, 87, 0, 0, 0, 0,
+ 88, 0, 77, 78, 0, 0, 0, 0, 0, 0,
+ 79, 0, 0, 80, 62, 0, 63, 0, 0, 0,
+ 0, 0, 0, 0, 0, 81, 0, 0, 0, 0,
+ 64, 150, 0, 82, 83, 0, 0, 84, 85, 0,
+ 0, 0, 67, 68, 0, 86, 0, 0, 0, 0,
+ 87, 70, 0, 0, 0, 88, 0, 71, 72, 0,
+ 73, 67, 68, 0, 0, 0, 74, 0, 0, 0,
+ 70, 0, 0, 0, 0, 76, 71, 72, 0, 73,
+ 0, 0, 0, 0, 0, 74, 0, 0, 77, 78,
+ 0, 0, 0, 0, 76, 0, 79, 0, 0, 80,
+ 0, 0, 0, 0, 0, 0, 0, 77, 241, 0,
+ 0, 81, 0, 0, 0, 79, 0, 0, 0, 82,
+ 0, 0, 0, 84, 85, 0, 0, 0, 0, 0,
+ 81, 86, 0, 0, 0, 0, 87, 0, 82, 0,
+ 0, 0, 84, 85, 0, 0, 0, 0, 0, 0,
+ 86, 0, 0, 0, 0, 87
};
static const yytype_int16 yycheck[] =
{
- 4, 122, 8, 118, 12, 135, 214, 11, 12, 114,
- 179, 41, 106, 235, 0, 10, 27, 122, 8, 200,
- 4, 8, 6, 7, 37, 4, 30, 6, 110, 160,
- 34, 35, 36, 37, 38, 205, 4, 8, 6, 69,
- 26, 135, 77, 23, 24, 29, 28, 21, 22, 59,
- 59, 172, 122, 126, 21, 22, 129, 87, 21, 22,
- 35, 75, 132, 38, 39, 170, 71, 172, 21, 22,
- 200, 106, 17, 18, 19, 61, 58, 171, 136, 10,
- 11, 12, 13, 14, 15, 16, 17, 18, 19, 93,
- 21, 22, 43, 151, 89, 100, 110, 105, 279, 29,
- 104, 105, 115, 107, 108, 236, 200, 137, 112, 113,
- 106, 121, 121, 283, 118, 54, 67, 130, 326, 130,
- 150, 111, 61, 119, 68, 131, 56, 106, 233, 74,
- 54, 135, 110, 108, 130, 0, 140, 61, 243, 244,
- 348, 310, 172, 74, 131, 149, 130, 98, 111, 279,
- 372, 130, 182, 133, 80, 185, 160, 131, 162, 163,
- 131, 255, 130, 332, 131, 116, 10, 4, 121, 6,
- 9, 69, 293, 159, 382, 65, 21, 22, 76, 50,
- 3, 296, 21, 22, 214, 279, 84, 32, 293, 68,
- 220, 69, 107, 83, 31, 132, 200, 201, 76, 43,
- 131, 205, 73, 81, 31, 103, 84, 130, 52, 9,
- 55, 44, 45, 343, 218, 64, 87, 88, 222, 388,
- 80, 21, 22, 5, 254, 103, 256, 130, 349, 233,
- 324, 261, 236, 131, 105, 4, 92, 6, 268, 354,
- 355, 122, 91, 92, 349, 89, 126, 7, 128, 343,
- 44, 45, 7, 124, 98, 376, 131, 132, 102, 289,
- 4, 366, 6, 271, 130, 114, 79, 271, 131, 132,
- 5, 376, 116, 131, 132, 279, 130, 21, 22, 283,
- 21, 22, 79, 131, 132, 131, 132, 31, 131, 132,
- 131, 132, 296, 323, 130, 5, 326, 418, 68, 131,
- 132, 131, 132, 131, 132, 335, 131, 132, 132, 313,
- 131, 132, 130, 418, 131, 132, 131, 132, 348, 131,
- 132, 131, 132, 131, 132, 131, 132, 123, 4, 5,
- 6, 7, 131, 132, 449, 130, 7, 7, 131, 343,
- 106, 130, 11, 130, 8, 21, 22, 23, 9, 464,
- 354, 355, 382, 468, 384, 20, 60, 33, 34, 131,
- 68, 132, 366, 131, 70, 41, 42, 131, 130, 82,
- 82, 130, 48, 49, 130, 51, 118, 60, 94, 119,
- 56, 57, 81, 413, 7, 94, 7, 63, 40, 419,
- 66, 94, 72, 130, 4, 5, 6, 7, 130, 5,
- 10, 405, 78, 79, 132, 27, 123, 89, 412, 130,
- 86, 21, 22, 89, 82, 130, 89, 36, 97, 130,
- 97, 93, 40, 33, 34, 101, 456, 431, 432, 130,
- 130, 41, 42, 109, 92, 95, 112, 113, 48, 49,
- 444, 51, 446, 132, 120, 449, 85, 57, 5, 125,
- 40, 7, 62, 63, 130, 131, 66, 102, 132, 90,
- 464, 96, 7, 130, 468, 131, 316, 130, 78, 79,
- 432, 130, 169, 446, 343, 218, 86, 105, 243, 89,
- 137, 244, 170, 124, 456, 141, 102, 4, 5, 6,
- 7, 101, 146, 10, 388, 129, 106, 183, 183, 109,
- -1, 403, 112, 113, 21, 22, -1, -1, -1, -1,
- 120, -1, -1, -1, -1, 125, 33, 34, -1, -1,
- 130, -1, -1, -1, 41, 42, -1, -1, -1, -1,
- -1, 48, 49, -1, 51, -1, -1, -1, -1, -1,
- 57, -1, -1, -1, -1, 62, 63, -1, -1, 66,
+ 4, 120, 218, 124, 12, 108, 138, 11, 12, 116,
+ 239, 182, 0, 41, 4, 27, 6, 124, 4, 209,
+ 6, 7, 4, 8, 6, 59, 30, 37, 8, 8,
+ 34, 35, 36, 37, 38, 138, 163, 112, 26, 8,
+ 78, 69, 67, 29, 10, 11, 12, 13, 14, 15,
+ 16, 17, 18, 19, 175, 21, 22, 10, 23, 24,
+ 88, 21, 22, 21, 22, 204, 173, 59, 175, 107,
+ 29, 174, 204, 61, 21, 22, 21, 22, 17, 18,
+ 19, 21, 22, 76, 35, 72, 69, 38, 39, 123,
+ 94, 10, 128, 28, 4, 131, 6, 56, 288, 107,
+ 81, 204, 106, 107, 132, 109, 110, 117, 133, 75,
+ 114, 115, 140, 240, 101, 331, 120, 107, 107, 112,
+ 132, 31, 132, 58, 43, 153, 133, 134, 113, 69,
+ 237, 123, 121, 52, 138, 112, 75, 90, 354, 143,
+ 247, 248, 132, 132, 315, 284, 132, 175, 152, 378,
+ 132, 0, 284, 133, 133, 50, 259, 185, 109, 163,
+ 188, 165, 166, 1, 133, 3, 337, 133, 113, 3,
+ 135, 90, 388, 133, 162, 133, 123, 298, 65, 74,
+ 99, 284, 301, 139, 103, 4, 4, 6, 6, 43,
+ 218, 298, 30, 88, 89, 70, 224, 84, 154, 118,
+ 204, 205, 77, 21, 22, 209, 124, 69, 46, 47,
+ 85, 106, 9, 31, 68, 53, 134, 349, 222, 108,
+ 58, 64, 226, 394, 21, 22, 329, 134, 54, 104,
+ 258, 126, 260, 237, 355, 61, 240, 265, 44, 45,
+ 78, 360, 361, 9, 272, 99, 349, 54, 355, 92,
+ 93, 31, 21, 22, 61, 21, 22, 128, 133, 130,
+ 132, 382, 100, 32, 118, 372, 294, 70, 276, 107,
+ 44, 45, 276, 116, 77, 382, 133, 134, 5, 82,
+ 284, 119, 85, 81, 288, 93, 55, 125, 133, 134,
+ 128, 21, 22, 133, 134, 132, 124, 301, 133, 134,
+ 328, 104, 7, 331, 133, 134, 427, 7, 133, 134,
+ 133, 134, 340, 80, 318, 5, 133, 134, 133, 134,
+ 427, 133, 134, 133, 134, 80, 354, 5, 133, 134,
+ 69, 133, 134, 133, 134, 4, 5, 6, 7, 132,
+ 459, 133, 134, 133, 134, 349, 133, 134, 133, 134,
+ 132, 132, 21, 22, 23, 474, 360, 361, 134, 478,
+ 388, 132, 390, 125, 33, 34, 7, 132, 372, 132,
+ 7, 11, 41, 42, 133, 107, 132, 8, 132, 48,
+ 49, 9, 51, 60, 20, 133, 69, 56, 57, 133,
+ 71, 134, 133, 421, 63, 132, 83, 66, 83, 132,
+ 428, 120, 60, 132, 121, 7, 95, 95, 82, 413,
+ 79, 80, 7, 95, 7, 40, 420, 73, 87, 132,
+ 132, 90, 5, 27, 134, 90, 125, 90, 132, 83,
+ 132, 132, 98, 102, 98, 94, 440, 441, 466, 132,
+ 36, 110, 111, 96, 7, 114, 115, 93, 132, 40,
+ 454, 86, 456, 122, 134, 459, 5, 132, 127, 4,
+ 5, 6, 7, 132, 133, 10, 40, 7, 91, 103,
+ 474, 133, 133, 97, 478, 132, 21, 22, 7, 134,
+ 132, 172, 321, 456, 441, 107, 222, 247, 33, 34,
+ 140, 349, 466, 144, 173, 248, 41, 42, 104, 126,
+ 149, 131, 394, 48, 49, 411, 51, -1, -1, -1,
+ 186, -1, 57, 186, -1, -1, -1, 62, 63, -1,
+ -1, 66, -1, -1, -1, -1, -1, -1, -1, -1,
+ -1, -1, -1, -1, 79, 80, -1, -1, -1, -1,
+ -1, -1, 87, -1, -1, 90, 4, 5, 6, 7,
+ -1, -1, 10, -1, -1, -1, -1, 102, -1, -1,
+ -1, -1, 107, 21, 22, 110, 111, -1, -1, 114,
+ 115, -1, -1, -1, -1, 33, 34, 122, -1, -1,
+ -1, -1, 127, 41, 42, -1, -1, 132, -1, -1,
+ 48, 49, -1, 51, -1, -1, -1, -1, -1, 57,
+ -1, -1, -1, -1, 62, 63, -1, -1, 66, -1,
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
- -1, 78, 79, -1, -1, -1, -1, -1, -1, 86,
- -1, -1, 89, -1, -1, -1, -1, -1, -1, -1,
- 4, 5, 6, 7, 101, -1, -1, -1, -1, -1,
- -1, -1, 109, -1, -1, 112, 113, 21, 22, 23,
- -1, -1, -1, 120, -1, -1, -1, -1, 125, 33,
- 34, -1, -1, 130, -1, -1, -1, 41, 42, -1,
- -1, -1, -1, -1, 48, 49, -1, 51, -1, -1,
- -1, -1, -1, 57, -1, -1, -1, -1, -1, 63,
- -1, -1, 66, -1, -1, -1, 4, 5, 6, 7,
- -1, -1, -1, -1, 78, 79, -1, -1, -1, -1,
- -1, -1, 86, 21, 22, 89, -1, -1, -1, -1,
- -1, -1, -1, -1, -1, 33, 34, 101, -1, -1,
- -1, -1, -1, 41, 42, 109, -1, -1, 112, 113,
- 48, 49, -1, 51, -1, -1, 120, -1, -1, 57,
- -1, 125, -1, -1, -1, 63, 130, -1, 66, -1,
- -1, -1, 4, 5, 6, 7, -1, -1, -1, -1,
- 78, 79, -1, -1, -1, -1, -1, -1, 86, 21,
- 22, 89, -1, -1, -1, -1, -1, -1, -1, -1,
- -1, 33, 34, 101, -1, -1, -1, -1, -1, 41,
- 42, 109, -1, -1, 112, 113, 48, 49, -1, 51,
- -1, -1, 120, 121, -1, 57, -1, 125, -1, -1,
- -1, 63, 130, -1, 66, -1, -1, -1, 4, 5,
- 6, 7, -1, -1, -1, -1, 78, 79, -1, -1,
- -1, -1, -1, -1, 86, 21, 22, 89, -1, -1,
- -1, -1, -1, -1, -1, -1, -1, 33, 34, 101,
- -1, -1, -1, -1, 106, 41, 42, 109, -1, -1,
- 112, 113, 48, 49, -1, 51, -1, -1, 120, -1,
- -1, 57, -1, 125, -1, -1, 62, 63, 130, -1,
- 66, -1, -1, -1, 4, 5, 6, 7, -1, -1,
- -1, -1, 78, 79, -1, -1, -1, -1, -1, -1,
- 86, 21, 22, 89, -1, -1, -1, -1, -1, -1,
- -1, -1, -1, 33, 34, 101, -1, -1, -1, -1,
- -1, 41, 42, 109, -1, -1, 112, 113, 48, 49,
- -1, 51, -1, -1, 120, -1, -1, 57, -1, 125,
- -1, -1, -1, 63, 130, -1, 66, -1, -1, -1,
- 4, 5, 6, 7, -1, -1, -1, -1, 78, 79,
- -1, -1, 1, -1, 3, -1, 86, 21, 22, 89,
- -1, -1, -1, -1, -1, -1, -1, -1, -1, 33,
- 34, 101, -1, -1, -1, -1, -1, 41, 42, 109,
- -1, 30, 112, 113, 48, 49, -1, 51, -1, -1,
- 120, -1, -1, 57, -1, 125, -1, 46, 47, 63,
- 130, -1, 66, -1, 53, -1, -1, -1, -1, 58,
- -1, -1, -1, -1, 78, 79, -1, -1, -1, -1,
- -1, -1, 86, -1, -1, 89, -1, -1, 77, -1,
- -1, -1, -1, -1, -1, -1, 5, 101, 7, -1,
- -1, -1, -1, -1, -1, 109, -1, -1, 112, 113,
- 99, -1, 21, 22, 33, 34, 120, 106, -1, -1,
- -1, 125, -1, 42, 33, 34, 130, -1, 117, 48,
- 49, -1, 51, 42, 123, -1, -1, 126, 57, 48,
- 49, -1, 51, -1, -1, -1, -1, 66, 57, -1,
- -1, -1, -1, -1, -1, -1, -1, 66, -1, 78,
- 79, -1, -1, -1, -1, -1, -1, 86, -1, 78,
- 79, -1, -1, -1, -1, -1, -1, 86, -1, -1,
- 89, -1, 101, -1, -1, -1, -1, -1, -1, -1,
- 109, -1, 101, 112, 113, -1, -1, -1, -1, -1,
- 109, 120, -1, 112, 113, -1, 125, -1, -1, -1,
- -1, 120, -1, -1, -1, -1, 125, 10, 11, 12,
- 13, 14, 15, 16, 17, 18, 19, -1, 21, 22,
- -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
- -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
- -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
- -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
- -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
- -1, 74
+ -1, 79, 80, -1, -1, -1, -1, -1, -1, 87,
+ -1, -1, 90, -1, 10, 11, 12, 13, 14, 15,
+ 16, 17, 18, 19, 102, 21, 22, -1, 4, 5,
+ 6, 7, 110, 111, -1, -1, 114, 115, -1, -1,
+ -1, -1, -1, -1, 122, 21, 22, 23, -1, 127,
+ -1, -1, -1, -1, 132, -1, -1, 33, 34, -1,
+ -1, -1, -1, -1, -1, 41, 42, -1, -1, -1,
+ -1, -1, 48, 49, -1, 51, -1, -1, -1, 75,
+ -1, 57, -1, 4, 5, 6, 7, 63, -1, -1,
+ 66, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+ 21, 22, -1, 79, 80, -1, -1, -1, -1, -1,
+ -1, 87, 33, 34, 90, -1, -1, -1, -1, -1,
+ 41, 42, -1, -1, -1, -1, 102, 48, 49, -1,
+ 51, -1, -1, -1, 110, 111, 57, -1, 114, 115,
+ -1, -1, 63, -1, -1, 66, 122, -1, -1, -1,
+ -1, 127, -1, -1, -1, -1, 132, -1, 79, 80,
+ -1, -1, -1, -1, -1, -1, 87, -1, -1, 90,
+ 4, 5, 6, 7, -1, -1, -1, -1, -1, -1,
+ -1, 102, -1, -1, -1, -1, -1, 21, 22, 110,
+ 111, -1, -1, 114, 115, -1, -1, -1, -1, 33,
+ 34, 122, 123, -1, -1, -1, 127, 41, 42, -1,
+ -1, 132, -1, -1, 48, 49, -1, 51, -1, -1,
+ -1, -1, -1, 57, -1, 4, 5, 6, 7, 63,
+ -1, -1, 66, -1, -1, -1, -1, -1, -1, -1,
+ -1, -1, 21, 22, -1, 79, 80, -1, -1, -1,
+ -1, -1, -1, 87, 33, 34, 90, -1, -1, -1,
+ -1, -1, 41, 42, -1, -1, -1, -1, 102, 48,
+ 49, -1, 51, 107, -1, -1, 110, 111, 57, -1,
+ 114, 115, -1, 62, 63, -1, -1, 66, 122, -1,
+ -1, -1, -1, 127, -1, -1, -1, -1, 132, -1,
+ 79, 80, -1, -1, -1, -1, -1, -1, 87, -1,
+ -1, 90, 4, 5, 6, 7, -1, -1, -1, -1,
+ -1, -1, -1, 102, -1, -1, -1, -1, -1, 21,
+ 22, 110, 111, -1, -1, 114, 115, -1, -1, -1,
+ -1, 33, 34, 122, -1, -1, -1, -1, 127, 41,
+ 42, -1, -1, 132, -1, -1, 48, 49, -1, 51,
+ -1, -1, -1, -1, -1, 57, -1, 4, 5, 6,
+ 7, 63, -1, -1, 66, -1, -1, -1, -1, -1,
+ -1, -1, -1, -1, 21, 22, -1, 79, 80, -1,
+ -1, -1, -1, -1, -1, 87, 33, 34, 90, -1,
+ -1, -1, -1, -1, 41, 42, -1, -1, -1, -1,
+ 102, 48, 49, -1, 51, -1, -1, -1, 110, 111,
+ 57, -1, 114, 115, -1, -1, 63, -1, -1, 66,
+ 122, -1, -1, -1, -1, 127, -1, -1, -1, -1,
+ 132, -1, 79, 80, -1, -1, -1, -1, -1, -1,
+ 87, -1, -1, 90, 5, -1, 7, -1, -1, -1,
+ -1, -1, -1, -1, -1, 102, -1, -1, -1, -1,
+ 21, 22, -1, 110, 111, -1, -1, 114, 115, -1,
+ -1, -1, 33, 34, -1, 122, -1, -1, -1, -1,
+ 127, 42, -1, -1, -1, 132, -1, 48, 49, -1,
+ 51, 33, 34, -1, -1, -1, 57, -1, -1, -1,
+ 42, -1, -1, -1, -1, 66, 48, 49, -1, 51,
+ -1, -1, -1, -1, -1, 57, -1, -1, 79, 80,
+ -1, -1, -1, -1, 66, -1, 87, -1, -1, 90,
+ -1, -1, -1, -1, -1, -1, -1, 79, 80, -1,
+ -1, 102, -1, -1, -1, 87, -1, -1, -1, 110,
+ -1, -1, -1, 114, 115, -1, -1, -1, -1, -1,
+ 102, 122, -1, -1, -1, -1, 127, -1, 110, -1,
+ -1, -1, 114, 115, -1, -1, -1, -1, -1, -1,
+ 122, -1, -1, -1, -1, 127
};
/* YYSTOS[STATE-NUM] -- The (internal number of the) accessing
symbol of state STATE-NUM. */
static const yytype_uint8 yystos[] =
{
- 0, 1, 3, 30, 46, 47, 53, 58, 77, 99,
- 106, 117, 123, 126, 135, 136, 137, 138, 139, 140,
- 141, 162, 163, 166, 167, 170, 171, 174, 227, 228,
- 110, 4, 6, 225, 75, 110, 68, 110, 80, 29,
- 56, 175, 225, 172, 173, 186, 225, 0, 126, 128,
- 77, 174, 126, 129, 3, 225, 68, 225, 225, 225,
- 225, 225, 5, 7, 21, 22, 23, 33, 34, 41,
- 42, 48, 49, 51, 57, 63, 66, 78, 79, 86,
- 89, 101, 109, 112, 113, 120, 125, 130, 144, 176,
- 177, 178, 180, 202, 203, 204, 205, 206, 207, 208,
- 215, 218, 221, 225, 107, 132, 31, 130, 80, 28,
- 58, 5, 92, 130, 122, 196, 197, 119, 130, 174,
- 7, 7, 121, 202, 211, 212, 130, 79, 130, 5,
- 130, 79, 174, 202, 5, 68, 179, 132, 21, 22,
- 31, 222, 225, 23, 24, 133, 223, 22, 205, 27,
- 130, 168, 169, 225, 173, 130, 180, 224, 225, 225,
- 44, 45, 44, 45, 123, 164, 225, 142, 143, 225,
- 10, 62, 130, 198, 199, 200, 201, 202, 218, 130,
- 224, 198, 121, 209, 210, 59, 212, 213, 7, 50,
- 73, 87, 88, 105, 124, 216, 216, 7, 131, 131,
- 130, 180, 183, 184, 187, 206, 225, 196, 178, 225,
- 203, 204, 225, 23, 56, 131, 202, 214, 132, 196,
- 11, 131, 132, 174, 142, 43, 67, 98, 116, 148,
- 225, 225, 130, 130, 151, 131, 132, 79, 144, 201,
- 180, 198, 202, 8, 9, 10, 11, 12, 13, 14,
- 15, 16, 17, 18, 19, 74, 220, 20, 215, 217,
- 131, 111, 202, 210, 213, 202, 60, 131, 68, 131,
- 183, 31, 185, 186, 69, 76, 84, 103, 182, 132,
- 185, 37, 115, 181, 70, 188, 131, 214, 131, 132,
- 169, 202, 225, 130, 82, 82, 130, 54, 61, 165,
- 218, 219, 225, 118, 148, 149, 150, 142, 10, 43,
- 52, 89, 98, 102, 116, 145, 146, 147, 131, 199,
- 200, 17, 18, 19, 74, 202, 130, 180, 202, 10,
- 89, 131, 132, 119, 202, 111, 60, 202, 131, 186,
- 94, 94, 94, 81, 183, 7, 7, 185, 40, 72,
- 189, 131, 202, 198, 130, 130, 224, 5, 64, 91,
- 92, 114, 226, 131, 132, 131, 132, 35, 38, 39,
- 108, 160, 132, 123, 152, 89, 130, 215, 82, 225,
- 145, 202, 130, 180, 9, 214, 89, 215, 130, 202,
- 131, 184, 97, 97, 214, 198, 93, 190, 131, 224,
- 224, 131, 54, 61, 218, 130, 161, 148, 36, 95,
- 153, 198, 130, 9, 214, 202, 131, 217, 92, 40,
- 85, 191, 131, 131, 5, 226, 155, 156, 157, 158,
- 159, 225, 130, 40, 131, 225, 202, 131, 131, 198,
- 192, 193, 202, 7, 102, 131, 132, 7, 29, 130,
- 225, 155, 71, 100, 154, 131, 132, 32, 55, 194,
- 225, 156, 224, 131, 130, 193, 90, 195, 130, 131,
- 224, 65, 83, 224, 131, 131, 96, 7
+ 0, 1, 3, 30, 46, 47, 53, 58, 78, 100,
+ 107, 119, 125, 128, 137, 138, 139, 140, 141, 142,
+ 143, 164, 165, 168, 169, 172, 173, 176, 230, 231,
+ 112, 4, 6, 228, 76, 112, 69, 112, 81, 29,
+ 56, 177, 228, 174, 175, 188, 228, 0, 128, 130,
+ 78, 176, 128, 131, 3, 228, 69, 228, 228, 228,
+ 228, 228, 5, 7, 21, 22, 23, 33, 34, 41,
+ 42, 48, 49, 51, 57, 63, 66, 79, 80, 87,
+ 90, 102, 110, 111, 114, 115, 122, 127, 132, 146,
+ 178, 179, 180, 182, 204, 205, 206, 207, 208, 209,
+ 210, 211, 218, 221, 224, 228, 108, 134, 31, 132,
+ 81, 28, 58, 5, 93, 132, 124, 198, 199, 121,
+ 132, 176, 7, 7, 123, 204, 214, 215, 132, 80,
+ 132, 5, 132, 132, 80, 176, 204, 5, 69, 181,
+ 134, 21, 22, 31, 225, 228, 23, 24, 135, 226,
+ 22, 207, 27, 132, 170, 171, 228, 175, 132, 182,
+ 227, 228, 228, 44, 45, 44, 45, 125, 166, 228,
+ 144, 145, 228, 10, 62, 132, 200, 201, 202, 203,
+ 204, 221, 132, 227, 200, 123, 212, 213, 59, 215,
+ 216, 7, 50, 74, 88, 89, 106, 126, 219, 219,
+ 204, 7, 133, 133, 132, 182, 185, 186, 189, 208,
+ 228, 198, 180, 228, 205, 206, 228, 23, 56, 133,
+ 204, 217, 134, 198, 11, 133, 134, 176, 144, 43,
+ 68, 99, 118, 150, 228, 228, 132, 132, 153, 133,
+ 134, 80, 146, 203, 182, 200, 204, 8, 9, 10,
+ 11, 12, 13, 14, 15, 16, 17, 18, 19, 75,
+ 223, 20, 218, 220, 133, 113, 204, 213, 216, 204,
+ 60, 133, 69, 69, 133, 185, 31, 187, 188, 70,
+ 77, 85, 104, 184, 134, 187, 37, 117, 183, 71,
+ 190, 133, 217, 133, 134, 171, 204, 228, 132, 83,
+ 83, 132, 54, 61, 167, 221, 222, 228, 120, 150,
+ 151, 152, 144, 10, 43, 52, 90, 99, 103, 118,
+ 147, 148, 149, 133, 201, 202, 17, 18, 19, 75,
+ 204, 132, 182, 204, 10, 90, 133, 134, 121, 204,
+ 113, 60, 204, 7, 133, 188, 95, 95, 95, 82,
+ 185, 7, 7, 187, 40, 73, 191, 133, 204, 200,
+ 132, 132, 227, 5, 64, 92, 93, 116, 229, 133,
+ 134, 133, 134, 35, 38, 39, 109, 162, 134, 125,
+ 154, 90, 132, 218, 83, 228, 147, 204, 132, 182,
+ 9, 217, 90, 218, 132, 204, 133, 67, 133, 186,
+ 98, 98, 217, 200, 94, 192, 133, 227, 227, 133,
+ 54, 61, 221, 132, 163, 150, 36, 96, 155, 200,
+ 132, 9, 217, 204, 133, 220, 7, 93, 40, 86,
+ 193, 133, 133, 5, 229, 157, 158, 159, 160, 161,
+ 228, 132, 40, 133, 228, 204, 133, 133, 133, 200,
+ 194, 195, 204, 7, 103, 133, 134, 7, 29, 132,
+ 228, 157, 72, 101, 156, 133, 134, 32, 55, 196,
+ 228, 158, 227, 133, 132, 195, 91, 197, 132, 133,
+ 227, 65, 84, 227, 133, 133, 97, 7
};
/* YYR1[YYN] -- Symbol number of symbol that rule YYN derives. */
static const yytype_uint8 yyr1[] =
{
- 0, 134, 135, 135, 135, 135, 135, 135, 136, 136,
- 136, 136, 136, 136, 136, 136, 136, 136, 137, 138,
- 138, 138, 138, 139, 140, 141, 142, 143, 143, 144,
- 144, 144, 144, 144, 144, 144, 144, 144, 144, 144,
- 144, 144, 144, 144, 144, 144, 144, 145, 145, 145,
- 145, 145, 145, 145, 146, 146, 147, 147, 148, 148,
- 148, 148, 149, 149, 150, 150, 151, 151, 152, 152,
- 153, 153, 154, 154, 155, 155, 156, 156, 156, 157,
- 157, 158, 159, 160, 160, 160, 160, 161, 161, 162,
- 162, 162, 162, 163, 164, 164, 165, 165, 165, 165,
- 166, 167, 168, 168, 169, 170, 170, 171, 172, 172,
- 173, 174, 175, 175, 175, 176, 176, 177, 177, 178,
- 178, 178, 179, 180, 181, 181, 181, 182, 182, 182,
- 182, 182, 182, 182, 182, 183, 183, 184, 184, 184,
- 184, 184, 184, 185, 185, 186, 186, 187, 187, 188,
- 188, 189, 189, 190, 190, 191, 191, 192, 192, 193,
- 194, 194, 194, 195, 195, 195, 196, 196, 197, 198,
- 198, 199, 199, 200, 200, 201, 201, 201, 201, 201,
- 201, 201, 201, 201, 201, 201, 202, 202, 203, 203,
- 204, 204, 205, 205, 205, 205, 205, 205, 205, 206,
- 206, 206, 206, 207, 208, 208, 209, 209, 210, 211,
- 211, 212, 213, 213, 214, 214, 215, 215, 215, 215,
- 215, 215, 215, 215, 216, 216, 216, 216, 216, 216,
- 217, 217, 218, 218, 219, 219, 220, 220, 220, 220,
- 220, 220, 220, 220, 220, 220, 221, 222, 222, 223,
- 223, 223, 224, 224, 225, 225, 226, 226, 226, 226,
- 227, 228, 228
+ 0, 136, 137, 137, 137, 137, 137, 137, 138, 138,
+ 138, 138, 138, 138, 138, 138, 138, 138, 139, 140,
+ 140, 140, 140, 141, 142, 143, 144, 145, 145, 146,
+ 146, 146, 146, 146, 146, 146, 146, 146, 146, 146,
+ 146, 146, 146, 146, 146, 146, 146, 147, 147, 147,
+ 147, 147, 147, 147, 148, 148, 149, 149, 150, 150,
+ 150, 150, 151, 151, 152, 152, 153, 153, 154, 154,
+ 155, 155, 156, 156, 157, 157, 158, 158, 158, 159,
+ 159, 160, 161, 162, 162, 162, 162, 163, 163, 164,
+ 164, 164, 164, 165, 166, 166, 167, 167, 167, 167,
+ 168, 169, 170, 170, 171, 172, 172, 173, 174, 174,
+ 175, 176, 177, 177, 177, 178, 178, 179, 179, 180,
+ 180, 180, 181, 182, 183, 183, 183, 184, 184, 184,
+ 184, 184, 184, 184, 184, 185, 185, 186, 186, 186,
+ 186, 186, 186, 187, 187, 188, 188, 189, 189, 190,
+ 190, 191, 191, 192, 192, 193, 193, 194, 194, 195,
+ 196, 196, 196, 197, 197, 197, 198, 198, 199, 200,
+ 200, 201, 201, 202, 202, 203, 203, 203, 203, 203,
+ 203, 203, 203, 203, 203, 203, 204, 204, 205, 205,
+ 206, 206, 207, 207, 207, 207, 207, 207, 207, 207,
+ 208, 208, 208, 208, 209, 210, 210, 211, 211, 212,
+ 212, 213, 214, 214, 215, 216, 216, 217, 217, 218,
+ 218, 218, 218, 218, 218, 218, 218, 219, 219, 219,
+ 219, 219, 219, 220, 220, 221, 221, 222, 222, 223,
+ 223, 223, 223, 223, 223, 223, 223, 223, 223, 224,
+ 225, 225, 226, 226, 226, 227, 227, 228, 228, 229,
+ 229, 229, 229, 230, 231, 231
};
/* YYR2[YYN] -- Number of symbols on the right hand side of rule YYN. */
@@ -1381,14 +1391,14 @@ static const yytype_uint8 yyr2[] =
0, 1, 1, 0, 2, 2, 0, 1, 2, 3,
1, 3, 1, 2, 1, 5, 6, 4, 3, 3,
3, 2, 3, 5, 4, 6, 3, 1, 3, 1,
- 2, 1, 1, 1, 1, 1, 1, 3, 1, 3,
- 4, 4, 5, 6, 5, 4, 1, 2, 4, 1,
- 2, 4, 0, 2, 1, 3, 1, 1, 2, 2,
- 1, 2, 3, 2, 1, 1, 1, 1, 1, 1,
- 1, 3, 1, 3, 1, 3, 1, 1, 1, 1,
- 1, 1, 1, 2, 1, 2, 1, 1, 1, 1,
- 1, 1, 1, 3, 1, 1, 1, 1, 1, 1,
- 2, 2, 0
+ 2, 1, 1, 1, 1, 1, 1, 1, 3, 1,
+ 3, 4, 4, 5, 6, 6, 8, 5, 4, 1,
+ 2, 4, 1, 2, 4, 0, 2, 1, 3, 1,
+ 1, 2, 2, 1, 2, 3, 2, 1, 1, 1,
+ 1, 1, 1, 1, 3, 1, 3, 1, 3, 1,
+ 1, 1, 1, 1, 1, 1, 2, 1, 2, 1,
+ 1, 1, 1, 1, 1, 1, 3, 1, 1, 1,
+ 1, 1, 1, 2, 2, 0
};
@@ -1885,603 +1895,593 @@ yydestruct (const char *yymsg, int yytype, YYSTYPE *yyvaluep, YYLTYPE *yylocatio
switch (yytype)
{
case 3: /* TOKEN_COMMAND */
-#line 573 "../SqlParser.ypp" /* yacc.c:1257 */
+#line 576 "../SqlParser.ypp" /* yacc.c:1257 */
{
if (((*yyvaluep).string_value_) != nullptr) {
delete ((*yyvaluep).string_value_);
}
}
-#line 1895 "SqlParser_gen.cpp" /* yacc.c:1257 */
+#line 1905 "SqlParser_gen.cpp" /* yacc.c:1257 */
break;
case 4: /* TOKEN_NAME */
-#line 573 "../SqlParser.ypp" /* yacc.c:1257 */
+#line 576 "../SqlParser.ypp" /* yacc.c:1257 */
{
if (((*yyvaluep).string_value_) != nullptr) {
delete ((*yyvaluep).string_value_);
}
}
-#line 1905 "SqlParser_gen.cpp" /* yacc.c:1257 */
+#line 1915 "SqlParser_gen.cpp" /* yacc.c:1257 */
break;
case 5: /* TOKEN_STRING_SINGLE_QUOTED */
-#line 573 "../SqlParser.ypp" /* yacc.c:1257 */
+#line 576 "../SqlParser.ypp" /* yacc.c:1257 */
{
if (((*yyvaluep).string_value_) != nullptr) {
delete ((*yyvaluep).string_value_);
}
}
-#line 1915 "SqlParser_gen.cpp" /* yacc.c:1257 */
+#line 1925 "SqlParser_gen.cpp" /* yacc.c:1257 */
break;
case 6: /* TOKEN_STRING_DOUBLE_QUOTED */
-#line 573 "../SqlParser.ypp" /* yacc.c:1257 */
+#line 576 "../SqlParser.ypp" /* yacc.c:1257 */
{
if (((*yyvaluep).string_value_) != nullptr) {
delete ((*yyvaluep).string_value_);
}
}
-#line 1925 "SqlParser_gen.cpp" /* yacc.c:1257 */
+#line 1935 "SqlParser_gen.cpp" /* yacc.c:1257 */
break;
case 7: /* TOKEN_UNSIGNED_NUMVAL */
-#line 573 "../SqlParser.ypp" /* yacc.c:1257 */
+#line 576 "../SqlParser.ypp" /* yacc.c:1257 */
{
if (((*yyvaluep).numeric_literal_value_) != nullptr) {
delete ((*yyvaluep).numeric_literal_value_);
}
}
-#line 1935 "SqlParser_gen.cpp" /* yacc.c:1257 */
+#line 1945 "SqlParser_gen.cpp" /* yacc.c:1257 */
break;
- case 136: /* sql_statement */
-#line 573 "../SqlParser.ypp" /* yacc.c:1257 */
+ case 138: /* sql_statement */
+#line 576 "../SqlParser.ypp" /* yacc.c:1257 */
{
if (((*yyvaluep).statement_) != nullptr) {
delete ((*yyvaluep).statement_);
}
}
-#line 1945 "SqlParser_gen.cpp" /* yacc.c:1257 */
+#line 1955 "SqlParser_gen.cpp" /* yacc.c:1257 */
break;
- case 137: /* quit_statement */
-#line 573 "../SqlParser.ypp" /* yacc.c:1257 */
+ case 139: /* quit_statement */
+#line 576 "../SqlParser.ypp" /* yacc.c:1257 */
{
if (((*yyvaluep).quit_statement_) != nullptr) {
delete ((*yyvaluep).quit_statement_);
}
}
-#line 1955 "SqlParser_gen.cpp" /* yacc.c:1257 */
+#line 1965 "SqlParser_gen.cpp" /* yacc.c:1257 */
break;
- case 138: /* alter_table_statement */
-#line 573 "../SqlParser.ypp" /* yacc.c:1257 */
+ case 140: /* alter_table_statement */
+#line 576 "../SqlParser.ypp" /* yacc.c:1257 */
{
if (((*yyvaluep).statement_) != nullptr) {
delete ((*yyvaluep).statement_);
}
}
-#line 1965 "SqlParser_gen.cpp" /* yacc.c:1257 */
+#line 1975 "SqlParser_gen.cpp" /* yacc.c:1257 */
break;
- case 139: /* create_table_statement */
-#line 573 "../SqlParser.ypp" /* yacc.c:1257 */
+ case 141: /* create_table_statement */
+#line 576 "../SqlParser.ypp" /* yacc.c:1257 */
{
if (((*yyvaluep).create_table_statement_) != nullptr) {
delete ((*yyvaluep).create_table_statement_);
}
}
-#line 1975 "SqlParser_gen.cpp" /* yacc.c:1257 */
+#line 1985 "SqlParser_gen.cpp" /* yacc.c:1257 */
break;
- case 140: /* create_index_statement */
-#line 573 "../SqlParser.ypp" /* yacc.c:1257 */
+ case 142: /* create_index_statement */
+#line 576 "../SqlParser.ypp" /* yacc.c:1257 */
{
if (((*yyvaluep).statement_) != nullptr) {
delete ((*yyvaluep).statement_);
}
}
-#line 1985 "SqlParser_gen.cpp" /* yacc.c:1257 */
+#line 1995 "SqlParser_gen.cpp" /* yacc.c:1257 */
break;
- case 141: /* drop_table_statement */
-#line 573 "../SqlParser.ypp" /* yacc.c:1257 */
+ case 143: /* drop_table_statement */
+#line 576 "../SqlParser.ypp" /* yacc.c:1257 */
{
if (((*yyvaluep).drop_table_statement_) != nullptr) {
delete ((*yyvaluep).drop_table_statement_);
}
}
-#line 1995 "SqlParser_gen.cpp" /* yacc.c:1257 */
+#line 2005 "SqlParser_gen.cpp" /* yacc.c:1257 */
break;
- case 142: /* column_def */
-#line 573 "../SqlParser.ypp" /* yacc.c:1257 */
+ case 144: /* column_def */
+#line 576 "../SqlParser.ypp" /* yacc.c:1257 */
{
if (((*yyvaluep).attribute_definition_) != nullptr) {
delete ((*yyvaluep).attribute_definition_);
}
}
-#line 2005 "SqlParser_gen.cpp" /* yacc.c:1257 */
+#line 2015 "SqlParser_gen.cpp" /* yacc.c:1257 */
break;
- case 143: /* column_def_commalist */
-#line 573 "../SqlParser.ypp" /* yacc.c:1257 */
+ case 145: /* column_def_commalist */
+#line 576 "../SqlParser.ypp" /* yacc.c:1257 */
{
if (((*yyvaluep).attribute_definition_list_) != nullptr) {
delete ((*yyvaluep).attribute_definition_list_);
}
}
-#line 2015 "SqlParser_gen.cpp" /* yacc.c:1257 */
+#line 2025 "SqlParser_gen.cpp" /* yacc.c:1257 */
break;
- case 144: /* data_type */
-#line 573 "../SqlParser.ypp" /* yacc.c:1257 */
+ case 146: /* data_type */
+#line 576 "../SqlParser.ypp" /* yacc.c:1257 */
{
if (((*yyvaluep).data_type_) != nullptr) {
delete ((*yyvaluep).data_type_);
}
}
-#line 2025 "SqlParser_gen.cpp" /* yacc.c:1257 */
+#line 2035 "SqlParser_gen.cpp" /* yacc.c:1257 */
break;
- case 145: /* column_constraint_def */
-#line 573 "../SqlParser.ypp" /* yacc.c:1257 */
+ case 147: /* column_constraint_def */
+#line 576 "../SqlParser.ypp" /* yacc.c:1257 */
{
if (((*yyvaluep).column_constraint_) != nullptr) {
delete ((*yyvaluep).column_constraint_);
}
}
-#line 2035 "SqlParser_gen.cpp" /* yacc.c:1257 */
+#line 2045 "SqlParser_gen.cpp" /* yacc.c:1257 */
break;
- case 146: /* column_constraint_def_list */
-#line 573 "../SqlParser.ypp" /* yacc.c:1257 */
+ case 148: /* column_constraint_def_list */
+#line 576 "../SqlParser.ypp" /* yacc.c:1257 */
{
if (((*yyvaluep).column_constraint_list_) != nullptr) {
delete ((*yyvaluep).column_constraint_list_);
}
}
-#line 2045 "SqlParser_gen.cpp" /* yacc.c:1257 */
+#line 2055 "SqlParser_gen.cpp" /* yacc.c:1257 */
break;
- case 147: /* opt_column_constraint_def_list */
-#line 573 "../SqlParser.ypp" /* yacc.c:1257 */
+ case 149: /* opt_column_constraint_def_list */
+#line 576 "../SqlParser.ypp" /* yacc.c:1257 */
{
if (((*yyvaluep).column_constraint_list_) != nullptr) {
delete ((*yyvaluep).column_constraint_list_);
}
}
-#line 2055 "SqlParser_gen.cpp" /* yacc.c:1257 */
+#line 2065 "SqlParser_gen.cpp" /* yacc.c:1257 */
break;
- case 151: /* opt_column_list */
-#line 573 "../SqlParser.ypp" /* yacc.c:1257 */
+ case 153: /* opt_column_list */
+#line 576 "../SqlParser.ypp" /* yacc.c:1257 */
{
if (((*yyvaluep).attribute_list_) != nullptr) {
delete ((*yyvaluep).attribute_list_);
}
}
-#line 2065 "SqlParser_gen.cpp" /* yacc.c:1257 */
+#line 2075 "SqlParser_gen.cpp" /*
<TRUNCATED>
[07/50] [abbrv] incubator-quickstep git commit: Set block slots
correctly. (#202)
Posted by zu...@apache.org.
Set block slots correctly. (#202)
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/abfc5f27
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/abfc5f27
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/abfc5f27
Branch: refs/heads/work-order-serialization
Commit: abfc5f275c500d40962dac70a60a4cd5881b77bc
Parents: 5b75e8e
Author: Zuyu ZHANG <zu...@users.noreply.github.com>
Authored: Fri Apr 29 12:01:19 2016 -0700
Committer: Jignesh Patel <pa...@users.noreply.github.com>
Committed: Fri Apr 29 12:01:19 2016 -0700
----------------------------------------------------------------------
query_optimizer/resolver/Resolver.cpp | 13 +++++++++----
query_optimizer/tests/physical_generator/Create.test | 6 +++---
query_optimizer/tests/resolver/Create.test | 15 ++++++++++++---
storage/StorageConstants.hpp | 3 +--
4 files changed, 25 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/abfc5f27/query_optimizer/resolver/Resolver.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/resolver/Resolver.cpp b/query_optimizer/resolver/Resolver.cpp
index 45ecf33..2667ee9 100644
--- a/query_optimizer/resolver/Resolver.cpp
+++ b/query_optimizer/resolver/Resolver.cpp
@@ -602,16 +602,21 @@ StorageBlockLayoutDescription* Resolver::resolveBlockProperties(
// Resolve the Block size (size -> # of slots).
std::int64_t slots = kDefaultBlockSizeInSlots;
if (block_properties->hasBlockSizeMb()) {
- std::int64_t blocksizemb = block_properties->getBlockSizeMbValue();
- if (blocksizemb == -1) {
+ const std::int64_t block_size_in_mega_bytes = block_properties->getBlockSizeMbValue();
+ if (block_size_in_mega_bytes == -1) {
// Indicates an error condition if the property is present but getter returns -1.
THROW_SQL_ERROR_AT(block_properties->getBlockSizeMb())
<< "The BLOCKSIZEMB property must be an integer.";
+ } else if ((block_size_in_mega_bytes * kAMegaByte) % kSlotSizeBytes != 0) {
+ THROW_SQL_ERROR_AT(block_properties->getBlockSizeMb())
+ << "The BLOCKSIZEMB property must be multiple times of "
+ << std::to_string(kSlotSizeBytes / kAMegaByte) << "MB.";
}
- slots = (blocksizemb * kAMegaByte) / kSlotSizeBytes;
+
+ slots = (block_size_in_mega_bytes * kAMegaByte) / kSlotSizeBytes;
DLOG(INFO) << "Resolver using BLOCKSIZEMB of " << slots << " slots"
<< " which is " << (slots * kSlotSizeBytes) << " bytes versus"
- << " user requested " << (blocksizemb * kAMegaByte) << " bytes.";
+ << " user requested " << (block_size_in_mega_bytes * kAMegaByte) << " bytes.";
const std::uint64_t max_size_slots = kBlockSizeUpperBoundBytes / kSlotSizeBytes;
const std::uint64_t min_size_slots = kBlockSizeLowerBoundBytes / kSlotSizeBytes;
if (static_cast<std::uint64_t>(slots) < min_size_slots ||
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/abfc5f27/query_optimizer/tests/physical_generator/Create.test
----------------------------------------------------------------------
diff --git a/query_optimizer/tests/physical_generator/Create.test b/query_optimizer/tests/physical_generator/Create.test
index 58e15fa..8e6c64b 100644
--- a/query_optimizer/tests/physical_generator/Create.test
+++ b/query_optimizer/tests/physical_generator/Create.test
@@ -1,5 +1,5 @@
# Copyright 2011-2015 Quickstep Technologies LLC.
-# Copyright 2015 Pivotal Software, Inc.
+# Copyright 2015-2016 Pivotal Software, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -86,8 +86,8 @@ TopLevelPlan
+-AttributeReference[id=10,name=col11,relation=foo,type=Char(5) NULL]
+-AttributeReference[id=11,name=col12,relation=foo,type=VarChar(5) NULL]
==
-CREATE TABLE foo (col1 INT) WITH BLOCKPROPERTIES
- (TYPE compressed_columnstore, SORT col1, COMPRESS ALL, BLOCKSIZEMB 5);
+CREATE TABLE foo (col1 INT) WITH BLOCKPROPERTIES
+ (TYPE compressed_columnstore, SORT col1, COMPRESS ALL, BLOCKSIZEMB 4);
--
[Optimized Logical Plan]
TopLevelPlan
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/abfc5f27/query_optimizer/tests/resolver/Create.test
----------------------------------------------------------------------
diff --git a/query_optimizer/tests/resolver/Create.test b/query_optimizer/tests/resolver/Create.test
index 18beacd..63f7ac9 100644
--- a/query_optimizer/tests/resolver/Create.test
+++ b/query_optimizer/tests/resolver/Create.test
@@ -1,5 +1,5 @@
# Copyright 2011-2015 Quickstep Technologies LLC.
-# Copyright 2015 Pivotal Software, Inc.
+# Copyright 2015-2016 Pivotal Software, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -123,7 +123,7 @@ BLOCKPROPERTIES (BLOCKSIZEMB 1...
==
# Rowstores cannot have a sorted attribute.
-CREATE TABLE foo (attr INT) WITH BLOCKPROPERTIES
+CREATE TABLE foo (attr INT) WITH BLOCKPROPERTIES
(TYPE rowstore, SORT attr);
--
ERROR: The SORT property does not apply to this block type. (2 : 22)
@@ -167,7 +167,7 @@ ERROR: The COMPRESS property does not apply to this block type. (2 : 7)
==
# Compress property is required for compressed blocks.
-CREATE TABLE foo (attr INT) WITH
+CREATE TABLE foo (attr INT) WITH
BLOCKPROPERTIES (TYPE compressed_rowstore);
--
ERROR: The COMPRESS property must be specified as ALL or a list of attributes. (2 : 1)
@@ -202,6 +202,15 @@ ERROR: The BLOCKSIZEMB property must be an integer. (2 : 17)
^
==
+# BLOCKSIZEMB property must be multiple times of the slot size.
+CREATE TABLE foo (attr INT) WITH BLOCKPROPERTIES
+(TYPE rowstore, BLOCKSIZEMB 25);
+--
+ERROR: The BLOCKSIZEMB property must be multiple times of 2MB. (2 : 17)
+(TYPE rowstore, BLOCKSIZEMB 25);
+ ^
+==
+
# BLOCKSIZEMB must be greater than the minimum (defined in StorageConstants.hpp).
CREATE TABLE foo (attr INT) WITH BLOCKPROPERTIES
(TYPE rowstore, BLOCKSIZEMB 0);
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/abfc5f27/storage/StorageConstants.hpp
----------------------------------------------------------------------
diff --git a/storage/StorageConstants.hpp b/storage/StorageConstants.hpp
index de54345..154d2f7 100644
--- a/storage/StorageConstants.hpp
+++ b/storage/StorageConstants.hpp
@@ -44,8 +44,7 @@ const std::uint64_t kAMegaByte = (1 << 20);
// the SQL clause BLOCKPROPERTIES.
const std::uint64_t kBlockSizeUpperBoundBytes = kAGigaByte;
-// 2 Megabytes.
-const std::uint64_t kBlockSizeLowerBoundBytes = kAMegaByte << 1;
+const std::uint64_t kBlockSizeLowerBoundBytes = kSlotSizeBytes;
// The default size of a new relation in terms of the number of slots.
const std::uint64_t kDefaultBlockSizeInSlots = 1;
[25/50] [abbrv] incubator-quickstep git commit: Refactor
hasNUMAPlacementScheme. (#215)
Posted by zu...@apache.org.
Refactor hasNUMAPlacementScheme. (#215)
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/ab79cf64
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/ab79cf64
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/ab79cf64
Branch: refs/heads/work-order-serialization
Commit: ab79cf646eedc931ab011bd1f78cb73ed87a0436
Parents: 8939c25
Author: Zuyu ZHANG <zu...@users.noreply.github.com>
Authored: Sun May 8 06:21:51 2016 -0700
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Mon May 30 15:46:32 2016 -0700
----------------------------------------------------------------------
catalog/CatalogRelation.hpp | 19 ++++++++++++-------
1 file changed, 12 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/ab79cf64/catalog/CatalogRelation.hpp
----------------------------------------------------------------------
diff --git a/catalog/CatalogRelation.hpp b/catalog/CatalogRelation.hpp
index 3701090..312f3b4 100644
--- a/catalog/CatalogRelation.hpp
+++ b/catalog/CatalogRelation.hpp
@@ -31,6 +31,11 @@
#include "catalog/CatalogRelationSchema.hpp"
#include "catalog/CatalogTypedefs.hpp"
#include "catalog/IndexScheme.hpp"
+
+#ifdef QUICKSTEP_HAVE_LIBNUMA
+#include "catalog/NUMAPlacementScheme.hpp"
+#endif // QUICKSTEP_HAVE_LIBNUMA
+
#include "catalog/PartitionScheme.hpp"
#include "storage/StorageBlockInfo.hpp"
#include "storage/StorageBlockLayout.hpp"
@@ -40,10 +45,6 @@
#include "threading/SpinSharedMutex.hpp"
#include "utility/Macros.hpp"
-#ifdef QUICKSTEP_HAVE_LIBNUMA
-#include "catalog/NUMAPlacementScheme.hpp"
-#endif
-
namespace quickstep {
class CatalogDatabase;
@@ -135,16 +136,20 @@ class CatalogRelation : public CatalogRelationSchema {
return partition_scheme_.get();
}
-#ifdef QUICKSTEP_HAVE_LIBNUMA
/**
* @brief Check if a NUMA placement scheme is available for the relation.
*
* @return True if the relation has a NUMA placement scheme, false otherwise.
**/
bool hasNUMAPlacementScheme() const {
+#ifdef QUICKSTEP_HAVE_LIBNUMA
return placement_scheme_ != nullptr;
+#else
+ return false;
+#endif // QUICKSTEP_HAVE_LIBNUMA
}
+#ifdef QUICKSTEP_HAVE_LIBNUMA
/**
* @brief Get the NUMA placement scheme of the catalog relation.
* @warning This is only safe if hasNUMAPlacementScheme() is true.
@@ -182,7 +187,7 @@ class CatalogRelation : public CatalogRelationSchema {
void setNUMAPlacementScheme(NUMAPlacementScheme *placement_scheme) {
placement_scheme_.reset(placement_scheme);
}
-#endif
+#endif // QUICKSTEP_HAVE_LIBNUMA
/**
* @brief Check if an index scheme is available for the relation.
@@ -397,7 +402,7 @@ class CatalogRelation : public CatalogRelationSchema {
// of the relation and the NUMA nodes/sockets. It also maintains a mapping
// between the blocks of the relation and the NUMA nodes..
std::unique_ptr<NUMAPlacementScheme> placement_scheme_;
-#endif
+#endif // QUICKSTEP_HAVE_LIBNUMA
friend class CatalogTest;
[50/50] [abbrv] incubator-quickstep git commit: Serialized WorkOrders
as proto.
Posted by zu...@apache.org.
Serialized WorkOrders as proto.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/3e35844f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/3e35844f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/3e35844f
Branch: refs/heads/work-order-serialization
Commit: 3e35844f8f3c879b6fa89e331a73b5413d83f419
Parents: 50b4e55
Author: Zuyu Zhang <zz...@pivotal.io>
Authored: Tue Apr 12 16:55:48 2016 -0700
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Mon May 30 16:17:09 2016 -0700
----------------------------------------------------------------------
query_execution/CMakeLists.txt | 6 +
query_execution/WorkOrderProtosContainer.hpp | 147 +++++++++++++++++++
query_execution/tests/Foreman_unittest.cpp | 6 +
query_execution/tests/QueryManager_unittest.cpp | 6 +
relational_operators/AggregationOperator.cpp | 33 +++++
relational_operators/AggregationOperator.hpp | 12 ++
relational_operators/BuildHashOperator.cpp | 39 +++++
relational_operators/BuildHashOperator.hpp | 12 ++
relational_operators/CMakeLists.txt | 36 ++++-
relational_operators/CreateIndexOperator.hpp | 9 ++
relational_operators/CreateTableOperator.hpp | 8 +
relational_operators/DeleteOperator.cpp | 37 ++++-
relational_operators/DeleteOperator.hpp | 26 +++-
relational_operators/DestroyHashOperator.cpp | 16 ++
relational_operators/DestroyHashOperator.hpp | 3 +
relational_operators/DropTableOperator.cpp | 22 +++
relational_operators/DropTableOperator.hpp | 3 +
.../FinalizeAggregationOperator.cpp | 19 +++
.../FinalizeAggregationOperator.hpp | 3 +
relational_operators/HashJoinOperator.cpp | 123 ++++++++++++++++
relational_operators/HashJoinOperator.hpp | 21 +++
relational_operators/InsertOperator.cpp | 18 +++
relational_operators/InsertOperator.hpp | 3 +
.../NestedLoopsJoinOperator.cpp | 141 ++++++++++++++++++
.../NestedLoopsJoinOperator.hpp | 51 +++++++
relational_operators/RelationalOperator.hpp | 22 +++
relational_operators/SampleOperator.cpp | 100 ++++++++++---
relational_operators/SampleOperator.hpp | 12 ++
relational_operators/SaveBlocksOperator.cpp | 17 +++
relational_operators/SaveBlocksOperator.hpp | 3 +
relational_operators/SelectOperator.cpp | 42 ++++++
relational_operators/SelectOperator.hpp | 12 ++
relational_operators/SortMergeRunOperator.cpp | 67 +++++++++
relational_operators/SortMergeRunOperator.hpp | 12 ++
.../SortRunGenerationOperator.cpp | 38 +++++
.../SortRunGenerationOperator.hpp | 12 ++
relational_operators/TableGeneratorOperator.cpp | 19 ++-
relational_operators/TableGeneratorOperator.hpp | 5 +-
relational_operators/TextScanOperator.cpp | 77 ++++++++++
relational_operators/TextScanOperator.hpp | 3 +
relational_operators/UpdateOperator.cpp | 22 +++
relational_operators/UpdateOperator.hpp | 3 +
42 files changed, 1238 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3e35844f/query_execution/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_execution/CMakeLists.txt b/query_execution/CMakeLists.txt
index 7d9d601..9b5c69f 100644
--- a/query_execution/CMakeLists.txt
+++ b/query_execution/CMakeLists.txt
@@ -36,6 +36,7 @@ add_library(quickstep_queryexecution_QueryExecutionState ../empty_src.cpp QueryE
add_library(quickstep_queryexecution_QueryExecutionTypedefs ../empty_src.cpp QueryExecutionTypedefs.hpp)
add_library(quickstep_queryexecution_QueryExecutionUtil ../empty_src.cpp QueryExecutionUtil.hpp)
add_library(quickstep_queryexecution_QueryManager QueryManager.cpp QueryManager.hpp)
+add_library(quickstep_queryexecution_WorkOrderProtosContainer ../empty_src.cpp WorkOrderProtosContainer.hpp)
add_library(quickstep_queryexecution_WorkOrdersContainer WorkOrdersContainer.cpp WorkOrdersContainer.hpp)
add_library(quickstep_queryexecution_Worker Worker.cpp Worker.hpp)
add_library(quickstep_queryexecution_WorkerDirectory ../empty_src.cpp WorkerDirectory.hpp)
@@ -151,6 +152,10 @@ target_link_libraries(quickstep_queryexecution_QueryManager
quickstep_utility_DAG
quickstep_utility_Macros
tmb)
+target_link_libraries(quickstep_queryexecution_WorkOrderProtosContainer
+ glog
+ quickstep_relationaloperators_WorkOrder_proto
+ quickstep_utility_Macros)
target_link_libraries(quickstep_queryexecution_WorkOrdersContainer
glog
quickstep_relationaloperators_WorkOrder
@@ -187,6 +192,7 @@ target_link_libraries(quickstep_queryexecution
quickstep_queryexecution_QueryExecutionTypedefs
quickstep_queryexecution_QueryExecutionUtil
quickstep_queryexecution_QueryManager
+ quickstep_queryexecution_WorkOrderProtosContainer
quickstep_queryexecution_WorkOrdersContainer
quickstep_queryexecution_Worker
quickstep_queryexecution_WorkerDirectory
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3e35844f/query_execution/WorkOrderProtosContainer.hpp
----------------------------------------------------------------------
diff --git a/query_execution/WorkOrderProtosContainer.hpp b/query_execution/WorkOrderProtosContainer.hpp
new file mode 100644
index 0000000..e0d6d22
--- /dev/null
+++ b/query_execution/WorkOrderProtosContainer.hpp
@@ -0,0 +1,147 @@
+/**
+ * Copyright 2015-2016 Pivotal Software, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#ifndef QUICKSTEP_QUERY_EXECUTION_WORKORDER_PROTOS_CONTAINER_HPP_
+#define QUICKSTEP_QUERY_EXECUTION_WORKORDER_PROTOS_CONTAINER_HPP_
+
+#include <cstddef>
+#include <memory>
+#include <queue>
+#include <vector>
+
+#include "relational_operators/WorkOrder.pb.h" // IWYU pragma: keep
+#include "utility/Macros.hpp"
+
+#include "glog/logging.h"
+
+namespace quickstep {
+
+/** \addtogroup QueryExecution
+ * @{
+ */
+
+ /**
+ * @brief A container used in the distributed version to hold the normal
+ * (non-rebuild) WorkOrder protos for a given query.
+ *
+ * @note This container stays alive during the lifetime of the query.
+ **/
+class WorkOrderProtosContainer {
+ public:
+ /**
+ * @brief Constructor
+ *
+ * @param num_operators Number of operators in the query DAG.
+ **/
+ explicit WorkOrderProtosContainer(const std::size_t num_operators)
+ : num_operators_(num_operators),
+ operator_containers_(num_operators_) {
+ DCHECK_NE(num_operators_, 0u);
+ }
+
+ /**
+ * @brief Destructor.
+ *
+ * @note If the query is executed normally, we should never encounter a
+ * situation where at the time of deletion the WorkOrderProtosContainer has
+ * pending WorkOrders.
+ **/
+ ~WorkOrderProtosContainer() {
+ for (std::size_t op = 0; op < num_operators_; ++op) {
+ if (hasWorkOrderProto(op)) {
+ LOG(WARNING) << "Destroying a WorkOrderProtosContainer that still has pending WorkOrder protos.";
+ break;
+ }
+ }
+ }
+
+ /**
+ * @brief Check if there are some pending WorkOrders for the given operator.
+ *
+ * @param operator_index Index of the operator.
+ *
+ * @return If there are pending WorkOrders.
+ **/
+ bool hasWorkOrderProto(const std::size_t operator_index) const {
+ DCHECK_LT(operator_index, num_operators_);
+ return !operator_containers_[operator_index].empty();
+ }
+
+ /**
+ * @brief Get a WorkOrder for a given operator.
+ *
+ * @param operator_index The index of the operator.
+ *
+ * @return Release a WorkOrder proto. If no WorkOrder proto is available,
+ * return nullptr.
+ **/
+ serialization::WorkOrder* getWorkOrderProto(const std::size_t operator_index) {
+ DCHECK_LT(operator_index, num_operators_);
+
+ if (operator_containers_[operator_index].empty()) {
+ return nullptr;
+ }
+
+ serialization::WorkOrder *proto =
+ operator_containers_[operator_index].front().release();
+ operator_containers_[operator_index].pop();
+
+ return proto;
+ }
+
+ /**
+ * @brief Add a WorkOrder generated from a given
+ * operator.
+ *
+ * @param workorder A pointer to the WorkOrder to be added.
+ * @param operator_index The index of the operator in the query DAG.
+ **/
+ void addWorkOrderProto(serialization::WorkOrder *proto,
+ const std::size_t operator_index) {
+ DCHECK(proto != nullptr);
+ DCHECK_LT(operator_index, num_operators_);
+
+ operator_containers_[operator_index].emplace(
+ std::unique_ptr<serialization::WorkOrder>(proto));
+ }
+
+ /**
+ * @brief Get the number of all pending WorkOrders
+ * for a given operator.
+ *
+ * @param operator_index The index of the operator.
+ *
+ * @return The number of pending WorkOrders.
+ **/
+ std::size_t getNumWorkOrderProtos(const std::size_t operator_index) const {
+ DCHECK_LT(operator_index, num_operators_);
+ return operator_containers_[operator_index].size();
+ }
+
+ private:
+ const std::size_t num_operators_;
+
+ std::vector<std::queue<std::unique_ptr<serialization::WorkOrder>>> operator_containers_;
+
+ DISALLOW_COPY_AND_ASSIGN(WorkOrderProtosContainer);
+};
+
+/** @} */
+
+} // namespace quickstep
+
+
+#endif // QUICKSTEP_QUERY_EXECUTION_WORKORDER_PROTOS_CONTAINER_HPP_
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3e35844f/query_execution/tests/Foreman_unittest.cpp
----------------------------------------------------------------------
diff --git a/query_execution/tests/Foreman_unittest.cpp b/query_execution/tests/Foreman_unittest.cpp
index 47cc641..500c57d 100644
--- a/query_execution/tests/Foreman_unittest.cpp
+++ b/query_execution/tests/Foreman_unittest.cpp
@@ -58,6 +58,8 @@ using tmb::client_id;
namespace quickstep {
+class WorkOrderProtosContainer;
+
class MockWorkOrder : public WorkOrder {
public:
explicit MockWorkOrder(const int op_index)
@@ -167,6 +169,10 @@ class MockOperator: public RelationalOperator {
return num_calls_get_workorders_ == max_getworkorder_iters_;
}
+ bool getAllWorkOrderProtos(WorkOrderProtosContainer *container) override {
+ return true;
+ }
+
void feedInputBlock(const block_id input_block_id,
const relation_id input_relation_id) override {
++num_calls_feedblock_;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3e35844f/query_execution/tests/QueryManager_unittest.cpp
----------------------------------------------------------------------
diff --git a/query_execution/tests/QueryManager_unittest.cpp b/query_execution/tests/QueryManager_unittest.cpp
index 1b9be48..4293569 100644
--- a/query_execution/tests/QueryManager_unittest.cpp
+++ b/query_execution/tests/QueryManager_unittest.cpp
@@ -59,6 +59,8 @@ using tmb::client_id;
namespace quickstep {
+class WorkOrderProtosContainer;
+
class MockWorkOrder : public WorkOrder {
public:
explicit MockWorkOrder(const int op_index)
@@ -168,6 +170,10 @@ class MockOperator: public RelationalOperator {
return num_calls_get_workorders_ == max_getworkorder_iters_;
}
+ bool getAllWorkOrderProtos(WorkOrderProtosContainer *container) override {
+ return true;
+ }
+
void feedInputBlock(const block_id input_block_id,
const relation_id input_relation_id) override {
++num_calls_feedblock_;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3e35844f/relational_operators/AggregationOperator.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/AggregationOperator.cpp b/relational_operators/AggregationOperator.cpp
index 1b935ee..28844f6 100644
--- a/relational_operators/AggregationOperator.cpp
+++ b/relational_operators/AggregationOperator.cpp
@@ -20,7 +20,9 @@
#include <vector>
#include "query_execution/QueryContext.hpp"
+#include "query_execution/WorkOrderProtosContainer.hpp"
#include "query_execution/WorkOrdersContainer.hpp"
+#include "relational_operators/WorkOrder.pb.h"
#include "storage/AggregationOperationState.hpp"
#include "storage/StorageBlockInfo.hpp"
@@ -57,6 +59,37 @@ bool AggregationOperator::getAllWorkOrders(
}
}
+bool AggregationOperator::getAllWorkOrderProtos(WorkOrderProtosContainer *container) {
+ if (input_relation_is_stored_) {
+ if (!started_) {
+ for (const block_id input_block_id : input_relation_block_ids_) {
+ container->addWorkOrderProto(createWorkOrderProto(input_block_id), op_index_);
+ }
+ started_ = true;
+ }
+ return true;
+ } else {
+ while (num_workorders_generated_ < input_relation_block_ids_.size()) {
+ container->addWorkOrderProto(
+ createWorkOrderProto(input_relation_block_ids_[num_workorders_generated_]),
+ op_index_);
+ ++num_workorders_generated_;
+ }
+ return done_feeding_input_relation_;
+ }
+}
+
+serialization::WorkOrder* AggregationOperator::createWorkOrderProto(const block_id block) {
+ serialization::WorkOrder *proto = new serialization::WorkOrder;
+ proto->set_work_order_type(serialization::AGGREGATION);
+
+ proto->SetExtension(serialization::AggregationWorkOrder::block_id, block);
+ proto->SetExtension(serialization::AggregationWorkOrder::aggr_state_index, aggr_state_index_);
+
+ return proto;
+}
+
+
void AggregationWorkOrder::execute() {
state_->aggregateBlock(input_block_id_);
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3e35844f/relational_operators/AggregationOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/AggregationOperator.hpp b/relational_operators/AggregationOperator.hpp
index 0e74dfc..15da776 100644
--- a/relational_operators/AggregationOperator.hpp
+++ b/relational_operators/AggregationOperator.hpp
@@ -38,8 +38,11 @@ namespace quickstep {
class AggregationOperationState;
class StorageManager;
+class WorkOrderProtosContainer;
class WorkOrdersContainer;
+namespace serialization { class WorkOrder; }
+
/** \addtogroup RelationalOperators
* @{
*/
@@ -77,6 +80,8 @@ class AggregationOperator : public RelationalOperator {
const tmb::client_id scheduler_client_id,
tmb::MessageBus *bus) override;
+ bool getAllWorkOrderProtos(WorkOrderProtosContainer *container) override;
+
void feedInputBlock(const block_id input_block_id, const relation_id input_relation_id) override {
input_relation_block_ids_.push_back(input_block_id);
}
@@ -88,6 +93,13 @@ class AggregationOperator : public RelationalOperator {
}
private:
+ /**
+ * @brief Create Work Order proto.
+ *
+ * @param block The block id used in the Work Order.
+ **/
+ serialization::WorkOrder* createWorkOrderProto(const block_id block);
+
const bool input_relation_is_stored_;
std::vector<block_id> input_relation_block_ids_;
const QueryContext::aggregation_state_id aggr_state_index_;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3e35844f/relational_operators/BuildHashOperator.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/BuildHashOperator.cpp b/relational_operators/BuildHashOperator.cpp
index df92159..f320e1a 100644
--- a/relational_operators/BuildHashOperator.cpp
+++ b/relational_operators/BuildHashOperator.cpp
@@ -22,7 +22,9 @@
#include "catalog/CatalogRelation.hpp"
#include "query_execution/QueryContext.hpp"
+#include "query_execution/WorkOrderProtosContainer.hpp"
#include "query_execution/WorkOrdersContainer.hpp"
+#include "relational_operators/WorkOrder.pb.h"
#include "storage/HashTable.hpp"
#include "storage/StorageBlock.hpp"
#include "storage/StorageBlockInfo.hpp"
@@ -97,6 +99,43 @@ bool BuildHashOperator::getAllWorkOrders(
}
}
+bool BuildHashOperator::getAllWorkOrderProtos(WorkOrderProtosContainer *container) {
+ if (input_relation_is_stored_) {
+ if (!started_) {
+ for (const block_id input_block_id : input_relation_block_ids_) {
+ container->addWorkOrderProto(createWorkOrderProto(input_block_id), op_index_);
+ }
+ started_ = true;
+ }
+ return true;
+ } else {
+ while (num_workorders_generated_ < input_relation_block_ids_.size()) {
+ container->addWorkOrderProto(
+ createWorkOrderProto(input_relation_block_ids_[num_workorders_generated_]),
+ op_index_);
+ ++num_workorders_generated_;
+ }
+ return done_feeding_input_relation_;
+ }
+}
+
+serialization::WorkOrder* BuildHashOperator::createWorkOrderProto(const block_id block) {
+ serialization::WorkOrder *proto = new serialization::WorkOrder;
+ proto->set_work_order_type(serialization::BUILD_HASH);
+
+ proto->SetExtension(serialization::BuildHashWorkOrder::relation_id, input_relation_.getID());
+ for (const attribute_id attr_id : join_key_attributes_) {
+ proto->AddExtension(serialization::BuildHashWorkOrder::join_key_attributes, attr_id);
+ }
+ proto->SetExtension(serialization::BuildHashWorkOrder::any_join_key_attributes_nullable,
+ any_join_key_attributes_nullable_);
+ proto->SetExtension(serialization::BuildHashWorkOrder::join_hash_table_index, hash_table_index_);
+ proto->SetExtension(serialization::BuildHashWorkOrder::block_id, block);
+
+ return proto;
+}
+
+
void BuildHashWorkOrder::execute() {
BlockReference block(
storage_manager_->getBlock(build_block_id_, input_relation_));
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3e35844f/relational_operators/BuildHashOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/BuildHashOperator.hpp b/relational_operators/BuildHashOperator.hpp
index f9d830f..c7fd9dd 100644
--- a/relational_operators/BuildHashOperator.hpp
+++ b/relational_operators/BuildHashOperator.hpp
@@ -39,6 +39,7 @@ namespace quickstep {
class CatalogRelationSchema;
class StorageManager;
+class WorkOrderProtosContainer;
class WorkOrdersContainer;
struct TupleReference;
@@ -46,6 +47,8 @@ struct TupleReference;
template <typename, bool, bool, bool, bool> class HashTable;
typedef HashTable<TupleReference, true, false, false, true> JoinHashTable;
+namespace serialization { class WorkOrder; }
+
/** \addtogroup RelationalOperators
* @{
*/
@@ -93,6 +96,8 @@ class BuildHashOperator : public RelationalOperator {
const tmb::client_id scheduler_client_id,
tmb::MessageBus *bus) override;
+ bool getAllWorkOrderProtos(WorkOrderProtosContainer *container) override;
+
void feedInputBlock(const block_id input_block_id,
const relation_id input_relation_id) override {
input_relation_block_ids_.push_back(input_block_id);
@@ -106,6 +111,13 @@ class BuildHashOperator : public RelationalOperator {
}
private:
+ /**
+ * @brief Create Work Order proto.
+ *
+ * @param block The block id used in the Work Order.
+ **/
+ serialization::WorkOrder* createWorkOrderProto(const block_id block);
+
const CatalogRelation &input_relation_;
const bool input_relation_is_stored_;
const std::vector<attribute_id> join_key_attributes_;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3e35844f/relational_operators/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/relational_operators/CMakeLists.txt b/relational_operators/CMakeLists.txt
index eec5300..7c623ac 100644
--- a/relational_operators/CMakeLists.txt
+++ b/relational_operators/CMakeLists.txt
@@ -71,9 +71,11 @@ target_link_libraries(quickstep_relationaloperators_AggregationOperator
quickstep_catalog_CatalogRelation
quickstep_catalog_CatalogTypedefs
quickstep_queryexecution_QueryContext
+ quickstep_queryexecution_WorkOrderProtosContainer
quickstep_queryexecution_WorkOrdersContainer
quickstep_relationaloperators_RelationalOperator
quickstep_relationaloperators_WorkOrder
+ quickstep_relationaloperators_WorkOrder_proto
quickstep_storage_AggregationOperationState
quickstep_storage_StorageBlockInfo
quickstep_utility_Macros
@@ -83,9 +85,11 @@ target_link_libraries(quickstep_relationaloperators_BuildHashOperator
quickstep_catalog_CatalogRelation
quickstep_catalog_CatalogTypedefs
quickstep_queryexecution_QueryContext
+ quickstep_queryexecution_WorkOrderProtosContainer
quickstep_queryexecution_WorkOrdersContainer
quickstep_relationaloperators_RelationalOperator
quickstep_relationaloperators_WorkOrder
+ quickstep_relationaloperators_WorkOrder_proto
quickstep_storage_HashTable
quickstep_storage_StorageBlock
quickstep_storage_StorageBlockInfo
@@ -118,9 +122,11 @@ target_link_libraries(quickstep_relationaloperators_DeleteOperator
quickstep_queryexecution_QueryExecutionMessages_proto
quickstep_queryexecution_QueryExecutionTypedefs
quickstep_queryexecution_QueryExecutionUtil
+ quickstep_queryexecution_WorkOrderProtosContainer
quickstep_queryexecution_WorkOrdersContainer
quickstep_relationaloperators_RelationalOperator
quickstep_relationaloperators_WorkOrder
+ quickstep_relationaloperators_WorkOrder_proto
quickstep_storage_StorageBlock
quickstep_storage_StorageBlockInfo
quickstep_storage_StorageManager
@@ -130,9 +136,11 @@ target_link_libraries(quickstep_relationaloperators_DeleteOperator
target_link_libraries(quickstep_relationaloperators_DestroyHashOperator
glog
quickstep_queryexecution_QueryContext
+ quickstep_queryexecution_WorkOrderProtosContainer
quickstep_queryexecution_WorkOrdersContainer
quickstep_relationaloperators_RelationalOperator
quickstep_relationaloperators_WorkOrder
+ quickstep_relationaloperators_WorkOrder_proto
quickstep_utility_Macros
tmb)
target_link_libraries(quickstep_relationaloperators_DropTableOperator
@@ -141,9 +149,11 @@ target_link_libraries(quickstep_relationaloperators_DropTableOperator
quickstep_catalog_CatalogDatabaseLite
quickstep_catalog_CatalogRelation
quickstep_catalog_CatalogTypedefs
+ quickstep_queryexecution_WorkOrderProtosContainer
quickstep_queryexecution_WorkOrdersContainer
quickstep_relationaloperators_RelationalOperator
quickstep_relationaloperators_WorkOrder
+ quickstep_relationaloperators_WorkOrder_proto
quickstep_storage_StorageBlockInfo
quickstep_storage_StorageManager
quickstep_utility_Macros
@@ -153,9 +163,11 @@ target_link_libraries(quickstep_relationaloperators_FinalizeAggregationOperator
quickstep_catalog_CatalogRelation
quickstep_catalog_CatalogTypedefs
quickstep_queryexecution_QueryContext
+ quickstep_queryexecution_WorkOrderProtosContainer
quickstep_queryexecution_WorkOrdersContainer
quickstep_relationaloperators_RelationalOperator
quickstep_relationaloperators_WorkOrder
+ quickstep_relationaloperators_WorkOrder_proto
quickstep_storage_AggregationOperationState
quickstep_utility_Macros
tmb)
@@ -168,9 +180,11 @@ target_link_libraries(quickstep_relationaloperators_HashJoinOperator
quickstep_expressions_predicate_Predicate
quickstep_expressions_scalar_Scalar
quickstep_queryexecution_QueryContext
+ quickstep_queryexecution_WorkOrderProtosContainer
quickstep_queryexecution_WorkOrdersContainer
quickstep_relationaloperators_RelationalOperator
quickstep_relationaloperators_WorkOrder
+ quickstep_relationaloperators_WorkOrder_proto
quickstep_storage_HashTable
quickstep_storage_InsertDestination
quickstep_storage_StorageBlock
@@ -192,9 +206,11 @@ target_link_libraries(quickstep_relationaloperators_InsertOperator
quickstep_catalog_CatalogRelation
quickstep_catalog_CatalogTypedefs
quickstep_queryexecution_QueryContext
+ quickstep_queryexecution_WorkOrderProtosContainer
quickstep_queryexecution_WorkOrdersContainer
quickstep_relationaloperators_RelationalOperator
quickstep_relationaloperators_WorkOrder
+ quickstep_relationaloperators_WorkOrder_proto
quickstep_storage_InsertDestination
quickstep_types_containers_Tuple
quickstep_utility_Macros
@@ -207,9 +223,11 @@ target_link_libraries(quickstep_relationaloperators_NestedLoopsJoinOperator
quickstep_expressions_predicate_Predicate
quickstep_expressions_scalar_Scalar
quickstep_queryexecution_QueryContext
+ quickstep_queryexecution_WorkOrderProtosContainer
quickstep_queryexecution_WorkOrdersContainer
quickstep_relationaloperators_RelationalOperator
quickstep_relationaloperators_WorkOrder
+ quickstep_relationaloperators_WorkOrder_proto
quickstep_storage_InsertDestination
quickstep_storage_StorageBlock
quickstep_storage_StorageBlockInfo
@@ -243,9 +261,11 @@ target_link_libraries(quickstep_relationaloperators_SampleOperator
quickstep_catalog_CatalogRelation
quickstep_catalog_CatalogTypedefs
quickstep_queryexecution_QueryContext
- quickstep_relationaloperators_RelationalOperator
+ quickstep_queryexecution_WorkOrderProtosContainer
quickstep_queryexecution_WorkOrdersContainer
+ quickstep_relationaloperators_RelationalOperator
quickstep_relationaloperators_WorkOrder
+ quickstep_relationaloperators_WorkOrder_proto
quickstep_storage_InsertDestination
quickstep_storage_StorageBlock
quickstep_storage_StorageBlockInfo
@@ -255,9 +275,11 @@ target_link_libraries(quickstep_relationaloperators_SampleOperator
target_link_libraries(quickstep_relationaloperators_SaveBlocksOperator
glog
quickstep_catalog_CatalogTypedefs
+ quickstep_queryexecution_WorkOrderProtosContainer
quickstep_queryexecution_WorkOrdersContainer
quickstep_relationaloperators_RelationalOperator
quickstep_relationaloperators_WorkOrder
+ quickstep_relationaloperators_WorkOrder_proto
quickstep_storage_StorageBlockInfo
quickstep_storage_StorageManager
quickstep_utility_Macros
@@ -268,9 +290,11 @@ target_link_libraries(quickstep_relationaloperators_SelectOperator
quickstep_catalog_CatalogTypedefs
quickstep_catalog_PartitionSchemeHeader
quickstep_queryexecution_QueryContext
+ quickstep_queryexecution_WorkOrderProtosContainer
quickstep_queryexecution_WorkOrdersContainer
quickstep_relationaloperators_RelationalOperator
quickstep_relationaloperators_WorkOrder
+ quickstep_relationaloperators_WorkOrder_proto
quickstep_storage_InsertDestination
quickstep_storage_StorageBlock
quickstep_storage_StorageBlockInfo
@@ -287,11 +311,13 @@ target_link_libraries(quickstep_relationaloperators_SortMergeRunOperator
quickstep_catalog_CatalogTypedefs
quickstep_queryexecution_QueryContext
quickstep_queryexecution_QueryExecutionTypedefs
+ quickstep_queryexecution_WorkOrderProtosContainer
quickstep_queryexecution_WorkOrdersContainer
quickstep_relationaloperators_RelationalOperator
quickstep_relationaloperators_SortMergeRunOperatorHelpers
quickstep_relationaloperators_SortMergeRunOperator_proto
quickstep_relationaloperators_WorkOrder
+ quickstep_relationaloperators_WorkOrder_proto
quickstep_storage_StorageBlockInfo
quickstep_threading_ThreadIDBasedMap
quickstep_utility_Macros
@@ -324,9 +350,11 @@ target_link_libraries(quickstep_relationaloperators_SortRunGenerationOperator
quickstep_catalog_CatalogRelation
quickstep_catalog_CatalogTypedefs
quickstep_queryexecution_QueryContext
+ quickstep_queryexecution_WorkOrderProtosContainer
quickstep_queryexecution_WorkOrdersContainer
quickstep_relationaloperators_RelationalOperator
quickstep_relationaloperators_WorkOrder
+ quickstep_relationaloperators_WorkOrder_proto
quickstep_storage_InsertDestination
quickstep_storage_StorageBlock
quickstep_storage_StorageBlockInfo
@@ -341,9 +369,11 @@ target_link_libraries(quickstep_relationaloperators_TableGeneratorOperator
quickstep_catalog_CatalogTypedefs
quickstep_expressions_tablegenerator_GeneratorFunctionHandle
quickstep_queryexecution_QueryContext
+ quickstep_queryexecution_WorkOrderProtosContainer
quickstep_queryexecution_WorkOrdersContainer
quickstep_relationaloperators_RelationalOperator
quickstep_relationaloperators_WorkOrder
+ quickstep_relationaloperators_WorkOrder_proto
quickstep_storage_InsertDestination
quickstep_storage_StorageBlockInfo
quickstep_types_containers_ColumnVectorsValueAccessor
@@ -360,10 +390,12 @@ target_link_libraries(quickstep_relationaloperators_TextScanOperator
quickstep_queryexecution_QueryExecutionMessages_proto
quickstep_queryexecution_QueryExecutionTypedefs
quickstep_queryexecution_QueryExecutionUtil
+ quickstep_queryexecution_WorkOrderProtosContainer
quickstep_queryexecution_WorkOrdersContainer
quickstep_relationaloperators_RelationalOperator
quickstep_relationaloperators_TextScanOperator_proto
quickstep_relationaloperators_WorkOrder
+ quickstep_relationaloperators_WorkOrder_proto
quickstep_storage_InsertDestination
quickstep_storage_StorageBlob
quickstep_storage_StorageBlockInfo
@@ -385,9 +417,11 @@ target_link_libraries(quickstep_relationaloperators_UpdateOperator
quickstep_queryexecution_QueryExecutionMessages_proto
quickstep_queryexecution_QueryExecutionTypedefs
quickstep_queryexecution_QueryExecutionUtil
+ quickstep_queryexecution_WorkOrderProtosContainer
quickstep_queryexecution_WorkOrdersContainer
quickstep_relationaloperators_RelationalOperator
quickstep_relationaloperators_WorkOrder
+ quickstep_relationaloperators_WorkOrder_proto
quickstep_storage_InsertDestination
quickstep_storage_StorageBlock
quickstep_storage_StorageBlockInfo
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3e35844f/relational_operators/CreateIndexOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/CreateIndexOperator.hpp b/relational_operators/CreateIndexOperator.hpp
index 2bfacc4..03f7fed 100644
--- a/relational_operators/CreateIndexOperator.hpp
+++ b/relational_operators/CreateIndexOperator.hpp
@@ -1,6 +1,7 @@
/**
* Copyright 2016, Quickstep Research Group, Computer Sciences Department,
* University of Wisconsin\u2014Madison.
+ * Copyright 2016 Pivotal Software, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -36,6 +37,7 @@ namespace quickstep {
class CatalogRelation;
class QueryContext;
class StorageManager;
+class WorkOrderProtosContainer;
class WorkOrdersContainer;
/** \addtogroup RelationalOperators
@@ -72,6 +74,13 @@ class CreateIndexOperator : public RelationalOperator {
const tmb::client_id scheduler_client_id,
tmb::MessageBus *bus) override;
+ /**
+ * @note no WorkOrder proto generated for this operator.
+ **/
+ bool getAllWorkOrderProtos(WorkOrderProtosContainer *container) override {
+ return true;
+ }
+
void updateCatalogOnCompletion() override;
private:
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3e35844f/relational_operators/CreateTableOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/CreateTableOperator.hpp b/relational_operators/CreateTableOperator.hpp
index 98f3253..eac6828 100644
--- a/relational_operators/CreateTableOperator.hpp
+++ b/relational_operators/CreateTableOperator.hpp
@@ -35,6 +35,7 @@ namespace quickstep {
class CatalogDatabase;
class QueryContext;
class StorageManager;
+class WorkOrderProtosContainer;
class WorkOrdersContainer;
/** \addtogroup RelationalOperators
@@ -70,6 +71,13 @@ class CreateTableOperator : public RelationalOperator {
const tmb::client_id scheduler_client_id,
tmb::MessageBus *bus) override;
+ /**
+ * @note no WorkOrder proto generated for this operator.
+ **/
+ bool getAllWorkOrderProtos(WorkOrderProtosContainer *container) override {
+ return true;
+ }
+
void updateCatalogOnCompletion() override;
private:
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3e35844f/relational_operators/DeleteOperator.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/DeleteOperator.cpp b/relational_operators/DeleteOperator.cpp
index 2c2c6de..4e02caf 100644
--- a/relational_operators/DeleteOperator.cpp
+++ b/relational_operators/DeleteOperator.cpp
@@ -26,13 +26,14 @@
#include "query_execution/QueryContext.hpp"
#include "query_execution/QueryExecutionMessages.pb.h"
#include "query_execution/QueryExecutionUtil.hpp"
+#include "query_execution/WorkOrderProtosContainer.hpp"
#include "query_execution/WorkOrdersContainer.hpp"
+#include "relational_operators/WorkOrder.pb.h"
#include "storage/StorageBlock.hpp"
#include "storage/StorageBlockInfo.hpp"
#include "storage/StorageManager.hpp"
#include "threading/ThreadIDBasedMap.hpp"
-
#include "glog/logging.h"
#include "tmb/id_typedefs.h"
@@ -83,6 +84,40 @@ bool DeleteOperator::getAllWorkOrders(
}
}
+bool DeleteOperator::getAllWorkOrderProtos(WorkOrderProtosContainer *container) {
+ if (relation_is_stored_) {
+ // If relation_ is stored, iterate over the list of blocks in relation_.
+ if (!started_) {
+ for (const block_id input_block_id : relation_block_ids_) {
+ container->addWorkOrderProto(createWorkOrderProto(input_block_id), op_index_);
+ }
+ started_ = true;
+ }
+ return true;
+ } else {
+ while (num_workorders_generated_ < relation_block_ids_.size()) {
+ container->addWorkOrderProto(
+ createWorkOrderProto(relation_block_ids_[num_workorders_generated_]),
+ op_index_);
+ ++num_workorders_generated_;
+ }
+ return done_feeding_input_relation_;
+ }
+}
+
+serialization::WorkOrder* DeleteOperator::createWorkOrderProto(const block_id block) {
+ serialization::WorkOrder *proto = new serialization::WorkOrder;
+ proto->set_work_order_type(serialization::DELETE);
+
+ proto->SetExtension(serialization::DeleteWorkOrder::operator_index, op_index_);
+ proto->SetExtension(serialization::DeleteWorkOrder::relation_id, relation_.getID());
+ proto->SetExtension(serialization::DeleteWorkOrder::predicate_index, predicate_index_);
+ proto->SetExtension(serialization::DeleteWorkOrder::block_id, block);
+
+ return proto;
+}
+
+
void DeleteWorkOrder::execute() {
MutableBlockReference block(
storage_manager_->getBlockMutable(input_block_id_, input_relation_));
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3e35844f/relational_operators/DeleteOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/DeleteOperator.hpp b/relational_operators/DeleteOperator.hpp
index 1d44552..6dc206b 100644
--- a/relational_operators/DeleteOperator.hpp
+++ b/relational_operators/DeleteOperator.hpp
@@ -41,8 +41,11 @@ namespace quickstep {
class CatalogRelationSchema;
class Predicate;
class StorageManager;
+class WorkOrderProtosContainer;
class WorkOrdersContainer;
+namespace serialization { class WorkOrder; }
+
/** \addtogroup RelationalOperators
* @{
*/
@@ -65,13 +68,13 @@ class DeleteOperator : public RelationalOperator {
DeleteOperator(const CatalogRelation &relation,
const QueryContext::predicate_id predicate_index,
const bool relation_is_stored)
- : relation_(relation),
- predicate_index_(predicate_index),
- relation_is_stored_(relation_is_stored),
- started_(false),
- relation_block_ids_(relation_is_stored ? relation.getBlocksSnapshot()
- : std::vector<block_id>()),
- num_workorders_generated_(0) {}
+ : relation_(relation),
+ predicate_index_(predicate_index),
+ relation_is_stored_(relation_is_stored),
+ started_(false),
+ relation_block_ids_(relation_is_stored ? relation.getBlocksSnapshot()
+ : std::vector<block_id>()),
+ num_workorders_generated_(0) {}
~DeleteOperator() override {}
@@ -81,6 +84,8 @@ class DeleteOperator : public RelationalOperator {
const tmb::client_id scheduler_client_id,
tmb::MessageBus *bus) override;
+ bool getAllWorkOrderProtos(WorkOrderProtosContainer *container) override;
+
const relation_id getOutputRelationID() const override {
return relation_.getID();
}
@@ -98,6 +103,13 @@ class DeleteOperator : public RelationalOperator {
}
private:
+ /**
+ * @brief Create Work Order proto.
+ *
+ * @param block The block id used in the Work Order.
+ **/
+ serialization::WorkOrder* createWorkOrderProto(const block_id block);
+
const CatalogRelation &relation_;
const QueryContext::predicate_id predicate_index_;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3e35844f/relational_operators/DestroyHashOperator.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/DestroyHashOperator.cpp b/relational_operators/DestroyHashOperator.cpp
index c2220d5..4567cf5 100644
--- a/relational_operators/DestroyHashOperator.cpp
+++ b/relational_operators/DestroyHashOperator.cpp
@@ -18,7 +18,9 @@
#include "relational_operators/DestroyHashOperator.hpp"
#include "query_execution/QueryContext.hpp"
+#include "query_execution/WorkOrderProtosContainer.hpp"
#include "query_execution/WorkOrdersContainer.hpp"
+#include "relational_operators/WorkOrder.pb.h"
#include "tmb/id_typedefs.h"
@@ -38,6 +40,20 @@ bool DestroyHashOperator::getAllWorkOrders(
return work_generated_;
}
+bool DestroyHashOperator::getAllWorkOrderProtos(WorkOrderProtosContainer *container) {
+ if (blocking_dependencies_met_ && !work_generated_) {
+ work_generated_ = true;
+
+ serialization::WorkOrder *proto = new serialization::WorkOrder;
+ proto->set_work_order_type(serialization::DESTROY_HASH);
+ proto->SetExtension(serialization::DestroyHashWorkOrder::join_hash_table_index, hash_table_index_);
+
+ container->addWorkOrderProto(proto, op_index_);
+ }
+ return work_generated_;
+}
+
+
void DestroyHashWorkOrder::execute() {
query_context_->destroyJoinHashTable(hash_table_index_);
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3e35844f/relational_operators/DestroyHashOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/DestroyHashOperator.hpp b/relational_operators/DestroyHashOperator.hpp
index 46331ba..67cbd4d 100644
--- a/relational_operators/DestroyHashOperator.hpp
+++ b/relational_operators/DestroyHashOperator.hpp
@@ -32,6 +32,7 @@ namespace tmb { class MessageBus; }
namespace quickstep {
class StorageManager;
+class WorkOrderProtosContainer;
class WorkOrdersContainer;
/** \addtogroup RelationalOperators
@@ -60,6 +61,8 @@ class DestroyHashOperator : public RelationalOperator {
const tmb::client_id scheduler_client_id,
tmb::MessageBus *bus) override;
+ bool getAllWorkOrderProtos(WorkOrderProtosContainer *container) override;
+
private:
const QueryContext::join_hash_table_id hash_table_index_;
bool work_generated_;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3e35844f/relational_operators/DropTableOperator.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/DropTableOperator.cpp b/relational_operators/DropTableOperator.cpp
index f3a3a2c..75ab221 100644
--- a/relational_operators/DropTableOperator.cpp
+++ b/relational_operators/DropTableOperator.cpp
@@ -24,7 +24,9 @@
#include "catalog/CatalogDatabaseLite.hpp"
#include "catalog/CatalogRelation.hpp"
#include "catalog/CatalogTypedefs.hpp"
+#include "query_execution/WorkOrderProtosContainer.hpp"
#include "query_execution/WorkOrdersContainer.hpp"
+#include "relational_operators/WorkOrder.pb.h"
#include "storage/StorageBlockInfo.hpp"
#include "storage/StorageManager.hpp"
@@ -54,6 +56,26 @@ bool DropTableOperator::getAllWorkOrders(
return work_generated_;
}
+bool DropTableOperator::getAllWorkOrderProtos(WorkOrderProtosContainer *container) {
+ if (blocking_dependencies_met_ && !work_generated_) {
+ work_generated_ = true;
+
+ serialization::WorkOrder *proto = new serialization::WorkOrder;
+ proto->set_work_order_type(serialization::DROP_TABLE);
+
+ std::vector<block_id> relation_blocks(relation_.getBlocksSnapshot());
+ for (const block_id relation_block : relation_blocks) {
+ proto->AddExtension(serialization::DropTableWorkOrder::block_ids, relation_block);
+ }
+
+ container->addWorkOrderProto(proto, op_index_);
+
+ database_->setStatus(CatalogDatabase::Status::kPendingBlockDeletions);
+ }
+
+ return work_generated_;
+}
+
void DropTableOperator::updateCatalogOnCompletion() {
const relation_id rel_id = relation_.getID();
if (only_drop_blocks_) {
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3e35844f/relational_operators/DropTableOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/DropTableOperator.hpp b/relational_operators/DropTableOperator.hpp
index bf9b1b1..ac3d96a 100644
--- a/relational_operators/DropTableOperator.hpp
+++ b/relational_operators/DropTableOperator.hpp
@@ -40,6 +40,7 @@ class CatalogDatabaseLite;
class CatalogRelation;
class QueryContext;
class StorageManager;
+class WorkOrderProtosContainer;
class WorkOrdersContainer;
/** \addtogroup RelationalOperators
@@ -75,6 +76,8 @@ class DropTableOperator : public RelationalOperator {
const tmb::client_id scheduler_client_id,
tmb::MessageBus *bus) override;
+ bool getAllWorkOrderProtos(WorkOrderProtosContainer *container) override;
+
void updateCatalogOnCompletion() override;
private:
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3e35844f/relational_operators/FinalizeAggregationOperator.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/FinalizeAggregationOperator.cpp b/relational_operators/FinalizeAggregationOperator.cpp
index 410ec69..7db36c0 100644
--- a/relational_operators/FinalizeAggregationOperator.cpp
+++ b/relational_operators/FinalizeAggregationOperator.cpp
@@ -18,7 +18,9 @@
#include "relational_operators/FinalizeAggregationOperator.hpp"
#include "query_execution/QueryContext.hpp"
+#include "query_execution/WorkOrderProtosContainer.hpp"
#include "query_execution/WorkOrdersContainer.hpp"
+#include "relational_operators/WorkOrder.pb.h"
#include "storage/AggregationOperationState.hpp"
#include "glog/logging.h"
@@ -45,6 +47,23 @@ bool FinalizeAggregationOperator::getAllWorkOrders(
return started_;
}
+bool FinalizeAggregationOperator::getAllWorkOrderProtos(WorkOrderProtosContainer *container) {
+ if (blocking_dependencies_met_ && !started_) {
+ started_ = true;
+
+ serialization::WorkOrder *proto = new serialization::WorkOrder;
+ proto->set_work_order_type(serialization::FINALIZE_AGGREGATION);
+ proto->SetExtension(serialization::FinalizeAggregationWorkOrder::aggr_state_index,
+ aggr_state_index_);
+ proto->SetExtension(serialization::FinalizeAggregationWorkOrder::insert_destination_index,
+ output_destination_index_);
+
+ container->addWorkOrderProto(proto, op_index_);
+ }
+ return started_;
+}
+
+
void FinalizeAggregationWorkOrder::execute() {
state_->finalizeAggregate(output_destination_);
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3e35844f/relational_operators/FinalizeAggregationOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/FinalizeAggregationOperator.hpp b/relational_operators/FinalizeAggregationOperator.hpp
index fb9608a..2254f6f 100644
--- a/relational_operators/FinalizeAggregationOperator.hpp
+++ b/relational_operators/FinalizeAggregationOperator.hpp
@@ -38,6 +38,7 @@ namespace quickstep {
class InsertDestination;
class StorageManager;
+class WorkOrderProtosContainer;
class WorkOrdersContainer;
/** \addtogroup RelationalOperators
@@ -74,6 +75,8 @@ class FinalizeAggregationOperator : public RelationalOperator {
const tmb::client_id scheduler_client_id,
tmb::MessageBus *bus) override;
+ bool getAllWorkOrderProtos(WorkOrderProtosContainer *container) override;
+
QueryContext::insert_destination_id getInsertDestinationID() const override {
return output_destination_index_;
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3e35844f/relational_operators/HashJoinOperator.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/HashJoinOperator.cpp b/relational_operators/HashJoinOperator.cpp
index aa03794..87ce7da 100644
--- a/relational_operators/HashJoinOperator.cpp
+++ b/relational_operators/HashJoinOperator.cpp
@@ -31,7 +31,9 @@
#include "expressions/predicate/Predicate.hpp"
#include "expressions/scalar/Scalar.hpp"
#include "query_execution/QueryContext.hpp"
+#include "query_execution/WorkOrderProtosContainer.hpp"
#include "query_execution/WorkOrdersContainer.hpp"
+#include "relational_operators/WorkOrder.pb.h"
#include "storage/HashTable.hpp"
#include "storage/InsertDestination.hpp"
#include "storage/StorageBlock.hpp"
@@ -386,6 +388,127 @@ bool HashJoinOperator::getAllOuterJoinWorkOrders(
return false;
}
+bool HashJoinOperator::getAllWorkOrderProtos(WorkOrderProtosContainer *container) {
+ switch (join_type_) {
+ case JoinType::kInnerJoin:
+ return getAllNonOuterJoinWorkOrderProtos(container, serialization::HashJoinWorkOrder::HASH_INNER_JOIN);
+ case JoinType::kLeftSemiJoin:
+ return getAllNonOuterJoinWorkOrderProtos(container, serialization::HashJoinWorkOrder::HASH_SEMI_JOIN);
+ case JoinType::kLeftAntiJoin:
+ return getAllNonOuterJoinWorkOrderProtos(container, serialization::HashJoinWorkOrder::HASH_ANTI_JOIN);
+ case JoinType::kLeftOuterJoin:
+ return getAllOuterJoinWorkOrderProtos(container);
+ default:
+ LOG(FATAL) << "Unknown join type in HashJoinOperator::getAllWorkOrderProtos()";
+ }
+}
+
+bool HashJoinOperator::getAllNonOuterJoinWorkOrderProtos(
+ WorkOrderProtosContainer *container,
+ const serialization::HashJoinWorkOrder::HashJoinWorkOrderType hash_join_type) {
+ // We wait until the building of global hash table is complete.
+ if (!blocking_dependencies_met_) {
+ return false;
+ }
+
+ if (probe_relation_is_stored_) {
+ if (!started_) {
+ for (const block_id probe_block_id : probe_relation_block_ids_) {
+ container->addWorkOrderProto(
+ createNonOuterJoinWorkOrderProto(hash_join_type, probe_block_id),
+ op_index_);
+ }
+ started_ = true;
+ }
+ return true;
+ } else {
+ while (num_workorders_generated_ < probe_relation_block_ids_.size()) {
+ container->addWorkOrderProto(
+ createNonOuterJoinWorkOrderProto(hash_join_type,
+ probe_relation_block_ids_[num_workorders_generated_]),
+ op_index_);
+ ++num_workorders_generated_;
+ }
+
+ return done_feeding_input_relation_;
+ }
+}
+
+serialization::WorkOrder* HashJoinOperator::createNonOuterJoinWorkOrderProto(
+ const serialization::HashJoinWorkOrder::HashJoinWorkOrderType hash_join_type,
+ const block_id block) {
+ serialization::WorkOrder *proto = new serialization::WorkOrder;
+ proto->set_work_order_type(serialization::HASH_JOIN);
+
+ proto->SetExtension(serialization::HashJoinWorkOrder::hash_join_work_order_type, hash_join_type);
+ proto->SetExtension(serialization::HashJoinWorkOrder::build_relation_id, build_relation_.getID());
+ proto->SetExtension(serialization::HashJoinWorkOrder::probe_relation_id, probe_relation_.getID());
+ for (const attribute_id attr_id : join_key_attributes_) {
+ proto->AddExtension(serialization::HashJoinWorkOrder::join_key_attributes, attr_id);
+ }
+ proto->SetExtension(serialization::HashJoinWorkOrder::any_join_key_attributes_nullable,
+ any_join_key_attributes_nullable_);
+ proto->SetExtension(serialization::HashJoinWorkOrder::insert_destination_index, output_destination_index_);
+ proto->SetExtension(serialization::HashJoinWorkOrder::join_hash_table_index, hash_table_index_);
+ proto->SetExtension(serialization::HashJoinWorkOrder::selection_index, selection_index_);
+ proto->SetExtension(serialization::HashJoinWorkOrder::block_id, block);
+ proto->SetExtension(serialization::HashJoinWorkOrder::residual_predicate_index, residual_predicate_index_);
+
+ return proto;
+}
+
+bool HashJoinOperator::getAllOuterJoinWorkOrderProtos(WorkOrderProtosContainer *container) {
+ // We wait until the building of global hash table is complete.
+ if (!blocking_dependencies_met_) {
+ return false;
+ }
+
+ if (probe_relation_is_stored_) {
+ if (!started_) {
+ for (const block_id probe_block_id : probe_relation_block_ids_) {
+ container->addWorkOrderProto(createOuterJoinWorkOrderProto(probe_block_id), op_index_);
+ }
+ started_ = true;
+ }
+ return true;
+ } else {
+ while (num_workorders_generated_ < probe_relation_block_ids_.size()) {
+ container->addWorkOrderProto(
+ createOuterJoinWorkOrderProto(probe_relation_block_ids_[num_workorders_generated_]),
+ op_index_);
+ ++num_workorders_generated_;
+ }
+
+ return done_feeding_input_relation_;
+ }
+}
+
+serialization::WorkOrder* HashJoinOperator::createOuterJoinWorkOrderProto(const block_id block) {
+ serialization::WorkOrder *proto = new serialization::WorkOrder;
+ proto->set_work_order_type(serialization::HASH_JOIN);
+
+ proto->SetExtension(serialization::HashJoinWorkOrder::hash_join_work_order_type,
+ serialization::HashJoinWorkOrder::HASH_OUTER_JOIN);
+ proto->SetExtension(serialization::HashJoinWorkOrder::build_relation_id, build_relation_.getID());
+ proto->SetExtension(serialization::HashJoinWorkOrder::probe_relation_id, probe_relation_.getID());
+ for (const attribute_id attr_id : join_key_attributes_) {
+ proto->AddExtension(serialization::HashJoinWorkOrder::join_key_attributes, attr_id);
+ }
+ proto->SetExtension(serialization::HashJoinWorkOrder::any_join_key_attributes_nullable,
+ any_join_key_attributes_nullable_);
+ proto->SetExtension(serialization::HashJoinWorkOrder::insert_destination_index, output_destination_index_);
+ proto->SetExtension(serialization::HashJoinWorkOrder::join_hash_table_index, hash_table_index_);
+ proto->SetExtension(serialization::HashJoinWorkOrder::selection_index, selection_index_);
+ proto->SetExtension(serialization::HashJoinWorkOrder::block_id, block);
+
+ for (const bool is_attribute_on_build : is_selection_on_build_) {
+ proto->AddExtension(serialization::HashJoinWorkOrder::is_selection_on_build, is_attribute_on_build);
+ }
+
+ return proto;
+}
+
+
void HashInnerJoinWorkOrder::execute() {
if (FLAGS_vector_based_joined_tuple_collector) {
executeWithCollectorType<VectorBasedJoinedTupleCollector>();
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3e35844f/relational_operators/HashJoinOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/HashJoinOperator.hpp b/relational_operators/HashJoinOperator.hpp
index fcc087a..58610ed 100644
--- a/relational_operators/HashJoinOperator.hpp
+++ b/relational_operators/HashJoinOperator.hpp
@@ -30,6 +30,7 @@
#include "query_execution/QueryContext.hpp"
#include "relational_operators/RelationalOperator.hpp"
#include "relational_operators/WorkOrder.hpp"
+#include "relational_operators/WorkOrder.pb.h"
#include "storage/HashTable.hpp"
#include "storage/StorageBlockInfo.hpp"
#include "utility/Macros.hpp"
@@ -47,6 +48,7 @@ class InsertDestination;
class Predicate;
class Scalar;
class StorageManager;
+class WorkOrderProtosContainer;
class WorkOrdersContainer;
/** \addtogroup RelationalOperators
@@ -157,6 +159,8 @@ class HashJoinOperator : public RelationalOperator {
const tmb::client_id scheduler_client_id,
tmb::MessageBus *bus) override;
+ bool getAllWorkOrderProtos(WorkOrderProtosContainer *container) override;
+
void feedInputBlock(const block_id input_block_id,
const relation_id input_relation_id) override {
DCHECK(input_relation_id == probe_relation_.getID());
@@ -198,6 +202,23 @@ class HashJoinOperator : public RelationalOperator {
QueryContext *query_context,
StorageManager *storage_manager);
+ bool getAllNonOuterJoinWorkOrderProtos(
+ WorkOrderProtosContainer *container,
+ const serialization::HashJoinWorkOrder::HashJoinWorkOrderType hash_join_type);
+
+ serialization::WorkOrder* createNonOuterJoinWorkOrderProto(
+ const serialization::HashJoinWorkOrder::HashJoinWorkOrderType hash_join_type,
+ const block_id block);
+
+ bool getAllOuterJoinWorkOrderProtos(WorkOrderProtosContainer *container);
+
+ /**
+ * @brief Create HashOuterJoinWorkOrder proto.
+ *
+ * @param block The block id used in the Work Order.
+ **/
+ serialization::WorkOrder* createOuterJoinWorkOrderProto(const block_id block);
+
const CatalogRelation &build_relation_;
const CatalogRelation &probe_relation_;
const bool probe_relation_is_stored_;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3e35844f/relational_operators/InsertOperator.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/InsertOperator.cpp b/relational_operators/InsertOperator.cpp
index 8d083e5..3dfde30 100644
--- a/relational_operators/InsertOperator.cpp
+++ b/relational_operators/InsertOperator.cpp
@@ -20,7 +20,9 @@
#include <memory>
#include "query_execution/QueryContext.hpp"
+#include "query_execution/WorkOrderProtosContainer.hpp"
#include "query_execution/WorkOrdersContainer.hpp"
+#include "relational_operators/WorkOrder.pb.h"
#include "storage/InsertDestination.hpp"
#include "glog/logging.h"
@@ -47,6 +49,22 @@ bool InsertOperator::getAllWorkOrders(
return work_generated_;
}
+bool InsertOperator::getAllWorkOrderProtos(WorkOrderProtosContainer *container) {
+ if (blocking_dependencies_met_ && !work_generated_) {
+ work_generated_ = true;
+
+ serialization::WorkOrder *proto = new serialization::WorkOrder;
+ proto->set_work_order_type(serialization::INSERT);
+ proto->SetExtension(serialization::InsertWorkOrder::insert_destination_index, output_destination_index_);
+ proto->SetExtension(serialization::InsertWorkOrder::tuple_index, tuple_index_);
+
+ container->addWorkOrderProto(proto, op_index_);
+ }
+
+ return work_generated_;
+}
+
+
void InsertWorkOrder::execute() {
output_destination_->insertTuple(*tuple_);
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3e35844f/relational_operators/InsertOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/InsertOperator.hpp b/relational_operators/InsertOperator.hpp
index 8a06c94..133e67f 100644
--- a/relational_operators/InsertOperator.hpp
+++ b/relational_operators/InsertOperator.hpp
@@ -38,6 +38,7 @@ namespace quickstep {
class InsertDestination;
class StorageManager;
+class WorkOrderProtosContainer;
class WorkOrdersContainer;
/** \addtogroup RelationalOperators
@@ -73,6 +74,8 @@ class InsertOperator : public RelationalOperator {
const tmb::client_id scheduler_client_id,
tmb::MessageBus *bus) override;
+ bool getAllWorkOrderProtos(WorkOrderProtosContainer *container) override;
+
QueryContext::insert_destination_id getInsertDestinationID() const override {
return output_destination_index_;
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3e35844f/relational_operators/NestedLoopsJoinOperator.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/NestedLoopsJoinOperator.cpp b/relational_operators/NestedLoopsJoinOperator.cpp
index 5cc498b..ca90df6 100644
--- a/relational_operators/NestedLoopsJoinOperator.cpp
+++ b/relational_operators/NestedLoopsJoinOperator.cpp
@@ -26,7 +26,9 @@
#include "expressions/predicate/Predicate.hpp"
#include "expressions/scalar/Scalar.hpp"
#include "query_execution/QueryContext.hpp"
+#include "query_execution/WorkOrderProtosContainer.hpp"
#include "query_execution/WorkOrdersContainer.hpp"
+#include "relational_operators/WorkOrder.pb.h"
#include "storage/InsertDestination.hpp"
#include "storage/StorageBlock.hpp"
#include "storage/StorageBlockInfo.hpp"
@@ -147,6 +149,72 @@ bool NestedLoopsJoinOperator::getAllWorkOrders(
}
}
+bool NestedLoopsJoinOperator::getAllWorkOrderProtos(WorkOrderProtosContainer *container) {
+ if (left_relation_is_stored_ && right_relation_is_stored_) {
+ // Make sure we generate workorders only once.
+ if (!all_workorders_generated_) {
+ for (const block_id left_block_id : left_relation_block_ids_) {
+ for (const block_id right_block_id : right_relation_block_ids_) {
+ container->addWorkOrderProto(createWorkOrderProto(left_block_id, right_block_id),
+ op_index_);
+ }
+ }
+ all_workorders_generated_ = true;
+ }
+ return true;
+ } else if (!(left_relation_is_stored_ || right_relation_is_stored_)) {
+ // Both relations are not stored.
+ const std::vector<block_id>::size_type new_left_blocks
+ = left_relation_block_ids_.size() - num_left_workorders_generated_;
+ const std::vector<block_id>::size_type new_right_blocks
+ = right_relation_block_ids_.size() - num_right_workorders_generated_;
+
+ std::size_t new_workorders = 0;
+ if (new_left_blocks > 0 && new_right_blocks > 0) {
+ // Blocks added to both left and right relations.
+ // First generate (left + new_left_blocks) * (new_right_blocks).
+ new_workorders =
+ getAllWorkOrderProtosHelperBothNotStored(container,
+ 0,
+ left_relation_block_ids_.size(),
+ num_right_workorders_generated_,
+ right_relation_block_ids_.size());
+
+ // Now generate new_left_blocks * (right).
+ new_workorders +=
+ getAllWorkOrderProtosHelperBothNotStored(container,
+ num_left_workorders_generated_,
+ left_relation_block_ids_.size(),
+ 0,
+ num_right_workorders_generated_);
+ } else if (new_left_blocks == 0 && new_right_blocks > 0) {
+ // Only new right blocks are added. Generate left * new_right_blocks.
+ new_workorders =
+ getAllWorkOrderProtosHelperBothNotStored(container,
+ 0,
+ left_relation_block_ids_.size(),
+ num_right_workorders_generated_,
+ right_relation_block_ids_.size());
+ } else if (new_left_blocks > 0 && new_right_blocks == 0) {
+ // Generate new_left_blocks * right
+ new_workorders =
+ getAllWorkOrderProtosHelperBothNotStored(container,
+ num_left_workorders_generated_,
+ left_relation_block_ids_.size(),
+ 0,
+ right_relation_block_ids_.size());
+ }
+ if (new_workorders > 0) {
+ num_left_workorders_generated_ = left_relation_block_ids_.size();
+ num_right_workorders_generated_ = right_relation_block_ids_.size();
+ }
+ return done_feeding_left_relation_ && done_feeding_right_relation_;
+ } else {
+ // Only one relation is a stored relation.
+ return getAllWorkOrderProtosHelperOneStored(container);
+ }
+}
+
std::size_t NestedLoopsJoinOperator::getAllWorkOrdersHelperBothNotStored(WorkOrdersContainer *container,
QueryContext *query_context,
StorageManager *storage_manager,
@@ -234,6 +302,79 @@ bool NestedLoopsJoinOperator::getAllWorkOrdersHelperOneStored(WorkOrdersContaine
}
}
+std::size_t NestedLoopsJoinOperator::getAllWorkOrderProtosHelperBothNotStored(
+ WorkOrderProtosContainer *container,
+ const std::vector<block_id>::size_type left_min,
+ const std::vector<block_id>::size_type left_max,
+ const std::vector<block_id>::size_type right_min,
+ const std::vector<block_id>::size_type right_max) {
+ DCHECK(!(left_relation_is_stored_ || right_relation_is_stored_));
+ DCHECK_LE(left_min, left_max);
+ DCHECK_LE(right_min, right_max);
+
+ for (std::vector<block_id>::size_type left_index = left_min;
+ left_index < left_max;
+ ++left_index) {
+ for (std::vector<block_id>::size_type right_index = right_min;
+ right_index < right_max;
+ ++right_index) {
+ container->addWorkOrderProto(
+ createWorkOrderProto(left_relation_block_ids_[left_index], right_relation_block_ids_[right_index]),
+ op_index_);
+ }
+ }
+ // Return the number of workorders produced.
+ return (left_max - left_min) * (right_max - right_min);
+}
+
+bool NestedLoopsJoinOperator::getAllWorkOrderProtosHelperOneStored(WorkOrderProtosContainer *container) {
+ DCHECK(left_relation_is_stored_ ^ right_relation_is_stored_);
+
+ if (left_relation_is_stored_) {
+ for (std::vector<block_id>::size_type right_index = num_right_workorders_generated_;
+ right_index < right_relation_block_ids_.size();
+ ++right_index) {
+ for (const block_id left_block_id : left_relation_block_ids_) {
+ container->addWorkOrderProto(
+ createWorkOrderProto(left_block_id, right_relation_block_ids_[right_index]),
+ op_index_);
+ }
+ }
+ num_right_workorders_generated_ = right_relation_block_ids_.size();
+ return done_feeding_right_relation_;
+ } else {
+ for (std::vector<block_id>::size_type left_index = num_left_workorders_generated_;
+ left_index < left_relation_block_ids_.size();
+ ++left_index) {
+ for (const block_id right_block_id : right_relation_block_ids_) {
+ container->addWorkOrderProto(
+ createWorkOrderProto(left_relation_block_ids_[left_index], right_block_id),
+ op_index_);
+ }
+ }
+ num_left_workorders_generated_ = left_relation_block_ids_.size();
+ return done_feeding_left_relation_;
+ }
+}
+
+serialization::WorkOrder* NestedLoopsJoinOperator::createWorkOrderProto(const block_id left_block,
+ const block_id right_block) {
+ serialization::WorkOrder *proto = new serialization::WorkOrder;
+ proto->set_work_order_type(serialization::NESTED_LOOP_JOIN);
+
+ proto->SetExtension(serialization::NestedLoopsJoinWorkOrder::left_relation_id, left_input_relation_.getID());
+ proto->SetExtension(serialization::NestedLoopsJoinWorkOrder::right_relation_id, right_input_relation_.getID());
+ proto->SetExtension(serialization::NestedLoopsJoinWorkOrder::left_block_id, left_block);
+ proto->SetExtension(serialization::NestedLoopsJoinWorkOrder::right_block_id, right_block);
+ proto->SetExtension(serialization::NestedLoopsJoinWorkOrder::insert_destination_index,
+ output_destination_index_);
+ proto->SetExtension(serialization::NestedLoopsJoinWorkOrder::join_predicate_index, join_predicate_index_);
+ proto->SetExtension(serialization::NestedLoopsJoinWorkOrder::selection_index, selection_index_);
+
+ return proto;
+}
+
+
template <bool LEFT_PACKED, bool RIGHT_PACKED>
void NestedLoopsJoinWorkOrder::executeHelper(const TupleStorageSubBlock &left_store,
const TupleStorageSubBlock &right_store) {
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3e35844f/relational_operators/NestedLoopsJoinOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/NestedLoopsJoinOperator.hpp b/relational_operators/NestedLoopsJoinOperator.hpp
index a52ca25..0d14af5 100644
--- a/relational_operators/NestedLoopsJoinOperator.hpp
+++ b/relational_operators/NestedLoopsJoinOperator.hpp
@@ -44,8 +44,11 @@ class Predicate;
class Scalar;
class StorageManager;
class TupleStorageSubBlock;
+class WorkOrderProtosContainer;
class WorkOrdersContainer;
+namespace serialization { class WorkOrder; }
+
/** \addtogroup RelationalOperators
* @{
*/
@@ -113,6 +116,8 @@ class NestedLoopsJoinOperator : public RelationalOperator {
const tmb::client_id scheduler_client_id,
tmb::MessageBus *bus) override;
+ bool getAllWorkOrderProtos(WorkOrderProtosContainer *container) override;
+
void doneFeedingInputBlocks(const relation_id rel_id) override {
if (rel_id == left_input_relation_.getID()) {
done_feeding_left_relation_ = true;
@@ -181,6 +186,52 @@ class NestedLoopsJoinOperator : public RelationalOperator {
QueryContext *query_context,
StorageManager *storage_manager);
+ /**
+ * @brief Pairs block IDs from left and right relation block IDs and generates
+ * NestedLoopsJoinWorkOrder protos and pushes them to the
+ * WorkOrderProtosContainer when both relations are not stored
+ * relations.
+ *
+ * @param container A pointer to the WorkOrderProtosContainer to store the
+ * resulting WorkOrder protos.
+ * @param left_min The starting index in left_relation_block_ids_ from where
+ * we begin generating NestedLoopsJoinWorkOrders.
+ * @param left_max The index in left_relation_block_ids_ until which we
+ * generate NestedLoopsJoinWorkOrders (excluding left_max).
+ * @param right_min The starting index in right_relation_block_ids_ from where
+ * we begin generating NestedLoopsJoinWorkOrders.
+ * @param right_max The index in right_relation_block_ids_ until which we
+ * generate NestedLoopsJoinWorkOrders. (excluding right_max).
+ *
+ * @return The number of workorder protos generated during the execution of this
+ * function.
+ **/
+ std::size_t getAllWorkOrderProtosHelperBothNotStored(WorkOrderProtosContainer *container,
+ const std::vector<block_id>::size_type left_min,
+ const std::vector<block_id>::size_type left_max,
+ const std::vector<block_id>::size_type right_min,
+ const std::vector<block_id>::size_type right_max);
+
+ /**
+ * @brief Pairs block IDs from left and right relation block IDs and generates
+ * NestedLoopsJoinWorkOrder protos and pushes them to the
+ * WorkOrderProtosContainer when only one relation is a stored relation.
+ *
+ * @param container A pointer to the WorkOrderProtosContainer to store the
+ * resulting WorkOrder protos.
+ *
+ * @return Whether all work orders have been generated.
+ **/
+ bool getAllWorkOrderProtosHelperOneStored(WorkOrderProtosContainer *container);
+
+ /**
+ * @brief Create Work Order proto.
+ *
+ * @param block The block id used in the Work Order.
+ **/
+ serialization::WorkOrder* createWorkOrderProto(const block_id left_block,
+ const block_id right_block);
+
const CatalogRelation &left_input_relation_;
const CatalogRelation &right_input_relation_;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3e35844f/relational_operators/RelationalOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/RelationalOperator.hpp b/relational_operators/RelationalOperator.hpp
index 75fde17..2b0373c 100644
--- a/relational_operators/RelationalOperator.hpp
+++ b/relational_operators/RelationalOperator.hpp
@@ -36,6 +36,7 @@ namespace tmb { class MessageBus; }
namespace quickstep {
class StorageManager;
+class WorkOrderProtosContainer;
class WorkOrdersContainer;
/** \addtogroup RelationalOperators
@@ -82,6 +83,27 @@ class RelationalOperator {
tmb::MessageBus *bus) = 0;
/**
+ * @brief For the distributed version, generate all the next WorkOrder protos
+ * for this RelationalOperator
+ *
+ * @note If a RelationalOperator has blocking dependencies, it should not
+ * generate workorders unless all of the blocking dependencies have been
+ * met.
+ *
+ * @note If a RelationalOperator is not parallelizeable on a block-level, then
+ * only one WorkOrder consisting of all the work for this
+ * RelationalOperator should be generated.
+ *
+ * @param container A pointer to a WorkOrderProtosContainer to be used to
+ * store the generated WorkOrder protos.
+ *
+ * @return Whether the operator has finished generating work order protos. If
+ * \c false, the execution engine will invoke this method after at
+ * least one pending work order has finished executing.
+ **/
+ virtual bool getAllWorkOrderProtos(WorkOrderProtosContainer *container) = 0;
+
+ /**
* @brief Update Catalog upon the completion of this RelationalOperator, if
* necessary.
*
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3e35844f/relational_operators/SampleOperator.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/SampleOperator.cpp b/relational_operators/SampleOperator.cpp
index 6842b28..52bbbd4 100644
--- a/relational_operators/SampleOperator.cpp
+++ b/relational_operators/SampleOperator.cpp
@@ -1,6 +1,7 @@
/**
* Copyright 2016, Quickstep Research Group, Computer Sciences Department,
* University of Wisconsin\u2014Madison.
+ * Copyright 2016 Pivotal Software, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -18,10 +19,13 @@
#include "relational_operators/SampleOperator.hpp"
#include <memory>
+#include <random>
#include <vector>
#include "query_execution/QueryContext.hpp"
+#include "query_execution/WorkOrderProtosContainer.hpp"
#include "query_execution/WorkOrdersContainer.hpp"
+#include "relational_operators/WorkOrder.pb.h"
#include "storage/InsertDestination.hpp"
#include "storage/StorageBlock.hpp"
#include "storage/StorageBlockInfo.hpp"
@@ -81,35 +85,97 @@ bool SampleOperator::getAllWorkOrders(
}
return started_;
} else {
- if (is_block_sample_) {
- while (num_workorders_generated_ < input_relation_block_ids_.size()) {
- if (distribution(generator) <= probability) {
- container->addNormalWorkOrder(
- new SampleWorkOrder(input_relation_,
- input_relation_block_ids_[num_workorders_generated_],
- is_block_sample_,
- percentage_,
- output_destination,
- storage_manager),
- op_index_);
- ++num_workorders_generated_;
- }
- }
- } else {
- while (num_workorders_generated_ < input_relation_block_ids_.size()) {
+ if (is_block_sample_) {
+ while (num_workorders_generated_ < input_relation_block_ids_.size()) {
+ if (distribution(generator) <= probability) {
container->addNormalWorkOrder(
new SampleWorkOrder(input_relation_,
input_relation_block_ids_[num_workorders_generated_],
is_block_sample_,
- percentage_, output_destination,
+ percentage_,
+ output_destination,
storage_manager),
op_index_);
++num_workorders_generated_;
}
}
+ } else {
+ while (num_workorders_generated_ < input_relation_block_ids_.size()) {
+ container->addNormalWorkOrder(
+ new SampleWorkOrder(input_relation_,
+ input_relation_block_ids_[num_workorders_generated_],
+ is_block_sample_,
+ percentage_,
+ output_destination,
+ storage_manager),
+ op_index_);
+ ++num_workorders_generated_;
+ }
+ }
+ return done_feeding_input_relation_;
+ }
+}
+
+bool SampleOperator::getAllWorkOrderProtos(WorkOrderProtosContainer *container) {
+ std::random_device random_device;
+ std::mt19937 generator(random_device());
+ std::uniform_real_distribution<> distribution(0, 1);
+ const double probability = static_cast<double>(percentage_) / 100;
+
+ if (input_relation_is_stored_) {
+ if (!started_) {
+ // If the sampling is by block choose blocks randomly
+ if (is_block_sample_) {
+ for (const block_id input_block_id : input_relation_block_ids_) {
+ if (distribution(generator) <= probability) {
+ container->addWorkOrderProto(createWorkOrderProto(input_block_id), op_index_);
+ }
+ }
+ } else {
+ // Add all the blocks for tuple sampling which would handle
+ // the sampling from each block
+ for (const block_id input_block_id : input_relation_block_ids_) {
+ container->addWorkOrderProto(createWorkOrderProto(input_block_id), op_index_);
+ }
+ }
+ started_ = true;
+ }
+ return true;
+ } else {
+ if (is_block_sample_) {
+ while (num_workorders_generated_ < input_relation_block_ids_.size()) {
+ if (distribution(generator) <= probability) {
+ container->addWorkOrderProto(
+ createWorkOrderProto(input_relation_block_ids_[num_workorders_generated_]),
+ op_index_);
+ ++num_workorders_generated_;
+ }
+ }
+ } else {
+ while (num_workorders_generated_ < input_relation_block_ids_.size()) {
+ container->addWorkOrderProto(
+ createWorkOrderProto(input_relation_block_ids_[num_workorders_generated_]),
+ op_index_);
+ ++num_workorders_generated_;
+ }
+ }
return done_feeding_input_relation_;
}
}
+
+serialization::WorkOrder* SampleOperator::createWorkOrderProto(const block_id block) {
+ serialization::WorkOrder *proto = new serialization::WorkOrder;
+ proto->set_work_order_type(serialization::SAMPLE);
+
+ proto->SetExtension(serialization::SampleWorkOrder::relation_id, input_relation_.getID());
+ proto->SetExtension(serialization::SampleWorkOrder::block_id, block);
+ proto->SetExtension(serialization::SampleWorkOrder::is_block_sample, is_block_sample_);
+ proto->SetExtension(serialization::SampleWorkOrder::percentage, percentage_);
+ proto->SetExtension(serialization::SampleWorkOrder::insert_destination_index, output_destination_index_);
+
+ return proto;
+}
+
void SampleWorkOrder::execute() {
BlockReference block(
storage_manager_->getBlock(input_block_id_, input_relation_));
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3e35844f/relational_operators/SampleOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/SampleOperator.hpp b/relational_operators/SampleOperator.hpp
index 305de34..8ebe17f 100644
--- a/relational_operators/SampleOperator.hpp
+++ b/relational_operators/SampleOperator.hpp
@@ -41,8 +41,11 @@ class CatalogDatabase;
class CatalogRelationSchema;
class InsertDestination;
class StorageManager;
+class WorkOrderProtosContainer;
class WorkOrdersContainer;
+namespace serialization { class WorkOrder; }
+
/** \addtogroup RelationalOperators
* @{
*/
@@ -91,6 +94,8 @@ class SampleOperator : public RelationalOperator {
const tmb::client_id scheduler_client_id,
tmb::MessageBus *bus) override;
+ bool getAllWorkOrderProtos(WorkOrderProtosContainer *container) override;
+
void feedInputBlock(const block_id input_block_id, const relation_id input_relation_id) override {
input_relation_block_ids_.push_back(input_block_id);
}
@@ -110,6 +115,13 @@ class SampleOperator : public RelationalOperator {
}
private:
+ /**
+ * @brief Create Work Order proto.
+ *
+ * @param block The block id used in the Work Order.
+ **/
+ serialization::WorkOrder* createWorkOrderProto(const block_id block);
+
const CatalogRelation &input_relation_;
const CatalogRelationSchema &output_relation_;
const QueryContext::insert_destination_id output_destination_index_;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3e35844f/relational_operators/SaveBlocksOperator.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/SaveBlocksOperator.cpp b/relational_operators/SaveBlocksOperator.cpp
index ac61407..c054b1a 100644
--- a/relational_operators/SaveBlocksOperator.cpp
+++ b/relational_operators/SaveBlocksOperator.cpp
@@ -19,7 +19,9 @@
#include <vector>
+#include "query_execution/WorkOrderProtosContainer.hpp"
#include "query_execution/WorkOrdersContainer.hpp"
+#include "relational_operators/WorkOrder.pb.h"
#include "storage/StorageBlockInfo.hpp"
#include "storage/StorageManager.hpp"
@@ -45,6 +47,21 @@ bool SaveBlocksOperator::getAllWorkOrders(
return done_feeding_input_relation_;
}
+bool SaveBlocksOperator::getAllWorkOrderProtos(WorkOrderProtosContainer *container) {
+ while (num_workorders_generated_ < destination_block_ids_.size()) {
+ serialization::WorkOrder *proto = new serialization::WorkOrder;
+ proto->set_work_order_type(serialization::SAVE_BLOCKS);
+ proto->SetExtension(serialization::SaveBlocksWorkOrder::block_id,
+ destination_block_ids_[num_workorders_generated_]);
+ proto->SetExtension(serialization::SaveBlocksWorkOrder::force, force_);
+
+ container->addWorkOrderProto(proto, op_index_);
+
+ ++num_workorders_generated_;
+ }
+ return done_feeding_input_relation_;
+}
+
void SaveBlocksOperator::feedInputBlock(const block_id input_block_id, const relation_id input_relation_id) {
destination_block_ids_.push_back(input_block_id);
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3e35844f/relational_operators/SaveBlocksOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/SaveBlocksOperator.hpp b/relational_operators/SaveBlocksOperator.hpp
index 49195ea..f50176f 100644
--- a/relational_operators/SaveBlocksOperator.hpp
+++ b/relational_operators/SaveBlocksOperator.hpp
@@ -36,6 +36,7 @@ namespace quickstep {
class QueryContext;
class StorageManager;
+class WorkOrderProtosContainer;
class WorkOrdersContainer;
/** \addtogroup RelationalOperators
@@ -65,6 +66,8 @@ class SaveBlocksOperator : public RelationalOperator {
const tmb::client_id scheduler_client_id,
tmb::MessageBus *bus) override;
+ bool getAllWorkOrderProtos(WorkOrderProtosContainer *container) override;
+
void feedInputBlock(const block_id input_block_id, const relation_id input_relation_id) override;
void feedInputBlocks(const relation_id rel_id, std::vector<block_id> *partially_filled_blocks) override {
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3e35844f/relational_operators/SelectOperator.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/SelectOperator.cpp b/relational_operators/SelectOperator.cpp
index 69bb434..97f9166 100644
--- a/relational_operators/SelectOperator.cpp
+++ b/relational_operators/SelectOperator.cpp
@@ -21,7 +21,9 @@
#include <vector>
#include "query_execution/QueryContext.hpp"
+#include "query_execution/WorkOrderProtosContainer.hpp"
#include "query_execution/WorkOrdersContainer.hpp"
+#include "relational_operators/WorkOrder.pb.h"
#include "storage/InsertDestination.hpp"
#include "storage/StorageBlock.hpp"
#include "storage/StorageBlockInfo.hpp"
@@ -167,6 +169,46 @@ bool SelectOperator::getAllWorkOrders(
}
}
+bool SelectOperator::getAllWorkOrderProtos(WorkOrderProtosContainer *container) {
+ if (input_relation_is_stored_) {
+ if (!started_) {
+ for (const block_id input_block_id : input_relation_block_ids_) {
+ container->addWorkOrderProto(createWorkOrderProto(input_block_id), op_index_);
+ }
+ started_ = true;
+ }
+ return true;
+ } else {
+ while (num_workorders_generated_ < input_relation_block_ids_.size()) {
+ container->addWorkOrderProto(
+ createWorkOrderProto(input_relation_block_ids_[num_workorders_generated_]),
+ op_index_);
+ ++num_workorders_generated_;
+ }
+ return done_feeding_input_relation_;
+ }
+}
+
+serialization::WorkOrder* SelectOperator::createWorkOrderProto(const block_id block) {
+ serialization::WorkOrder *proto = new serialization::WorkOrder;
+ proto->set_work_order_type(serialization::SELECT);
+
+ proto->SetExtension(serialization::SelectWorkOrder::relation_id, input_relation_.getID());
+ proto->SetExtension(serialization::SelectWorkOrder::insert_destination_index, output_destination_index_);
+ proto->SetExtension(serialization::SelectWorkOrder::predicate_index, predicate_index_);
+ proto->SetExtension(serialization::SelectWorkOrder::block_id, block);
+ proto->SetExtension(serialization::SelectWorkOrder::simple_projection, simple_projection_);
+ if (simple_projection_) {
+ for (const attribute_id attr_id : simple_selection_) {
+ proto->AddExtension(serialization::SelectWorkOrder::simple_selection, attr_id);
+ }
+ }
+ proto->SetExtension(serialization::SelectWorkOrder::selection_index, selection_index_);
+
+ return proto;
+}
+
+
void SelectWorkOrder::execute() {
BlockReference block(
storage_manager_->getBlock(input_block_id_, input_relation_, getPreferredNUMANodes()[0]));
[30/50] [abbrv] incubator-quickstep git commit: Fix bug in the SMA
code (#223)
Posted by zu...@apache.org.
Fix bug in the SMA code (#223)
* Fix bug in the SMA code so that the SMA predicate evaluation is only
applied if at least one of the operands in the predicate is a
static value.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/1fa81a8b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/1fa81a8b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/1fa81a8b
Branch: refs/heads/work-order-serialization
Commit: 1fa81a8bd2efc7e7638fb0952c422c20dd9ce2fa
Parents: ba25b13
Author: Jignesh Patel <pa...@users.noreply.github.com>
Authored: Mon May 16 12:04:32 2016 -0500
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Mon May 30 15:47:27 2016 -0700
----------------------------------------------------------------------
storage/SMAIndexSubBlock.cpp | 11 ++++++++---
1 file changed, 8 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/1fa81a8b/storage/SMAIndexSubBlock.cpp
----------------------------------------------------------------------
diff --git a/storage/SMAIndexSubBlock.cpp b/storage/SMAIndexSubBlock.cpp
index 2a0e4f9..aa9bc54 100644
--- a/storage/SMAIndexSubBlock.cpp
+++ b/storage/SMAIndexSubBlock.cpp
@@ -621,9 +621,14 @@ Selectivity SMAIndexSubBlock::getSelectivityForPredicate(const ComparisonPredica
predicate_cost_t SMAIndexSubBlock::estimatePredicateEvaluationCost(
const ComparisonPredicate &predicate) const {
DCHECK(initialized_);
- Selectivity selectivity = getSelectivityForPredicate(predicate);
- if (selectivity == Selectivity::kAll || selectivity == Selectivity::kNone) {
- return predicate_cost::kConstantTime;
+
+ // Check that at least one of the operands has a static value.
+ if (predicate.getLeftOperand().hasStaticValue() ||
+ predicate.getRightOperand().hasStaticValue()) {
+ Selectivity selectivity = getSelectivityForPredicate(predicate);
+ if (selectivity == Selectivity::kAll || selectivity == Selectivity::kNone) {
+ return predicate_cost::kConstantTime;
+ }
}
return predicate_cost::kInfinite;
}
[21/50] [abbrv] incubator-quickstep git commit: Added support for the
substring function. (#211)
Posted by zu...@apache.org.
Added support for the substring function. (#211)
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/767b2ef1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/767b2ef1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/767b2ef1
Branch: refs/heads/work-order-serialization
Commit: 767b2ef1a9ea570c8d51808559c800c01e0a385e
Parents: 6c9108a
Author: Jianqiao Zhu <ji...@cs.wisc.edu>
Authored: Thu May 5 11:17:37 2016 -0500
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Mon May 30 15:46:31 2016 -0700
----------------------------------------------------------------------
parser/ParseBasicExpressions.cpp | 31 +
parser/ParseBasicExpressions.hpp | 77 +
parser/ParseExpression.hpp | 1 +
parser/SqlLexer.lpp | 2 +
parser/SqlParser.ypp | 16 +
parser/preprocessed/SqlLexer_gen.cpp | 1173 +++----
parser/preprocessed/SqlLexer_gen.hpp | 2 +-
parser/preprocessed/SqlParser_gen.cpp | 2920 +++++++++---------
parser/preprocessed/SqlParser_gen.hpp | 126 +-
query_optimizer/resolver/CMakeLists.txt | 1 +
query_optimizer/resolver/Resolver.cpp | 32 +
.../tests/execution_generator/Select.test | 24 +
query_optimizer/tests/resolver/Select.test | 78 +
types/operations/Operation.proto | 9 +
.../operations/unary_operations/CMakeLists.txt | 22 +
.../unary_operations/SubstringOperation.cpp | 214 ++
.../unary_operations/SubstringOperation.hpp | 234 ++
.../unary_operations/UnaryOperation.cpp | 4 +
.../unary_operations/UnaryOperationFactory.cpp | 12 +
.../unary_operations/UnaryOperationID.cpp | 6 +-
.../unary_operations/UnaryOperationID.hpp | 3 +
21 files changed, 2905 insertions(+), 2082 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/767b2ef1/parser/ParseBasicExpressions.cpp
----------------------------------------------------------------------
diff --git a/parser/ParseBasicExpressions.cpp b/parser/ParseBasicExpressions.cpp
index 2b1d7e0..a9d84ea 100644
--- a/parser/ParseBasicExpressions.cpp
+++ b/parser/ParseBasicExpressions.cpp
@@ -189,4 +189,35 @@ void ParseExtractFunction::getFieldStringItems(
non_container_child_fields->push_back(date_expression_.get());
}
+std::string ParseSubstringFunction::generateName() const {
+ std::string name;
+ name.append("SUBSTRING(");
+ name.append(operand_->generateName());
+ name.append(" FROM ");
+ name.append(std::to_string(start_position_));
+ if (length_ != kDefaultLength) {
+ name.append(" FOR ");
+ name.append(std::to_string(length_));
+ }
+ name.push_back(')');
+ return name;
+}
+
+void ParseSubstringFunction::getFieldStringItems(
+ std::vector<std::string> *inline_field_names,
+ std::vector<std::string> *inline_field_values,
+ std::vector<std::string> *non_container_child_field_names,
+ std::vector<const ParseTreeNode*> *non_container_child_fields,
+ std::vector<std::string> *container_child_field_names,
+ std::vector<std::vector<const ParseTreeNode*>> *container_child_fields) const {
+ inline_field_names->push_back("start_position");
+ inline_field_values->push_back(std::to_string(start_position_));
+
+ inline_field_names->push_back("length");
+ inline_field_values->push_back(std::to_string(length_));
+
+ non_container_child_field_names->push_back("operand");
+ non_container_child_fields->push_back(operand_.get());
+}
+
} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/767b2ef1/parser/ParseBasicExpressions.hpp
----------------------------------------------------------------------
diff --git a/parser/ParseBasicExpressions.hpp b/parser/ParseBasicExpressions.hpp
index 1886c8b..dea25d7 100644
--- a/parser/ParseBasicExpressions.hpp
+++ b/parser/ParseBasicExpressions.hpp
@@ -20,6 +20,8 @@
#ifndef QUICKSTEP_PARSER_PARSE_BASIC_EXPRESSIONS_HPP_
#define QUICKSTEP_PARSER_PARSE_BASIC_EXPRESSIONS_HPP_
+#include <cstddef>
+#include <limits>
#include <memory>
#include <string>
#include <vector>
@@ -511,6 +513,81 @@ class ParseExtractFunction : public ParseExpression {
DISALLOW_COPY_AND_ASSIGN(ParseExtractFunction);
};
+
+/**
+ * @brief Parsed representation of the substring function.
+ */
+class ParseSubstringFunction : public ParseExpression {
+ public:
+ static constexpr std::size_t kDefaultLength = std::numeric_limits<std::size_t>::max();
+
+ /**
+ * @brief Constructor.
+ *
+ * @param line_number The line number of the first token of the function call.
+ * @param column_number The column number of the first token of the function call.
+ * @param operand The operand of the substring.
+ * @param start_position The 1-based starting position of the substring.
+ * @param length Optional substring length.
+ */
+ ParseSubstringFunction(const int line_number,
+ const int column_number,
+ ParseExpression *operand,
+ const std::size_t start_position,
+ const std::size_t length = kDefaultLength)
+ : ParseExpression(line_number, column_number),
+ operand_(operand),
+ start_position_(start_position),
+ length_(length) {}
+
+ ExpressionType getExpressionType() const override {
+ return kSubstring;
+ }
+
+ std::string getName() const override {
+ return "Substring";
+ }
+
+ /**
+ * @return The operand of the substring.
+ */
+ const ParseExpression* operand() const {
+ return operand_.get();
+ }
+
+ /**
+ * @return The 1-based starting position of the substring.
+ */
+ std::size_t start_position() const {
+ return start_position_;
+ }
+
+ /**
+ * @return Then substring length.
+ */
+ std::size_t length() const {
+ return length_;
+ }
+
+ std::string generateName() const override;
+
+ protected:
+ void getFieldStringItems(
+ std::vector<std::string> *inline_field_names,
+ std::vector<std::string> *inline_field_values,
+ std::vector<std::string> *non_container_child_field_names,
+ std::vector<const ParseTreeNode*> *non_container_child_fields,
+ std::vector<std::string> *container_child_field_names,
+ std::vector<std::vector<const ParseTreeNode*>> *container_child_fields) const override;
+
+ private:
+ std::unique_ptr<ParseExpression> operand_;
+ const std::size_t start_position_;
+ const std::size_t length_;
+
+ DISALLOW_COPY_AND_ASSIGN(ParseSubstringFunction);
+};
+
/** @} */
} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/767b2ef1/parser/ParseExpression.hpp
----------------------------------------------------------------------
diff --git a/parser/ParseExpression.hpp b/parser/ParseExpression.hpp
index e959e72..3541f83 100644
--- a/parser/ParseExpression.hpp
+++ b/parser/ParseExpression.hpp
@@ -45,6 +45,7 @@ class ParseExpression : public ParseTreeNode {
kSearchedCaseExpression,
kSimpleCaseExpression,
kSubqueryExpression,
+ kSubstring,
kUnaryExpression,
};
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/767b2ef1/parser/SqlLexer.lpp
----------------------------------------------------------------------
diff --git a/parser/SqlLexer.lpp b/parser/SqlLexer.lpp
index a399723..ac1c708 100644
--- a/parser/SqlLexer.lpp
+++ b/parser/SqlLexer.lpp
@@ -209,6 +209,7 @@ unsigned_numeric_literal {exact_numeric_literal}|{approximate_numeric_literal}
"false" return TOKEN_FALSE;
"first" return TOKEN_FIRST;
"float" return TOKEN_FLOAT;
+ "for" return TOKEN_FOR;
"foreign" return TOKEN_FOREIGN;
"from" return TOKEN_FROM;
"full" return TOKEN_FULL;
@@ -258,6 +259,7 @@ unsigned_numeric_literal {exact_numeric_literal}|{approximate_numeric_literal}
"set" return TOKEN_SET;
"sma" return TOKEN_SMA;
"smallint" return TOKEN_SMALLINT;
+ "substring" return TOKEN_SUBSTRING;
"table" return TOKEN_TABLE;
"then" return TOKEN_THEN;
"time" return TOKEN_TIME;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/767b2ef1/parser/SqlParser.ypp
----------------------------------------------------------------------
diff --git a/parser/SqlParser.ypp b/parser/SqlParser.ypp
index 1202d66..b07c48e 100644
--- a/parser/SqlParser.ypp
+++ b/parser/SqlParser.ypp
@@ -273,6 +273,7 @@ void NotSupported(const YYLTYPE *location, yyscan_t yyscanner, const std::string
%token TOKEN_FALSE;
%token TOKEN_FIRST;
%token TOKEN_FLOAT;
+%token TOKEN_FOR;
%token TOKEN_FOREIGN;
%token TOKEN_FROM;
%token TOKEN_FULL;
@@ -319,6 +320,7 @@ void NotSupported(const YYLTYPE *location, yyscan_t yyscanner, const std::string
%token TOKEN_SET;
%token TOKEN_SMA;
%token TOKEN_SMALLINT;
+%token TOKEN_SUBSTRING;
%token TOKEN_TABLE;
%token TOKEN_THEN;
%token TOKEN_TIME;
@@ -361,6 +363,7 @@ void NotSupported(const YYLTYPE *location, yyscan_t yyscanner, const std::string
case_expression
opt_else_clause
extract_function
+ substr_function
%type <attribute_>
attribute_ref
@@ -1505,6 +1508,9 @@ expression_base:
| extract_function {
$$ = $1;
}
+ | substr_function {
+ $$ = $1;
+ }
| case_expression {
$$ = $1;
}
@@ -1536,6 +1542,16 @@ extract_function:
$$ = new quickstep::ParseExtractFunction(@1.first_line, @1.first_column, $3, $5);
};
+substr_function:
+ TOKEN_SUBSTRING '(' add_expression TOKEN_FROM TOKEN_UNSIGNED_NUMVAL ')' {
+ $$ = new quickstep::ParseSubstringFunction(
+ @1.first_line, @1.first_column, $3, $5->long_value());
+ }
+ | TOKEN_SUBSTRING '(' add_expression TOKEN_FROM TOKEN_UNSIGNED_NUMVAL TOKEN_FOR TOKEN_UNSIGNED_NUMVAL ')' {
+ $$ = new quickstep::ParseSubstringFunction(
+ @1.first_line, @1.first_column, $3, $5->long_value(), $7->long_value());
+ };
+
case_expression:
TOKEN_CASE add_expression simple_when_clause_list opt_else_clause TOKEN_END {
$$ = new quickstep::ParseSimpleCaseExpression(@1.first_line, @1.first_column, $2, $3, $4);
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/767b2ef1/parser/preprocessed/SqlLexer_gen.cpp
----------------------------------------------------------------------
diff --git a/parser/preprocessed/SqlLexer_gen.cpp b/parser/preprocessed/SqlLexer_gen.cpp
index d836988..db20491 100644
--- a/parser/preprocessed/SqlLexer_gen.cpp
+++ b/parser/preprocessed/SqlLexer_gen.cpp
@@ -381,8 +381,8 @@ static void yy_fatal_error (yyconst char msg[] ,yyscan_t yyscanner );
*yy_cp = '\0'; \
yyg->yy_c_buf_p = yy_cp;
-#define YY_NUM_RULES 150
-#define YY_END_OF_BUFFER 151
+#define YY_NUM_RULES 152
+#define YY_END_OF_BUFFER 153
/* This struct is not used in this scanner,
but its presence is necessary. */
struct yy_trans_info
@@ -390,68 +390,69 @@ struct yy_trans_info
flex_int32_t yy_verify;
flex_int32_t yy_nxt;
};
-static yyconst flex_int16_t yy_accept[545] =
+static yyconst flex_int16_t yy_accept[553] =
{ 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 151, 2, 2, 149, 149, 148, 147, 149,
- 126, 122, 125, 122, 122, 145, 118, 115, 119, 144,
- 144, 144, 144, 144, 144, 144, 144, 144, 144, 144,
- 144, 144, 144, 144, 144, 144, 144, 144, 144, 144,
- 144, 144, 144, 144, 123, 4, 5, 5, 3, 141,
- 141, 138, 142, 142, 136, 143, 143, 140, 1, 148,
- 116, 146, 145, 145, 145, 0, 120, 117, 121, 144,
- 144, 144, 144, 10, 144, 144, 144, 22, 144, 144,
- 144, 144, 144, 144, 144, 144, 144, 144, 124, 144,
-
- 144, 144, 144, 144, 144, 144, 144, 144, 144, 144,
- 144, 144, 57, 65, 144, 144, 144, 144, 144, 144,
- 144, 144, 144, 144, 144, 79, 80, 144, 144, 144,
- 144, 144, 144, 144, 144, 144, 144, 144, 144, 144,
- 144, 144, 144, 144, 144, 144, 144, 144, 144, 144,
- 4, 5, 3, 141, 137, 142, 135, 135, 127, 129,
- 130, 131, 132, 133, 134, 135, 143, 139, 146, 145,
- 0, 145, 6, 7, 144, 9, 11, 144, 144, 15,
- 144, 144, 144, 144, 144, 144, 144, 144, 144, 144,
- 32, 144, 144, 144, 144, 144, 144, 144, 144, 43,
-
- 144, 144, 144, 144, 144, 144, 144, 144, 144, 144,
- 144, 144, 144, 144, 144, 144, 61, 144, 67, 144,
- 144, 144, 144, 144, 144, 144, 75, 144, 78, 144,
- 144, 144, 144, 144, 144, 144, 144, 144, 144, 144,
- 144, 144, 144, 96, 97, 144, 144, 144, 144, 144,
- 144, 144, 144, 144, 144, 144, 144, 144, 127, 129,
- 128, 144, 144, 144, 144, 144, 144, 144, 20, 23,
- 144, 144, 144, 28, 144, 144, 30, 144, 144, 144,
- 144, 37, 144, 144, 41, 42, 144, 144, 144, 144,
- 144, 144, 144, 51, 52, 144, 54, 144, 56, 144,
-
- 144, 144, 144, 64, 66, 68, 69, 70, 144, 72,
- 144, 144, 76, 144, 144, 144, 144, 144, 87, 144,
- 89, 144, 144, 144, 144, 144, 144, 144, 144, 100,
- 101, 103, 144, 144, 144, 144, 144, 144, 110, 144,
- 112, 113, 127, 128, 8, 144, 144, 144, 144, 144,
- 144, 144, 25, 144, 144, 144, 144, 144, 144, 144,
- 144, 144, 144, 144, 144, 144, 144, 144, 47, 48,
- 49, 144, 53, 144, 58, 59, 144, 144, 144, 71,
- 144, 74, 77, 81, 82, 144, 144, 144, 88, 144,
- 144, 92, 144, 144, 144, 144, 99, 144, 144, 144,
-
- 144, 107, 144, 144, 111, 144, 144, 144, 14, 144,
- 144, 144, 144, 144, 26, 144, 29, 144, 144, 144,
- 144, 35, 144, 144, 144, 40, 144, 45, 144, 144,
- 55, 60, 144, 144, 73, 144, 144, 144, 144, 91,
- 144, 94, 95, 144, 144, 144, 105, 106, 108, 144,
- 144, 144, 13, 144, 144, 144, 144, 144, 144, 21,
- 144, 33, 34, 144, 144, 144, 144, 46, 50, 62,
- 144, 144, 85, 86, 144, 144, 144, 144, 144, 109,
- 144, 144, 144, 144, 144, 144, 144, 144, 31, 144,
- 144, 39, 144, 63, 144, 144, 144, 98, 144, 144,
-
- 144, 12, 144, 144, 144, 144, 24, 144, 36, 144,
- 144, 83, 144, 144, 102, 144, 114, 16, 144, 144,
- 144, 27, 38, 144, 84, 90, 144, 144, 144, 18,
- 19, 144, 144, 104, 144, 144, 144, 144, 144, 93,
- 144, 44, 17, 0
+ 0, 0, 153, 2, 2, 151, 151, 150, 149, 151,
+ 128, 124, 127, 124, 124, 147, 120, 117, 121, 146,
+ 146, 146, 146, 146, 146, 146, 146, 146, 146, 146,
+ 146, 146, 146, 146, 146, 146, 146, 146, 146, 146,
+ 146, 146, 146, 146, 125, 4, 5, 5, 3, 143,
+ 143, 140, 144, 144, 138, 145, 145, 142, 1, 150,
+ 118, 148, 147, 147, 147, 0, 122, 119, 123, 146,
+ 146, 146, 146, 10, 146, 146, 146, 22, 146, 146,
+ 146, 146, 146, 146, 146, 146, 146, 146, 126, 146,
+
+ 146, 146, 146, 146, 146, 146, 146, 146, 146, 146,
+ 146, 146, 58, 66, 146, 146, 146, 146, 146, 146,
+ 146, 146, 146, 146, 146, 80, 81, 146, 146, 146,
+ 146, 146, 146, 146, 146, 146, 146, 146, 146, 146,
+ 146, 146, 146, 146, 146, 146, 146, 146, 146, 146,
+ 146, 4, 5, 3, 143, 139, 144, 137, 137, 129,
+ 131, 132, 133, 134, 135, 136, 137, 145, 141, 148,
+ 147, 0, 147, 6, 7, 146, 9, 11, 146, 146,
+ 15, 146, 146, 146, 146, 146, 146, 146, 146, 146,
+ 146, 32, 146, 146, 146, 146, 146, 146, 146, 146,
+
+ 43, 146, 146, 146, 146, 146, 146, 50, 146, 146,
+ 146, 146, 146, 146, 146, 146, 146, 62, 146, 68,
+ 146, 146, 146, 146, 146, 146, 146, 76, 146, 79,
+ 146, 146, 146, 146, 146, 146, 146, 146, 146, 146,
+ 146, 146, 146, 146, 97, 98, 146, 146, 146, 146,
+ 146, 146, 146, 146, 146, 146, 146, 146, 146, 146,
+ 129, 131, 130, 146, 146, 146, 146, 146, 146, 146,
+ 20, 23, 146, 146, 146, 28, 146, 146, 30, 146,
+ 146, 146, 146, 37, 146, 146, 41, 42, 146, 146,
+ 146, 146, 146, 146, 146, 52, 53, 146, 55, 146,
+
+ 57, 146, 146, 146, 146, 65, 67, 69, 70, 71,
+ 146, 73, 146, 146, 77, 146, 146, 146, 146, 146,
+ 88, 146, 90, 146, 146, 146, 146, 146, 146, 146,
+ 146, 146, 102, 103, 105, 146, 146, 146, 146, 146,
+ 146, 112, 146, 114, 115, 129, 130, 8, 146, 146,
+ 146, 146, 146, 146, 146, 25, 146, 146, 146, 146,
+ 146, 146, 146, 146, 146, 146, 146, 146, 146, 146,
+ 146, 47, 48, 49, 146, 54, 146, 59, 60, 146,
+ 146, 146, 72, 146, 75, 78, 82, 83, 146, 146,
+ 146, 89, 146, 146, 93, 146, 146, 146, 146, 146,
+
+ 101, 146, 146, 146, 146, 109, 146, 146, 113, 146,
+ 146, 146, 14, 146, 146, 146, 146, 146, 26, 146,
+ 29, 146, 146, 146, 146, 35, 146, 146, 146, 40,
+ 146, 45, 146, 146, 56, 61, 146, 146, 74, 146,
+ 146, 146, 146, 92, 146, 95, 96, 146, 146, 146,
+ 146, 107, 108, 110, 146, 146, 146, 13, 146, 146,
+ 146, 146, 146, 146, 21, 146, 33, 34, 146, 146,
+ 146, 146, 46, 51, 63, 146, 146, 86, 87, 146,
+ 146, 146, 146, 146, 146, 111, 146, 146, 146, 146,
+ 146, 146, 146, 146, 31, 146, 146, 39, 146, 64,
+
+ 146, 146, 146, 99, 146, 146, 146, 146, 12, 146,
+ 146, 146, 146, 24, 146, 36, 146, 146, 84, 146,
+ 146, 100, 104, 146, 116, 16, 146, 146, 146, 27,
+ 38, 146, 85, 91, 146, 146, 146, 18, 19, 146,
+ 146, 106, 146, 146, 146, 146, 146, 94, 146, 44,
+ 17, 0
} ;
static yyconst YY_CHAR yy_ec[256] =
@@ -498,143 +499,145 @@ static yyconst YY_CHAR yy_meta[72] =
8
} ;
-static yyconst flex_uint16_t yy_base[560] =
+static yyconst flex_uint16_t yy_base[568] =
{ 0,
0, 1, 46, 0, 117, 163, 2, 3, 128, 132,
- 6, 10, 257, 1212, 1212, 0, 1212, 13, 1212, 233,
- 1212, 1212, 1212, 208, 6, 130, 4, 1212, 195, 124,
+ 6, 10, 257, 1219, 1219, 0, 1219, 13, 1219, 233,
+ 1219, 1219, 1219, 208, 6, 130, 4, 1219, 195, 124,
161, 170, 178, 207, 260, 92, 167, 161, 96, 107,
- 219, 214, 212, 224, 236, 92, 279, 171, 278, 281,
- 128, 227, 0, 125, 1212, 184, 4, 19, 0, 0,
+ 219, 214, 212, 224, 236, 92, 279, 272, 278, 281,
+ 128, 168, 0, 125, 1219, 184, 4, 19, 0, 0,
0, 146, 0, 0, 343, 0, 0, 145, 0, 22,
- 1212, 0, 297, 316, 338, 18, 1212, 1212, 1212, 0,
- 170, 227, 173, 178, 224, 299, 270, 0, 270, 335,
- 330, 286, 320, 327, 376, 308, 316, 326, 1212, 335,
-
- 351, 355, 371, 348, 346, 353, 359, 370, 382, 383,
- 380, 379, 399, 0, 392, 379, 386, 401, 399, 401,
- 402, 407, 402, 413, 420, 0, 431, 417, 420, 422,
- 434, 437, 435, 451, 446, 433, 456, 459, 459, 457,
- 450, 444, 454, 462, 469, 465, 465, 474, 460, 483,
- 148, 29, 0, 0, 1212, 0, 1212, 1212, 22, 24,
- 1212, 1212, 1212, 1212, 1212, 0, 0, 1212, 0, 515,
- 26, 28, 0, 0, 488, 0, 490, 473, 489, 478,
- 501, 502, 496, 512, 496, 499, 494, 520, 503, 521,
- 0, 519, 528, 526, 529, 514, 535, 522, 534, 0,
-
- 539, 522, 524, 532, 534, 553, 551, 546, 550, 544,
- 564, 564, 556, 570, 571, 572, 574, 564, 0, 561,
- 564, 581, 578, 583, 571, 573, 0, 583, 0, 591,
- 592, 578, 596, 587, 589, 604, 600, 609, 612, 612,
- 98, 608, 625, 0, 619, 620, 619, 629, 630, 624,
- 620, 638, 628, 623, 642, 633, 640, 632, 30, 125,
- 0, 635, 640, 650, 642, 652, 647, 654, 0, 668,
- 659, 659, 655, 0, 658, 663, 668, 676, 669, 671,
- 679, 688, 685, 683, 0, 0, 681, 680, 701, 698,
- 685, 686, 699, 0, 0, 693, 0, 697, 0, 688,
-
- 695, 696, 709, 0, 0, 0, 0, 0, 695, 0,
- 705, 720, 711, 715, 718, 730, 741, 746, 0, 743,
- 0, 731, 726, 731, 748, 739, 752, 746, 755, 0,
- 742, 0, 758, 743, 746, 760, 764, 762, 0, 766,
- 0, 759, 136, 1212, 0, 769, 769, 763, 784, 772,
- 780, 791, 0, 783, 786, 800, 801, 798, 807, 797,
- 805, 802, 799, 802, 813, 814, 802, 819, 0, 0,
- 0, 817, 0, 818, 0, 0, 807, 823, 807, 0,
- 825, 0, 0, 0, 0, 811, 818, 823, 0, 838,
- 828, 0, 841, 845, 832, 846, 0, 842, 844, 859,
-
- 860, 0, 847, 866, 0, 853, 860, 857, 0, 852,
- 858, 876, 870, 860, 0, 881, 0, 878, 872, 874,
- 867, 0, 868, 885, 887, 0, 93, 0, 879, 887,
- 0, 0, 884, 903, 0, 898, 890, 888, 906, 0,
- 909, 0, 0, 908, 922, 923, 0, 0, 0, 907,
- 912, 913, 0, 920, 917, 921, 923, 932, 929, 0,
- 935, 0, 0, 936, 934, 924, 926, 0, 0, 0,
- 934, 932, 0, 0, 945, 948, 939, 947, 949, 0,
- 945, 961, 957, 962, 963, 960, 963, 968, 0, 965,
- 970, 0, 965, 0, 973, 985, 979, 0, 977, 979,
-
- 988, 0, 991, 994, 989, 997, 0, 983, 0, 997,
- 987, 987, 996, 1008, 0, 1006, 0, 0, 1002, 1018,
- 1008, 0, 0, 1020, 0, 0, 1016, 1032, 1018, 0,
- 0, 1025, 1035, 0, 1032, 1035, 1025, 1040, 1029, 0,
- 1031, 0, 0, 1212, 1096, 1106, 1116, 1126, 1136, 1140,
- 1143, 1149, 1157, 1167, 1177, 1187, 1197, 1202, 1204
+ 1219, 0, 307, 337, 341, 18, 1219, 1219, 1219, 0,
+ 170, 224, 177, 181, 215, 269, 224, 0, 263, 326,
+ 336, 286, 290, 333, 382, 321, 322, 329, 1219, 328,
+
+ 347, 351, 346, 350, 346, 353, 352, 374, 386, 385,
+ 383, 382, 402, 0, 395, 382, 389, 403, 399, 397,
+ 399, 401, 399, 408, 418, 0, 421, 406, 410, 425,
+ 437, 438, 438, 455, 451, 436, 457, 462, 462, 463,
+ 461, 454, 447, 454, 462, 469, 465, 466, 474, 460,
+ 481, 148, 29, 0, 0, 1219, 0, 1219, 1219, 22,
+ 24, 1219, 1219, 1219, 1219, 1219, 0, 0, 1219, 0,
+ 509, 26, 28, 0, 0, 478, 0, 481, 478, 501,
+ 493, 514, 513, 501, 517, 500, 503, 498, 523, 505,
+ 521, 0, 518, 527, 526, 529, 513, 532, 520, 532,
+
+ 0, 537, 521, 523, 523, 525, 558, 563, 559, 562,
+ 554, 568, 568, 560, 574, 575, 576, 577, 569, 0,
+ 564, 565, 581, 578, 582, 569, 571, 0, 581, 0,
+ 589, 590, 576, 595, 586, 580, 595, 605, 621, 625,
+ 624, 98, 618, 629, 0, 623, 617, 625, 624, 634,
+ 635, 629, 625, 642, 630, 624, 644, 634, 641, 632,
+ 30, 125, 0, 633, 639, 649, 642, 652, 647, 647,
+ 0, 674, 673, 675, 670, 0, 671, 674, 673, 681,
+ 674, 676, 684, 693, 690, 688, 0, 0, 685, 682,
+ 702, 700, 686, 687, 699, 0, 0, 693, 0, 696,
+
+ 0, 687, 694, 696, 726, 0, 0, 0, 0, 0,
+ 696, 0, 698, 734, 727, 730, 731, 741, 746, 751,
+ 0, 748, 0, 736, 731, 736, 753, 744, 756, 748,
+ 741, 758, 0, 745, 0, 760, 745, 747, 761, 764,
+ 762, 0, 767, 0, 761, 136, 1219, 0, 772, 786,
+ 780, 801, 788, 800, 805, 0, 795, 792, 806, 807,
+ 804, 813, 803, 811, 808, 804, 805, 816, 817, 804,
+ 821, 0, 0, 0, 818, 0, 819, 0, 0, 807,
+ 823, 808, 0, 827, 0, 0, 0, 0, 814, 835,
+ 840, 0, 855, 846, 0, 859, 861, 846, 858, 850,
+
+ 0, 849, 851, 866, 867, 0, 854, 873, 0, 860,
+ 867, 863, 0, 857, 862, 880, 873, 863, 0, 883,
+ 0, 880, 873, 876, 870, 0, 872, 903, 913, 0,
+ 93, 0, 898, 906, 0, 0, 904, 922, 0, 915,
+ 905, 901, 913, 0, 916, 0, 0, 915, 921, 930,
+ 931, 0, 0, 0, 915, 920, 921, 0, 928, 923,
+ 926, 928, 936, 933, 0, 938, 0, 0, 939, 937,
+ 928, 931, 0, 0, 0, 953, 959, 0, 0, 973,
+ 969, 960, 967, 969, 970, 0, 964, 978, 972, 971,
+ 972, 969, 972, 977, 0, 974, 979, 0, 975, 0,
+
+ 982, 992, 985, 0, 992, 984, 985, 994, 0, 997,
+ 1001, 996, 1018, 0, 1012, 0, 1027, 1018, 1019, 1020,
+ 1031, 0, 0, 1029, 0, 0, 1024, 1038, 1026, 0,
+ 0, 1036, 0, 0, 1026, 1042, 1028, 0, 0, 1035,
+ 1045, 0, 1042, 1046, 1036, 1050, 1037, 0, 1038, 0,
+ 0, 1219, 1103, 1113, 1123, 1133, 1143, 1147, 1150, 1156,
+ 1164, 1174, 1184, 1194, 1204, 1209, 1211
} ;
-static yyconst flex_int16_t yy_def[560] =
+static yyconst flex_int16_t yy_def[568] =
{ 0,
- 545, 545, 544, 3, 546, 546, 547, 547, 548, 548,
- 549, 549, 544, 544, 544, 550, 544, 544, 544, 544,
- 544, 544, 544, 544, 544, 544, 544, 544, 544, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 544, 544, 544, 544, 552, 553,
- 553, 544, 554, 554, 555, 556, 556, 544, 550, 544,
- 544, 557, 544, 544, 544, 544, 544, 544, 544, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 544, 551,
-
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 544, 544, 552, 553, 544, 554, 544, 544, 544, 544,
- 544, 544, 544, 544, 544, 558, 556, 544, 557, 544,
- 544, 544, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
-
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 544, 544,
- 559, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
-
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 544, 544, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
-
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
-
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 551, 551, 551, 551, 551, 551, 551,
- 551, 551, 551, 0, 544, 544, 544, 544, 544, 544,
- 544, 544, 544, 544, 544, 544, 544, 544, 544
+ 553, 553, 552, 3, 554, 554, 555, 555, 556, 556,
+ 557, 557, 552, 552, 552, 558, 552, 552, 552, 552,
+ 552, 552, 552, 552, 552, 552, 552, 552, 552, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 552, 552, 552, 552, 560, 561,
+ 561, 552, 562, 562, 563, 564, 564, 552, 558, 552,
+ 552, 565, 552, 552, 552, 552, 552, 552, 552, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 552, 559,
+
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 552, 552, 560, 561, 552, 562, 552, 552, 552,
+ 552, 552, 552, 552, 552, 552, 566, 564, 552, 565,
+ 552, 552, 552, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 552, 552, 567, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 552, 552, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 559, 559, 559, 559, 559, 559, 559, 559, 559,
+ 559, 0, 552, 552, 552, 552, 552, 552, 552, 552,
+ 552, 552, 552, 552, 552, 552, 552
} ;
-static yyconst flex_uint16_t yy_nxt[1284] =
+static yyconst flex_uint16_t yy_nxt[1291] =
{ 0,
- 544, 544, 15, 15, 61, 61, 152, 152, 67, 62,
- 62, 68, 67, 544, 70, 68, 70, 73, 73, 77,
- 78, 152, 152, 70, 544, 70, 171, 171, 544, 172,
- 172, 152, 152, 259, 260, 260, 260, 172, 172, 172,
- 172, 343, 260, 544, 16, 16, 17, 18, 19, 18,
+ 552, 552, 15, 15, 61, 61, 153, 153, 67, 62,
+ 62, 68, 67, 552, 70, 68, 70, 73, 73, 77,
+ 78, 153, 153, 70, 552, 70, 172, 172, 552, 173,
+ 173, 153, 153, 261, 262, 262, 262, 173, 173, 173,
+ 173, 346, 262, 552, 16, 16, 17, 18, 19, 18,
20, 21, 22, 23, 22, 24, 25, 26, 26, 17,
27, 28, 29, 30, 31, 32, 33, 34, 35, 36,
37, 38, 39, 40, 41, 42, 43, 44, 45, 46,
@@ -644,142 +647,142 @@ static yyconst flex_uint16_t yy_nxt[1284] =
38, 39, 40, 41, 42, 43, 44, 45, 46, 47,
48, 49, 50, 51, 52, 53, 54, 17, 56, 57,
58, 17, 17, 17, 17, 17, 110, 115, 116, 132,
- 64, 17, 17, 17, 64, 62, 260, 260, 467, 62,
- 74, 75, 75, 325, 81, 147, 150, 260, 260, 151,
- 168, 76, 82, 155, 83, 110, 115, 116, 132, 84,
+ 64, 17, 17, 17, 64, 62, 262, 262, 472, 62,
+ 74, 75, 75, 327, 81, 148, 151, 262, 262, 152,
+ 169, 76, 82, 156, 83, 110, 115, 116, 132, 84,
17, 17, 17, 17, 56, 57, 58, 17, 17, 17,
- 17, 17, 65, 81, 147, 150, 65, 17, 17, 17,
- 76, 82, 85, 83, 111, 151, 86, 89, 84, 87,
- 173, 113, 137, 176, 90, 94, 114, 177, 112, 95,
+ 17, 17, 65, 81, 148, 151, 65, 17, 17, 17,
+ 76, 82, 85, 83, 111, 152, 86, 89, 84, 87,
+ 174, 113, 149, 150, 90, 94, 114, 177, 112, 95,
- 138, 91, 88, 96, 92, 93, 17, 17, 17, 97,
- 79, 85, 98, 111, 99, 86, 89, 72, 87, 173,
- 113, 137, 176, 90, 94, 114, 177, 112, 95, 138,
+ 178, 91, 88, 96, 92, 93, 17, 17, 17, 97,
+ 79, 85, 98, 111, 99, 86, 89, 72, 87, 174,
+ 113, 149, 150, 90, 94, 114, 177, 112, 95, 178,
91, 88, 96, 92, 93, 100, 117, 101, 97, 121,
118, 98, 102, 123, 119, 122, 125, 103, 71, 124,
- 120, 148, 149, 129, 126, 174, 544, 130, 127, 544,
- 178, 128, 544, 175, 100, 117, 101, 544, 121, 118,
+ 120, 179, 175, 129, 126, 182, 552, 130, 127, 552,
+ 176, 128, 552, 552, 100, 117, 101, 552, 121, 118,
131, 102, 123, 119, 122, 125, 103, 104, 124, 120,
- 148, 149, 129, 126, 174, 105, 130, 127, 106, 178,
- 128, 107, 175, 544, 108, 139, 133, 109, 544, 131,
-
- 134, 181, 140, 141, 135, 182, 104, 188, 73, 73,
- 136, 144, 142, 145, 105, 143, 146, 106, 76, 544,
- 107, 544, 179, 108, 139, 133, 109, 170, 170, 134,
- 181, 140, 141, 135, 182, 180, 188, 76, 189, 136,
- 144, 142, 145, 196, 143, 146, 158, 76, 74, 75,
- 75, 179, 183, 197, 159, 160, 184, 198, 185, 76,
- 186, 161, 187, 190, 180, 162, 76, 189, 191, 544,
- 199, 200, 196, 163, 201, 544, 204, 164, 544, 165,
- 205, 183, 197, 166, 206, 184, 198, 185, 76, 186,
- 161, 187, 190, 207, 162, 192, 202, 191, 193, 199,
-
- 200, 208, 163, 201, 194, 204, 164, 203, 165, 205,
- 209, 195, 166, 206, 210, 211, 213, 218, 212, 214,
- 219, 220, 207, 221, 192, 202, 222, 193, 223, 215,
- 208, 224, 225, 194, 216, 217, 203, 226, 227, 209,
- 195, 228, 229, 210, 211, 213, 218, 212, 214, 219,
- 220, 230, 221, 231, 232, 222, 233, 223, 215, 234,
- 224, 225, 235, 216, 217, 236, 226, 227, 237, 240,
- 228, 229, 241, 238, 239, 242, 245, 246, 247, 248,
- 230, 249, 231, 232, 243, 233, 250, 251, 234, 252,
- 253, 235, 244, 254, 236, 256, 257, 237, 240, 255,
-
- 258, 241, 238, 239, 242, 245, 246, 247, 248, 262,
- 249, 263, 264, 243, 265, 250, 251, 266, 252, 253,
- 267, 244, 254, 269, 256, 257, 170, 170, 255, 258,
- 270, 271, 268, 272, 273, 274, 76, 275, 262, 276,
- 263, 264, 277, 265, 278, 279, 266, 280, 282, 267,
- 283, 281, 269, 284, 285, 286, 287, 288, 289, 270,
- 271, 268, 272, 273, 274, 76, 275, 290, 276, 291,
- 292, 277, 293, 278, 279, 294, 280, 282, 295, 283,
- 281, 296, 284, 285, 286, 287, 288, 289, 297, 298,
- 299, 300, 301, 302, 305, 303, 290, 306, 291, 292,
-
- 307, 293, 308, 309, 294, 304, 310, 295, 311, 312,
- 296, 313, 314, 315, 316, 317, 318, 297, 298, 299,
- 300, 301, 302, 305, 303, 319, 306, 320, 321, 307,
- 322, 308, 309, 323, 304, 310, 324, 311, 312, 326,
- 313, 314, 315, 316, 317, 318, 327, 328, 329, 330,
- 331, 332, 333, 334, 319, 335, 320, 321, 336, 322,
- 337, 338, 323, 339, 341, 324, 342, 340, 326, 345,
- 346, 347, 348, 349, 350, 327, 328, 329, 330, 331,
- 332, 333, 334, 351, 335, 352, 353, 336, 354, 337,
- 338, 355, 339, 341, 356, 342, 340, 357, 345, 346,
-
- 347, 348, 349, 350, 358, 359, 360, 361, 362, 363,
- 364, 365, 351, 366, 352, 353, 367, 354, 368, 369,
- 355, 370, 371, 356, 372, 373, 357, 374, 375, 376,
- 377, 380, 378, 358, 359, 360, 361, 362, 363, 364,
- 365, 381, 366, 379, 382, 367, 383, 368, 369, 384,
- 370, 371, 385, 372, 373, 386, 374, 375, 376, 377,
- 380, 378, 387, 388, 389, 390, 391, 392, 393, 394,
- 381, 395, 379, 382, 396, 383, 397, 398, 384, 399,
- 400, 385, 401, 402, 386, 403, 404, 405, 406, 407,
- 408, 387, 388, 389, 390, 391, 392, 393, 394, 409,
-
- 395, 410, 413, 396, 411, 397, 398, 412, 399, 400,
- 414, 401, 402, 415, 403, 404, 405, 406, 407, 408,
- 416, 417, 418, 419, 420, 421, 422, 423, 409, 424,
- 410, 413, 425, 411, 426, 427, 412, 428, 429, 414,
- 430, 431, 415, 432, 433, 434, 435, 436, 437, 416,
- 417, 418, 419, 420, 421, 422, 423, 438, 424, 439,
- 440, 425, 441, 426, 427, 442, 428, 429, 443, 430,
- 431, 444, 432, 433, 434, 435, 436, 437, 445, 446,
- 447, 448, 449, 450, 451, 452, 438, 453, 439, 440,
- 454, 441, 455, 456, 442, 457, 458, 443, 459, 460,
-
- 444, 461, 462, 463, 464, 465, 466, 445, 446, 447,
- 448, 449, 450, 451, 452, 468, 453, 469, 470, 454,
- 471, 455, 456, 472, 457, 458, 473, 459, 460, 474,
- 461, 462, 463, 464, 465, 466, 475, 476, 477, 478,
- 479, 480, 481, 482, 468, 483, 469, 470, 484, 471,
- 485, 486, 472, 487, 488, 473, 489, 490, 474, 491,
- 492, 493, 494, 495, 496, 475, 476, 477, 478, 479,
- 480, 481, 482, 497, 483, 498, 499, 484, 500, 485,
- 486, 501, 487, 488, 502, 489, 490, 503, 491, 492,
- 493, 494, 495, 496, 504, 505, 506, 507, 508, 509,
-
- 510, 511, 497, 512, 498, 499, 513, 500, 514, 515,
- 501, 516, 517, 502, 518, 519, 503, 520, 521, 522,
- 523, 524, 525, 504, 505, 506, 507, 508, 509, 510,
- 511, 526, 512, 527, 528, 513, 529, 514, 515, 530,
- 516, 517, 531, 518, 519, 532, 520, 521, 522, 523,
- 524, 525, 533, 534, 535, 536, 537, 538, 539, 540,
- 526, 541, 527, 528, 542, 529, 543, 544, 530, 544,
- 544, 531, 544, 544, 532, 544, 544, 544, 544, 544,
- 544, 533, 534, 535, 536, 537, 538, 539, 540, 544,
- 541, 544, 544, 542, 544, 543, 14, 14, 14, 14,
-
- 14, 14, 14, 14, 14, 14, 59, 59, 59, 59,
- 59, 59, 59, 59, 59, 59, 60, 60, 60, 60,
- 60, 60, 60, 60, 60, 60, 63, 63, 63, 63,
- 63, 63, 63, 63, 63, 63, 66, 66, 66, 66,
- 66, 66, 66, 66, 66, 66, 69, 69, 80, 80,
- 80, 544, 80, 153, 153, 153, 153, 154, 154, 154,
- 544, 154, 154, 154, 154, 154, 154, 156, 156, 156,
- 544, 156, 156, 156, 156, 544, 156, 157, 157, 157,
- 157, 157, 157, 157, 157, 157, 157, 167, 167, 544,
- 167, 167, 167, 167, 167, 167, 167, 169, 544, 169,
-
- 169, 169, 169, 169, 169, 169, 169, 261, 261, 344,
- 344, 13, 544, 544, 544, 544, 544, 544, 544, 544,
- 544, 544, 544, 544, 544, 544, 544, 544, 544, 544,
- 544, 544, 544, 544, 544, 544, 544, 544, 544, 544,
- 544, 544, 544, 544, 544, 544, 544, 544, 544, 544,
- 544, 544, 544, 544, 544, 544, 544, 544, 544, 544,
- 544, 544, 544, 544, 544, 544, 544, 544, 544, 544,
- 544, 544, 544, 544, 544, 544, 544, 544, 544, 544,
- 544, 544, 544
+ 179, 175, 129, 126, 182, 105, 130, 127, 106, 176,
+ 128, 107, 180, 137, 108, 140, 133, 109, 183, 131,
+
+ 134, 138, 141, 142, 135, 181, 104, 189, 190, 139,
+ 136, 145, 143, 146, 105, 144, 147, 106, 73, 73,
+ 107, 180, 137, 108, 140, 133, 109, 183, 76, 134,
+ 138, 141, 142, 135, 181, 552, 189, 190, 139, 136,
+ 145, 143, 146, 184, 144, 147, 159, 185, 171, 171,
+ 552, 74, 75, 75, 160, 161, 197, 76, 76, 198,
+ 199, 162, 76, 200, 186, 163, 187, 201, 188, 191,
+ 202, 203, 184, 164, 192, 552, 185, 165, 205, 166,
+ 206, 552, 204, 167, 207, 197, 208, 76, 198, 199,
+ 162, 76, 200, 186, 163, 187, 201, 188, 191, 202,
+
+ 203, 193, 164, 192, 194, 209, 165, 205, 166, 206,
+ 195, 204, 167, 207, 210, 208, 211, 196, 212, 214,
+ 219, 213, 215, 220, 221, 222, 223, 225, 224, 226,
+ 193, 227, 216, 194, 209, 228, 229, 217, 218, 195,
+ 230, 231, 232, 210, 233, 211, 196, 212, 214, 219,
+ 213, 215, 220, 221, 222, 223, 225, 224, 226, 234,
+ 227, 216, 235, 236, 228, 229, 217, 218, 237, 230,
+ 231, 232, 238, 233, 241, 242, 243, 239, 240, 246,
+ 247, 248, 249, 250, 251, 244, 252, 253, 234, 254,
+ 255, 235, 236, 245, 256, 258, 259, 237, 260, 264,
+
+ 257, 238, 265, 241, 242, 243, 239, 240, 246, 247,
+ 248, 249, 250, 251, 244, 252, 253, 266, 254, 255,
+ 171, 171, 245, 256, 258, 259, 267, 260, 264, 257,
+ 76, 265, 268, 269, 271, 272, 273, 274, 275, 276,
+ 277, 278, 279, 280, 281, 270, 266, 282, 284, 285,
+ 286, 283, 287, 288, 289, 267, 290, 291, 292, 76,
+ 293, 268, 269, 271, 272, 273, 274, 275, 276, 277,
+ 278, 279, 280, 281, 270, 294, 282, 284, 285, 286,
+ 283, 287, 288, 289, 295, 290, 291, 292, 296, 293,
+ 297, 298, 299, 300, 301, 302, 303, 304, 305, 307,
+
+ 308, 309, 310, 311, 294, 312, 313, 314, 306, 315,
+ 316, 317, 318, 295, 319, 320, 321, 296, 322, 297,
+ 298, 299, 300, 301, 302, 303, 304, 305, 307, 308,
+ 309, 310, 311, 323, 312, 313, 314, 306, 315, 316,
+ 317, 318, 324, 319, 320, 321, 325, 322, 326, 328,
+ 329, 330, 331, 332, 333, 334, 335, 336, 337, 338,
+ 339, 340, 323, 341, 342, 344, 345, 348, 343, 349,
+ 350, 324, 351, 352, 353, 325, 354, 326, 328, 329,
+ 330, 331, 332, 333, 334, 335, 336, 337, 338, 339,
+ 340, 355, 341, 342, 344, 345, 348, 343, 349, 350,
+
+ 356, 351, 352, 353, 357, 354, 358, 359, 360, 361,
+ 362, 363, 364, 365, 366, 367, 368, 369, 370, 371,
+ 355, 372, 373, 374, 375, 376, 377, 378, 379, 356,
+ 380, 552, 383, 357, 384, 358, 359, 360, 361, 362,
+ 363, 364, 365, 366, 367, 368, 369, 370, 371, 381,
+ 372, 373, 374, 375, 376, 377, 378, 379, 385, 380,
+ 382, 383, 386, 384, 387, 388, 389, 390, 391, 392,
+ 393, 394, 395, 396, 397, 398, 399, 400, 381, 401,
+ 402, 403, 404, 405, 406, 407, 408, 385, 409, 382,
+ 410, 386, 411, 387, 388, 389, 390, 391, 392, 393,
+
+ 394, 395, 396, 397, 398, 399, 400, 412, 401, 402,
+ 403, 404, 405, 406, 407, 408, 413, 409, 414, 410,
+ 415, 411, 417, 416, 418, 419, 420, 421, 422, 423,
+ 424, 425, 426, 427, 428, 429, 412, 430, 431, 432,
+ 433, 434, 435, 436, 437, 413, 438, 414, 439, 415,
+ 440, 417, 416, 418, 419, 420, 421, 422, 423, 424,
+ 425, 426, 427, 428, 429, 441, 430, 431, 432, 433,
+ 434, 435, 436, 437, 442, 438, 443, 439, 444, 440,
+ 445, 446, 447, 448, 449, 450, 451, 452, 453, 454,
+ 455, 456, 457, 458, 441, 459, 460, 461, 462, 463,
+
+ 464, 465, 466, 442, 467, 443, 468, 444, 469, 445,
+ 446, 447, 448, 449, 450, 451, 452, 453, 454, 455,
+ 456, 457, 458, 470, 459, 460, 461, 462, 463, 464,
+ 465, 466, 471, 467, 473, 468, 474, 469, 475, 476,
+ 477, 478, 479, 480, 481, 482, 483, 484, 485, 486,
+ 487, 488, 470, 489, 490, 491, 492, 493, 494, 495,
+ 496, 471, 497, 473, 498, 474, 499, 475, 476, 477,
+ 478, 479, 480, 481, 482, 483, 484, 485, 486, 487,
+ 488, 500, 489, 490, 491, 492, 493, 494, 495, 496,
+ 501, 497, 502, 498, 503, 499, 504, 505, 506, 507,
+
+ 508, 509, 510, 511, 512, 513, 514, 515, 516, 517,
+ 500, 518, 519, 520, 521, 522, 523, 524, 525, 501,
+ 526, 502, 527, 503, 528, 504, 505, 506, 507, 508,
+ 509, 510, 511, 512, 513, 514, 515, 516, 517, 529,
+ 518, 519, 520, 521, 522, 523, 524, 525, 530, 526,
+ 531, 527, 532, 528, 533, 534, 535, 536, 537, 538,
+ 539, 540, 541, 542, 543, 544, 545, 546, 529, 547,
+ 548, 549, 550, 551, 552, 552, 552, 530, 552, 531,
+ 552, 532, 552, 533, 534, 535, 536, 537, 538, 539,
+ 540, 541, 542, 543, 544, 545, 546, 552, 547, 548,
+
+ 549, 550, 551, 14, 14, 14, 14, 14, 14, 14,
+ 14, 14, 14, 59, 59, 59, 59, 59, 59, 59,
+ 59, 59, 59, 60, 60, 60, 60, 60, 60, 60,
+ 60, 60, 60, 63, 63, 63, 63, 63, 63, 63,
+ 63, 63, 63, 66, 66, 66, 66, 66, 66, 66,
+ 66, 66, 66, 69, 69, 80, 80, 80, 552, 80,
+ 154, 154, 154, 154, 155, 155, 155, 552, 155, 155,
+ 155, 155, 155, 155, 157, 157, 157, 552, 157, 157,
+ 157, 157, 552, 157, 158, 158, 158, 158, 158, 158,
+ 158, 158, 158, 158, 168, 168, 552, 168, 168, 168,
+
+ 168, 168, 168, 168, 170, 552, 170, 170, 170, 170,
+ 170, 170, 170, 170, 263, 263, 347, 347, 13, 552,
+ 552, 552, 552, 552, 552, 552, 552, 552, 552, 552,
+ 552, 552, 552, 552, 552, 552, 552, 552, 552, 552,
+ 552, 552, 552, 552, 552, 552, 552, 552, 552, 552,
+ 552, 552, 552, 552, 552, 552, 552, 552, 552, 552,
+ 552, 552, 552, 552, 552, 552, 552, 552, 552, 552,
+ 552, 552, 552, 552, 552, 552, 552, 552, 552, 552,
+ 552, 552, 552, 552, 552, 552, 552, 552, 552, 552
} ;
-static yyconst flex_int16_t yy_chk[1284] =
+static yyconst flex_int16_t yy_chk[1291] =
{ 0,
0, 0, 1, 2, 7, 8, 57, 57, 11, 7,
8, 11, 12, 0, 18, 12, 18, 25, 25, 27,
27, 58, 58, 70, 0, 70, 76, 76, 0, 76,
- 76, 152, 152, 159, 159, 160, 160, 171, 171, 172,
- 172, 259, 259, 0, 1, 2, 3, 3, 3, 3,
+ 76, 153, 153, 160, 160, 161, 161, 172, 172, 173,
+ 173, 261, 261, 0, 1, 2, 3, 3, 3, 3,
3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
@@ -789,137 +792,137 @@ static yyconst flex_int16_t yy_chk[1284] =
3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
3, 3, 3, 3, 3, 3, 3, 5, 5, 5,
5, 5, 5, 5, 5, 5, 36, 39, 40, 46,
- 9, 5, 5, 5, 10, 9, 260, 260, 427, 10,
- 26, 26, 26, 241, 30, 51, 54, 343, 343, 151,
+ 9, 5, 5, 5, 10, 9, 262, 262, 431, 10,
+ 26, 26, 26, 242, 30, 51, 54, 346, 346, 152,
68, 26, 30, 62, 30, 36, 39, 40, 46, 30,
5, 5, 5, 6, 6, 6, 6, 6, 6, 6,
6, 6, 9, 30, 51, 54, 10, 6, 6, 6,
26, 30, 31, 30, 37, 56, 31, 32, 30, 31,
- 81, 38, 48, 83, 32, 33, 38, 84, 37, 33,
+ 81, 38, 52, 52, 32, 33, 38, 83, 37, 33,
- 48, 32, 31, 33, 32, 32, 6, 6, 6, 33,
+ 84, 32, 31, 33, 32, 32, 6, 6, 6, 33,
29, 31, 33, 37, 34, 31, 32, 24, 31, 81,
- 38, 48, 83, 32, 33, 38, 84, 37, 33, 48,
+ 38, 52, 52, 32, 33, 38, 83, 37, 33, 84,
32, 31, 33, 32, 32, 34, 41, 34, 33, 42,
41, 33, 34, 43, 41, 42, 44, 34, 20, 43,
- 41, 52, 52, 45, 44, 82, 13, 45, 44, 0,
- 85, 44, 0, 82, 34, 41, 34, 0, 42, 41,
+ 41, 85, 82, 45, 44, 87, 13, 45, 44, 0,
+ 82, 44, 0, 0, 34, 41, 34, 0, 42, 41,
45, 34, 43, 41, 42, 44, 34, 35, 43, 41,
- 52, 52, 45, 44, 82, 35, 45, 44, 35, 85,
- 44, 35, 82, 0, 35, 49, 47, 35, 0, 45,
-
- 47, 87, 49, 49, 47, 89, 35, 92, 73, 73,
- 47, 50, 49, 50, 35, 49, 50, 35, 73, 0,
- 35, 0, 86, 35, 49, 47, 35, 74, 74, 47,
- 87, 49, 49, 47, 89, 86, 92, 74, 93, 47,
- 50, 49, 50, 96, 49, 50, 65, 73, 75, 75,
- 75, 86, 90, 97, 65, 65, 90, 98, 91, 75,
- 91, 65, 91, 94, 86, 65, 74, 93, 94, 0,
- 100, 101, 96, 65, 102, 0, 104, 65, 0, 65,
- 105, 90, 97, 65, 106, 90, 98, 91, 75, 91,
- 65, 91, 94, 107, 65, 95, 103, 94, 95, 100,
-
- 101, 108, 65, 102, 95, 104, 65, 103, 65, 105,
- 109, 95, 65, 106, 110, 111, 112, 115, 111, 113,
- 116, 117, 107, 118, 95, 103, 119, 95, 119, 113,
- 108, 120, 121, 95, 113, 113, 103, 122, 123, 109,
- 95, 124, 125, 110, 111, 112, 115, 111, 113, 116,
- 117, 127, 118, 128, 129, 119, 130, 119, 113, 131,
- 120, 121, 132, 113, 113, 133, 122, 123, 134, 135,
- 124, 125, 136, 134, 134, 137, 138, 139, 140, 141,
- 127, 142, 128, 129, 137, 130, 143, 144, 131, 145,
- 146, 132, 137, 147, 133, 148, 149, 134, 135, 147,
-
- 150, 136, 134, 134, 137, 138, 139, 140, 141, 175,
- 142, 177, 178, 137, 179, 143, 144, 180, 145, 146,
- 181, 137, 147, 182, 148, 149, 170, 170, 147, 150,
- 183, 184, 181, 185, 186, 187, 170, 188, 175, 189,
- 177, 178, 190, 179, 192, 193, 180, 194, 195, 181,
- 196, 194, 182, 197, 198, 199, 201, 202, 203, 183,
- 184, 181, 185, 186, 187, 170, 188, 204, 189, 205,
- 206, 190, 207, 192, 193, 208, 194, 195, 209, 196,
- 194, 210, 197, 198, 199, 201, 202, 203, 211, 212,
- 213, 214, 215, 216, 218, 217, 204, 220, 205, 206,
-
- 221, 207, 222, 223, 208, 217, 224, 209, 225, 226,
- 210, 228, 230, 231, 232, 233, 234, 211, 212, 213,
- 214, 215, 216, 218, 217, 235, 220, 236, 237, 221,
- 238, 222, 223, 239, 217, 224, 240, 225, 226, 242,
- 228, 230, 231, 232, 233, 234, 243, 245, 246, 247,
- 248, 249, 250, 251, 235, 252, 236, 237, 253, 238,
- 254, 255, 239, 256, 257, 240, 258, 256, 242, 262,
- 263, 264, 265, 266, 267, 243, 245, 246, 247, 248,
- 249, 250, 251, 268, 252, 270, 271, 253, 272, 254,
- 255, 273, 256, 257, 275, 258, 256, 276, 262, 263,
-
- 264, 265, 266, 267, 277, 278, 279, 280, 281, 282,
- 283, 284, 268, 287, 270, 271, 288, 272, 289, 290,
- 273, 291, 292, 275, 293, 296, 276, 298, 300, 301,
- 302, 309, 303, 277, 278, 279, 280, 281, 282, 283,
- 284, 311, 287, 303, 312, 288, 313, 289, 290, 314,
- 291, 292, 315, 293, 296, 316, 298, 300, 301, 302,
- 309, 303, 317, 318, 320, 322, 323, 324, 325, 326,
- 311, 327, 303, 312, 328, 313, 329, 331, 314, 333,
- 334, 315, 335, 336, 316, 337, 338, 340, 342, 346,
- 347, 317, 318, 320, 322, 323, 324, 325, 326, 348,
-
- 327, 349, 351, 328, 350, 329, 331, 350, 333, 334,
- 352, 335, 336, 354, 337, 338, 340, 342, 346, 347,
- 355, 356, 357, 358, 359, 360, 361, 362, 348, 363,
- 349, 351, 364, 350, 365, 366, 350, 367, 368, 352,
- 372, 374, 354, 377, 378, 379, 381, 386, 387, 355,
- 356, 357, 358, 359, 360, 361, 362, 388, 363, 390,
- 391, 364, 393, 365, 366, 394, 367, 368, 395, 372,
- 374, 396, 377, 378, 379, 381, 386, 387, 398, 399,
- 400, 401, 403, 404, 406, 407, 388, 408, 390, 391,
- 410, 393, 411, 412, 394, 413, 414, 395, 416, 418,
-
- 396, 419, 420, 421, 423, 424, 425, 398, 399, 400,
- 401, 403, 404, 406, 407, 429, 408, 430, 433, 410,
- 434, 411, 412, 436, 413, 414, 437, 416, 418, 438,
- 419, 420, 421, 423, 424, 425, 439, 441, 444, 445,
- 446, 450, 451, 452, 429, 454, 430, 433, 455, 434,
- 456, 457, 436, 458, 459, 437, 461, 464, 438, 465,
- 466, 467, 471, 472, 475, 439, 441, 444, 445, 446,
- 450, 451, 452, 476, 454, 477, 478, 455, 479, 456,
- 457, 481, 458, 459, 482, 461, 464, 483, 465, 466,
- 467, 471, 472, 475, 484, 485, 486, 487, 488, 490,
-
- 491, 493, 476, 495, 477, 478, 496, 479, 497, 499,
- 481, 500, 501, 482, 503, 504, 483, 505, 506, 508,
- 510, 511, 512, 484, 485, 486, 487, 488, 490, 491,
- 493, 513, 495, 514, 516, 496, 519, 497, 499, 520,
- 500, 501, 521, 503, 504, 524, 505, 506, 508, 510,
- 511, 512, 527, 528, 529, 532, 533, 535, 536, 537,
- 513, 538, 514, 516, 539, 519, 541, 0, 520, 0,
- 0, 521, 0, 0, 524, 0, 0, 0, 0, 0,
- 0, 527, 528, 529, 532, 533, 535, 536, 537, 0,
- 538, 0, 0, 539, 0, 541, 545, 545, 545, 545,
-
- 545, 545, 545, 545, 545, 545, 546, 546, 546, 546,
- 546, 546, 546, 546, 546, 546, 547, 547, 547, 547,
- 547, 547, 547, 547, 547, 547, 548, 548, 548, 548,
- 548, 548, 548, 548, 548, 548, 549, 549, 549, 549,
- 549, 549, 549, 549, 549, 549, 550, 550, 551, 551,
- 551, 0, 551, 552, 552, 552, 552, 553, 553, 553,
- 0, 553, 553, 553, 553, 553, 553, 554, 554, 554,
- 0, 554, 554, 554, 554, 0, 554, 555, 555, 555,
- 555, 555, 555, 555, 555, 555, 555, 556, 556, 0,
- 556, 556, 556, 556, 556, 556, 556, 557, 0, 557,
-
- 557, 557, 557, 557, 557, 557, 557, 558, 558, 559,
- 559, 544, 544, 544, 544, 544, 544, 544, 544, 544,
- 544, 544, 544, 544, 544, 544, 544, 544, 544, 544,
- 544, 544, 544, 544, 544, 544, 544, 544, 544, 544,
- 544, 544, 544, 544, 544, 544, 544, 544, 544, 544,
- 544, 544, 544, 544, 544, 544, 544, 544, 544, 544,
- 544, 544, 544, 544, 544, 544, 544, 544, 544, 544,
- 544, 544, 544, 544, 544, 544, 544, 544, 544, 544,
- 544, 544, 544
+ 85, 82, 45, 44, 87, 35, 45, 44, 35, 82,
+ 44, 35, 86, 48, 35, 49, 47, 35, 89, 45,
+
+ 47, 48, 49, 49, 47, 86, 35, 92, 93, 48,
+ 47, 50, 49, 50, 35, 49, 50, 35, 73, 73,
+ 35, 86, 48, 35, 49, 47, 35, 89, 73, 47,
+ 48, 49, 49, 47, 86, 0, 92, 93, 48, 47,
+ 50, 49, 50, 90, 49, 50, 65, 90, 74, 74,
+ 0, 75, 75, 75, 65, 65, 96, 73, 74, 97,
+ 98, 65, 75, 100, 91, 65, 91, 101, 91, 94,
+ 102, 103, 90, 65, 94, 0, 90, 65, 104, 65,
+ 105, 0, 103, 65, 106, 96, 107, 74, 97, 98,
+ 65, 75, 100, 91, 65, 91, 101, 91, 94, 102,
+
+ 103, 95, 65, 94, 95, 108, 65, 104, 65, 105,
+ 95, 103, 65, 106, 109, 107, 110, 95, 111, 112,
+ 115, 111, 113, 116, 117, 118, 119, 120, 119, 121,
+ 95, 122, 113, 95, 108, 123, 124, 113, 113, 95,
+ 125, 127, 128, 109, 129, 110, 95, 111, 112, 115,
+ 111, 113, 116, 117, 118, 119, 120, 119, 121, 130,
+ 122, 113, 131, 132, 123, 124, 113, 113, 133, 125,
+ 127, 128, 134, 129, 135, 136, 137, 134, 134, 138,
+ 139, 140, 141, 142, 143, 137, 144, 145, 130, 146,
+ 147, 131, 132, 137, 148, 149, 150, 133, 151, 176,
+
+ 148, 134, 178, 135, 136, 137, 134, 134, 138, 139,
+ 140, 141, 142, 143, 137, 144, 145, 179, 146, 147,
+ 171, 171, 137, 148, 149, 150, 180, 151, 176, 148,
+ 171, 178, 181, 182, 183, 184, 185, 186, 187, 188,
+ 189, 190, 191, 193, 194, 182, 179, 195, 196, 197,
+ 198, 195, 199, 200, 202, 180, 203, 204, 205, 171,
+ 206, 181, 182, 183, 184, 185, 186, 187, 188, 189,
+ 190, 191, 193, 194, 182, 207, 195, 196, 197, 198,
+ 195, 199, 200, 202, 208, 203, 204, 205, 209, 206,
+ 210, 211, 212, 213, 214, 215, 216, 217, 218, 219,
+
+ 221, 222, 223, 224, 207, 225, 226, 227, 218, 229,
+ 231, 232, 233, 208, 234, 235, 236, 209, 237, 210,
+ 211, 212, 213, 214, 215, 216, 217, 218, 219, 221,
+ 222, 223, 224, 238, 225, 226, 227, 218, 229, 231,
+ 232, 233, 239, 234, 235, 236, 240, 237, 241, 243,
+ 244, 246, 247, 248, 249, 250, 251, 252, 253, 254,
+ 255, 256, 238, 257, 258, 259, 260, 264, 258, 265,
+ 266, 239, 267, 268, 269, 240, 270, 241, 243, 244,
+ 246, 247, 248, 249, 250, 251, 252, 253, 254, 255,
+ 256, 272, 257, 258, 259, 260, 264, 258, 265, 266,
+
+ 273, 267, 268, 269, 274, 270, 275, 277, 278, 279,
+ 280, 281, 282, 283, 284, 285, 286, 289, 290, 291,
+ 272, 292, 293, 294, 295, 298, 300, 302, 303, 273,
+ 304, 0, 311, 274, 313, 275, 277, 278, 279, 280,
+ 281, 282, 283, 284, 285, 286, 289, 290, 291, 305,
+ 292, 293, 294, 295, 298, 300, 302, 303, 314, 304,
+ 305, 311, 315, 313, 316, 317, 318, 319, 320, 322,
+ 324, 325, 326, 327, 328, 329, 330, 331, 305, 332,
+ 334, 336, 337, 338, 339, 340, 341, 314, 343, 305,
+ 345, 315, 349, 316, 317, 318, 319, 320, 322, 324,
+
+ 325, 326, 327, 328, 329, 330, 331, 350, 332, 334,
+ 336, 337, 338, 339, 340, 341, 351, 343, 352, 345,
+ 353, 349, 354, 353, 355, 357, 358, 359, 360, 361,
+ 362, 363, 364, 365, 366, 367, 350, 368, 369, 370,
+ 371, 375, 377, 380, 381, 351, 382, 352, 384, 353,
+ 389, 354, 353, 355, 357, 358, 359, 360, 361, 362,
+ 363, 364, 365, 366, 367, 390, 368, 369, 370, 371,
+ 375, 377, 380, 381, 391, 382, 393, 384, 394, 389,
+ 396, 397, 398, 399, 400, 402, 403, 404, 405, 407,
+ 408, 410, 411, 412, 390, 414, 415, 416, 417, 418,
+
+ 420, 422, 423, 391, 424, 393, 425, 394, 427, 396,
+ 397, 398, 399, 400, 402, 403, 404, 405, 407, 408,
+ 410, 411, 412, 428, 414, 415, 416, 417, 418, 420,
+ 422, 423, 429, 424, 433, 425, 434, 427, 437, 438,
+ 440, 441, 442, 443, 445, 448, 449, 450, 451, 455,
+ 456, 457, 428, 459, 460, 461, 462, 463, 464, 466,
+ 469, 429, 470, 433, 471, 434, 472, 437, 438, 440,
+ 441, 442, 443, 445, 448, 449, 450, 451, 455, 456,
+ 457, 476, 459, 460, 461, 462, 463, 464, 466, 469,
+ 477, 470, 480, 471, 481, 472, 482, 483, 484, 485,
+
+ 487, 488, 489, 490, 491, 492, 493, 494, 496, 497,
+ 476, 499, 501, 502, 503, 505, 506, 507, 508, 477,
+ 510, 480, 511, 481, 512, 482, 483, 484, 485, 487,
+ 488, 489, 490, 491, 492, 493, 494, 496, 497, 513,
+ 499, 501, 502, 503, 505, 506, 507, 508, 515, 510,
+ 517, 511, 518, 512, 519, 520, 521, 524, 527, 528,
+ 529, 532, 535, 536, 537, 540, 541, 543, 513, 544,
+ 545, 546, 547, 549, 0, 0, 0, 515, 0, 517,
+ 0, 518, 0, 519, 520, 521, 524, 527, 528, 529,
+ 532, 535, 536, 537, 540, 541, 543, 0, 544, 545,
+
+ 546, 547, 549, 553, 553, 553, 553, 553, 553, 553,
+ 553, 553, 553, 554, 554, 554, 554, 554, 554, 554,
+ 554, 554, 554, 555, 555, 555, 555, 555, 555, 555,
+ 555, 555, 555, 556, 556, 556, 556, 556, 556, 556,
+ 556, 556, 556, 557, 557, 557, 557, 557, 557, 557,
+ 557, 557, 557, 558, 558, 559, 559, 559, 0, 559,
+ 560, 560, 560, 560, 561, 561, 561, 0, 561, 561,
+ 561, 561, 561, 561, 562, 562, 562, 0, 562, 562,
+ 562, 562, 0, 562, 563, 563, 563, 563, 563, 563,
+ 563, 563, 563, 563, 564, 564, 0, 564, 564, 564,
+
+ 564, 564, 564, 564, 565, 0, 565, 565, 565, 565,
+ 565, 565, 565, 565, 566, 566, 567, 567, 552, 552,
+ 552, 552, 552, 552, 552, 552, 552, 552, 552, 552,
+ 552, 552, 552, 552, 552, 552, 552, 552, 552, 552,
+ 552, 552, 552, 552, 552, 552, 552, 552, 552, 552,
+ 552, 552, 552, 552, 552, 552, 552, 552, 552, 552,
+ 552, 552, 552, 552, 552, 552, 552, 552, 552, 552,
+ 552, 552, 552, 552, 552, 552, 552, 552, 552, 552,
+ 552, 552, 552, 552, 552, 552, 552, 552, 552, 552
} ;
/* Table of booleans, true if rule could match eol. */
-static yyconst flex_int32_t yy_rule_can_match_eol[151] =
+static yyconst flex_int32_t yy_rule_can_match_eol[153] =
{ 0,
0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
@@ -927,8 +930,8 @@ static yyconst flex_int32_t yy_rule_can_match_eol[151] =
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0,
- 0, 1, 1, 1, 0, 0, 0, 1, 0, 0, 0, };
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0,
+ 0, 0, 0, 1, 1, 1, 0, 0, 0, 1, 0, 0, 0, };
/* The intent behind this definition is that it'll catch
* any uses of REJECT which flex missed.
@@ -1045,7 +1048,7 @@ class UnaryOperation;
-#line 1049 "SqlLexer_gen.cpp"
+#line 1052 "SqlLexer_gen.cpp"
#define INITIAL 0
#define CONDITION_SQL 1
@@ -1336,7 +1339,7 @@ YY_DECL
#line 128 "../SqlLexer.lpp"
-#line 1340 "SqlLexer_gen.cpp"
+#line 1343 "SqlLexer_gen.cpp"
while ( /*CONSTCOND*/1 ) /* loops until end-of-file is reached */
{
@@ -1363,13 +1366,13 @@ yy_match:
while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state )
{
yy_current_state = (int) yy_def[yy_current_state];
- if ( yy_current_state >= 545 )
+ if ( yy_current_state >= 553 )
yy_c = yy_meta[(unsigned int) yy_c];
}
yy_current_state = yy_nxt[yy_base[yy_current_state] + (unsigned int) yy_c];
++yy_cp;
}
- while ( yy_current_state != 544 );
+ while ( yy_current_state != 552 );
yy_cp = yyg->yy_last_accepting_cpos;
yy_current_state = yyg->yy_last_accepting_state;
@@ -1678,62 +1681,62 @@ return TOKEN_FLOAT;
case 50:
YY_RULE_SETUP
#line 212 "../SqlLexer.lpp"
-return TOKEN_FOREIGN;
+return TOKEN_FOR;
YY_BREAK
case 51:
YY_RULE_SETUP
#line 213 "../SqlLexer.lpp"
-return TOKEN_FROM;
+return TOKEN_FOREIGN;
YY_BREAK
case 52:
YY_RULE_SETUP
#line 214 "../SqlLexer.lpp"
-return TOKEN_FULL;
+return TOKEN_FROM;
YY_BREAK
case 53:
YY_RULE_SETUP
#line 215 "../SqlLexer.lpp"
-return TOKEN_GROUP;
+return TOKEN_FULL;
YY_BREAK
case 54:
YY_RULE_SETUP
#line 216 "../SqlLexer.lpp"
-return TOKEN_HASH;
+return TOKEN_GROUP;
YY_BREAK
case 55:
YY_RULE_SETUP
#line 217 "../SqlLexer.lpp"
-return TOKEN_HAVING;
+return TOKEN_HASH;
YY_BREAK
case 56:
YY_RULE_SETUP
#line 218 "../SqlLexer.lpp"
-return TOKEN_HOUR;
+return TOKEN_HAVING;
YY_BREAK
case 57:
YY_RULE_SETUP
#line 219 "../SqlLexer.lpp"
-return TOKEN_IN;
+return TOKEN_HOUR;
YY_BREAK
case 58:
YY_RULE_SETUP
#line 220 "../SqlLexer.lpp"
-return TOKEN_INDEX;
+return TOKEN_IN;
YY_BREAK
case 59:
YY_RULE_SETUP
#line 221 "../SqlLexer.lpp"
-return TOKEN_INNER;
+return TOKEN_INDEX;
YY_BREAK
case 60:
YY_RULE_SETUP
#line 222 "../SqlLexer.lpp"
-return TOKEN_INSERT;
+return TOKEN_INNER;
YY_BREAK
case 61:
YY_RULE_SETUP
#line 223 "../SqlLexer.lpp"
-return TOKEN_INTEGER;
+return TOKEN_INSERT;
YY_BREAK
case 62:
YY_RULE_SETUP
@@ -1743,331 +1746,341 @@ return TOKEN_INTEGER;
case 63:
YY_RULE_SETUP
#line 225 "../SqlLexer.lpp"
-return TOKEN_INTERVAL;
+return TOKEN_INTEGER;
YY_BREAK
case 64:
YY_RULE_SETUP
#line 226 "../SqlLexer.lpp"
-return TOKEN_INTO;
+return TOKEN_INTERVAL;
YY_BREAK
case 65:
YY_RULE_SETUP
#line 227 "../SqlLexer.lpp"
-return TOKEN_IS;
+return TOKEN_INTO;
YY_BREAK
case 66:
YY_RULE_SETUP
#line 228 "../SqlLexer.lpp"
-return TOKEN_JOIN;
+return TOKEN_IS;
YY_BREAK
case 67:
YY_RULE_SETUP
#line 229 "../SqlLexer.lpp"
-return TOKEN_KEY;
+return TOKEN_JOIN;
YY_BREAK
case 68:
YY_RULE_SETUP
#line 230 "../SqlLexer.lpp"
-return TOKEN_LAST;
+return TOKEN_KEY;
YY_BREAK
case 69:
YY_RULE_SETUP
#line 231 "../SqlLexer.lpp"
-return TOKEN_LEFT;
+return TOKEN_LAST;
YY_BREAK
case 70:
YY_RULE_SETUP
#line 232 "../SqlLexer.lpp"
-return TOKEN_LIKE;
+return TOKEN_LEFT;
YY_BREAK
case 71:
YY_RULE_SETUP
#line 233 "../SqlLexer.lpp"
-return TOKEN_LIMIT;
+return TOKEN_LIKE;
YY_BREAK
case 72:
YY_RULE_SETUP
#line 234 "../SqlLexer.lpp"
-return TOKEN_LONG;
+return TOKEN_LIMIT;
YY_BREAK
case 73:
YY_RULE_SETUP
#line 235 "../SqlLexer.lpp"
-return TOKEN_MINUTE;
+return TOKEN_LONG;
YY_BREAK
case 74:
YY_RULE_SETUP
#line 236 "../SqlLexer.lpp"
-return TOKEN_MONTH;
+return TOKEN_MINUTE;
YY_BREAK
case 75:
YY_RULE_SETUP
#line 237 "../SqlLexer.lpp"
-return TOKEN_NOT;
+return TOKEN_MONTH;
YY_BREAK
case 76:
YY_RULE_SETUP
#line 238 "../SqlLexer.lpp"
-return TOKEN_NULL;
+return TOKEN_NOT;
YY_BREAK
case 77:
YY_RULE_SETUP
#line 239 "../SqlLexer.lpp"
-return TOKEN_NULLS;
+return TOKEN_NULL;
YY_BREAK
case 78:
YY_RULE_SETUP
#line 240 "../SqlLexer.lpp"
-return TOKEN_OFF;
+return TOKEN_NULLS;
YY_BREAK
case 79:
YY_RULE_SETUP
#line 241 "../SqlLexer.lpp"
-return TOKEN_ON;
+return TOKEN_OFF;
YY_BREAK
case 80:
YY_RULE_SETUP
#line 242 "../SqlLexer.lpp"
-return TOKEN_OR;
+return TOKEN_ON;
YY_BREAK
case 81:
YY_RULE_SETUP
#line 243 "../SqlLexer.lpp"
-return TOKEN_ORDER;
+return TOKEN_OR;
YY_BREAK
case 82:
YY_RULE_SETUP
#line 244 "../SqlLexer.lpp"
-return TOKEN_OUTER;
+return TOKEN_ORDER;
YY_BREAK
case 83:
YY_RULE_SETUP
#line 245 "../SqlLexer.lpp"
-return TOKEN_PARTITION;
+return TOKEN_OUTER;
YY_BREAK
case 84:
YY_RULE_SETUP
#line 246 "../SqlLexer.lpp"
-return TOKEN_PARTITIONS;
+return TOKEN_PARTITION;
YY_BREAK
case 85:
YY_RULE_SETUP
#line 247 "../SqlLexer.lpp"
-return TOKEN_PERCENT;
+return TOKEN_PARTITIONS;
YY_BREAK
case 86:
YY_RULE_SETUP
#line 248 "../SqlLexer.lpp"
-return TOKEN_PRIMARY;
+return TOKEN_PERCENT;
YY_BREAK
case 87:
YY_RULE_SETUP
#line 249 "../SqlLexer.lpp"
-return TOKEN_QUIT;
+return TOKEN_PRIMARY;
YY_BREAK
case 88:
YY_RULE_SETUP
#line 250 "../SqlLexer.lpp"
-return TOKEN_RANGE;
+return TOKEN_QUIT;
YY_BREAK
case 89:
YY_RULE_SETUP
#line 251 "../SqlLexer.lpp"
-return TOKEN_REAL;
+return TOKEN_RANGE;
YY_BREAK
case 90:
YY_RULE_SETUP
#line 252 "../SqlLexer.lpp"
-return TOKEN_REFERENCES;
+return TOKEN_REAL;
YY_BREAK
case 91:
YY_RULE_SETUP
#line 253 "../SqlLexer.lpp"
-return TOKEN_REGEXP;
+return TOKEN_REFERENCES;
YY_BREAK
case 92:
YY_RULE_SETUP
#line 254 "../SqlLexer.lpp"
-return TOKEN_RIGHT;
+return TOKEN_REGEXP;
YY_BREAK
case 93:
YY_RULE_SETUP
#line 255 "../SqlLexer.lpp"
-return TOKEN_ROW_DELIMITER;
+return TOKEN_RIGHT;
YY_BREAK
case 94:
YY_RULE_SETUP
#line 256 "../SqlLexer.lpp"
-return TOKEN_SECOND;
+return TOKEN_ROW_DELIMITER;
YY_BREAK
case 95:
YY_RULE_SETUP
#line 257 "../SqlLexer.lpp"
-return TOKEN_SELECT;
+return TOKEN_SECOND;
YY_BREAK
case 96:
YY_RULE_SETUP
#line 258 "../SqlLexer.lpp"
-return TOKEN_SET;
+return TOKEN_SELECT;
YY_BREAK
case 97:
YY_RULE_SETUP
#line 259 "../SqlLexer.lpp"
-return TOKEN_SMA;
+return TOKEN_SET;
YY_BREAK
case 98:
YY_RULE_SETUP
#line 260 "../SqlLexer.lpp"
-return TOKEN_SMALLINT;
+return TOKEN_SMA;
YY_BREAK
case 99:
YY_RULE_SETUP
#line 261 "../SqlLexer.lpp"
-return TOKEN_TABLE;
+return TOKEN_SMALLINT;
YY_BREAK
case 100:
YY_RULE_SETUP
#line 262 "../SqlLexer.lpp"
-return TOKEN_THEN;
+return TOKEN_SUBSTRING;
YY_BREAK
case 101:
YY_RULE_SETUP
#line 263 "../SqlLexer.lpp"
-return TOKEN_TIME;
+return TOKEN_TABLE;
YY_BREAK
case 102:
YY_RULE_SETUP
#line 264 "../SqlLexer.lpp"
-return TOKEN_TIMESTAMP;
+return TOKEN_THEN;
YY_BREAK
case 103:
YY_RULE_SETUP
#line 265 "../SqlLexer.lpp"
-return TOKEN_TRUE;
+return TOKEN_TIME;
YY_BREAK
case 104:
YY_RULE_SETUP
#line 266 "../SqlLexer.lpp"
-return TOKEN_TUPLESAMPLE;
+return TOKEN_TIMESTAMP;
YY_BREAK
case 105:
YY_RULE_SETUP
#line 267 "../SqlLexer.lpp"
-return TOKEN_UNIQUE;
+return TOKEN_TRUE;
YY_BREAK
case 106:
YY_RULE_SETUP
#line 268 "../SqlLexer.lpp"
-return TOKEN_UPDATE;
+return TOKEN_TUPLESAMPLE;
YY_BREAK
case 107:
YY_RULE_SETUP
#line 269 "../SqlLexer.lpp"
-return TOKEN_USING;
+return TOKEN_UNIQUE;
YY_BREAK
case 108:
YY_RULE_SETUP
#line 270 "../SqlLexer.lpp"
-return TOKEN_VALUES;
+return TOKEN_UPDATE;
YY_BREAK
case 109:
YY_RULE_SETUP
#line 271 "../SqlLexer.lpp"
-return TOKEN_VARCHAR;
+return TOKEN_USING;
YY_BREAK
case 110:
YY_RULE_SETUP
#line 272 "../SqlLexer.lpp"
-return TOKEN_WHEN;
+return TOKEN_VALUES;
YY_BREAK
case 111:
YY_RULE_SETUP
#line 273 "../SqlLexer.lpp"
-return TOKEN_WHERE;
+return TOKEN_VARCHAR;
YY_BREAK
case 112:
YY_RULE_SETUP
#line 274 "../SqlLexer.lpp"
-return TOKEN_WITH;
+return TOKEN_WHEN;
YY_BREAK
case 113:
YY_RULE_SETUP
#line 275 "../SqlLexer.lpp"
-return TOKEN_YEAR;
+return TOKEN_WHERE;
YY_BREAK
case 114:
YY_RULE_SETUP
#line 276 "../SqlLexer.lpp"
-return TOKEN_YEARMONTH;
+return TOKEN_WITH;
YY_BREAK
case 115:
YY_RULE_SETUP
-#line 278 "../SqlLexer.lpp"
-return TOKEN_EQ;
+#line 277 "../SqlLexer.lpp"
+return TOKEN_YEAR;
YY_BREAK
case 116:
YY_RULE_SETUP
-#line 279 "../SqlLexer.lpp"
-return TOKEN_NEQ;
+#line 278 "../SqlLexer.lpp"
+return TOKEN_YEARMONTH;
YY_BREAK
case 117:
YY_RULE_SETUP
#line 280 "../SqlLexer.lpp"
-return TOKEN_NEQ;
+return TOKEN_EQ;
YY_BREAK
case 118:
YY_RULE_SETUP
#line 281 "../SqlLexer.lpp"
-return TOKEN_LT;
+return TOKEN_NEQ;
YY_BREAK
case 119:
YY_RULE_SETUP
#line 282 "../SqlLexer.lpp"
-return TOKEN_GT;
+return TOKEN_NEQ;
YY_BREAK
case 120:
YY_RULE_SETUP
#line 283 "../SqlLexer.lpp"
-return TOKEN_LEQ;
+return TOKEN_LT;
YY_BREAK
case 121:
YY_RULE_SETUP
#line 284 "../SqlLexer.lpp"
-return TOKEN_GEQ;
+return TOKEN_GT;
YY_BREAK
case 122:
YY_RULE_SETUP
+#line 285 "../SqlLexer.lpp"
+return TOKEN_LEQ;
+ YY_BREAK
+case 123:
+YY_RULE_SETUP
#line 286 "../SqlLexer.lpp"
+return TOKEN_GEQ;
+ YY_BREAK
+case 124:
+YY_RULE_SETUP
+#line 288 "../SqlLexer.lpp"
return yytext[0];
YY_BREAK
-case 123:
+case 125:
YY_RULE_SETUP
-#line 287 "../SqlLexer.lpp"
+#line 289 "../SqlLexer.lpp"
return yytext[0];
YY_BREAK
/**
* Quoted strings. Prefacing a string with an 'e' or 'E' causes escape
* sequences to be processed (as in PostgreSQL).
**/
-case 124:
+case 126:
YY_RULE_SETUP
-#line 293 "../SqlLexer.lpp"
+#line 295 "../SqlLexer.lpp"
{
yylval->string_value_ = new quickstep::ParseString(yylloc->first_line, yylloc->first_column);
BEGIN(CONDITION_STRING_SINGLE_QUOTED_ESCAPED);
}
YY_BREAK
-case 125:
+case 127:
YY_RULE_SETUP
-#line 298 "../SqlLexer.lpp"
+#line 300 "../SqlLexer.lpp"
{
yylval->string_value_ = new quickstep::ParseString(yylloc->first_line, yylloc->first_column);
BEGIN(CONDITION_STRING_SINGLE_QUOTED);
}
YY_BREAK
-case 126:
+case 128:
YY_RULE_SETUP
-#line 303 "../SqlLexer.lpp"
+#line 305 "../SqlLexer.lpp"
{
yylval->string_value_ = new quickstep::ParseString(yylloc->first_line, yylloc->first_column);
BEGIN(CONDITION_STRING_DOUBLE_QUOTED);
@@ -2079,7 +2092,7 @@ YY_RULE_SETUP
case YY_STATE_EOF(CONDITION_STRING_SINGLE_QUOTED):
case YY_STATE_EOF(CONDITION_STRING_SINGLE_QUOTED_ESCAPED):
case YY_STATE_EOF(CONDITION_STRING_DOUBLE_QUOTED):
-#line 312 "../SqlLexer.lpp"
+#line 314 "../SqlLexer.lpp"
{
delete yylval->string_value_;
BEGIN(INITIAL);
@@ -2090,9 +2103,9 @@ case YY_STATE_EOF(CONDITION_STRING_DOUBLE_QUOTED):
/* Process escape sequences. */
-case 127:
+case 129:
YY_RULE_SETUP
-#line 322 "../SqlLexer.lpp"
+#line 324 "../SqlLexer.lpp"
{
/* Octal code */
unsigned int code;
@@ -2106,9 +2119,9 @@ YY_RULE_SETUP
yylval->string_value_->push_back(code);
}
YY_BREAK
-case 128:
+case 130:
YY_RULE_SETUP
-#line 334 "../SqlLexer.lpp"
+#line 336 "../SqlLexer.lpp"
{
/* Hexadecimal code */
unsigned int code;
@@ -2116,9 +2129,9 @@ YY_RULE_SETUP
yylval->string_value_->push_back(code);
}
YY_BREAK
-case 129:
+case 131:
YY_RULE_SETUP
-#line 340 "../SqlLexer.lpp"
+#line 342 "../SqlLexer.lpp"
{
/* A numeric escape sequence that isn't correctly specified. */
delete yylval->string_value_;
@@ -2127,58 +2140,58 @@ YY_RULE_SETUP
return TOKEN_LEX_ERROR;
}
YY_BREAK
-case 130:
+case 132:
YY_RULE_SETUP
-#line 347 "../SqlLexer.lpp"
+#line 349 "../SqlLexer.lpp"
{
/* Backspace */
yylval->string_value_->push_back('\b');
}
YY_BREAK
-case 131:
+case 133:
YY_RULE_SETUP
-#line 351 "../SqlLexer.lpp"
+#line 353 "../SqlLexer.lpp"
{
/* Form-feed */
yylval->string_value_->push_back('\f');
}
YY_BREAK
-case 132:
+case 134:
YY_RULE_SETUP
-#line 355 "../SqlLexer.lpp"
+#line 357 "../SqlLexer.lpp"
{
/* Newline */
yylval->string_value_->push_back('\n');
}
YY_BREAK
-case 133:
+case 135:
YY_RULE_SETUP
-#line 359 "../SqlLexer.lpp"
+#line 361 "../SqlLexer.lpp"
{
/* Carriage-return */
yylval->string_value_->push_back('\r');
}
YY_BREAK
-case 134:
+case 136:
YY_RULE_SETUP
-#line 363 "../SqlLexer.lpp"
+#line 365 "../SqlLexer.lpp"
{
/* Horizontal Tab */
yylval->string_value_->push_back('\t');
}
YY_BREAK
-case 135:
-/* rule 135 can match eol */
+case 137:
+/* rule 137 can match eol */
YY_RULE_SETUP
-#line 367 "../SqlLexer.lpp"
+#line 369 "../SqlLexer.lpp"
{
/* Any other character (including actual newline or carriage return) */
yylval->string_value_->push_back(yytext[1]);
}
YY_BREAK
-case 136:
+case 138:
YY_RULE_SETUP
-#line 371 "../SqlLexer.lpp"
+#line 373 "../SqlLexer.lpp"
{
/* This should only be encountered right before an EOF. */
delete yylval->string_value_;
@@ -2189,17 +2202,17 @@ YY_RULE_SETUP
YY_BREAK
-case 137:
+case 139:
YY_RULE_SETUP
-#line 381 "../SqlLexer.lpp"
+#line 383 "../SqlLexer.lpp"
{
/* Two quotes in a row become a single quote (this is specified by the SQL standard). */
yylval->string_value_->push_back('\'');
}
YY_BREAK
-case 138:
+case 140:
YY_RULE_SETUP
-#line 385 "../SqlLexer.lpp"
+#line 387 "../SqlLexer.lpp"
{
/* End string */
BEGIN(CONDITION_SQL);
@@ -2208,17 +2221,17 @@ YY_RULE_SETUP
YY_BREAK
-case 139:
+case 141:
YY_RULE_SETUP
-#line 393 "../SqlLexer.lpp"
+#line 395 "../SqlLexer.lpp"
{
/* Two quotes in a row become a single quote (this is specified by the SQL standard). */
yylval->string_value_->push_back('"');
}
YY_BREAK
-case 140:
+case 142:
YY_RULE_SETUP
-#line 397 "../SqlLexer.lpp"
+#line 399 "../SqlLexer.lpp"
{
/* End string */
BEGIN(CONDITION_SQL);
@@ -2226,94 +2239,94 @@ YY_RULE_SETUP
}
YY_BREAK
-case 141:
-/* rule 141 can match eol */
+case 143:
+/* rule 143 can match eol */
YY_RULE_SETUP
-#line 404 "../SqlLexer.lpp"
+#line 406 "../SqlLexer.lpp"
{
/* Scan up to a quote. */
yylval->string_value_->append(yytext, yyleng);
}
YY_BREAK
-case 142:
-/* rule 142 can match eol */
+case 144:
+/* rule 144 can match eol */
YY_RULE_SETUP
-#line 409 "../SqlLexer.lpp"
+#line 411 "../SqlLexer.lpp"
{
/* Scan up to a quote or escape sequence. */
yylval->string_value_->append(yytext, yyleng);
}
YY_BREAK
-case 143:
-/* rule 143 can match eol */
+case 145:
+/* rule 145 can match eol */
YY_RULE_SETUP
-#line 414 "../SqlLexer.lpp"
+#line 416 "../SqlLexer.lpp"
{
/* Scan up to a quote. */
yylval->string_value_->append(yytext, yyleng);
}
YY_BREAK
-case 144:
+case 146:
YY_RULE_SETUP
-#line 420 "../SqlLexer.lpp"
+#line 422 "../SqlLexer.lpp"
{
yylval->string_value_ = new quickstep::ParseString(
yylloc->first_line, yylloc->first_column, std::string(yytext, yyleng));
return TOKEN_NAME;
}
YY_BREAK
-case 145:
+case 147:
YY_RULE_SETUP
-#line 426 "../SqlLexer.lpp"
+#line 428 "../SqlLexer.lpp"
{
yylval->numeric_literal_value_ = new quickstep::NumericParseLiteralValue(
yylloc->first_line, yylloc->first_column, yytext);
return TOKEN_UNSIGNED_NUMVAL;
}
YY_BREAK
-case 146:
+case 148:
YY_RULE_SETUP
-#line 432 "../SqlLexer.lpp"
+#line 434 "../SqlLexer.lpp"
/* comment */
YY_BREAK
-case 147:
-/* rule 147 can match eol */
+case 149:
+/* rule 149 can match eol */
YY_RULE_SETUP
-#line 434 "../SqlLexer.lpp"
+#line 436 "../SqlLexer.lpp"
{ yycolumn = 0; }
YY_BREAK
-case 148:
+case 150:
YY_RULE_SETUP
-#line 436 "../SqlLexer.lpp"
+#line 438 "../SqlLexer.lpp"
; /* ignore white space */
YY_BREAK
/* CONDITION_SQL */
case YY_STATE_EOF(INITIAL):
case YY_STATE_EOF(CONDITION_COMMAND):
case YY_STATE_EOF(CONDITION_SQL):
-#line 440 "../SqlLexer.lpp"
+#line 442 "../SqlLexer.lpp"
{
/* All conditions except for mutli-state string extracting conditions. */
BEGIN(INITIAL);
return TOKEN_EOF;
}
YY_BREAK
-case 149:
+case 151:
YY_RULE_SETUP
-#line 446 "../SqlLexer.lpp"
+#line 448 "../SqlLexer.lpp"
{
BEGIN(INITIAL);
quickstep_yyerror(NULL, yyscanner, NULL, "illegal character");
return TOKEN_LEX_ERROR;
}
YY_BREAK
-case 150:
+case 152:
YY_RULE_SETUP
-#line 452 "../SqlLexer.lpp"
+#line 454 "../SqlLexer.lpp"
YY_FATAL_ERROR( "flex scanner jammed" );
YY_BREAK
-#line 2317 "SqlLexer_gen.cpp"
+#line 2330 "SqlLexer_gen.cpp"
case YY_END_OF_BUFFER:
{
@@ -2607,7 +2620,7 @@ static int yy_get_next_buffer (yyscan_t yyscanner)
while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state )
{
yy_current_state = (int) yy_def[yy_current_state];
- if ( yy_current_state >= 545 )
+ if ( yy_current_state >= 553 )
yy_c = yy_meta[(unsigned int) yy_c];
}
yy_current_state = yy_nxt[yy_base[yy_current_state] + (unsigned int) yy_c];
@@ -2636,11 +2649,11 @@ static int yy_get_next_buffer (yyscan_t yyscanner)
while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state )
{
yy_current_state = (int) yy_def[yy_current_state];
- if ( yy_current_state >= 545 )
+ if ( yy_current_state >= 553 )
yy_c = yy_meta[(unsigned int) yy_c];
}
yy_current_state = yy_nxt[yy_base[yy_current_state] + (unsigned int) yy_c];
- yy_is_jam = (yy_current_state == 544);
+ yy_is_jam = (yy_current_state == 552);
(void)yyg;
return yy_is_jam ? 0 : yy_current_state;
@@ -3474,7 +3487,7 @@ void quickstep_yyfree (void * ptr , yyscan_t yyscanner)
#define YYTABLES_NAME "yytables"
-#line 452 "../SqlLexer.lpp"
+#line 454 "../SqlLexer.lpp"
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/767b2ef1/parser/preprocessed/SqlLexer_gen.hpp
----------------------------------------------------------------------
diff --git a/parser/preprocessed/SqlLexer_gen.hpp b/parser/preprocessed/SqlLexer_gen.hpp
index d629f04..c14559b 100644
--- a/parser/preprocessed/SqlLexer_gen.hpp
+++ b/parser/preprocessed/SqlLexer_gen.hpp
@@ -360,7 +360,7 @@ extern int quickstep_yylex \
#undef YY_DECL
#endif
-#line 452 "../SqlLexer.lpp"
+#line 454 "../SqlLexer.lpp"
#line 367 "SqlLexer_gen.hpp"
[22/50] [abbrv] incubator-quickstep git commit: Added print_query
gflag (#213)
Posted by zu...@apache.org.
Added print_query gflag (#213)
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/8939c253
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/8939c253
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/8939c253
Branch: refs/heads/work-order-serialization
Commit: 8939c25397c55149ed14834e7b4a41d88c6f9d00
Parents: 32e7c1b
Author: Jianqiao Zhu <ji...@cs.wisc.edu>
Authored: Sat May 7 09:44:56 2016 -0500
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Mon May 30 15:46:31 2016 -0700
----------------------------------------------------------------------
cli/QuickstepCli.cpp | 10 ++++++++++
1 file changed, 10 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/8939c253/cli/QuickstepCli.cpp
----------------------------------------------------------------------
diff --git a/cli/QuickstepCli.cpp b/cli/QuickstepCli.cpp
index ec195f7..4c0a166 100644
--- a/cli/QuickstepCli.cpp
+++ b/cli/QuickstepCli.cpp
@@ -1,6 +1,8 @@
/**
* Copyright 2011-2015 Quickstep Technologies LLC.
* Copyright 2015-2016 Pivotal Software, Inc.
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -144,6 +146,10 @@ DEFINE_string(worker_affinities, "",
"means that they will all be runable on any CPU according to "
"the kernel's own scheduling policy).");
DEFINE_bool(initialize_db, false, "If true, initialize a database.");
+DEFINE_bool(print_query, false,
+ "Print each input query statement. This is useful when running a "
+ "large number of queries in a batch.");
+
} // namespace quickstep
int main(int argc, char* argv[]) {
@@ -336,6 +342,10 @@ int main(int argc, char* argv[]) {
break;
}
+ if (quickstep::FLAGS_print_query) {
+ printf("\n%s\n", command_string->c_str());
+ }
+
parser_wrapper->feedNextBuffer(command_string);
bool quitting = false;
[33/50] [abbrv] incubator-quickstep git commit: Change the default
value of Joined Tuple Collector (#226)
Posted by zu...@apache.org.
Change the default value of Joined Tuple Collector (#226)
On NUMA boxes, the vector based joined tuple collector slows down queries.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/a25da390
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/a25da390
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/a25da390
Branch: refs/heads/work-order-serialization
Commit: a25da39001017360907ee523d94b48cb4417fe24
Parents: 3c84537
Author: Jignesh Patel <pa...@users.noreply.github.com>
Authored: Wed May 18 11:56:53 2016 -0500
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Mon May 30 15:47:51 2016 -0700
----------------------------------------------------------------------
relational_operators/HashJoinOperator.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/a25da390/relational_operators/HashJoinOperator.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/HashJoinOperator.cpp b/relational_operators/HashJoinOperator.cpp
index 104a02d..aa03794 100644
--- a/relational_operators/HashJoinOperator.cpp
+++ b/relational_operators/HashJoinOperator.cpp
@@ -59,7 +59,7 @@ namespace quickstep {
namespace {
-DEFINE_bool(vector_based_joined_tuple_collector, true,
+DEFINE_bool(vector_based_joined_tuple_collector, false,
"If true, use simple vector-based joined tuple collector in "
"hash join, with a final sort pass to group joined tuple pairs "
"by inner block. If false, use unordered_map based collector that "
[16/50] [abbrv] incubator-quickstep git commit: Transaction Part 4:
LockManager, CycleDetector and DeadLockDetector. (#187)
Posted by zu...@apache.org.
Transaction Part 4: LockManager, CycleDetector and DeadLockDetector. (#187)
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/bbaff7a2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/bbaff7a2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/bbaff7a2
Branch: refs/heads/work-order-serialization
Commit: bbaff7a2aa84b0792a33ca652da701c1b3498008
Parents: a3889a3
Author: Hakan Memisoglu <ha...@gmail.com>
Authored: Wed May 4 23:30:51 2016 -0500
Committer: Zuyu ZHANG <zu...@users.noreply.github.com>
Committed: Wed May 4 21:30:51 2016 -0700
----------------------------------------------------------------------
transaction/AccessMode.hpp | 76 +++++-
transaction/CMakeLists.txt | 59 ++++-
transaction/CycleDetector.cpp | 120 ++++++++++
transaction/CycleDetector.hpp | 83 +++++++
transaction/DeadLockDetector.cpp | 177 ++++++++++++++
transaction/DeadLockDetector.hpp | 156 ++++++++++++
transaction/DirectedGraph.hpp | 56 ++---
transaction/LockManager.cpp | 237 +++++++++++++++++++
transaction/LockManager.hpp | 128 ++++++++++
transaction/LockTable.cpp | 22 +-
transaction/LockTable.hpp | 36 +--
transaction/ResourceId.hpp | 9 +-
transaction/StronglyConnectedComponents.cpp | 1 -
transaction/Transaction.cpp | 48 ----
transaction/TransactionTable.cpp | 8 +-
transaction/TransactionTable.hpp | 11 +-
transaction/tests/AccessMode_unittest.cpp | 12 +-
transaction/tests/CycleDetector_unittest.cpp | 157 ++++++++++++
transaction/tests/DeadLockDetector_unittest.cpp | 96 ++++++++
transaction/tests/DirectedGraph_unittest.cpp | 113 +++++----
transaction/tests/LockRequest_unittest.cpp | 4 +-
transaction/tests/LockTable_unittest.cpp | 63 ++---
transaction/tests/Lock_unittest.cpp | 13 +-
.../StronglyConnectedComponents_unittest.cpp | 15 +-
transaction/tests/TransactionTable_unittest.cpp | 102 ++++----
25 files changed, 1512 insertions(+), 290 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bbaff7a2/transaction/AccessMode.hpp
----------------------------------------------------------------------
diff --git a/transaction/AccessMode.hpp b/transaction/AccessMode.hpp
index 34ace36..bb06689 100644
--- a/transaction/AccessMode.hpp
+++ b/transaction/AccessMode.hpp
@@ -32,12 +32,12 @@ namespace transaction {
* @brief Represents mode type. Possible options are NL, IS, IX, S, SIX, X.
**/
enum class AccessModeType : std::uint8_t {
- kNoLock = 0,
- kIsLock,
- kIxLock,
- kSLock,
- kSixLock,
- kXLock,
+ kNoLockMode = 0,
+ kIsLockMode,
+ kIxLockMode,
+ kSLockMode,
+ kSixLockMode,
+ kXLockMode,
kNumAccessModeTypes,
};
@@ -55,6 +55,60 @@ class AccessMode {
: access_mode_(access_mode) {}
/**
+ * @brief Factory method for NoLockMode.
+ *
+ * @return NoLockMode instance.
+ **/
+ static AccessMode NoLockMode() {
+ return AccessMode(AccessModeType::kNoLockMode);
+ }
+
+ /**
+ * @brief Factory method for IsLockMode.
+ *
+ * @return IsLockMode instance.
+ **/
+ static AccessMode IsLockMode() {
+ return AccessMode(AccessModeType::kIsLockMode);
+ }
+
+ /**
+ * @brief Factory method for IxLockMode.
+ *
+ * @return IxLockMode instance.
+ **/
+ static AccessMode IxLockMode() {
+ return AccessMode(AccessModeType::kIxLockMode);
+ }
+
+ /**
+ * @brief Factory method for SixLockMode.
+ *
+ * @return SixLockMode instance.
+ **/
+ static AccessMode SixLockMode() {
+ return AccessMode(AccessModeType::kSixLockMode);
+ }
+
+ /**
+ * @brief Factory method for SLockMode.
+ *
+ * @return SLockMode instance.
+ **/
+ static AccessMode SLockMode() {
+ return AccessMode(AccessModeType::kSLockMode);
+ }
+
+ /**
+ * @brief Factory method for XLockMode.
+ *
+ * @return XLockMode instance.
+ **/
+ static AccessMode XLockMode() {
+ return AccessMode(AccessModeType::kXLockMode);
+ }
+
+ /**
* @brief Checks whether this access mode is compatible with the other.
*
* @param other Other access mode that will be checked against to this one.
@@ -74,7 +128,7 @@ class AccessMode {
* @return True if it is IS mode, false otherwise.
**/
inline bool isIntentionShareLock() const {
- return access_mode_ == AccessModeType::kIsLock;
+ return access_mode_ == AccessModeType::kIsLockMode;
}
/**
@@ -83,7 +137,7 @@ class AccessMode {
* @return True if it is IX mode, false otherwise.
**/
inline bool isIntentionExclusiveLock() const {
- return access_mode_ == AccessModeType::kIxLock;
+ return access_mode_ == AccessModeType::kIxLockMode;
}
/**
@@ -92,7 +146,7 @@ class AccessMode {
* @return True if it is SIX mode, false otherwise.
**/
inline bool isShareAndIntentionExclusiveLock() const {
- return access_mode_ == AccessModeType::kSixLock;
+ return access_mode_ == AccessModeType::kSixLockMode;
}
/**
@@ -101,7 +155,7 @@ class AccessMode {
* @return True if it is S mode, false otherwise.
**/
inline bool isShareLock() const {
- return access_mode_ == AccessModeType::kSLock;
+ return access_mode_ == AccessModeType::kSLockMode;
}
/**
@@ -110,7 +164,7 @@ class AccessMode {
* @return True if it is X mode, false otherwise.
**/
inline bool isExclusiveLock() const {
- return access_mode_ == AccessModeType::kXLock;
+ return access_mode_ == AccessModeType::kXLockMode;
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bbaff7a2/transaction/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/transaction/CMakeLists.txt b/transaction/CMakeLists.txt
index 05fc96a..c6c87b6 100644
--- a/transaction/CMakeLists.txt
+++ b/transaction/CMakeLists.txt
@@ -16,12 +16,21 @@
add_library(quickstep_transaction_AccessMode
AccessMode.cpp
AccessMode.hpp)
+add_library(quickstep_transaction_CycleDetector
+ CycleDetector.cpp
+ CycleDetector.hpp)
+add_library(quickstep_transaction_DeadLockDetector
+ DeadLockDetector.cpp
+ DeadLockDetector.cpp)
add_library(quickstep_transaction_DirectedGraph
../empty_src.cpp
DirectedGraph.hpp)
add_library(quickstep_transaction_Lock
../empty_src.cpp
Lock.hpp)
+add_library(quickstep_transaction_LockManager
+ LockManager.hpp
+ LockManager.cpp)
add_library(quickstep_transaction_LockRequest
../empty_src.cpp
LockRequest.hpp)
@@ -40,7 +49,17 @@ add_library(quickstep_transaction_Transaction
add_library(quickstep_transaction_TransactionTable
TransactionTable.cpp
TransactionTable.hpp)
-
+
+target_link_libraries(quickstep_transaction_CycleDetector
+ quickstep_transaction_DirectedGraph
+ quickstep_transaction_StronglyConnectedComponents
+ quickstep_utility_Macros)
+target_link_libraries(quickstep_transaction_DeadLockDetector
+ glog
+ quickstep_transaction_CycleDetector
+ quickstep_transaction_DirectedGraph
+ quickstep_transaction_LockTable
+ quickstep_transaction_Transaction)
target_link_libraries(quickstep_transaction_DirectedGraph
glog
quickstep_transaction_Transaction
@@ -48,6 +67,18 @@ target_link_libraries(quickstep_transaction_DirectedGraph
target_link_libraries(quickstep_transaction_Lock
quickstep_transaction_AccessMode
quickstep_transaction_ResourceId)
+target_link_libraries(quickstep_transaction_LockManager
+ gflags_nothreads-static
+ glog
+ quickstep_utility_ThreadSafeQueue
+ quickstep_threading_Thread
+ quickstep_transaction_AccessMode
+ quickstep_transaction_DeadLockDetector
+ quickstep_transaction_LockRequest
+ quickstep_transaction_LockTable
+ quickstep_transaction_ResourceId
+ quickstep_transaction_Transaction
+ quickstep_transaction_TransactionTable)
target_link_libraries(quickstep_transaction_LockRequest
quickstep_transaction_AccessMode
quickstep_transaction_ResourceId
@@ -80,8 +111,11 @@ add_library(quickstep_transaction
TransactionModule.hpp)
target_link_libraries(quickstep_transaction
quickstep_transaction_AccessMode
+ quickstep_transaction_CycleDetector
+ quickstep_transaction_DeadLockDetector
quickstep_transaction_DirectedGraph
quickstep_transaction_Lock
+ quickstep_transaction_LockManager
quickstep_transaction_LockRequest
quickstep_transaction_LockTable
quickstep_transaction_ResourceId
@@ -97,6 +131,29 @@ target_link_libraries(AccessMode_unittest
quickstep_transaction_AccessMode)
add_test(AccessMode_unittest AccessMode_unittest)
+add_executable(CycleDetector_unittest
+ "${CMAKE_CURRENT_SOURCE_DIR}/tests/CycleDetector_unittest.cpp")
+target_link_libraries(CycleDetector_unittest
+ gtest
+ gtest_main
+ quickstep_transaction_CycleDetector
+ quickstep_transaction_DirectedGraph
+ quickstep_transaction_StronglyConnectedComponents)
+add_test(CycleDetector_unittest CycleDetector_unittest)
+
+add_executable(DeadLockDetector_unittest
+ "${CMAKE_CURRENT_SOURCE_DIR}/tests/DeadLockDetector_unittest.cpp")
+target_link_libraries(DeadLockDetector_unittest
+ gtest
+ gtest_main
+ quickstep_threading_Thread
+ quickstep_transaction_AccessMode
+ quickstep_transaction_DeadLockDetector
+ quickstep_transaction_LockTable
+ quickstep_transaction_ResourceId
+ quickstep_transaction_Transaction)
+add_test(DeadLockDetector_unittest DeadLockDetector_unittest)
+
add_executable(DirectedGraph_unittest
"${CMAKE_CURRENT_SOURCE_DIR}/tests/DirectedGraph_unittest.cpp")
target_link_libraries(DirectedGraph_unittest
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bbaff7a2/transaction/CycleDetector.cpp
----------------------------------------------------------------------
diff --git a/transaction/CycleDetector.cpp b/transaction/CycleDetector.cpp
new file mode 100644
index 0000000..b12897f
--- /dev/null
+++ b/transaction/CycleDetector.cpp
@@ -0,0 +1,120 @@
+/**
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#include "transaction/CycleDetector.hpp"
+
+#include <cstdint>
+#include <memory>
+#include <stack>
+#include <unordered_map>
+#include <unordered_set>
+#include <utility>
+#include <vector>
+
+#include "transaction/DirectedGraph.hpp"
+#include "transaction/StronglyConnectedComponents.hpp"
+
+namespace quickstep {
+namespace transaction {
+
+CycleDetector::CycleDetector(DirectedGraph *wait_for_graph)
+ : wait_for_graph_(wait_for_graph),
+ strongly_connected_components_(
+ std::make_unique<StronglyConnectedComponents>(*wait_for_graph)) {
+}
+
+std::vector<DirectedGraph::node_id>
+CycleDetector::chooseVictimsToBreakCycle() const {
+ std::vector<DirectedGraph::node_id> nodes_to_kill;
+ const std::unordered_map<std::uint64_t, std::vector<DirectedGraph::node_id>>
+ component_mapping = strongly_connected_components_->getComponentMapping();
+ for (const auto &entry : component_mapping) {
+ // One node means no cycle.
+ if (entry.second.size() == 1) {
+ continue;
+ }
+ const std::vector<DirectedGraph::node_id> nodes =
+ chooseVictimsInComponent(entry.second);
+ nodes_to_kill.insert(nodes_to_kill.end(), nodes.begin(), nodes.end());
+ }
+ return nodes_to_kill;
+}
+
+std::vector<DirectedGraph::node_id> CycleDetector::chooseVictimsInComponent(
+ const std::vector<DirectedGraph::node_id> &nodes) const {
+ std::vector<DirectedGraph::node_id> targets;
+ // Convert it to set to ensure defensively that the elements are unique.
+ std::unordered_set<DirectedGraph::node_id> nodes_set(nodes.begin(),
+ nodes.end());
+
+ while (true) {
+ if (!hasCycle(nodes_set)) {
+ break;
+ }
+ // Connected component still has a cycle, therefore choose a
+ // victim and keep trying to remove nodes until there is no cycle.
+ const DirectedGraph::node_id victim = chooseVictim(nodes_set);
+ // Remove the victim node from the connected component.
+ nodes_set.erase(victim);
+ // Removed node is a victim now.
+ targets.push_back(victim);
+ }
+ return targets;
+}
+
+bool CycleDetector::hasCycle(
+ const std::unordered_set<DirectedGraph::node_id> &nodes) const {
+ // Keeps track of the nodes the algorithms visited.
+ std::unordered_set<DirectedGraph::node_id> visited;
+ for (const DirectedGraph::node_id node_id : nodes) {
+ // If it is visited, then pass to the next one.
+ if (visited.count(node_id) == 1) {
+ continue;
+ }
+ // Save the backtracking information.
+ std::stack<DirectedGraph::node_id> to_visit;
+ // Mark this id as "to be visited".
+ to_visit.push(node_id);
+ // Start to visit nodes until it is done.
+ while (!to_visit.empty()) {
+ const DirectedGraph::node_id current_node = to_visit.top();
+ to_visit.pop();
+ // Mark the node coming from stack as "visited".
+ visited.insert(current_node);
+ // For all adjacent nodes of this "visited" node,
+ const std::vector<DirectedGraph::node_id> adjacents
+ = wait_for_graph_->getAdjacentNodes(current_node);
+ for (const DirectedGraph::node_id adjacent : adjacents) {
+ if (visited.count(adjacent) == 1) {
+ // If this adjacent node is a node we already visited, then
+ // there is a cycle.
+ return true;
+ } else if (nodes.count(adjacent) == 1 && visited.count(adjacent) == 0) {
+ // Otherwise, if it is a node that we did not visit before
+ // mark this nodes as "to be visited".
+ to_visit.push(adjacent);
+ }
+ }
+ }
+ }
+ // If we have already visited all nodes and could not find a cycle,
+ // then we should return "no cycle" result.
+ return false;
+}
+
+} // namespace transaction
+} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bbaff7a2/transaction/CycleDetector.hpp
----------------------------------------------------------------------
diff --git a/transaction/CycleDetector.hpp b/transaction/CycleDetector.hpp
new file mode 100644
index 0000000..6865e2d
--- /dev/null
+++ b/transaction/CycleDetector.hpp
@@ -0,0 +1,83 @@
+/**
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#ifndef QUICKSTEP_TRANSACTION_CYCLE_DETECTOR_HPP_
+#define QUICKSTEP_TRANSACTION_CYCLE_DETECTOR_HPP_
+
+#include <memory>
+#include <unordered_set>
+#include <vector>
+
+#include "transaction/DirectedGraph.hpp"
+#include "transaction/StronglyConnectedComponents.hpp"
+#include "utility/Macros.hpp"
+
+namespace quickstep {
+namespace transaction {
+
+/** \addtogroup Transaction
+ * @{
+ */
+
+/**
+ * @brief Class for running cycle detection algorithm on directed graph.
+ */
+class CycleDetector {
+ public:
+ /**
+ * @brief Constructor for DirectedGraph.
+ *
+ * @param wait_for_graph Pointer to a directed wait-for graph.
+ */
+ explicit CycleDetector(DirectedGraph *wait_for_graph);
+
+ /**
+ * @brief Calculate which nodes should be killed to eliminate all cycles
+ * in the graph.
+ *
+ * @return Vector of node ids that should be killed to break all cycles.
+ */
+ std::vector<DirectedGraph::node_id> chooseVictimsToBreakCycle() const;
+
+ private:
+ std::vector<DirectedGraph::node_id> chooseVictimsInComponent(
+ const std::vector<DirectedGraph::node_id> &nodes) const;
+
+ inline DirectedGraph::node_id chooseVictim(
+ const std::unordered_set<DirectedGraph::node_id> &nodes_set) const {
+ // TODO(Hakan): This is very inefficient scheme, however in the
+ // future, we can use the transaction's priority
+ // as the victim selection parameter.
+ return *(nodes_set.begin());
+ }
+
+ // Checks whether the nodes in the set make a cycle.
+ bool hasCycle(const std::unordered_set<DirectedGraph::node_id> &within) const;
+
+ DirectedGraph *wait_for_graph_;
+
+ std::unique_ptr<StronglyConnectedComponents> strongly_connected_components_;
+
+ DISALLOW_COPY_AND_ASSIGN(CycleDetector);
+};
+
+/** @} */
+
+} // namespace transaction
+} // namespace quickstep
+
+#endif // QUICKSTEP_TRANSACTION_CYCLE_DETECTOR_HPP_
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bbaff7a2/transaction/DeadLockDetector.cpp
----------------------------------------------------------------------
diff --git a/transaction/DeadLockDetector.cpp b/transaction/DeadLockDetector.cpp
new file mode 100644
index 0000000..26ab115
--- /dev/null
+++ b/transaction/DeadLockDetector.cpp
@@ -0,0 +1,177 @@
+/**
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#include "transaction/DeadLockDetector.hpp"
+
+#include <algorithm>
+#include <atomic>
+#include <chrono>
+#include <memory>
+#include <thread> // NOLINT(build/c++11)
+#include <utility>
+#include <vector>
+
+#include "transaction/CycleDetector.hpp"
+#include "transaction/DirectedGraph.hpp"
+#include "transaction/LockTable.hpp"
+#include "transaction/Transaction.hpp"
+
+#include "glog/logging.h"
+
+namespace quickstep {
+namespace transaction {
+
+constexpr std::int64_t DeadLockDetector::kSleepDurationInSeconds;
+
+DeadLockDetector::DeadLockDetector(LockTable *lock_table,
+ std::atomic<DeadLockDetectorStatus> *status,
+ std::vector<DirectedGraph::node_id> *victims)
+ : tid_node_mapping_(std::make_unique<transaction_id_node_map>()),
+ lock_table_(lock_table),
+ status_(status),
+ victims_(victims) {
+}
+
+void DeadLockDetector::run() {
+ while (true) {
+ if (status_->load() == DeadLockDetectorStatus::kQuit) {
+ // DeadLockDetector should stop.
+ return;
+ }
+ while (status_->load() == DeadLockDetectorStatus::kDone) {
+ // LockTable has not process the previous batch yet.
+ }
+
+ // TODO(Hakan): Implement logging mechanism for deadlock detection
+ // start and end times.
+ std::vector<DirectedGraph::node_id> victim_new_batch = getAllVictims();
+
+ // Swap new batch with old batch to make LockTable to see new victims.
+ std::swap(victim_new_batch, *victims_);
+
+ // Signal LockTable that new batch is ready.
+ status_->store(DeadLockDetectorStatus::kDone);
+
+ // DeadLockDetector should run once in a predefined interval.
+ std::this_thread::sleep_for(
+ std::chrono::seconds(kSleepDurationInSeconds));
+ }
+}
+
+void DeadLockDetector::addPendingInfo(const transaction_id pending,
+ const transaction_id owner) {
+ const DirectedGraph::node_id pending_node_id = getNodeId(pending);
+ const DirectedGraph::node_id owner_node_id = getNodeId(owner);
+
+ // TODO(Hakan): Check first whether link is already created. Use checked
+ // version for adding an edge.
+ wait_for_graph_->addEdgeUnchecked(pending_node_id, owner_node_id);
+}
+
+void DeadLockDetector::deletePendingInfo(const transaction_id pending,
+ const transaction_id owner) {
+ LOG(FATAL) << "Not implemented";
+}
+
+bool DeadLockDetector::isDependent(const transaction_id pending,
+ const transaction_id owner) const {
+ LOG(FATAL) << "Not implemented";
+}
+
+std::vector<transaction_id>
+DeadLockDetector::getAllDependents(const transaction_id owner) const {
+ LOG(FATAL) << "Not implemented";
+}
+
+std::vector<transaction_id>
+DeadLockDetector::getAllDependees(transaction_id pending) {
+ const DirectedGraph::node_id pending_node_id = getNodeId(pending);
+ const std::vector<DirectedGraph::node_id> nodes
+ = wait_for_graph_->getAdjacentNodes(pending_node_id);
+ std::vector<transaction_id> transactions;
+ transactions.reserve(nodes.size());
+ for (const DirectedGraph::node_id node_id : nodes) {
+ const transaction_id tid = wait_for_graph_->getDataFromNode(node_id);
+ transactions.push_back(tid);
+ }
+ return transactions;
+}
+
+DirectedGraph::node_id DeadLockDetector::getNodeId(const transaction_id tid) {
+ DirectedGraph::node_id node_id;
+ if (tid_node_mapping_->count(tid) == 0) {
+ // If it is not created, create it.
+ node_id = addNode(tid);
+ } else {
+ // Otherwise find it in the map.
+ node_id = (*tid_node_mapping_)[tid];
+ }
+ return node_id;
+}
+
+
+DirectedGraph::node_id DeadLockDetector::addNode(const transaction_id tid) {
+ const DirectedGraph::node_id node_id =
+ wait_for_graph_->addNodeUnchecked(tid);
+ tid_node_mapping_->emplace(tid, node_id);
+ return node_id;
+}
+
+std::vector<transaction_id> DeadLockDetector::getAllVictims() {
+ std::vector<transaction_id> result_victims;
+ wait_for_graph_.reset(new DirectedGraph());
+
+ // Critical region on LockTable starts here.
+ lock_table_->latchShared();
+ for (const auto &lock_control_block : *lock_table_) {
+ const LockTable::lock_own_list &own_list = lock_control_block.second.first;
+ const LockTable::lock_pending_list &pending_list =
+ lock_control_block.second.second;
+
+ for (const auto &owned_lock_info : own_list) {
+ const transaction_id owned_transaction = owned_lock_info.first;
+ const DirectedGraph::node_id owned_node = getNodeId(owned_transaction);
+
+ for (const auto &pending_lock_info : pending_list) {
+ const transaction_id pending_transaction = pending_lock_info.first;
+ const DirectedGraph::node_id pending_node = getNodeId(pending_transaction);
+
+ wait_for_graph_->addEdgeUnchecked(pending_node, owned_node);
+ }
+ }
+ }
+
+ lock_table_->unlatchShared();
+ // Critical region on LockTable ends here.
+
+ const CycleDetector cycle_detector(wait_for_graph_.get());
+ const std::vector<DirectedGraph::node_id> victim_nodes =
+ cycle_detector.chooseVictimsToBreakCycle();
+ for (const DirectedGraph::node_id node_id : victim_nodes) {
+ const transaction_id victim_tid = wait_for_graph_->getDataFromNode(node_id);
+ result_victims.push_back(victim_tid);
+ }
+
+ // Destroy the wait graph. It will be reconstructed kSleepDurationSeconds
+ // seconds later.
+ wait_for_graph_.reset();
+
+ return result_victims;
+}
+
+} // namespace transaction
+} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bbaff7a2/transaction/DeadLockDetector.hpp
----------------------------------------------------------------------
diff --git a/transaction/DeadLockDetector.hpp b/transaction/DeadLockDetector.hpp
new file mode 100644
index 0000000..6897afb
--- /dev/null
+++ b/transaction/DeadLockDetector.hpp
@@ -0,0 +1,156 @@
+/**
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#ifndef QUICKSTEP_TRANSACTION_DEAD_LOCK_DETECTOR_HPP_
+#define QUICKSTEP_TRANSACTION_DEAD_LOCK_DETECTOR_HPP_
+
+#include <atomic>
+#include <cstdint>
+#include <memory>
+#include <unordered_map>
+#include <vector>
+
+#include "threading/Thread.hpp"
+#include "transaction/DirectedGraph.hpp"
+#include "transaction/Transaction.hpp"
+
+namespace quickstep {
+namespace transaction {
+
+class LockTable;
+
+/** \addtogroup Transaction
+ * @{
+ */
+
+/**
+ * @brief Notification mechanism between LockManager and DeadLockDetector.
+ **/
+enum class DeadLockDetectorStatus {
+ kNotReady = 0,
+ kDone,
+ kQuit,
+};
+
+/**
+ * @brief Class for deadlock detection on wait-for graph.
+ **/
+class DeadLockDetector : public Thread {
+ public:
+ typedef std::unordered_map<transaction_id, DirectedGraph::node_id>
+ transaction_id_node_map;
+
+ /**
+ * @brief Constructor for DeadLockDetector.
+ *
+ * @param lock_table Pointer to lock table, which this class gets the
+ * necessary information.
+ * @param status Pointer to status object which will act as a message
+ * passing algorithm between LockManager.
+ * @param victims Message passing buffer betwen DeadLockDetector and
+ * LockManager.
+ **/
+ DeadLockDetector(LockTable *lock_table,
+ std::atomic<DeadLockDetectorStatus> *status,
+ std::vector<DirectedGraph::node_id> *victims);
+
+ void run() override;
+
+ /**
+ * @brief Adds pending information based on a resource conflict.
+ *
+ * @param pending Id of the transaction that waits for the resource lock.
+ * @param owner Id of the transaction that owns the resource lock.
+ */
+ void addPendingInfo(const transaction_id pending,
+ const transaction_id owner);
+
+ /**
+ * @brief Deletes pending information on a resource.
+ *
+ * @param pending Id of the transaction that waits for the resource lock.
+ * @param owner Id of the transaction that owns the resource lock.
+ *
+ * @warning This method is not implemented yet.
+ */
+ void deletePendingInfo(const transaction_id pending,
+ const transaction_id owner);
+
+ /**
+ * @brief Check whether first transaction waits for the latter.
+ *
+ * @param pending Id of the transaction which will be checked whether
+ * it waits for the other.
+ * @param owner Id of the transaction which will be checked whether
+ * it is waited by the first.
+ *
+ * @warning This method is not implemented yet.
+ */
+ bool isDependent(const transaction_id pending,
+ const transaction_id owner) const;
+
+ /**
+ * @brief Gives the ids of transactions that wait for the owner transaction.
+ *
+ * @param owner Id of the transaction whose the penders will be returned.
+ * @return Vector of transaction ids that wait for owner.
+ */
+ std::vector<transaction_id> getAllDependents(const transaction_id owner) const;
+
+ /**
+ * @brief Gives the ids of transaction that the pending transaction waits for.
+ * @warning This method is not implemented yet.
+
+ * @param pending Id of the transaction that is pending.
+ * @return Vector of transaction ids which the pending transaction waits for.
+ */
+ std::vector<transaction_id> getAllDependees(transaction_id pending);
+
+ /**
+ * @brief Gives the list of victims whose the terminations will end the cycle.
+ *
+ * @return Vector of victim transaction ids.
+ */
+ std::vector<transaction_id> getAllVictims();
+
+ private:
+ static constexpr std::int64_t kSleepDurationInSeconds = 5;
+
+ DirectedGraph::node_id getNodeId(const transaction_id tid);
+
+ DirectedGraph::node_id addNode(const transaction_id tid);
+
+ // Owned pointer to wait-for graph.
+ std::unique_ptr<DirectedGraph> wait_for_graph_;
+
+ // Mapping from TransactioId to NodeId in graph.
+ std::unique_ptr<transaction_id_node_map> tid_node_mapping_;
+
+ // Pointer to lock table. Dependence edges will be created
+ // by the information got from lock table.
+ LockTable *lock_table_;
+
+ std::atomic<DeadLockDetectorStatus> *status_;
+ std::vector<DirectedGraph::node_id> *victims_;
+};
+
+/** @} */
+
+} // namespace transaction
+} // namespace quickstep
+
+#endif // QUICKSTEP_TRANSACTION_DEAD_LOCK_DETECTOR_HPP_
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bbaff7a2/transaction/DirectedGraph.hpp
----------------------------------------------------------------------
diff --git a/transaction/DirectedGraph.hpp b/transaction/DirectedGraph.hpp
index 89ce9c6..16b551a 100644
--- a/transaction/DirectedGraph.hpp
+++ b/transaction/DirectedGraph.hpp
@@ -21,8 +21,8 @@
#include <algorithm>
#include <cstddef>
#include <cstdint>
+#include <iterator>
#include <memory>
-#include <stack>
#include <unordered_set>
#include <vector>
@@ -39,7 +39,7 @@ namespace transaction {
*/
/**
- * @brief Class for representing a directed graph. Vertices are transaction
+ * @brief Class for representing a directed graph. Vertices are transaction
* ids, edges are wait-for relations.
**/
class DirectedGraph {
@@ -54,34 +54,31 @@ class DirectedGraph {
/**
* @brief Adds a new node to the graph with the given transaction id.
* It does not check whether the transaction id is valid or not.
- * @warning Pointer ownership will pass to the graph, therefore it
- * should not be deleted.
*
- * @param data Pointer to the transaction id that will be contained
+ * @param transaction_id_payload Transaction id that will be contained
* in the node.
* @return Id of the newly created node.
**/
- inline node_id addNodeUnchecked(transaction_id *data) {
- nodes_.emplace_back(data);
+ inline
+ node_id addNodeUnchecked(const transaction_id transaction_id_payload) {
+ nodes_.emplace_back(transaction_id_payload);
return nodes_.size() - 1;
}
/**
* @brief Adds a new node to the graph with the given transaction id.
* It checks whether the transaction id is valid or not.
- * @warning Pointer ownership will pass to the graph, therefore it
- * should not be deleted.
*
- * @param data Pointer to the transaction id that will be contained
+ * @param transaction_id_payload Transaction id that will be contained
* in the node.
* @return Id of the newly created node.
**/
- inline node_id addNodeCheckExists(transaction_id *data) {
- for (std::vector<DirectedGraphNode>::const_iterator
- it = nodes_.cbegin(); it != nodes_.cend(); ++it) {
- CHECK(*data != it->getData());
+ node_id addNodeCheckExists(const transaction_id transaction_id_payload) {
+ for (const auto &node : nodes_) {
+ CHECK(transaction_id_payload != node.getData());
}
- nodes_.emplace_back(data);
+
+ nodes_.emplace_back(transaction_id_payload);
return nodes_.size() - 1;
}
@@ -91,10 +88,10 @@ class DirectedGraph {
* @warning Does not check arguments are legit. It may cause
* out of range errors.
*
- * @param fromNode The node that edge is orginated.
+ * @param fromNode The node that edge is originated.
* @param toNode The node that edge is ended.
**/
- inline void addEdgeUnchecked(node_id from_node, node_id to_node) {
+ void addEdgeUnchecked(node_id from_node, node_id to_node) {
nodes_[from_node].addOutgoingEdge(to_node);
}
@@ -105,7 +102,7 @@ class DirectedGraph {
* @param fromNode The node that edge is orginated.
* @param toNode The node that edge is ended.
**/
- inline void addEdgeCheckExists(node_id from_node, node_id to_node) {
+ void addEdgeCheckExists(node_id from_node, node_id to_node) {
CHECK(from_node < getNumNodes() && to_node < getNumNodes());
nodes_[from_node].addOutgoingEdge(to_node);
}
@@ -119,7 +116,7 @@ class DirectedGraph {
* @param toNode Id of the node that edge is ended.
* @return True if there is an edge, false otherwise.
**/
- inline bool hasEdge(node_id from_node, node_id to_node) const {
+ bool hasEdge(node_id from_node, node_id to_node) const {
DCHECK(from_node < getNumNodes() && to_node < getNumNodes());
return nodes_[from_node].hasOutgoingEdge(to_node);
}
@@ -130,7 +127,7 @@ class DirectedGraph {
* @param node Id of the node that the data is got from.
* @return Id of the transaction that this node contains.
**/
- inline transaction_id getDataFromNode(node_id node) const {
+ transaction_id getDataFromNode(node_id node) const {
DCHECK(node < getNumNodes());
return nodes_[node].getData();
}
@@ -140,7 +137,7 @@ class DirectedGraph {
*
* @return The number of nodes the graph has.
**/
- inline std::size_t getNumNodes() const {
+ std::size_t getNumNodes() const {
return nodes_.size();
}
@@ -158,18 +155,18 @@ class DirectedGraph {
// Class for representing a graph node.
class DirectedGraphNode {
public:
- explicit DirectedGraphNode(transaction_id *data)
- : data_(data) {}
+ explicit DirectedGraphNode(const transaction_id payload)
+ : transaction_id_payload_(payload) {}
- inline void addOutgoingEdge(node_id to_node) {
+ void addOutgoingEdge(node_id to_node) {
outgoing_edges_.insert(to_node);
}
- inline bool hasOutgoingEdge(node_id to_node) const {
+ bool hasOutgoingEdge(node_id to_node) const {
return outgoing_edges_.count(to_node) == 1;
}
- inline std::vector<node_id> getOutgoingEdges() const {
+ std::vector<node_id> getOutgoingEdges() const {
// TODO(hakan): Benchmark this version and the alternative which the
// function returns const reference and the uniqueness
// is imposed in the outgoing_edges_ as a vector.
@@ -179,13 +176,12 @@ class DirectedGraph {
return result;
}
- inline transaction_id getData() const {
- return *(data_.get());
+ transaction_id getData() const {
+ return transaction_id_payload_;
}
private:
- // Owner pointer to transaction id.
- std::unique_ptr<transaction_id> data_;
+ const transaction_id transaction_id_payload_;
// Endpoint nodes of outgoing edges originated from this node.
std::unordered_set<node_id> outgoing_edges_;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bbaff7a2/transaction/LockManager.cpp
----------------------------------------------------------------------
diff --git a/transaction/LockManager.cpp b/transaction/LockManager.cpp
new file mode 100644
index 0000000..da6181a
--- /dev/null
+++ b/transaction/LockManager.cpp
@@ -0,0 +1,237 @@
+/**
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#include "transaction/LockManager.hpp"
+
+#include <cstdint>
+#include <stack>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "transaction/AccessMode.hpp"
+#include "transaction/DeadLockDetector.hpp"
+#include "transaction/LockRequest.hpp"
+#include "transaction/LockTable.hpp"
+#include "transaction/ResourceId.hpp"
+#include "transaction/Transaction.hpp"
+#include "transaction/TransactionTable.hpp"
+#include "utility/ThreadSafeQueue.hpp"
+
+#include "gflags/gflags.h"
+#include "glog/logging.h"
+
+namespace quickstep {
+namespace transaction {
+
+DEFINE_uint64(max_try_incoming, 10000,
+ "The maximum number of tries that lock manager checks incoming "
+ "request buffer until the buffer is empty.");
+
+DEFINE_uint64(max_try_inner, 6000,
+ "The maximum number of tries that lock manager checks inner "
+ "request buffer until the buffer is empty.");
+
+LockManager::LockManager(ThreadSafeQueue<LockRequest> *incoming_requests,
+ ThreadSafeQueue<LockRequest> *permitted_requests)
+ : lock_table_(std::make_unique<LockTable>()),
+ transaction_table_(std::make_unique<TransactionTable>()),
+ detector_status_(DeadLockDetectorStatus::kNotReady),
+ deadlock_detector_(std::make_unique<DeadLockDetector>(lock_table_.get(),
+ &detector_status_,
+ &victim_result_)),
+ incoming_requests_(*incoming_requests),
+ permitted_requests_(*permitted_requests),
+ inner_pending_requests_() {
+}
+
+LockManager::~LockManager() {
+ deadlock_detector_->join();
+}
+
+void LockManager::run() {
+ deadlock_detector_->start();
+
+ const std::uint64_t kMaxTryIncoming =
+ static_cast<std::uint64_t>(FLAGS_max_try_incoming);
+ const std::uint64_t kMaxTryInner =
+ static_cast<std::uint64_t>(FLAGS_max_try_incoming);
+
+ while (true) {
+ for (std::uint64_t tries = 0; tries < kMaxTryIncoming; ++tries) {
+ if (!incoming_requests_.empty()) {
+ const LockRequest request = incoming_requests_.popOne();
+ if (request.getRequestType() == RequestType::kReleaseLocks) {
+ CHECK(releaseAllLocks(request.getTransactionId()))
+ << "Unexpected condition occured.";
+
+ } else if (acquireLock(request.getTransactionId(),
+ request.getResourceId(),
+ request.getAccessMode())) {
+ LOG(INFO) << "Transaction "
+ << std::to_string(request.getTransactionId())
+ << " is waiting " + request.getResourceId().toString();
+
+ inner_pending_requests_.push(request);
+ } else {
+ LOG(INFO) << "Transaction "
+ << std::to_string(request.getTransactionId())
+ << " acquired " + request.getResourceId().toString();
+
+ permitted_requests_.push(request);
+ }
+ }
+ }
+
+ for (std::uint64_t tries = 0; tries < kMaxTryInner; ++tries) {
+ if (!inner_pending_requests_.empty()) {
+ const LockRequest request = inner_pending_requests_.front();
+
+ if (acquireLock(request.getTransactionId(), request.getResourceId(),
+ request.getAccessMode())) {
+ inner_pending_requests_.pop();
+ permitted_requests_.push(request);
+ }
+ }
+ }
+
+ // Resolve deadlocks.
+ killVictims();
+ }
+}
+
+bool LockManager::acquireLock(const transaction_id tid,
+ const ResourceId &rid,
+ const AccessMode &access_mode) {
+ std::stack<std::pair<ResourceId, AccessMode>> stack;
+ ResourceId current_rid = rid;
+ AccessMode current_access_mode = access_mode;
+ stack.push(std::make_pair(current_rid, current_access_mode));
+
+ while (current_rid.hasParent()) {
+ current_rid = current_rid.getParentResourceId();
+ current_access_mode = (current_access_mode.isShareLock() ||
+ current_access_mode.isIntentionShareLock())
+ ? AccessMode(AccessMode::IsLockMode())
+ : AccessMode(AccessMode::IxLockMode());
+
+ stack.push(std::make_pair(current_rid, current_access_mode));
+ }
+
+ lock_table_->latchExclusive();
+
+ while (!stack.empty()) {
+ const std::pair<ResourceId, AccessMode> pair_to_pick = stack.top();
+ const ResourceId rid_to_pick = pair_to_pick.first;
+ const AccessMode access_mode_to_pick = pair_to_pick.second;
+
+ if (!acquireLockInternal(tid, rid_to_pick, access_mode_to_pick)) {
+ lock_table_->unlatchExclusive();
+ return false;
+ }
+ stack.pop();
+ }
+ lock_table_->unlatchExclusive();
+ return true;
+}
+
+bool LockManager::releaseAllLocks(const transaction_id tid,
+ const bool latch_table) {
+ const std::vector<ResourceId> resource_ids
+ = transaction_table_->getResourceIdList(tid);
+ const TransactionTableResult transaction_deleted
+ = transaction_table_->deleteTransaction(tid);
+
+ CHECK(transaction_deleted != TransactionTableResult::kTransactionDeleteError)
+ << "In LockManager.releaseAllLocks: Transaction could not be deleted!";
+
+ if (latch_table) {
+ lock_table_->latchExclusive();
+ }
+ for (const auto &resource_id : resource_ids) {
+ const LockTableResult lock_deleted = lock_table_->deleteLock(tid, resource_id);
+
+ LOG(INFO) << "Transaction "
+ << std::to_string(tid)
+ << " released lock:"
+ << resource_id.toString();
+ CHECK(lock_deleted != LockTableResult::kDeleteError)
+ << "In LockManager.releaseAllLock lock could not be deleted from "
+ "LockTable";
+ }
+ if (latch_table) {
+ lock_table_->unlatchExclusive();
+ }
+ return true;
+}
+
+bool LockManager::acquireLockInternal(const transaction_id tid,
+ const ResourceId &rid,
+ const AccessMode &access_mode) {
+ const LockTableResult lock_result = lock_table_->putLock(tid, rid, access_mode);
+ CHECK(lock_result != LockTableResult::kPutError)
+ << "Unexpected result in LockManager.acquireLockInternal";
+
+ switch (lock_result) {
+ case LockTableResult::kAlreadyInOwned: {
+ return true;
+ }
+ case LockTableResult::kPlacedInOwned: {
+ const TransactionTableResult transaction_result
+ = transaction_table_->putOwnEntry(tid, rid, access_mode);
+ CHECK(transaction_result == TransactionTableResult::kPlacedInOwned)
+ << "Unexpected result in LockManager.acquireLockInternal: "
+ "Mismatch of table results: LockTable entry is owned, "
+ "whereas TransactionTable entry is not owned.";
+ return true;
+ }
+ case LockTableResult::kAlreadyInPending: {
+ return false;
+ }
+ case LockTableResult::kPlacedInPending: {
+ const TransactionTableResult transaction_result =
+ transaction_table_->putPendingEntry(tid, rid, access_mode);
+ CHECK(transaction_result == TransactionTableResult::kPlacedInPending)
+ << "Unexpected result in LockManager.acquireLockInternal: "
+ "Mismatch of table results: LockTable entry is pending, "
+ "whereas TransactionTable entry is not pending";
+ return false;
+ }
+ default: {
+ return false;
+ }
+ }
+}
+
+void LockManager::killVictims() {
+ if (detector_status_.load() == DeadLockDetectorStatus::kDone) {
+ lock_table_->latchExclusive();
+ for (const auto victim_transaction_id : victim_result_) {
+ releaseAllLocks(victim_transaction_id, false);
+ // TODO(Hakan): Find a way to kill transaction, so that requests with this
+ // tid should be ignored.
+ LOG(INFO) << "Killed transaction "
+ << std::to_string(victim_transaction_id);
+ }
+ lock_table_->unlatchExclusive();
+ }
+ victim_result_.clear();
+ detector_status_.store(DeadLockDetectorStatus::kNotReady);
+}
+
+} // namespace transaction
+} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bbaff7a2/transaction/LockManager.hpp
----------------------------------------------------------------------
diff --git a/transaction/LockManager.hpp b/transaction/LockManager.hpp
new file mode 100644
index 0000000..40ee6c8
--- /dev/null
+++ b/transaction/LockManager.hpp
@@ -0,0 +1,128 @@
+/**
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#ifndef QUICKSTEP_TRANSACTION_LOCK_MANAGER_HPP_
+#define QUICKSTEP_TRANSACTION_LOCK_MANAGER_HPP_
+
+#include <atomic>
+#include <memory>
+#include <queue>
+#include <vector>
+
+#include "threading/Thread.hpp"
+#include "transaction/DeadLockDetector.hpp"
+#include "transaction/LockRequest.hpp"
+#include "transaction/Transaction.hpp"
+
+namespace quickstep {
+
+template <typename T> class ThreadSafeQueue;
+
+namespace transaction {
+
+class AccessMode;
+class LockTable;
+class ResourceId;
+class TransactionTable;
+
+/** \addtogroup Transaction
+ * @{
+ **/
+
+/**
+ * @brief Class for centralized location of acquisition and releasing
+ * of resource locks.
+ **/
+class LockManager : public Thread {
+ public:
+ /**
+ * @brief Constructor
+ *
+ * @param incoming_requests Queue for the lock requests that are waiting
+ * for the permission.
+ * @param permitted_requests Queue for the lock requests that are granted
+ * permission.
+ **/
+ LockManager(ThreadSafeQueue<LockRequest> *incoming_requests,
+ ThreadSafeQueue<LockRequest> *permitted_requests);
+
+ /**
+ * @brief Destructor for LockManager. It handles the thread
+ * joins that it owns.
+ **/
+ ~LockManager();
+
+ /**
+ * @brief Method for defining the LockManager's thread main logic.
+ *
+ * @warning Users must not use this function directly. Instead use
+ * start() method inherited from the Thread class.
+ **/
+ void run() override;
+
+ /**
+ * @brief Acquires the lock on resource with specified access mode.
+ *
+ * @param tid Id of the transaction which the resource lock is acquired for.
+ * @param rid Id of the resource on which the resource lock is acquired.
+ * @param access_mode Permissible access mode on resource.
+ *
+ * @return True if it can acquire the lock from root to leaf lock hierarchy,
+ * false otherwise.
+ **/
+ bool acquireLock(const transaction_id tid,
+ const ResourceId &rid,
+ const AccessMode &access_mode);
+
+ /**
+ * @brief Releases all locks hold by the transaction.
+ *
+ * @param tid Id of the transaction whose locks will be released.
+ * @param latch_table If it is true, the method latch the whole
+ * lock table, which is default.
+ * @return True if tid releases all of its locks.
+ **/
+ bool releaseAllLocks(const transaction_id tid,
+ const bool latch_table = true);
+
+ /**
+ * @brief Release the locks acquired by the transactions contained
+ * in victim buffer to break the deadlock.
+ **/
+ void killVictims();
+
+ private:
+ bool acquireLockInternal(const transaction_id tid,
+ const ResourceId &rid,
+ const AccessMode &access_mode);
+
+ std::unique_ptr<LockTable> lock_table_;
+ std::unique_ptr<TransactionTable> transaction_table_;
+ std::atomic<DeadLockDetectorStatus> detector_status_;
+ std::vector<transaction_id> victim_result_;
+ std::unique_ptr<DeadLockDetector> deadlock_detector_;
+ ThreadSafeQueue<LockRequest> &incoming_requests_;
+ ThreadSafeQueue<LockRequest> &permitted_requests_;
+ std::queue<LockRequest> inner_pending_requests_;
+};
+
+/** @} */
+
+} // namespace transaction
+} // namespace quickstep
+
+#endif // QUICKSTEP_TRANSACTION_LOCK_MANAGER_HPP_
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bbaff7a2/transaction/LockTable.cpp
----------------------------------------------------------------------
diff --git a/transaction/LockTable.cpp b/transaction/LockTable.cpp
index 77986f6..7b568fc 100644
--- a/transaction/LockTable.cpp
+++ b/transaction/LockTable.cpp
@@ -18,15 +18,11 @@
#include "transaction/LockTable.hpp"
#include <list>
-#include <unordered_map>
#include <utility>
-#include "threading/SharedMutex.hpp"
#include "transaction/AccessMode.hpp"
#include "transaction/Lock.hpp"
-#include "transaction/ResourceId.hpp"
#include "transaction/Transaction.hpp"
-#include "utility/Macros.hpp"
namespace quickstep {
namespace transaction {
@@ -34,7 +30,7 @@ namespace transaction {
LockTableResult
LockTable::putLock(const transaction_id tid,
const ResourceId &rid,
- const AccessMode access_mode) {
+ const AccessMode &access_mode) {
// TODO(hakan): Lock upgrade is not supported.
lock_list_pair &lock_list_pair = internal_map_[rid];
@@ -47,7 +43,7 @@ LockTable::putLock(const transaction_id tid,
for (lock_own_list::const_iterator it = lock_own_list.cbegin();
it != lock_own_list.cend(); ++it) {
if (it->first == tid && it->second.getAccessMode() == access_mode) {
- return LockTableResult::kALREADY_IN_OWNED;
+ return LockTableResult::kAlreadyInOwned;
}
}
@@ -56,7 +52,7 @@ LockTable::putLock(const transaction_id tid,
for (lock_pending_list::const_iterator it = lock_pending_list.cbegin();
it != lock_pending_list.cend(); ++it) {
if (it->first == tid && it->second.getAccessMode() == access_mode) {
- return LockTableResult::kALREADY_IN_PENDING;
+ return LockTableResult::kAlreadyInPending;
}
}
@@ -68,18 +64,18 @@ LockTable::putLock(const transaction_id tid,
if (!access_mode.isCompatible(it->second.getAccessMode())) {
lock_pending_list.push_back(std::make_pair(tid,
Lock(rid, access_mode)));
- return LockTableResult::kPLACED_IN_PENDING;
+ return LockTableResult::kPlacedInPending;
}
}
lock_own_list.push_back(std::make_pair(tid, Lock(rid, access_mode)));
- return LockTableResult::kPLACED_IN_OWNED;
+ return LockTableResult::kPlacedInOwned;
} else {
// If the pending list is not empty, even if the lock request is compatible
// with other owned lock entries, we put the new request into the pending
// list to eliminate starvation.
lock_pending_list.push_back(std::make_pair(tid, Lock(rid, access_mode)));
- return LockTableResult::kPLACED_IN_PENDING;
+ return LockTableResult::kPlacedInPending;
}
}
@@ -105,7 +101,7 @@ LockTable::deleteLock(const transaction_id tid,
// compatible with the remaining owned entries.
movePendingToOwned(rid);
- return LockTableResult::kDEL_FROM_OWNED;
+ return LockTableResult::kDeleteFromOwned;
}
}
@@ -116,13 +112,13 @@ LockTable::deleteLock(const transaction_id tid,
if (it->first == tid) {
// If it exists, erase it from pending list.
lock_pending_list.erase(it);
- return LockTableResult::kDEL_FROM_PENDING;
+ return LockTableResult::kDeleteFromPending;
}
}
// Execution reaches here, if we cannot find the corresponding lock entry
// in the both list.
- return LockTableResult::kDEL_ERROR;
+ return LockTableResult::kDeleteError;
}
void LockTable::movePendingToOwned(const ResourceId &rid) {
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bbaff7a2/transaction/LockTable.hpp
----------------------------------------------------------------------
diff --git a/transaction/LockTable.hpp b/transaction/LockTable.hpp
index 5a0612e..529db12 100644
--- a/transaction/LockTable.hpp
+++ b/transaction/LockTable.hpp
@@ -23,7 +23,6 @@
#include <utility>
#include "threading/SharedMutex.hpp"
-#include "transaction/AccessMode.hpp"
#include "transaction/Lock.hpp"
#include "transaction/ResourceId.hpp"
#include "transaction/Transaction.hpp"
@@ -32,6 +31,8 @@
namespace quickstep {
namespace transaction {
+class AccessMode;
+
/** \addtogroup Transaction
* @{
*/
@@ -40,14 +41,14 @@ namespace transaction {
* @brief Represents different results for LockTable's methods.
**/
enum class LockTableResult {
- kPLACED_IN_OWNED = 0,
- kPLACED_IN_PENDING,
- kALREADY_IN_OWNED,
- kALREADY_IN_PENDING,
- kDEL_FROM_OWNED,
- kDEL_FROM_PENDING,
- kDEL_ERROR,
- kPUT_ERROR,
+ kPlacedInOwned = 0,
+ kPlacedInPending,
+ kAlreadyInOwned,
+ kAlreadyInPending,
+ kDeleteFromOwned,
+ kDeleteFromPending,
+ kDeleteError,
+ kPutError,
};
/**
@@ -79,32 +80,31 @@ class LockTable {
* @param rid Id of the resource to be locked.
* @param access_mode Access mode of the lock.
*
- * @return LockTableResult::kPLACED_IN_OWNED if lock is granted,
- * LockTableResult::kPLACED_IN_PENDING if lock is not granted,
- * LockTableResult::kALREADY_IN_OWNED if lock has been
+ * @return LockTableResult::kPlacedInOwned if lock is granted,
+ * LockTableResult::kPlacedInPending if lock is not granted,
+ * LockTableResult::kAlreadyInOwned if lock has been
* already granted,
- * LockTableResult::kALREADY_IN_PENDING if lock has been
+ * LockTableResult::kAlreadyInPending if lock has been
* already pending.
**/
LockTableResult putLock(const transaction_id tid,
const ResourceId &rid,
- const AccessMode access_mode);
+ const AccessMode &access_mode);
/**
* @brief Deletes the lock entry.
*
* @param tid Id of the transaction that owns or awaits.
* @param rid Id of resource that the lock covers.
*
- * @return LockTableResult::kDEL_FROM_OWNED if the lock is deleted from
+ * @return LockTableResult::kDelFromOwned if the lock is deleted from
* owned list,
- * LockTableResult::kDEL_FROM_PENDING if the lock is deleted from
+ * LockTableResult::kDelFromPending if the lock is deleted from
* pending list,
- * LockTableResult::kDEL_ERROR if the lock cannot be found
+ * LockTableResult::kDelError if the lock cannot be found
**/
LockTableResult deleteLock(const transaction_id tid,
const ResourceId &rid);
-
/**
* @brief Iterator for begin position.
*
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bbaff7a2/transaction/ResourceId.hpp
----------------------------------------------------------------------
diff --git a/transaction/ResourceId.hpp b/transaction/ResourceId.hpp
index b9d1cdf..3a770dd 100644
--- a/transaction/ResourceId.hpp
+++ b/transaction/ResourceId.hpp
@@ -19,7 +19,6 @@
#define QUICKSTEP_TRANSACTION_RESOURCE_ID_HPP_
#include <cstddef>
-#include <limits>
#include <string>
#include "catalog/CatalogTypedefs.hpp"
@@ -202,10 +201,10 @@ class ResourceId {
return tuple_id_ == kTupleIdPlaceholder;
}
- const database_id db_id_;
- const relation_id rel_id_;
- const block_id block_id_;
- const tuple_id tuple_id_;
+ database_id db_id_;
+ relation_id rel_id_;
+ block_id block_id_;
+ tuple_id tuple_id_;
};
/** @} */
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bbaff7a2/transaction/StronglyConnectedComponents.cpp
----------------------------------------------------------------------
diff --git a/transaction/StronglyConnectedComponents.cpp b/transaction/StronglyConnectedComponents.cpp
index 89daf46..f50ed85 100644
--- a/transaction/StronglyConnectedComponents.cpp
+++ b/transaction/StronglyConnectedComponents.cpp
@@ -17,7 +17,6 @@
#include "transaction/StronglyConnectedComponents.hpp"
-#include <cstddef>
#include <cstdint>
#include <stack>
#include <unordered_map>
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bbaff7a2/transaction/Transaction.cpp
----------------------------------------------------------------------
diff --git a/transaction/Transaction.cpp b/transaction/Transaction.cpp
deleted file mode 100644
index 3478d01..0000000
--- a/transaction/Transaction.cpp
+++ /dev/null
@@ -1,48 +0,0 @@
-/**
- * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
- * University of Wisconsin\u2014Madison.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- **/
-
-#include "transaction/Transaction.hpp"
-
-#include <functional>
-
-namespace quickstep {
-
-namespace transaction {
-
-TransactionId Transaction::getTransactionId() const {
- return tid_;
-}
-
-void Transaction::setStatus(TransactionStatus status) {
- status_ = status;
-}
-
-TransactionStatus Transaction::getStatus() const {
- return status_;
-}
-
-bool Transaction::operator==(const Transaction &other) const {
- return tid_ == other.tid_;
-}
-
-std::size_t Transaction::TransactionHasher::operator()(const Transaction &transaction) const {
- return std::hash<TransactionId>()(transaction.tid_);
-}
-
-} // namespace transaction
-
-} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bbaff7a2/transaction/TransactionTable.cpp
----------------------------------------------------------------------
diff --git a/transaction/TransactionTable.cpp b/transaction/TransactionTable.cpp
index 993703a..3e37439 100644
--- a/transaction/TransactionTable.cpp
+++ b/transaction/TransactionTable.cpp
@@ -33,7 +33,7 @@ namespace transaction {
TransactionTableResult
TransactionTable::putOwnEntry(const transaction_id tid,
const ResourceId &rid,
- const AccessMode access_mode) {
+ const AccessMode &access_mode) {
transaction_list_pair &transaction_list_pair = internal_map_[tid];
transaction_own_list &transaction_own_list = transaction_list_pair.first;
@@ -45,7 +45,7 @@ TransactionTable::putOwnEntry(const transaction_id tid,
TransactionTableResult
TransactionTable::putPendingEntry(const transaction_id tid,
const ResourceId &rid,
- const AccessMode access_mode) {
+ const AccessMode &access_mode) {
transaction_list_pair &transaction_list_pair = internal_map_[tid];
transaction_pending_list &transaction_pending_list
= transaction_list_pair.second;
@@ -59,7 +59,7 @@ TransactionTable::putPendingEntry(const transaction_id tid,
TransactionTableResult
TransactionTable::deleteOwnEntry(const transaction_id tid,
const ResourceId &rid,
- const AccessMode access_mode) {
+ const AccessMode &access_mode) {
transaction_list_pair &transaction_list_pair = internal_map_[tid];
transaction_own_list &transaction_own_list = transaction_list_pair.first;
@@ -79,7 +79,7 @@ TransactionTable::deleteOwnEntry(const transaction_id tid,
TransactionTableResult
TransactionTable::deletePendingEntry(const transaction_id tid,
const ResourceId &rid,
- const AccessMode access_mode) {
+ const AccessMode &access_mode) {
transaction_list_pair &transaction_list_pair = internal_map_[tid];
transaction_pending_list &transaction_pending_list
= transaction_list_pair.second;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bbaff7a2/transaction/TransactionTable.hpp
----------------------------------------------------------------------
diff --git a/transaction/TransactionTable.hpp b/transaction/TransactionTable.hpp
index a5e1da4..29df536 100644
--- a/transaction/TransactionTable.hpp
+++ b/transaction/TransactionTable.hpp
@@ -23,7 +23,6 @@
#include <utility>
#include <vector>
-#include "transaction/AccessMode.hpp"
#include "transaction/Lock.hpp"
#include "transaction/ResourceId.hpp"
#include "transaction/Transaction.hpp"
@@ -32,6 +31,8 @@
namespace quickstep {
namespace transaction {
+class AccessMode;
+
/** \addtogroup Transaction
* @{
*/
@@ -81,7 +82,7 @@ class TransactionTable {
**/
TransactionTableResult putOwnEntry(const transaction_id tid,
const ResourceId &rid,
- const AccessMode access_mode);
+ const AccessMode &access_mode);
/**
* @brief Puts a pending entry of the given resource id in the given
@@ -95,7 +96,7 @@ class TransactionTable {
**/
TransactionTableResult putPendingEntry(const transaction_id tid,
const ResourceId &rid,
- const AccessMode access_mode);
+ const AccessMode &access_mode);
/**
* @brief Deletes the owned entry corresponding to the resource id
@@ -110,7 +111,7 @@ class TransactionTable {
**/
TransactionTableResult deleteOwnEntry(const transaction_id tid,
const ResourceId &rid,
- const AccessMode access_mode);
+ const AccessMode &access_mode);
/**
* @brief Deletes the pending entry corresponding to the resource id
@@ -124,7 +125,7 @@ class TransactionTable {
**/
TransactionTableResult deletePendingEntry(const transaction_id tid,
const ResourceId &rid,
- const AccessMode access_mode);
+ const AccessMode &access_mode);
/**
* @brief Returns a vector of resource ids which the corresponding transaction
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bbaff7a2/transaction/tests/AccessMode_unittest.cpp
----------------------------------------------------------------------
diff --git a/transaction/tests/AccessMode_unittest.cpp b/transaction/tests/AccessMode_unittest.cpp
index fa51525..3287fb0 100644
--- a/transaction/tests/AccessMode_unittest.cpp
+++ b/transaction/tests/AccessMode_unittest.cpp
@@ -25,12 +25,12 @@ namespace transaction {
class AccessModeTest : public ::testing::Test {
protected:
AccessModeTest()
- : nl_mode_(AccessModeType::kNoLock),
- is_mode_(AccessModeType::kIsLock),
- ix_mode_(AccessModeType::kIxLock),
- s_mode_(AccessModeType::kSLock),
- six_mode_(AccessModeType::kSixLock),
- x_mode_(AccessModeType::kXLock) {
+ : nl_mode_(AccessMode::NoLockMode()),
+ is_mode_(AccessMode::IsLockMode()),
+ ix_mode_(AccessMode::IxLockMode()),
+ s_mode_(AccessMode::SLockMode()),
+ six_mode_(AccessMode::SixLockMode()),
+ x_mode_(AccessMode::XLockMode()) {
}
const AccessMode nl_mode_;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bbaff7a2/transaction/tests/CycleDetector_unittest.cpp
----------------------------------------------------------------------
diff --git a/transaction/tests/CycleDetector_unittest.cpp b/transaction/tests/CycleDetector_unittest.cpp
new file mode 100644
index 0000000..6edaa63
--- /dev/null
+++ b/transaction/tests/CycleDetector_unittest.cpp
@@ -0,0 +1,157 @@
+/**
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#include "transaction/CycleDetector.hpp"
+
+#include <cstdint>
+#include <memory>
+#include <stack>
+#include <unordered_set>
+#include <utility>
+#include <vector>
+
+#include "transaction/DirectedGraph.hpp"
+#include "transaction/Transaction.hpp"
+
+#include "gtest/gtest.h"
+
+namespace quickstep {
+namespace transaction {
+
+class CycleDetectorTest : public testing::Test {
+ protected:
+ const std::uint64_t kNumberOfTransactions = 12;
+
+ CycleDetectorTest()
+ : wait_for_graph_(std::make_unique<DirectedGraph>()) {
+ }
+
+ virtual void SetUp() {
+ std::vector<transaction_id> transactions(kNumberOfTransactions);
+ for (std::uint64_t i = 0; i < kNumberOfTransactions; ++i) {
+ transactions.push_back(transaction_id(i));
+ }
+
+ std::vector<DirectedGraph::node_id> node_ids;
+ for (std::uint64_t i = 0; i < kNumberOfTransactions; ++i) {
+ node_ids.push_back(wait_for_graph_->addNodeUnchecked(transactions[i]));
+ }
+ }
+
+ void initializeCycleDetector() {
+ for (const auto &edge : edges_) {
+ wait_for_graph_->addEdgeUnchecked(edge.first, edge.second);
+ }
+
+ cycle_detector_.reset(new CycleDetector(wait_for_graph_.get()));
+ }
+
+ void checkVictims(
+ const std::unordered_set<DirectedGraph::node_id> &expected_victims) {
+ const std::vector<DirectedGraph::node_id> victims =
+ cycle_detector_->chooseVictimsToBreakCycle();
+
+ std::unordered_set<DirectedGraph::node_id> remaining_nodes;
+
+ for (DirectedGraph::node_id node = 0; node < wait_for_graph_->getNumNodes();
+ ++node) {
+ if (std::find(victims.begin(), victims.end(), node) == victims.end()) {
+ // Node is not in victims, then insert it to remaining set.
+ remaining_nodes.insert(node);
+ }
+ }
+
+ for (const auto node : remaining_nodes) {
+ ASSERT_FALSE(isSelfReachableNode(node, remaining_nodes));
+ }
+ }
+
+ bool isSelfReachableNode(
+ const DirectedGraph::node_id start_node,
+ const std::unordered_set<DirectedGraph::node_id> &node_set) {
+ std::unordered_set<DirectedGraph::node_id> marked_nodes;
+ std::stack<DirectedGraph::node_id> to_be_visied_nodes;
+
+ const std::vector<DirectedGraph::node_id> neighbors_of_start_node =
+ wait_for_graph_->getAdjacentNodes(start_node);
+ for (const auto node : neighbors_of_start_node) {
+ marked_nodes.insert(node);
+ to_be_visied_nodes.push(node);
+ }
+
+ while (!to_be_visied_nodes.empty()) {
+ const DirectedGraph::node_id current_node = to_be_visied_nodes.top();
+ to_be_visied_nodes.pop();
+ if (current_node == start_node) {
+ return true;
+ }
+ if (node_set.count(current_node) == 1 &&
+ marked_nodes.count(current_node) == 0) {
+ // Means, we did not visited this node yet, and it is in the node set,
+ // so we should process it (mark it and push all of its neighbors
+ // into stack).
+ marked_nodes.insert(current_node);
+ const auto neighbors = wait_for_graph_->getAdjacentNodes(current_node);
+ for (const auto neighbor : neighbors) {
+ to_be_visied_nodes.push(neighbor);
+ }
+ }
+ }
+ return false;
+ }
+
+ std::vector<std::pair<DirectedGraph::node_id, DirectedGraph::node_id>> edges_;
+ std::unique_ptr<DirectedGraph> wait_for_graph_;
+ std::unique_ptr<CycleDetector> cycle_detector_;
+};
+
+TEST_F(CycleDetectorTest, Interleaving) {
+ edges_ = {{0, 1},
+ {1, 0}};
+
+ initializeCycleDetector();
+
+ std::unordered_set<DirectedGraph::node_id> expected_victims = {1};
+
+ checkVictims(expected_victims);
+}
+
+TEST_F(CycleDetectorTest, MultipleCycle) {
+ // This edge contains lots of cycles of degree 1, 2 and 3.
+ edges_ = {{0, 1},
+ {1, 2}, {1, 3}, {1, 4},
+ {2, 5},
+ {3, 4}, {3, 6},
+ {4, 1}, {4, 5}, {4, 6},
+ {5, 2}, {5, 7},
+ {6, 7}, {6, 9},
+ {7, 6},
+ {8, 6},
+ {9, 8}, {9, 10},
+ {10, 11},
+ {11, 9}};
+
+ initializeCycleDetector();
+
+ std::unordered_set<DirectedGraph::node_id> expected_victims
+ = {4, 5, 7, 8, 9, 10, 11};
+
+ checkVictims(expected_victims);
+}
+
+} // namespace transaction
+} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bbaff7a2/transaction/tests/DeadLockDetector_unittest.cpp
----------------------------------------------------------------------
diff --git a/transaction/tests/DeadLockDetector_unittest.cpp b/transaction/tests/DeadLockDetector_unittest.cpp
new file mode 100644
index 0000000..bc65ef5
--- /dev/null
+++ b/transaction/tests/DeadLockDetector_unittest.cpp
@@ -0,0 +1,96 @@
+/**
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#include "transaction/DeadLockDetector.hpp"
+
+#include <atomic>
+#include <memory>
+#include <vector>
+
+#include "transaction/AccessMode.hpp"
+#include "transaction/DirectedGraph.hpp"
+#include "transaction/LockTable.hpp"
+#include "transaction/ResourceId.hpp"
+#include "transaction/Transaction.hpp"
+
+#include "gtest/gtest.h"
+
+namespace quickstep {
+namespace transaction {
+
+class DeadLockDetectorTest : public ::testing::Test {
+ protected:
+ DeadLockDetectorTest()
+ : lock_table_(std::make_unique<LockTable>()),
+ status_(DeadLockDetectorStatus::kDone) {
+ }
+
+ std::unique_ptr<LockTable> lock_table_;
+ std::atomic<DeadLockDetectorStatus> status_;
+ std::vector<DirectedGraph::node_id> victims_;
+};
+
+TEST_F(DeadLockDetectorTest, SimpleCycle) {
+ const transaction_id transaction_one(1), transaction_two(2);
+ const ResourceId resource_one(1, 2), resource_two(4, 5);
+
+ const AccessMode x_lock_mode(AccessMode::XLockMode());
+
+ // Produce a conflicting schedule.
+ // Transaction 1 will acquire X lock on resource 1.
+ lock_table_->putLock(transaction_one,
+ resource_one,
+ x_lock_mode);
+
+ // Transaction 2 will acquire X lock on resource 2.
+ lock_table_->putLock(transaction_two,
+ resource_two,
+ x_lock_mode);
+
+ // Transaction 1 will try to acquire X lock on resource 2,
+ // but it will fail since Transaction 2 has already acquired
+ // X lock on resource 2.
+ lock_table_->putLock(transaction_one,
+ resource_two,
+ x_lock_mode);
+
+ // Transaction 2 will try to acquire X lock on resource 1,
+ // but it will fail since Transaction 1 has already acquired
+ // X lock on resource 2.
+ lock_table_->putLock(transaction_two,
+ resource_one,
+ x_lock_mode);
+
+ // Run deadlock detector.
+ DeadLockDetector deadlock_detector(lock_table_.get(), &status_, &victims_);
+ status_.store(DeadLockDetectorStatus::kNotReady);
+
+ deadlock_detector.start();
+
+ // Signal deadlock detector.
+ while (status_.load() == DeadLockDetectorStatus::kNotReady) {
+ }
+
+ status_.store(DeadLockDetectorStatus::kQuit);
+ deadlock_detector.join();
+
+ // Victim size must be 1.
+ ASSERT_EQ(1u, victims_.size());
+}
+
+} // namespace transaction
+} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bbaff7a2/transaction/tests/DirectedGraph_unittest.cpp
----------------------------------------------------------------------
diff --git a/transaction/tests/DirectedGraph_unittest.cpp b/transaction/tests/DirectedGraph_unittest.cpp
index 43ad972..00fe276 100644
--- a/transaction/tests/DirectedGraph_unittest.cpp
+++ b/transaction/tests/DirectedGraph_unittest.cpp
@@ -26,101 +26,96 @@
namespace quickstep {
namespace transaction {
-TEST(DirectedGraphTest, AddNode) {
- // Prepare the data, but do not include in the graph.
- DirectedGraph wait_for_graph;
- transaction_id *tid3 = new transaction_id(3);
- transaction_id *tid4 = new transaction_id(4);
- transaction_id *tid5 = new transaction_id(5);
- transaction_id *tid6 = new transaction_id(6);
-
+class DirectedGraphTest : public ::testing::Test {
+ protected:
+ DirectedGraphTest()
+ : tid3_(3),
+ tid4_(4),
+ tid5_(5),
+ tid6_(6) {
+ }
+
+ DirectedGraph wait_for_graph_;
+ transaction_id tid3_;
+ transaction_id tid4_;
+ transaction_id tid5_;
+ transaction_id tid6_;
+};
+
+TEST_F(DirectedGraphTest, AddNode) {
// The nodes are not added yet, total no of nodesshould be zero.
- EXPECT_EQ(0u, wait_for_graph.getNumNodes());
+ EXPECT_EQ(0u, wait_for_graph_.getNumNodes());
- wait_for_graph.addNodeUnchecked(tid3);
+ wait_for_graph_.addNodeUnchecked(tid3_);
// One node is added.
- EXPECT_EQ(1u, wait_for_graph.getNumNodes());
+ EXPECT_EQ(1u, wait_for_graph_.getNumNodes());
- wait_for_graph.addNodeUnchecked(tid4);
+ wait_for_graph_.addNodeUnchecked(tid4_);
// Another node is added.
- EXPECT_EQ(2u, wait_for_graph.getNumNodes());
+ EXPECT_EQ(2u, wait_for_graph_.getNumNodes());
- wait_for_graph.addNodeUnchecked(tid5);
- wait_for_graph.addNodeUnchecked(tid6);
+ wait_for_graph_.addNodeUnchecked(tid5_);
+ wait_for_graph_.addNodeUnchecked(tid6_);
// Total no of nodes should be 4 right now.
- EXPECT_EQ(4u, wait_for_graph.getNumNodes());
+ EXPECT_EQ(4u, wait_for_graph_.getNumNodes());
}
-TEST(DirectedGraphTest, AddEdge) {
- // Prepare the graph.
- DirectedGraph wait_for_graph;
- transaction_id *tid3 = new transaction_id(3);
- transaction_id *tid4 = new transaction_id(4);
- transaction_id *tid5 = new transaction_id(5);
- transaction_id *tid6 = new transaction_id(6);
-
- DirectedGraph::node_id nid3 = wait_for_graph.addNodeUnchecked(tid3);
- DirectedGraph::node_id nid6 = wait_for_graph.addNodeUnchecked(tid6);
- DirectedGraph::node_id nid4 = wait_for_graph.addNodeUnchecked(tid4);
- DirectedGraph::node_id nid5 = wait_for_graph.addNodeUnchecked(tid5);
+TEST_F(DirectedGraphTest, AddEdge) {
+ DirectedGraph::node_id nid3 = wait_for_graph_.addNodeUnchecked(tid3_);
+ DirectedGraph::node_id nid6 = wait_for_graph_.addNodeUnchecked(tid6_);
+ DirectedGraph::node_id nid4 = wait_for_graph_.addNodeUnchecked(tid4_);
+ DirectedGraph::node_id nid5 = wait_for_graph_.addNodeUnchecked(tid5_);
// Add edges.
- wait_for_graph.addEdgeUnchecked(nid3, nid5);
- wait_for_graph.addEdgeUnchecked(nid6, nid4);
- wait_for_graph.addEdgeUnchecked(nid3, nid6);
- wait_for_graph.addEdgeUnchecked(nid4, nid6);
+ wait_for_graph_.addEdgeUnchecked(nid3, nid5);
+ wait_for_graph_.addEdgeUnchecked(nid6, nid4);
+ wait_for_graph_.addEdgeUnchecked(nid3, nid6);
+ wait_for_graph_.addEdgeUnchecked(nid4, nid6);
// Check whether the edges are already there.
- EXPECT_TRUE(wait_for_graph.hasEdge(nid3, nid5));
- EXPECT_TRUE(wait_for_graph.hasEdge(nid6, nid4));
- EXPECT_TRUE(wait_for_graph.hasEdge(nid4, nid6));
- EXPECT_TRUE(wait_for_graph.hasEdge(nid3, nid6));
+ EXPECT_TRUE(wait_for_graph_.hasEdge(nid3, nid5));
+ EXPECT_TRUE(wait_for_graph_.hasEdge(nid6, nid4));
+ EXPECT_TRUE(wait_for_graph_.hasEdge(nid4, nid6));
+ EXPECT_TRUE(wait_for_graph_.hasEdge(nid3, nid6));
// Check non-existent edges.
- EXPECT_FALSE(wait_for_graph.hasEdge(nid5, nid3));
- EXPECT_FALSE(wait_for_graph.hasEdge(nid6, nid3));
- EXPECT_FALSE(wait_for_graph.hasEdge(nid4, nid5));
+ EXPECT_FALSE(wait_for_graph_.hasEdge(nid5, nid3));
+ EXPECT_FALSE(wait_for_graph_.hasEdge(nid6, nid3));
+ EXPECT_FALSE(wait_for_graph_.hasEdge(nid4, nid5));
}
-TEST(DirectedGraphTest, GetAdjacentNodes) {
- // Prepare the graph.
- DirectedGraph wait_for_graph;
- transaction_id *tid3 = new transaction_id(3);
- transaction_id *tid4 = new transaction_id(4);
- transaction_id *tid5 = new transaction_id(5);
- transaction_id *tid6 = new transaction_id(6);
-
+TEST_F(DirectedGraphTest, GetAdjacentNodes) {
// Add 4 disconnected nodes to the graph.
- DirectedGraph::node_id nid3 = wait_for_graph.addNodeUnchecked(tid3);
- DirectedGraph::node_id nid6 = wait_for_graph.addNodeUnchecked(tid6);
- DirectedGraph::node_id nid4 = wait_for_graph.addNodeUnchecked(tid4);
- DirectedGraph::node_id nid5 = wait_for_graph.addNodeUnchecked(tid5);
+ DirectedGraph::node_id nid3 = wait_for_graph_.addNodeUnchecked(tid3_);
+ DirectedGraph::node_id nid6 = wait_for_graph_.addNodeUnchecked(tid6_);
+ DirectedGraph::node_id nid4 = wait_for_graph_.addNodeUnchecked(tid4_);
+ DirectedGraph::node_id nid5 = wait_for_graph_.addNodeUnchecked(tid5_);
- std::vector<DirectedGraph::node_id> result1 = wait_for_graph.getAdjacentNodes(nid3);
+ std::vector<DirectedGraph::node_id> result1 = wait_for_graph_.getAdjacentNodes(nid3);
// nid3 has no edge to other nodes.
EXPECT_EQ(0u, result1.size());
// Now nid3 has outgoing edge to nid4 and nid5.
- wait_for_graph.addEdgeUnchecked(nid3, nid4);
- wait_for_graph.addEdgeUnchecked(nid3, nid5);
+ wait_for_graph_.addEdgeUnchecked(nid3, nid4);
+ wait_for_graph_.addEdgeUnchecked(nid3, nid5);
- std::vector<DirectedGraph::node_id> result2 = wait_for_graph.getAdjacentNodes(nid3);
+ std::vector<DirectedGraph::node_id> result2 = wait_for_graph_.getAdjacentNodes(nid3);
// Therefore, number of outgoing edges from nid3 is 2.
EXPECT_EQ(2u, result2.size());
// Add an edge from nid3 to nid6.
- wait_for_graph.addEdgeUnchecked(nid3, nid6);
- std::vector<DirectedGraph::node_id> result3 = wait_for_graph.getAdjacentNodes(nid3);
+ wait_for_graph_.addEdgeUnchecked(nid3, nid6);
+ std::vector<DirectedGraph::node_id> result3 = wait_for_graph_.getAdjacentNodes(nid3);
// Now there are 3 outgoing edges.
EXPECT_EQ(3u, result3.size());
// Again add edge from nid3 to nid6.
- wait_for_graph.addEdgeUnchecked(nid3, nid6);
- std::vector<DirectedGraph::node_id> result4 = wait_for_graph.getAdjacentNodes(nid3);
+ wait_for_graph_.addEdgeUnchecked(nid3, nid6);
+ std::vector<DirectedGraph::node_id> result4 = wait_for_graph_.getAdjacentNodes(nid3);
// Since we have already add same edge before, number of edges are still 3.
EXPECT_EQ(3u, result4.size());
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bbaff7a2/transaction/tests/LockRequest_unittest.cpp
----------------------------------------------------------------------
diff --git a/transaction/tests/LockRequest_unittest.cpp b/transaction/tests/LockRequest_unittest.cpp
index 0e4138a..77047d9 100644
--- a/transaction/tests/LockRequest_unittest.cpp
+++ b/transaction/tests/LockRequest_unittest.cpp
@@ -31,7 +31,7 @@ class LockRequestTest : public ::testing::Test {
LockRequestTest()
: lock_request_(transaction_id(3),
ResourceId(5),
- AccessMode(AccessModeType::kSLock),
+ AccessMode::SLockMode(),
RequestType::kAcquireLock) {
}
@@ -41,7 +41,7 @@ class LockRequestTest : public ::testing::Test {
TEST_F(LockRequestTest, CheckGetters) {
EXPECT_EQ(transaction_id(3), lock_request_.getTransactionId());
EXPECT_EQ(ResourceId(5), lock_request_.getResourceId());
- EXPECT_EQ(AccessMode(AccessModeType::kSLock), lock_request_.getAccessMode());
+ EXPECT_EQ(AccessMode::SLockMode(), lock_request_.getAccessMode());
EXPECT_EQ(RequestType::kAcquireLock, lock_request_.getRequestType());
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bbaff7a2/transaction/tests/LockTable_unittest.cpp
----------------------------------------------------------------------
diff --git a/transaction/tests/LockTable_unittest.cpp b/transaction/tests/LockTable_unittest.cpp
index 577cb79..1aed0b8 100644
--- a/transaction/tests/LockTable_unittest.cpp
+++ b/transaction/tests/LockTable_unittest.cpp
@@ -41,64 +41,73 @@ class LockTableTest : public ::testing::Test {
};
TEST_F(LockTableTest, CompatibleRequestsFromDifferentTransactions) {
- EXPECT_EQ(lock_table_.putLock(tid_1_,
+ const AccessMode is_lock_mode = AccessMode::IsLockMode();
+ const AccessMode s_lock_mode = AccessMode::SLockMode();
+
+ EXPECT_EQ(LockTableResult::kPlacedInOwned,
+ lock_table_.putLock(tid_1_,
ResourceId(2),
- AccessMode(AccessModeType::kIsLock)),
- LockTableResult::kPLACED_IN_OWNED);
+ is_lock_mode));
// Acquire the same lock mode on same resource.
- EXPECT_EQ(lock_table_.putLock(tid_1_,
+ EXPECT_EQ(LockTableResult::kAlreadyInOwned,
+ lock_table_.putLock(tid_1_,
ResourceId(2),
- AccessMode(AccessModeType::kIsLock)),
- LockTableResult::kALREADY_IN_OWNED);
+ is_lock_mode));
// Another transaction acquires compatible lock on the same resource.
- EXPECT_EQ(lock_table_.putLock(tid_2_,
+ EXPECT_EQ(LockTableResult::kPlacedInOwned,
+ lock_table_.putLock(tid_2_,
ResourceId(2),
- AccessMode(AccessModeType::kSLock)),
- LockTableResult::kPLACED_IN_OWNED);
+ s_lock_mode));
}
TEST_F(LockTableTest, IncompatibleRequestsFromDifferentTransactions) {
- EXPECT_EQ(lock_table_.putLock(tid_1_,
+ const AccessMode is_lock_mode = AccessMode::IsLockMode();
+ const AccessMode x_lock_mode = AccessMode::XLockMode();
+
+ EXPECT_EQ(LockTableResult::kPlacedInOwned,
+ lock_table_.putLock(tid_1_,
ResourceId(2),
- AccessMode(AccessModeType::kIsLock)),
- LockTableResult::kPLACED_IN_OWNED);
+ is_lock_mode));
// Acquire the same lock mode on same resource.
- EXPECT_EQ(lock_table_.putLock(tid_1_,
+ EXPECT_EQ(LockTableResult::kAlreadyInOwned,
+ lock_table_.putLock(tid_1_,
ResourceId(2),
- AccessMode(AccessModeType::kIsLock)),
- LockTableResult::kALREADY_IN_OWNED);
+ is_lock_mode));
// Another transaction acquires incompatible lock on the same resource.
- EXPECT_EQ(lock_table_.putLock(tid_2_,
+ EXPECT_EQ(LockTableResult::kPlacedInPending,
+ lock_table_.putLock(tid_2_,
ResourceId(2),
- AccessMode(AccessModeType::kXLock)),
- LockTableResult::kPLACED_IN_PENDING);
+ x_lock_mode));
}
TEST_F(LockTableTest, StarvationProtection) {
- EXPECT_EQ(lock_table_.putLock(tid_1_,
+ const AccessMode is_lock_mode = AccessMode::IsLockMode();
+ const AccessMode x_lock_mode = AccessMode::XLockMode();
+
+ EXPECT_EQ(LockTableResult::kPlacedInOwned,
+ lock_table_.putLock(tid_1_,
ResourceId(2),
- AccessMode(AccessModeType::kIsLock)),
- LockTableResult::kPLACED_IN_OWNED);
+ is_lock_mode));
// Another transaction requests incompatible lock on the same resource.
// It should wait for the previous transaction.
- EXPECT_EQ(lock_table_.putLock(tid_2_,
+ EXPECT_EQ(LockTableResult::kPlacedInPending,
+ lock_table_.putLock(tid_2_,
ResourceId(2),
- AccessMode(AccessModeType::kXLock)),
- LockTableResult::kPLACED_IN_PENDING);
+ x_lock_mode));
// Another third transaction requests a compatible lock on the same resource.
// Normally, it should acquire the lock, however, there is a pending
// transaction waiting on the same resource. To prevent starvation, we should
// put in the pending list.
- EXPECT_EQ(lock_table_.putLock(tid_3_,
+ EXPECT_EQ(LockTableResult::kPlacedInPending,
+ lock_table_.putLock(tid_3_,
ResourceId(2),
- AccessMode(AccessModeType::kIsLock)),
- LockTableResult::kPLACED_IN_PENDING);
+ is_lock_mode));
}
} // namespace transaction
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/bbaff7a2/transaction/tests/Lock_unittest.cpp
----------------------------------------------------------------------
diff --git a/transaction/tests/Lock_unittest.cpp b/transaction/tests/Lock_unittest.cpp
index 2ab8b3e..59a5e7a 100644
--- a/transaction/tests/Lock_unittest.cpp
+++ b/transaction/tests/Lock_unittest.cpp
@@ -18,6 +18,7 @@
#include "transaction/Lock.hpp"
#include <cstddef>
+#include <memory>
#include <vector>
#include "transaction/AccessMode.hpp"
@@ -31,12 +32,12 @@ namespace transaction {
class LockTest : public ::testing::Test {
protected:
LockTest()
- : modes_({AccessMode(AccessModeType::kNoLock),
- AccessMode(AccessModeType::kIsLock),
- AccessMode(AccessModeType::kIxLock),
- AccessMode(AccessModeType::kSLock),
- AccessMode(AccessModeType::kSixLock),
- AccessMode(AccessModeType::kXLock)}),
+ : modes_({AccessMode::NoLockMode(),
+ AccessMode::IsLockMode(),
+ AccessMode::IxLockMode(),
+ AccessMode::SLockMode(),
+ AccessMode::SixLockMode(),
+ AccessMode::XLockMode()}),
resource_a_(3, 10, 2, 5),
resource_b_(4, 5, 3, 2),
locks_on_resource_a_({Lock(resource_a_, modes_[0]),
[34/50] [abbrv] incubator-quickstep git commit: Initial support for
collecting table statistics: number of distinct values (#227)
Posted by zu...@apache.org.
Initial support for collecting table statistics: number of distinct values (#227)
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/df4a05d7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/df4a05d7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/df4a05d7
Branch: refs/heads/work-order-serialization
Commit: df4a05d7ea95cc65c93015e85eaf6edb824816d4
Parents: a25da39
Author: Jianqiao Zhu <ji...@cs.wisc.edu>
Authored: Thu May 19 10:58:43 2016 -0500
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Mon May 30 15:47:51 2016 -0700
----------------------------------------------------------------------
catalog/CMakeLists.txt | 11 +++
catalog/Catalog.proto | 13 ++-
catalog/CatalogRelation.cpp | 11 +++
catalog/CatalogRelation.hpp | 24 ++++-
catalog/CatalogRelationStatistics.cpp | 49 ++++++++++
catalog/CatalogRelationStatistics.hpp | 122 +++++++++++++++++++++++
cli/CMakeLists.txt | 21 +++-
cli/CommandExecutor.cpp | 138 ++++++++++++++++++++++++++-
cli/CommandExecutor.hpp | 17 ++--
cli/QuickstepCli.cpp | 2 +
cli/tests/CommandExecutorTestRunner.cpp | 2 +
query_optimizer/ExecutionGenerator.cpp | 6 +-
query_optimizer/QueryProcessor.hpp | 10 ++
13 files changed, 410 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/df4a05d7/catalog/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/catalog/CMakeLists.txt b/catalog/CMakeLists.txt
index 94da838..64b4f16 100644
--- a/catalog/CMakeLists.txt
+++ b/catalog/CMakeLists.txt
@@ -1,5 +1,7 @@
# Copyright 2011-2015 Quickstep Technologies LLC.
# Copyright 2015-2016 Pivotal Software, Inc.
+# Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+# University of Wisconsin\u2014Madison.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -36,6 +38,9 @@ add_library(quickstep_catalog_CatalogRelation CatalogRelation.cpp CatalogRelatio
add_library(quickstep_catalog_CatalogRelationSchema
CatalogRelationSchema.cpp
CatalogRelationSchema.hpp)
+add_library(quickstep_catalog_CatalogRelationStatistics
+ CatalogRelationStatistics.cpp
+ CatalogRelationStatistics.hpp)
add_library(quickstep_catalog_CatalogTypedefs ../empty_src.cpp CatalogTypedefs.hpp)
add_library(quickstep_catalog_IndexScheme IndexScheme.cpp IndexScheme.hpp)
if(QUICKSTEP_HAVE_LIBNUMA)
@@ -98,6 +103,7 @@ target_link_libraries(quickstep_catalog_CatalogRelation
glog
quickstep_catalog_CatalogAttribute
quickstep_catalog_CatalogRelationSchema
+ quickstep_catalog_CatalogRelationStatistics
quickstep_catalog_CatalogTypedefs
quickstep_catalog_Catalog_proto
quickstep_catalog_IndexScheme
@@ -111,6 +117,10 @@ target_link_libraries(quickstep_catalog_CatalogRelation
quickstep_threading_SpinSharedMutex
quickstep_utility_Macros
quickstep_utility_PtrVector)
+target_link_libraries(quickstep_catalog_CatalogRelationStatistics
+ quickstep_catalog_CatalogTypedefs
+ quickstep_catalog_Catalog_proto
+ quickstep_utility_Macros)
target_link_libraries(quickstep_catalog_IndexScheme
glog
quickstep_catalog_Catalog_proto
@@ -173,6 +183,7 @@ target_link_libraries(quickstep_catalog
quickstep_catalog_CatalogErrors
quickstep_catalog_CatalogRelation
quickstep_catalog_CatalogRelationSchema
+ quickstep_catalog_CatalogRelationStatistics
quickstep_catalog_CatalogTypedefs
quickstep_catalog_IndexScheme
quickstep_catalog_PartitionScheme
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/df4a05d7/catalog/Catalog.proto
----------------------------------------------------------------------
diff --git a/catalog/Catalog.proto b/catalog/Catalog.proto
index 81e28cf..ce4bc2e 100644
--- a/catalog/Catalog.proto
+++ b/catalog/Catalog.proto
@@ -1,7 +1,7 @@
// Copyright 2011-2015 Quickstep Technologies LLC.
// Copyright 2015-2016 Pivotal Software, Inc.
// Copyright 2016, Quickstep Research Group, Computer Sciences Department,
-// University of Wisconsin\u2014Madison.
+// University of Wisconsin\u2014Madison.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -80,6 +80,16 @@ message IndexScheme {
repeated IndexEntry index_entries = 1;
}
+message CatalogRelationStatistics {
+ optional fixed64 num_tuples = 1;
+
+ message NumDistinctValuesEntry {
+ required int32 attr_id = 1;
+ required fixed64 num_distinct_values = 2;
+ }
+ repeated NumDistinctValuesEntry num_distinct_values_map = 2;
+}
+
message CatalogRelationSchema {
required int32 relation_id = 1;
required string name = 2;
@@ -99,6 +109,7 @@ message CatalogRelation {
optional IndexScheme index_scheme = 18;
optional PartitionScheme partition_scheme = 19;
optional NUMAPlacementScheme placement_scheme = 20;
+ optional CatalogRelationStatistics statistics = 21;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/df4a05d7/catalog/CatalogRelation.cpp
----------------------------------------------------------------------
diff --git a/catalog/CatalogRelation.cpp b/catalog/CatalogRelation.cpp
index 36f82d9..01aebb5 100644
--- a/catalog/CatalogRelation.cpp
+++ b/catalog/CatalogRelation.cpp
@@ -132,6 +132,14 @@ CatalogRelation::CatalogRelation(const serialization::CatalogRelationSchema &pro
}
default_layout_.reset(new StorageBlockLayout(*this, proto_default_layout));
+
+ if (proto.HasExtension(serialization::CatalogRelation::statistics)) {
+ statistics_.reset(
+ new CatalogRelationStatistics(
+ proto.GetExtension(serialization::CatalogRelation::statistics)));
+ } else {
+ statistics_.reset(new CatalogRelationStatistics());
+ }
}
serialization::CatalogRelationSchema CatalogRelation::getProto() const {
@@ -177,6 +185,9 @@ serialization::CatalogRelationSchema CatalogRelation::getProto() const {
#endif
}
+ proto.MutableExtension(serialization::CatalogRelation::statistics)
+ ->MergeFrom(statistics_->getProto());
+
return proto;
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/df4a05d7/catalog/CatalogRelation.hpp
----------------------------------------------------------------------
diff --git a/catalog/CatalogRelation.hpp b/catalog/CatalogRelation.hpp
index 312f3b4..e0d5350 100644
--- a/catalog/CatalogRelation.hpp
+++ b/catalog/CatalogRelation.hpp
@@ -29,6 +29,7 @@
#include "catalog/Catalog.pb.h"
#include "catalog/CatalogConfig.h"
#include "catalog/CatalogRelationSchema.hpp"
+#include "catalog/CatalogRelationStatistics.hpp"
#include "catalog/CatalogTypedefs.hpp"
#include "catalog/IndexScheme.hpp"
@@ -79,7 +80,8 @@ class CatalogRelation : public CatalogRelationSchema {
const relation_id id = -1,
bool temporary = false)
: CatalogRelationSchema(parent, name, id, temporary),
- default_layout_(nullptr) {
+ default_layout_(nullptr),
+ statistics_(new CatalogRelationStatistics()) {
}
/**
@@ -377,6 +379,24 @@ class CatalogRelation : public CatalogRelationSchema {
* getDefaultStorageBlockLayout().estimateTuplesPerBlock();
}
+ /**
+ * @brief Get an immutable reference to the statistics of this catalog relation.
+ *
+ * @return A reference to the statistics of this catalog relation.
+ */
+ const CatalogRelationStatistics& getStatistics() const {
+ return *statistics_;
+ }
+
+ /**
+ * @brief Get a mutable pointer to the statistics of this catalog relation.
+ *
+ * @return A pointer to the statistics of this catalog relation.
+ */
+ CatalogRelationStatistics* getStatisticsMutable() {
+ return statistics_.get();
+ }
+
private:
// A list of blocks belonged to the relation.
std::vector<block_id> blocks_;
@@ -397,6 +417,8 @@ class CatalogRelation : public CatalogRelationSchema {
// Mutex for locking the index scheme.
alignas(kCacheLineBytes) mutable SpinSharedMutex<false> index_scheme_mutex_;
+ std::unique_ptr<CatalogRelationStatistics> statistics_;
+
#ifdef QUICKSTEP_HAVE_LIBNUMA
// NUMA placement scheme object which has the mapping between the partitions
// of the relation and the NUMA nodes/sockets. It also maintains a mapping
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/df4a05d7/catalog/CatalogRelationStatistics.cpp
----------------------------------------------------------------------
diff --git a/catalog/CatalogRelationStatistics.cpp b/catalog/CatalogRelationStatistics.cpp
new file mode 100644
index 0000000..2bd92b4
--- /dev/null
+++ b/catalog/CatalogRelationStatistics.cpp
@@ -0,0 +1,49 @@
+/**
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#include "catalog/CatalogRelationStatistics.hpp"
+
+#include "catalog/Catalog.pb.h"
+
+namespace quickstep {
+
+CatalogRelationStatistics::CatalogRelationStatistics(
+ const serialization::CatalogRelationStatistics &proto) {
+ if (proto.has_num_tuples()) {
+ num_tuples_ = proto.num_tuples();
+ }
+ for (int i = 0; i < proto.num_distinct_values_map_size(); ++i) {
+ const auto &entry = proto.num_distinct_values_map(i);
+ num_distinct_values_map_.emplace(entry.attr_id(),
+ entry.num_distinct_values());
+ }
+}
+
+serialization::CatalogRelationStatistics CatalogRelationStatistics::getProto() const {
+ serialization::CatalogRelationStatistics proto;
+ if (num_tuples_ != 0) {
+ proto.set_num_tuples(num_tuples_);
+ }
+ for (const auto &pair : num_distinct_values_map_) {
+ auto entry = proto.add_num_distinct_values_map();
+ entry->set_attr_id(pair.first);
+ entry->set_num_distinct_values(pair.second);
+ }
+ return proto;
+}
+
+} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/df4a05d7/catalog/CatalogRelationStatistics.hpp
----------------------------------------------------------------------
diff --git a/catalog/CatalogRelationStatistics.hpp b/catalog/CatalogRelationStatistics.hpp
new file mode 100644
index 0000000..572d141
--- /dev/null
+++ b/catalog/CatalogRelationStatistics.hpp
@@ -0,0 +1,122 @@
+/**
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#ifndef QUICKSTEP_CATALOG_CATALOG_RELATION_STATISTICS_HPP_
+#define QUICKSTEP_CATALOG_CATALOG_RELATION_STATISTICS_HPP_
+
+#include <cstddef>
+#include <unordered_map>
+#include <utility>
+
+#include "catalog/Catalog.pb.h"
+#include "catalog/CatalogTypedefs.hpp"
+#include "utility/Macros.hpp"
+
+namespace quickstep {
+
+/** \addtogroup Catalog
+ * @{
+ */
+
+/**
+ * @brief Statistics of a catalog relation. E.g. total number of tuples,
+ * number of distinct values for each column.
+ **/
+class CatalogRelationStatistics {
+ public:
+ /**
+ * @brief Constructor.
+ **/
+ CatalogRelationStatistics()
+ : num_tuples_(0) {}
+
+ /**
+ * @brief Reconstruct a CatalogRelationStatistics object from its serialized
+ * Protocol Buffer form.
+ *
+ * @param proto The Protocol Buffer serialization of a CatalogRelationStatistics
+ * object, previously produced by getProto().
+ **/
+ explicit CatalogRelationStatistics(const serialization::CatalogRelationStatistics &proto);
+
+ /**
+ * @brief Serialize the CatalogRelationStatistics object as Protocol Buffer.
+ *
+ * @return The Protocol Buffer representation of the CatalogRelationStatistics
+ * object.
+ **/
+ serialization::CatalogRelationStatistics getProto() const;
+
+ /**
+ * @brief Set the number of tuples statistic.
+ *
+ * @param num_tuples The number of tuples statistic.
+ */
+ void setNumTuples(std::size_t num_tuples) {
+ num_tuples_ = num_tuples;
+ }
+
+ /**
+ * @brief Get the number of tuples statistic.
+ *
+ * @return The number of tuples. Returns 0 if the statistic is not set.
+ */
+ std::size_t getNumTuples() const {
+ return num_tuples_;
+ }
+
+ /**
+ * @brief Set the number of distinct values statistic for a column (catalog attribute).
+ *
+ * @param attr_id The id of the column.
+ * @param num_distinct_values The number of distinct values statistic.
+ */
+ void setNumDistinctValues(attribute_id attr_id, std::size_t num_distinct_values) {
+ num_distinct_values_map_[attr_id] = num_distinct_values;
+ }
+
+ /**
+ * @brief Get the number of distinct values statistic for a column (catalog attribute).
+ *
+ * @param The id of the column.
+ * @return The number of distinct values statistic for the column. Returns 0
+ * if the statistic is not set.
+ */
+ std::size_t getNumDistinctValues(attribute_id attr_id) const {
+ const auto it = num_distinct_values_map_.find(attr_id);
+ if (it == num_distinct_values_map_.end()) {
+ return static_cast<std::size_t>(0);
+ } else {
+ return it->second;
+ }
+ }
+
+ private:
+ // Total number of tuples in the relation.
+ std::size_t num_tuples_;
+
+ // Number of distinct values for each column.
+ std::unordered_map<attribute_id, std::size_t> num_distinct_values_map_;
+
+ DISALLOW_COPY_AND_ASSIGN(CatalogRelationStatistics);
+};
+
+/** @} */
+
+} // namespace quickstep
+
+#endif // QUICKSTEP_CATALOG_CATALOG_RELATION_STATISTICS_HPP_
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/df4a05d7/cli/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/cli/CMakeLists.txt b/cli/CMakeLists.txt
index 761b6d8..8fee7a4 100644
--- a/cli/CMakeLists.txt
+++ b/cli/CMakeLists.txt
@@ -1,5 +1,7 @@
# Copyright 2011-2015 Quickstep Technologies LLC.
# Copyright 2015 Pivotal Software, Inc.
+# Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+# University of Wisconsin\u2014Madison.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -73,11 +75,24 @@ target_link_libraries(quickstep_cli_CommandExecutor
quickstep_catalog_CatalogDatabase
quickstep_catalog_CatalogRelation
quickstep_catalog_CatalogRelationSchema
- quickstep_cli_PrintToScreen
+ quickstep_cli_DropRelation
+ quickstep_cli_PrintToScreen
quickstep_parser_ParseStatement
+ quickstep_parser_SqlParserWrapper
+ quickstep_queryexecution_Foreman
+ quickstep_queryoptimizer_QueryHandle
+ quickstep_queryoptimizer_QueryPlan
+ quickstep_queryoptimizer_QueryProcessor
+ quickstep_storage_StorageBlock
quickstep_storage_StorageBlockInfo
- quickstep_utility_Macros
- quickstep_utility_PtrVector
+ quickstep_storage_StorageManager
+ quickstep_storage_TupleIdSequence
+ quickstep_storage_TupleStorageSubBlock
+ quickstep_parser_ParseString
+ quickstep_types_Type
+ quickstep_types_TypeID
+ quickstep_types_TypedValue
+ quickstep_utility_PtrVector
quickstep_utility_SqlError)
target_link_libraries(quickstep_cli_DefaultsConfigurator
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/df4a05d7/cli/CommandExecutor.cpp
----------------------------------------------------------------------
diff --git a/cli/CommandExecutor.cpp b/cli/CommandExecutor.cpp
index 026922a..3cb3f86 100644
--- a/cli/CommandExecutor.cpp
+++ b/cli/CommandExecutor.cpp
@@ -19,6 +19,7 @@
#include <algorithm>
#include <cstddef>
+#include <cstdint>
#include <cstdio>
#include <memory>
#include <string>
@@ -28,14 +29,26 @@
#include "catalog/CatalogDatabase.hpp"
#include "catalog/CatalogRelation.hpp"
#include "catalog/CatalogRelationSchema.hpp"
+#include "cli/DropRelation.hpp"
#include "cli/PrintToScreen.hpp"
#include "parser/ParseStatement.hpp"
+#include "parser/ParseString.hpp"
+#include "parser/SqlParserWrapper.hpp"
+#include "query_execution/Foreman.hpp"
+#include "query_optimizer/QueryHandle.hpp"
+#include "query_optimizer/QueryPlan.hpp"
+#include "query_optimizer/QueryProcessor.hpp"
+#include "storage/StorageBlock.hpp"
#include "storage/StorageBlockInfo.hpp"
+#include "storage/StorageManager.hpp"
+#include "storage/TupleIdSequence.hpp"
+#include "storage/TupleStorageSubBlock.hpp"
+#include "types/Type.hpp"
+#include "types/TypeID.hpp"
+#include "types/TypedValue.hpp"
#include "utility/PtrVector.hpp"
-#include "utility/Macros.hpp"
#include "utility/SqlError.hpp"
-#include "gflags/gflags.h"
#include "glog/logging.h"
using std::fprintf;
@@ -195,11 +208,130 @@ void executeDescribeTable(
}
}
+/**
+ * @brief A helper function that executes a SQL query to obtain a scalar result.
+ */
+inline TypedValue executeQueryForSingleResult(const std::string &query_string,
+ StorageManager *storage_manager,
+ QueryProcessor *query_processor,
+ SqlParserWrapper *parser_wrapper,
+ Foreman *foreman) {
+ parser_wrapper->feedNextBuffer(new std::string(query_string));
+
+ ParseResult result = parser_wrapper->getNextStatement();
+ DCHECK(result.condition == ParseResult::kSuccess);
+
+ // Generate the query plan.
+ std::unique_ptr<QueryHandle> query_handle(
+ query_processor->generateQueryHandle(*result.parsed_statement));
+ DCHECK(query_handle->getQueryPlanMutable() != nullptr);
+
+ // Use foreman to execute the query plan.
+ foreman->setQueryPlan(query_handle->getQueryPlanMutable()->getQueryPlanDAGMutable());
+ foreman->reconstructQueryContextFromProto(query_handle->getQueryContextProto());
+
+ foreman->start();
+ foreman->join();
+
+ // Retrieve the scalar result from the result relation.
+ const CatalogRelation *query_result_relation = query_handle->getQueryResultRelation();
+ DCHECK(query_result_relation != nullptr);
+
+ TypedValue value;
+ {
+ std::vector<block_id> blocks = query_result_relation->getBlocksSnapshot();
+ DCHECK_EQ(1u, blocks.size());
+ BlockReference block = storage_manager->getBlock(blocks[0], *query_result_relation);
+ const TupleStorageSubBlock &tuple_store = block->getTupleStorageSubBlock();
+ DCHECK_EQ(1, tuple_store.numTuples());
+ DCHECK_EQ(1u, tuple_store.getRelation().size());
+
+ if (tuple_store.isPacked()) {
+ value = tuple_store.getAttributeValueTyped(0, 0);
+ } else {
+ std::unique_ptr<TupleIdSequence> existence_map(tuple_store.getExistenceMap());
+ value = tuple_store.getAttributeValueTyped(*existence_map->begin(), 0);
+ }
+ value.ensureNotReference();
+ }
+
+ // Drop the result relation.
+ DropRelation::Drop(*query_result_relation,
+ query_processor->getDefaultDatabase(),
+ query_processor->getStorageManager());
+
+ return value;
+}
+
+void executeAnalyze(QueryProcessor *query_processor,
+ Foreman *foreman,
+ FILE *out) {
+ const CatalogDatabase &database = *query_processor->getDefaultDatabase();
+ StorageManager *storage_manager = query_processor->getStorageManager();
+
+ std::unique_ptr<SqlParserWrapper> parser_wrapper(new SqlParserWrapper());
+ std::vector<std::reference_wrapper<const CatalogRelation>> relations(
+ database.begin(), database.end());
+
+ // Analyze each relation in the database.
+ for (const CatalogRelation &relation : relations) {
+ fprintf(out, "Analyzing %s ... ", relation.getName().c_str());
+ fflush(out);
+
+ CatalogRelation *mutable_relation =
+ query_processor->getDefaultDatabase()->getRelationByIdMutable(relation.getID());
+
+ // Get the number of distinct values for each column.
+ for (const CatalogAttribute &attribute : relation) {
+ std::string query_string = "SELECT COUNT(DISTINCT ";
+ query_string.append(attribute.getName());
+ query_string.append(") FROM ");
+ query_string.append(relation.getName());
+ query_string.append(";");
+
+ TypedValue num_distinct_values =
+ executeQueryForSingleResult(query_string,
+ storage_manager,
+ query_processor,
+ parser_wrapper.get(),
+ foreman);
+
+ DCHECK(num_distinct_values.getTypeID() == TypeID::kLong);
+ mutable_relation->getStatisticsMutable()->setNumDistinctValues(
+ attribute.getID(),
+ num_distinct_values.getLiteral<std::int64_t>());
+ }
+
+ // Get the number of tuples for the relation.
+ std::string query_string = "SELECT COUNT(*) FROM ";
+ query_string.append(relation.getName());
+ query_string.append(";");
+
+ TypedValue num_tuples =
+ executeQueryForSingleResult(query_string,
+ storage_manager,
+ query_processor,
+ parser_wrapper.get(),
+ foreman);
+
+ DCHECK(num_tuples.getTypeID() == TypeID::kLong);
+ mutable_relation->getStatisticsMutable()->setNumTuples(
+ num_tuples.getLiteral<std::int64_t>());
+
+ fprintf(out, "done\n");
+ fflush(out);
+ }
+ query_processor->markCatalogAltered();
+ query_processor->saveCatalog();
+}
+
} // namespace
void executeCommand(const ParseStatement &statement,
const CatalogDatabase &catalog_database,
StorageManager *storage_manager,
+ QueryProcessor *query_processor,
+ Foreman *foreman,
FILE *out) {
const ParseCommand &command = static_cast<const ParseCommand &>(statement);
const PtrVector<ParseString> *arguments = command.arguments();
@@ -212,6 +344,8 @@ void executeCommand(const ParseStatement &statement,
} else {
executeDescribeTable(arguments, catalog_database, out);
}
+ } else if (command_str == C::kAnalyzeCommand) {
+ executeAnalyze(query_processor, foreman, out);
} else {
THROW_SQL_ERROR_AT(command.command()) << "Invalid Command";
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/df4a05d7/cli/CommandExecutor.hpp
----------------------------------------------------------------------
diff --git a/cli/CommandExecutor.hpp b/cli/CommandExecutor.hpp
index f367ca1..c819981 100644
--- a/cli/CommandExecutor.hpp
+++ b/cli/CommandExecutor.hpp
@@ -19,13 +19,8 @@
#define QUICKSTEP_CLI_COMMAND_COMMAND_EXECUTOR_HPP_
#include <cstdio>
-#include <limits>
#include <string>
-#include "parser/ParseStatement.hpp"
-#include "storage/StorageBlockInfo.hpp"
-#include "utility/Macros.hpp"
-
using std::fprintf;
using std::fputc;
using std::string;
@@ -33,11 +28,13 @@ using std::string;
namespace quickstep {
class CatalogDatabase;
-class CatalogAttribute;
-class CatalogRelation;
+class Foreman;
+class ParseStatement;
+class QueryProcessor;
class StorageManager;
namespace cli {
+
/** \addtogroup CLI
* @{
*/
@@ -49,17 +46,23 @@ constexpr int kInitMaxColumnWidth = 6;
constexpr char kDescribeDatabaseCommand[] = "\\dt";
constexpr char kDescribeTableCommand[] = "\\d";
+constexpr char kAnalyzeCommand[] = "\\analyze";
/**
* @brief Executes the command by calling the command handler.
*
* @param statement The parsed statement from the cli.
* @param catalog_database The catalog information about the current database.
+ * @param storage_manager The current StorageManager.
+ * @param query_processor The query processor to generate plans for SQL queries.
+ * @param foreman The foreman to execute query plans.
* @param out The stream where the output of the command has to be redirected to.
*/
void executeCommand(const ParseStatement &statement,
const CatalogDatabase &catalog_database,
StorageManager *storage_manager,
+ QueryProcessor *query_processor,
+ Foreman *foreman,
FILE *out);
/** @} */
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/df4a05d7/cli/QuickstepCli.cpp
----------------------------------------------------------------------
diff --git a/cli/QuickstepCli.cpp b/cli/QuickstepCli.cpp
index b7b28ba..558d6eb 100644
--- a/cli/QuickstepCli.cpp
+++ b/cli/QuickstepCli.cpp
@@ -367,6 +367,8 @@ int main(int argc, char* argv[]) {
*result.parsed_statement,
*(query_processor->getDefaultDatabase()),
query_processor->getStorageManager(),
+ query_processor.get(),
+ &foreman,
stdout);
} catch (const quickstep::SqlError &sql_error) {
fprintf(stderr, "%s",
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/df4a05d7/cli/tests/CommandExecutorTestRunner.cpp
----------------------------------------------------------------------
diff --git a/cli/tests/CommandExecutorTestRunner.cpp b/cli/tests/CommandExecutorTestRunner.cpp
index 73d2092..9cd493e 100644
--- a/cli/tests/CommandExecutorTestRunner.cpp
+++ b/cli/tests/CommandExecutorTestRunner.cpp
@@ -88,6 +88,8 @@ void CommandExecutorTestRunner::runTestCase(
*result.parsed_statement,
*(test_database_loader_.catalog_database()),
test_database_loader_.storage_manager(),
+ nullptr,
+ nullptr,
output_stream.file());
} else {
QueryHandle query_handle(optimizer_context.query_id());
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/df4a05d7/query_optimizer/ExecutionGenerator.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/ExecutionGenerator.cpp b/query_optimizer/ExecutionGenerator.cpp
index 7209cfa..c590b6e 100644
--- a/query_optimizer/ExecutionGenerator.cpp
+++ b/query_optimizer/ExecutionGenerator.cpp
@@ -1389,11 +1389,13 @@ void ExecutionGenerator::convertAggregate(
// Add distinctify hash table impl type if it is a DISTINCT aggregation.
if (unnamed_aggregate_expression->is_distinct()) {
- if (group_by_types.empty()) {
+ const std::vector<E::ScalarPtr> &arguments = unnamed_aggregate_expression->getArguments();
+ DCHECK_GE(arguments.size(), 1u);
+ if (group_by_types.empty() && arguments.size() == 1) {
aggr_state_proto->add_distinctify_hash_table_impl_types(
SimplifyHashTableImplTypeProto(
HashTableImplTypeProtoFromString(FLAGS_aggregate_hashtable_type),
- {&unnamed_aggregate_expression->getValueType()}));
+ {&arguments[0]->getValueType()}));
} else {
aggr_state_proto->add_distinctify_hash_table_impl_types(
HashTableImplTypeProtoFromString(FLAGS_aggregate_hashtable_type));
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/df4a05d7/query_optimizer/QueryProcessor.hpp
----------------------------------------------------------------------
diff --git a/query_optimizer/QueryProcessor.hpp b/query_optimizer/QueryProcessor.hpp
index 4514f45..32739dc 100644
--- a/query_optimizer/QueryProcessor.hpp
+++ b/query_optimizer/QueryProcessor.hpp
@@ -1,6 +1,8 @@
/**
* Copyright 2011-2015 Quickstep Technologies LLC.
* Copyright 2015-2016 Pivotal Software, Inc.
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -159,6 +161,14 @@ class QueryProcessor {
void saveCatalog();
/**
+ * @brief Set \p catalog_altered_ to true to indicate that the catalog
+ * has been altered.
+ */
+ void markCatalogAltered() {
+ catalog_altered_ = true;
+ }
+
+ /**
* @brief Get the default database in the Catalog held by this
* QueryProcessor.
**/
[17/50] [abbrv] incubator-quickstep git commit: Change default
aggregate_hashtable_type from LinearOpenAddressing to SeparateChaining (#210)
Posted by zu...@apache.org.
Change default aggregate_hashtable_type from LinearOpenAddressing to SeparateChaining (#210)
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/6c9108a6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/6c9108a6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/6c9108a6
Branch: refs/heads/work-order-serialization
Commit: 6c9108a6d468345baa0bab2f9901cf48b84363fa
Parents: bbaff7a
Author: Jianqiao Zhu <ji...@cs.wisc.edu>
Authored: Thu May 5 09:58:45 2016 -0500
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Mon May 30 15:46:17 2016 -0700
----------------------------------------------------------------------
query_optimizer/ExecutionGenerator.cpp | 69 ++++++++++++--------
.../tests/execution_generator/Select.test | 23 +++----
storage/AggregationOperationState.cpp | 28 +++++++-
storage/AggregationOperationState.hpp | 5 ++
storage/AggregationOperationState.proto | 5 ++
5 files changed, 90 insertions(+), 40 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/6c9108a6/query_optimizer/ExecutionGenerator.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/ExecutionGenerator.cpp b/query_optimizer/ExecutionGenerator.cpp
index c34f084..077d35d 100644
--- a/query_optimizer/ExecutionGenerator.cpp
+++ b/query_optimizer/ExecutionGenerator.cpp
@@ -135,7 +135,7 @@ static const volatile bool join_hashtable_type_dummy
= gflags::RegisterFlagValidator(&FLAGS_join_hashtable_type,
&ValidateHashTableImplTypeString);
-DEFINE_string(aggregate_hashtable_type, "LinearOpenAddressing",
+DEFINE_string(aggregate_hashtable_type, "SeparateChaining",
"HashTable implementation to use for aggregates with GROUP BY "
"(valid options are SeparateChaining or LinearOpenAddressing)");
static const volatile bool aggregate_hashtable_type_dummy
@@ -1285,6 +1285,35 @@ void ExecutionGenerator::convertAggregate(
findRelationInfoOutputByPhysical(physical_plan->input());
aggr_state_proto->set_relation_id(input_relation_info->relation->getID());
+ std::vector<const Type*> group_by_types;
+ for (const E::NamedExpressionPtr &grouping_expression : physical_plan->grouping_expressions()) {
+ unique_ptr<const Scalar> execution_group_by_expression;
+ E::AliasPtr alias;
+ if (E::SomeAlias::MatchesWithConditionalCast(grouping_expression, &alias)) {
+ E::ScalarPtr scalar;
+ // NOTE(zuyu): For aggregate expressions, all child expressions of an
+ // Alias should be a Scalar.
+ CHECK(E::SomeScalar::MatchesWithConditionalCast(alias->expression(), &scalar))
+ << alias->toString();
+ execution_group_by_expression.reset(scalar->concretize(attribute_substitution_map_));
+ } else {
+ execution_group_by_expression.reset(
+ grouping_expression->concretize(attribute_substitution_map_));
+ }
+ aggr_state_proto->add_group_by_expressions()->CopyFrom(execution_group_by_expression->getProto());
+ group_by_types.push_back(&execution_group_by_expression->getType());
+ }
+
+ if (!group_by_types.empty()) {
+ // SimplifyHashTableImplTypeProto() switches the hash table implementation
+ // from SeparateChaining to SimpleScalarSeparateChaining when there is a
+ // single scalar key type with a reversible hash function.
+ aggr_state_proto->set_hash_table_impl_type(
+ SimplifyHashTableImplTypeProto(
+ HashTableImplTypeProtoFromString(FLAGS_aggregate_hashtable_type),
+ group_by_types));
+ }
+
for (const E::AliasPtr &named_aggregate_expression : physical_plan->aggregate_expressions()) {
const E::AggregateFunctionPtr unnamed_aggregate_expression =
std::static_pointer_cast<const E::AggregateFunction>(named_aggregate_expression->expression());
@@ -1304,25 +1333,19 @@ void ExecutionGenerator::convertAggregate(
// Set whether it is a DISTINCT aggregation.
aggr_proto->set_is_distinct(unnamed_aggregate_expression->is_distinct());
- }
- std::vector<const Type*> group_by_types;
- for (const E::NamedExpressionPtr &grouping_expression : physical_plan->grouping_expressions()) {
- unique_ptr<const Scalar> execution_group_by_expression;
- E::AliasPtr alias;
- if (E::SomeAlias::MatchesWithConditionalCast(grouping_expression, &alias)) {
- E::ScalarPtr scalar;
- // NOTE(zuyu): For aggregate expressions, all child expressions of an
- // Alias should be a Scalar.
- CHECK(E::SomeScalar::MatchesWithConditionalCast(alias->expression(), &scalar))
- << alias->toString();
- execution_group_by_expression.reset(scalar->concretize(attribute_substitution_map_));
- } else {
- execution_group_by_expression.reset(
- grouping_expression->concretize(attribute_substitution_map_));
+ // Add distinctify hash table impl type if it is a DISTINCT aggregation.
+ if (unnamed_aggregate_expression->is_distinct()) {
+ if (group_by_types.empty()) {
+ aggr_state_proto->add_distinctify_hash_table_impl_types(
+ SimplifyHashTableImplTypeProto(
+ HashTableImplTypeProtoFromString(FLAGS_aggregate_hashtable_type),
+ {&unnamed_aggregate_expression->getValueType()}));
+ } else {
+ aggr_state_proto->add_distinctify_hash_table_impl_types(
+ HashTableImplTypeProtoFromString(FLAGS_aggregate_hashtable_type));
+ }
}
- aggr_state_proto->add_group_by_expressions()->CopyFrom(execution_group_by_expression->getProto());
- group_by_types.push_back(&execution_group_by_expression->getType());
}
if (physical_plan->filter_predicate() != nullptr) {
@@ -1332,16 +1355,6 @@ void ExecutionGenerator::convertAggregate(
aggr_state_proto->set_estimated_num_entries(cost_model_->estimateCardinality(physical_plan));
- if (!group_by_types.empty()) {
- // SimplifyHashTableImplTypeProto() switches the hash table implementation
- // from SeparateChaining to SimpleScalarSeparateChaining when there is a
- // single scalar key type with a reversible hash function.
- aggr_state_proto->set_hash_table_impl_type(
- SimplifyHashTableImplTypeProto(
- HashTableImplTypeProtoFromString(FLAGS_aggregate_hashtable_type),
- group_by_types));
- }
-
const QueryPlan::DAGNodeIndex aggregation_operator_index =
execution_plan_->addRelationalOperator(
new AggregationOperator(
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/6c9108a6/query_optimizer/tests/execution_generator/Select.test
----------------------------------------------------------------------
diff --git a/query_optimizer/tests/execution_generator/Select.test b/query_optimizer/tests/execution_generator/Select.test
index a08b012..390b7b6 100644
--- a/query_optimizer/tests/execution_generator/Select.test
+++ b/query_optimizer/tests/execution_generator/Select.test
@@ -535,21 +535,12 @@ WHERE int_col = 2;
SELECT int_col
FROM test
-GROUP BY int_col;
+GROUP BY int_col
+ORDER BY int_col;
--
+-----------+
|int_col |
+-----------+
-| 2|
-| 4|
-| 6|
-| 8|
-| 12|
-| 14|
-| 16|
-| 18|
-| 22|
-| 24|
| -23|
| -21|
| -19|
@@ -562,6 +553,16 @@ GROUP BY int_col;
| -5|
| -3|
| -1|
+| 2|
+| 4|
+| 6|
+| 8|
+| 12|
+| 14|
+| 16|
+| 18|
+| 22|
+| 24|
+-----------+
==
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/6c9108a6/storage/AggregationOperationState.cpp
----------------------------------------------------------------------
diff --git a/storage/AggregationOperationState.cpp b/storage/AggregationOperationState.cpp
index a3a669c..d209ceb 100644
--- a/storage/AggregationOperationState.cpp
+++ b/storage/AggregationOperationState.cpp
@@ -1,6 +1,8 @@
/**
* Copyright 2011-2015 Quickstep Technologies LLC.
* Copyright 2015-2016 Pivotal Software, Inc.
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -64,6 +66,7 @@ AggregationOperationState::AggregationOperationState(
const Predicate *predicate,
const std::size_t estimated_num_entries,
const HashTableImplType hash_table_impl_type,
+ const std::vector<HashTableImplType> &distinctify_hash_table_impl_types,
StorageManager *storage_manager)
: input_relation_(input_relation),
predicate_(predicate),
@@ -101,6 +104,8 @@ AggregationOperationState::AggregationOperationState(
std::vector<std::vector<std::unique_ptr<const Scalar>>>::const_iterator args_it
= arguments_.begin();
std::vector<bool>::const_iterator is_distinct_it = is_distinct_.begin();
+ std::vector<HashTableImplType>::const_iterator distinctify_hash_table_impl_types_it
+ = distinctify_hash_table_impl_types.begin();
for (; agg_func_it != aggregate_functions.end(); ++agg_func_it, ++args_it, ++is_distinct_it) {
// Get the Types of this aggregate's arguments so that we can create an
// AggregationHandle.
@@ -161,10 +166,11 @@ AggregationOperationState::AggregationOperationState(
// query optimization, if it worths.
distinctify_hashtables_.emplace_back(
handles_.back()->createDistinctifyHashTable(
- hash_table_impl_type,
+ *distinctify_hash_table_impl_types_it,
key_types,
estimated_num_entries,
storage_manager));
+ ++distinctify_hash_table_impl_types_it;
} else {
distinctify_hashtables_.emplace_back(nullptr);
}
@@ -182,6 +188,8 @@ AggregationOperationState* AggregationOperationState::ReconstructFromProto(
std::vector<const AggregateFunction*> aggregate_functions;
std::vector<std::vector<std::unique_ptr<const Scalar>>> arguments;
std::vector<bool> is_distinct;
+ std::vector<HashTableImplType> distinctify_hash_table_impl_types;
+ std::size_t distinctify_hash_table_impl_type_index = 0;
for (int agg_idx = 0; agg_idx < proto.aggregates_size(); ++agg_idx) {
const serialization::Aggregate &agg_proto = proto.aggregates(agg_idx);
@@ -197,6 +205,13 @@ AggregationOperationState* AggregationOperationState::ReconstructFromProto(
}
is_distinct.emplace_back(agg_proto.is_distinct());
+
+ if (agg_proto.is_distinct()) {
+ distinctify_hash_table_impl_types.emplace_back(
+ HashTableImplTypeFromProto(
+ proto.distinctify_hash_table_impl_types(distinctify_hash_table_impl_type_index)));
+ ++distinctify_hash_table_impl_type_index;
+ }
}
std::vector<std::unique_ptr<const Scalar>> group_by_expressions;
@@ -223,6 +238,7 @@ AggregationOperationState* AggregationOperationState::ReconstructFromProto(
predicate.release(),
proto.estimated_num_entries(),
HashTableImplTypeFromProto(proto.hash_table_impl_type()),
+ distinctify_hash_table_impl_types,
storage_manager);
}
@@ -234,6 +250,8 @@ bool AggregationOperationState::ProtoIsValid(const serialization::AggregationOpe
return false;
}
+ std::size_t num_distinctify_hash_tables = proto.distinctify_hash_table_impl_types_size();
+ std::size_t distinctify_hash_table_impl_type_index = 0;
for (int i = 0; i < proto.aggregates_size(); ++i) {
if (!AggregateFunctionFactory::ProtoIsValid(proto.aggregates(i).function())) {
return false;
@@ -251,6 +269,14 @@ bool AggregationOperationState::ProtoIsValid(const serialization::AggregationOpe
return false;
}
}
+
+ if (proto.aggregates(i).is_distinct()) {
+ if (distinctify_hash_table_impl_type_index >= num_distinctify_hash_tables ||
+ !serialization::HashTableImplType_IsValid(
+ proto.distinctify_hash_table_impl_types(distinctify_hash_table_impl_type_index))) {
+ return false;
+ }
+ }
}
for (int i = 0; i < proto.group_by_expressions_size(); ++i) {
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/6c9108a6/storage/AggregationOperationState.hpp
----------------------------------------------------------------------
diff --git a/storage/AggregationOperationState.hpp b/storage/AggregationOperationState.hpp
index b883ed1..c3a1278 100644
--- a/storage/AggregationOperationState.hpp
+++ b/storage/AggregationOperationState.hpp
@@ -1,6 +1,8 @@
/**
* Copyright 2011-2015 Quickstep Technologies LLC.
* Copyright 2015-2016 Pivotal Software, Inc.
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -93,6 +95,8 @@ class AggregationOperationState {
* in the input relation.
* @param hash_table_impl_type The HashTable implementation to use for
* GROUP BY. Ignored if group_by is empty.
+ * @param distinctify_hash_table_impl_type The HashTable implementation to use
+ * for the distinctify phase of each DISTINCT aggregation.
* @param storage_manager The StorageManager to use for allocating hash
* tables. Single aggregation state (when GROUP BY list is not
* specified) is not allocated using memory from storage manager.
@@ -105,6 +109,7 @@ class AggregationOperationState {
const Predicate *predicate,
const std::size_t estimated_num_entries,
const HashTableImplType hash_table_impl_type,
+ const std::vector<HashTableImplType> &distinctify_hash_table_impl_types,
StorageManager *storage_manager);
~AggregationOperationState() {}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/6c9108a6/storage/AggregationOperationState.proto
----------------------------------------------------------------------
diff --git a/storage/AggregationOperationState.proto b/storage/AggregationOperationState.proto
index 031f782..bf78e3a 100644
--- a/storage/AggregationOperationState.proto
+++ b/storage/AggregationOperationState.proto
@@ -1,5 +1,7 @@
// Copyright 2011-2015 Quickstep Technologies LLC.
// Copyright 2015-2016 Pivotal Software, Inc.
+// Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+// University of Wisconsin\u2014Madison.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -37,4 +39,7 @@ message AggregationOperationState {
// NOTE(chasseur): 'hash_table_impl_type' is marked optional, but it is
// needed if 'group_by_expressions' is non-empty, and ignored otherwise.
optional HashTableImplType hash_table_impl_type = 6;
+
+ // Each DISTINCT aggregation has its distinctify hash table impl type.
+ repeated HashTableImplType distinctify_hash_table_impl_types = 7;
}
[44/50] [abbrv] incubator-quickstep git commit: Fixed Travis CI link
in Readme.
Posted by zu...@apache.org.
Fixed Travis CI link in Readme.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/8f8a03a2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/8f8a03a2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/8f8a03a2
Branch: refs/heads/work-order-serialization
Commit: 8f8a03a2f91592ebc07389268a47219d458832d6
Parents: fe80c01
Author: Zuyu Zhang <zz...@pivotal.io>
Authored: Mon May 16 13:39:12 2016 -0700
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Mon May 30 15:47:53 2016 -0700
----------------------------------------------------------------------
README.md | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/8f8a03a2/README.md
----------------------------------------------------------------------
diff --git a/README.md b/README.md
index 7cea80c..04d5d66 100644
--- a/README.md
+++ b/README.md
@@ -2,8 +2,8 @@
[![Travis Widget]][Travis]
-[Travis]: https://travis-ci.org/pivotalsoftware/quickstep
-[Travis Widget]: https://travis-ci.org/pivotalsoftware/quickstep.svg?branch=master
+[Travis]: https://travis-ci.org/apache/incubator-quickstep
+[Travis Widget]: https://travis-ci.org/apache/incubator-quickstep.svg?branch=master
Apache Quickstep is an experimental high-performance database engine designed with the
aim of Data at Bare-Metal Speed. It began life in 2011 as a
[06/50] [abbrv] incubator-quickstep git commit: Increased the size of
the sharded lock manager to a large primary number (#198)
Posted by zu...@apache.org.
Increased the size of the sharded lock manager to a large primary number (#198)
Having small number of entries in the sharded lock manager introduces artificial conflict causing the buffer pool to grow. The size of an entry in the lock_manager_ is small, so don't be so stingy with the size of the lock manager. By default, we want to run well on large memory boxes.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/5b75e8ec
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/5b75e8ec
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/5b75e8ec
Branch: refs/heads/work-order-serialization
Commit: 5b75e8ec24366e2379624804db9cb118cbe0bcd3
Parents: c5460f4
Author: Jignesh Patel <pa...@users.noreply.github.com>
Authored: Thu Apr 28 16:11:22 2016 -0500
Committer: Zuyu ZHANG <zu...@users.noreply.github.com>
Committed: Thu Apr 28 14:11:22 2016 -0700
----------------------------------------------------------------------
storage/StorageManager.hpp | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5b75e8ec/storage/StorageManager.hpp
----------------------------------------------------------------------
diff --git a/storage/StorageManager.hpp b/storage/StorageManager.hpp
index 0b68b76..52326c2 100644
--- a/storage/StorageManager.hpp
+++ b/storage/StorageManager.hpp
@@ -479,7 +479,11 @@ class StorageManager {
// (2) If it is not safe to evict a block, then either that block's
// reference count is greater than 0 or a shared lock is held on the
// block's lock shard.
- static constexpr std::size_t kLockManagerNumShards = 256;
+ // TODO(jmp): Would be good to set this more intelligently in the future
+ // based on the hardware concurrency, the amount of main memory
+ // and slot size. For now pick the largest prime that is less
+ // than 8K.
+ static constexpr std::size_t kLockManagerNumShards = 0x2000-1;
ShardedLockManager<block_id, kLockManagerNumShards, SpinSharedMutex<false>> lock_manager_;
FRIEND_TEST(StorageManagerTest, DifferentNUMANodeBlobTestWithEviction);
[29/50] [abbrv] incubator-quickstep git commit: Fixed a g++ compiler
warning (#221)
Posted by zu...@apache.org.
Fixed a g++ compiler warning (#221)
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/ba25b132
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/ba25b132
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/ba25b132
Branch: refs/heads/work-order-serialization
Commit: ba25b1326e8063f41a4d89b62b6860b6ef8d0e52
Parents: ee54e1d
Author: Marc S <cr...@users.noreply.github.com>
Authored: Wed May 11 14:34:37 2016 -0500
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Mon May 30 15:47:27 2016 -0700
----------------------------------------------------------------------
query_optimizer/ExecutionGenerator.cpp | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/ba25b132/query_optimizer/ExecutionGenerator.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/ExecutionGenerator.cpp b/query_optimizer/ExecutionGenerator.cpp
index 7f26e85..7209cfa 100644
--- a/query_optimizer/ExecutionGenerator.cpp
+++ b/query_optimizer/ExecutionGenerator.cpp
@@ -588,8 +588,8 @@ void ExecutionGenerator::convertHashJoin(const P::HashJoinPtr &physical_plan) {
std::vector<attribute_id> probe_original_attribute_ids;
std::vector<attribute_id> build_original_attribute_ids;
- const CatalogRelation *referenced_stored_probe_relation;
- const CatalogRelation *referenced_stored_build_relation;
+ const CatalogRelation *referenced_stored_probe_relation = nullptr;
+ const CatalogRelation *referenced_stored_build_relation = nullptr;
bool any_probe_attributes_nullable = false;
bool any_build_attributes_nullable = false;
[49/50] [abbrv] incubator-quickstep git commit: Serialized WorkOrders
as proto.
Posted by zu...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3e35844f/relational_operators/SelectOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/SelectOperator.hpp b/relational_operators/SelectOperator.hpp
index 76f4cb6..b2a4601 100644
--- a/relational_operators/SelectOperator.hpp
+++ b/relational_operators/SelectOperator.hpp
@@ -49,8 +49,11 @@ class InsertDestination;
class Predicate;
class Scalar;
class StorageManager;
+class WorkOrderProtosContainer;
class WorkOrdersContainer;
+namespace serialization { class WorkOrder; }
+
/** \addtogroup RelationalOperators
* @{
*/
@@ -182,6 +185,8 @@ class SelectOperator : public RelationalOperator {
const tmb::client_id scheduler_client_id,
tmb::MessageBus *bus) override;
+ bool getAllWorkOrderProtos(WorkOrderProtosContainer *container) override;
+
void feedInputBlock(const block_id input_block_id, const relation_id input_relation_id) override {
if (input_relation_.hasPartitionScheme()) {
const partition_id part_id =
@@ -233,6 +238,13 @@ class SelectOperator : public RelationalOperator {
InsertDestination *output_destination);
private:
+ /**
+ * @brief Create Work Order proto.
+ *
+ * @param block The block id used in the Work Order.
+ **/
+ serialization::WorkOrder* createWorkOrderProto(const block_id block);
+
const CatalogRelation &input_relation_;
const CatalogRelation &output_relation_;
const QueryContext::insert_destination_id output_destination_index_;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3e35844f/relational_operators/SortMergeRunOperator.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/SortMergeRunOperator.cpp b/relational_operators/SortMergeRunOperator.cpp
index 7427d44..cff2a3c 100644
--- a/relational_operators/SortMergeRunOperator.cpp
+++ b/relational_operators/SortMergeRunOperator.cpp
@@ -23,9 +23,11 @@
#include <vector>
#include "query_execution/QueryExecutionTypedefs.hpp"
+#include "query_execution/WorkOrderProtosContainer.hpp"
#include "query_execution/WorkOrdersContainer.hpp"
#include "relational_operators/SortMergeRunOperator.pb.h"
#include "relational_operators/SortMergeRunOperatorHelpers.hpp"
+#include "relational_operators/WorkOrder.pb.h"
#include "threading/ThreadIDBasedMap.hpp"
#include "glog/logging.h"
@@ -69,6 +71,71 @@ bool SortMergeRunOperator::getAllWorkOrders(
return generateWorkOrders(container, query_context, storage_manager, scheduler_client_id, bus);
}
+bool SortMergeRunOperator::getAllWorkOrderProtos(WorkOrderProtosContainer *container) {
+ if (input_relation_is_stored_) {
+ // Input blocks (or runs) are from base relation. Only possible when base
+ // relation is stored sorted.
+ if (!started_) {
+ // Initialize merge tree completely, since all input runs are known.
+ merge_tree_.initializeTree(input_relation_block_ids_.size());
+ started_ = true;
+ initializeInputRuns();
+ }
+ } else {
+ // Input blocks (or runs) are pipelined from the sorted run generation
+ // operator.
+ if (!started_ && !input_stream_done_) {
+ // Initialize merge tree for first pipeline mode.
+ merge_tree_.initializeForPipeline();
+ started_ = true;
+ initializeInputRuns();
+ }
+ }
+
+ // Get merge jobs from merge tree.
+ std::vector<MergeTree::MergeJob> jobs;
+ const bool done_generating = merge_tree_.getMergeJobs(&jobs);
+
+ for (std::vector<MergeTree::MergeJob>::size_type job_id = 0;
+ job_id < jobs.size();
+ ++job_id) {
+ // Add work order for each merge job.
+ container->addWorkOrderProto(createWorkOrderProto(&jobs[job_id]), op_index_);
+ }
+
+ return done_generating;
+}
+
+serialization::WorkOrder* SortMergeRunOperator::createWorkOrderProto(
+ merge_run_operator::MergeTree::MergeJob *job) {
+ DCHECK(job != nullptr);
+ DCHECK(!job->runs.empty());
+
+ serialization::WorkOrder *proto = new serialization::WorkOrder;
+ proto->set_work_order_type(serialization::SORT_MERGE_RUN);
+
+ proto->SetExtension(serialization::SortMergeRunWorkOrder::operator_index, op_index_);
+ proto->SetExtension(serialization::SortMergeRunWorkOrder::sort_config_index, sort_config_index_);
+
+ for (const merge_run_operator::Run &run : job->runs) {
+ serialization::Run *run_proto = proto->AddExtension(serialization::SortMergeRunWorkOrder::runs);
+ for (const block_id block : run) {
+ run_proto->add_blocks(block);
+ }
+ }
+
+ proto->SetExtension(serialization::SortMergeRunWorkOrder::top_k, top_k_);
+ proto->SetExtension(serialization::SortMergeRunWorkOrder::merge_level, job->level);
+ proto->SetExtension(serialization::SortMergeRunWorkOrder::relation_id,
+ job->level > 0 ? run_relation_.getID()
+ : input_relation_.getID());
+ proto->SetExtension(serialization::SortMergeRunWorkOrder::insert_destination_index,
+ job->is_final_level ? output_destination_index_
+ : run_block_destination_index_);
+
+ return proto;
+}
+
WorkOrder *SortMergeRunOperator::createWorkOrder(
merge_run_operator::MergeTree::MergeJob *job,
QueryContext *query_context,
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3e35844f/relational_operators/SortMergeRunOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/SortMergeRunOperator.hpp b/relational_operators/SortMergeRunOperator.hpp
index f92affe..c7e466f 100644
--- a/relational_operators/SortMergeRunOperator.hpp
+++ b/relational_operators/SortMergeRunOperator.hpp
@@ -44,8 +44,11 @@ namespace quickstep {
class CatalogRelationSchema;
class InsertDestination;
class StorageManager;
+class WorkOrderProtosContainer;
class WorkOrdersContainer;
+namespace serialization { class WorkOrder; }
+
/**
* @defgroup SortMergeRun Merging Sorted Runs
* @ingroup Sort
@@ -128,6 +131,8 @@ class SortMergeRunOperator : public RelationalOperator {
const tmb::client_id scheduler_client_id,
tmb::MessageBus *bus) override;
+ bool getAllWorkOrderProtos(WorkOrderProtosContainer *container) override;
+
void feedInputBlock(const block_id input_block_id,
const relation_id input_relation_id) override {
input_relation_block_ids_.push_back(input_block_id);
@@ -178,6 +183,13 @@ class SortMergeRunOperator : public RelationalOperator {
const tmb::client_id scheduler_client_id,
tmb::MessageBus *bus);
+ /**
+ * @brief Create Work Order proto.
+ *
+ * @param job The merge job.
+ **/
+ serialization::WorkOrder* createWorkOrderProto(merge_run_operator::MergeTree::MergeJob *job);
+
const CatalogRelation &input_relation_;
const CatalogRelation &output_relation_;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3e35844f/relational_operators/SortRunGenerationOperator.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/SortRunGenerationOperator.cpp b/relational_operators/SortRunGenerationOperator.cpp
index 9bb3f51..1c0ba3c 100644
--- a/relational_operators/SortRunGenerationOperator.cpp
+++ b/relational_operators/SortRunGenerationOperator.cpp
@@ -21,7 +21,9 @@
#include "catalog/CatalogRelation.hpp"
#include "query_execution/QueryContext.hpp"
+#include "query_execution/WorkOrderProtosContainer.hpp"
#include "query_execution/WorkOrdersContainer.hpp"
+#include "relational_operators/WorkOrder.pb.h"
#include "storage/InsertDestination.hpp"
#include "storage/StorageBlock.hpp"
#include "storage/StorageManager.hpp"
@@ -78,6 +80,42 @@ bool SortRunGenerationOperator::getAllWorkOrders(
}
}
+bool SortRunGenerationOperator::getAllWorkOrderProtos(WorkOrderProtosContainer *container) {
+ if (input_relation_is_stored_) {
+ // Input blocks are from a base relation.
+ if (!started_) {
+ for (const block_id input_block_id : input_relation_block_ids_) {
+ container->addWorkOrderProto(createWorkOrderProto(input_block_id), op_index_);
+ }
+ started_ = true;
+ }
+ return true;
+ } else {
+ // Input blocks are pipelined.
+ while (num_workorders_generated_ < input_relation_block_ids_.size()) {
+ container->addWorkOrderProto(
+ createWorkOrderProto(input_relation_block_ids_[num_workorders_generated_]),
+ op_index_);
+ ++num_workorders_generated_;
+ }
+ return done_feeding_input_relation_;
+ }
+}
+
+serialization::WorkOrder* SortRunGenerationOperator::createWorkOrderProto(const block_id block) {
+ serialization::WorkOrder *proto = new serialization::WorkOrder;
+ proto->set_work_order_type(serialization::SORT_RUN_GENERATION);
+
+ proto->SetExtension(serialization::SortRunGenerationWorkOrder::sort_config_index, sort_config_index_);
+ proto->SetExtension(serialization::SortRunGenerationWorkOrder::relation_id, input_relation_.getID());
+ proto->SetExtension(serialization::SortRunGenerationWorkOrder::insert_destination_index,
+ output_destination_index_);
+ proto->SetExtension(serialization::SortRunGenerationWorkOrder::block_id, block);
+
+ return proto;
+}
+
+
void SortRunGenerationWorkOrder::execute() {
BlockReference block(
storage_manager_->getBlock(input_block_id_, input_relation_));
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3e35844f/relational_operators/SortRunGenerationOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/SortRunGenerationOperator.hpp b/relational_operators/SortRunGenerationOperator.hpp
index 04290a9..d7fbf2d 100644
--- a/relational_operators/SortRunGenerationOperator.hpp
+++ b/relational_operators/SortRunGenerationOperator.hpp
@@ -40,8 +40,11 @@ namespace quickstep {
class CatalogRelationSchema;
class InsertDestination;
class StorageManager;
+class WorkOrderProtosContainer;
class WorkOrdersContainer;
+namespace serialization { class WorkOrder; }
+
/**
* \defgroup Sort Sorting
* \ingroup RelationalOperators
@@ -107,6 +110,8 @@ class SortRunGenerationOperator : public RelationalOperator {
const tmb::client_id scheduler_client_id,
tmb::MessageBus *bus) override;
+ bool getAllWorkOrderProtos(WorkOrderProtosContainer *container) override;
+
void feedInputBlock(const block_id input_block_id, const relation_id input_relation_id) override {
DCHECK(input_relation_id == input_relation_.getID());
input_relation_block_ids_.push_back(input_block_id);
@@ -128,6 +133,13 @@ class SortRunGenerationOperator : public RelationalOperator {
}
private:
+ /**
+ * @brief Create Work Order proto.
+ *
+ * @param block The block id used in the Work Order.
+ **/
+ serialization::WorkOrder* createWorkOrderProto(const block_id block);
+
const CatalogRelation &input_relation_;
const CatalogRelation &output_relation_;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3e35844f/relational_operators/TableGeneratorOperator.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/TableGeneratorOperator.cpp b/relational_operators/TableGeneratorOperator.cpp
index 886d05f..480ec23 100644
--- a/relational_operators/TableGeneratorOperator.cpp
+++ b/relational_operators/TableGeneratorOperator.cpp
@@ -1,6 +1,6 @@
/**
* Copyright 2016, Quickstep Research Group, Computer Sciences Department,
- * University of Wisconsin\u2014Madison.
+ * University of Wisconsin\u2014Madison.
* Copyright 2016 Pivotal Software, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
@@ -20,7 +20,9 @@
#include "expressions/table_generator/GeneratorFunctionHandle.hpp"
#include "query_execution/QueryContext.hpp"
+#include "query_execution/WorkOrderProtosContainer.hpp"
#include "query_execution/WorkOrdersContainer.hpp"
+#include "relational_operators/WorkOrder.pb.h"
#include "storage/InsertDestination.hpp"
#include "types/containers/ColumnVectorsValueAccessor.hpp"
@@ -50,6 +52,21 @@ bool TableGeneratorOperator::getAllWorkOrders(
return started_;
}
+bool TableGeneratorOperator::getAllWorkOrderProtos(WorkOrderProtosContainer *container) {
+ if (!started_) {
+ serialization::WorkOrder *proto = new serialization::WorkOrder;
+ proto->set_work_order_type(serialization::TABLE_GENERATOR);
+
+ proto->SetExtension(serialization::TableGeneratorWorkOrder::generator_function_index, generator_function_index_);
+ proto->SetExtension(serialization::TableGeneratorWorkOrder::insert_destination_index, output_destination_index_);
+
+ container->addWorkOrderProto(proto, op_index_);
+ started_ = true;
+ }
+ return true;
+}
+
+
void TableGeneratorWorkOrder::execute() {
ColumnVectorsValueAccessor temp_result;
function_handle_.populateColumns(&temp_result);
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3e35844f/relational_operators/TableGeneratorOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/TableGeneratorOperator.hpp b/relational_operators/TableGeneratorOperator.hpp
index a26b227..d126f08 100644
--- a/relational_operators/TableGeneratorOperator.hpp
+++ b/relational_operators/TableGeneratorOperator.hpp
@@ -1,6 +1,6 @@
/**
* Copyright 2016, Quickstep Research Group, Computer Sciences Department,
- * University of Wisconsin\u2014Madison.
+ * University of Wisconsin\u2014Madison.
* Copyright 2016 Pivotal Software, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
@@ -40,6 +40,7 @@ namespace quickstep {
class GeneratorFunctionHandle;
class InsertDestination;
class StorageManager;
+class WorkOrderProtosContainer;
class WorkOrdersContainer;
/** \addtogroup RelationalOperators
@@ -79,6 +80,8 @@ class TableGeneratorOperator : public RelationalOperator {
const tmb::client_id scheduler_client_id,
tmb::MessageBus *bus) override;
+ bool getAllWorkOrderProtos(WorkOrderProtosContainer *container) override;
+
void feedInputBlock(const block_id input_block_id, const relation_id input_relation_id) override {
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3e35844f/relational_operators/TextScanOperator.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/TextScanOperator.cpp b/relational_operators/TextScanOperator.cpp
index 5ede6f7..939859b 100644
--- a/relational_operators/TextScanOperator.cpp
+++ b/relational_operators/TextScanOperator.cpp
@@ -33,8 +33,10 @@
#include "query_execution/QueryContext.hpp"
#include "query_execution/QueryExecutionMessages.pb.h"
#include "query_execution/QueryExecutionUtil.hpp"
+#include "query_execution/WorkOrderProtosContainer.hpp"
#include "query_execution/WorkOrdersContainer.hpp"
#include "relational_operators/TextScanOperator.pb.h"
+#include "relational_operators/WorkOrder.pb.h"
#include "storage/InsertDestination.hpp"
#include "storage/StorageBlob.hpp"
#include "storage/StorageBlockInfo.hpp"
@@ -214,6 +216,81 @@ bool TextScanOperator::getAllWorkOrders(
}
}
+bool TextScanOperator::getAllWorkOrderProtos(WorkOrderProtosContainer *container) {
+ const std::vector<std::string> files = utility::file::GlobExpand(file_pattern_);
+ if (parallelize_load_) {
+ // Parallel implementation: Split work orders are generated for each file
+ // being bulk-loaded. (More than one file can be loaded, because we support
+ // glob() semantics in file name.) These work orders read the input file,
+ // and split them in the blobs that can be parsed independently.
+ if (blocking_dependencies_met_) {
+ if (!work_generated_) {
+ work_generated_ = true;
+
+ // First, generate text-split work orders.
+ for (const string &file : files) {
+ serialization::WorkOrder *proto = new serialization::WorkOrder;
+ proto->set_work_order_type(serialization::TEXT_SPLIT);
+
+ proto->SetExtension(serialization::TextSplitWorkOrder::operator_index, op_index_);
+ proto->SetExtension(serialization::TextSplitWorkOrder::filename, file);
+ proto->SetExtension(serialization::TextSplitWorkOrder::process_escape_sequences,
+ process_escape_sequences_);
+
+ container->addWorkOrderProto(proto, op_index_);
+
+ ++num_split_work_orders_;
+ }
+ return false;
+ } else {
+ // Check if there are blobs to parse.
+ while (!text_blob_queue_.empty()) {
+ const TextBlob blob_work = text_blob_queue_.popOne();
+
+ serialization::WorkOrder *proto = new serialization::WorkOrder;
+ proto->set_work_order_type(serialization::TEXT_SCAN);
+
+ proto->SetExtension(serialization::TextScanWorkOrder::field_terminator, field_terminator_);
+ proto->SetExtension(serialization::TextScanWorkOrder::process_escape_sequences,
+ process_escape_sequences_);
+ proto->SetExtension(serialization::TextScanWorkOrder::insert_destination_index,
+ output_destination_index_);
+
+ serialization::TextBlob *text_blob_proto =
+ proto->MutableExtension(serialization::TextScanWorkOrder::text_blob);
+ text_blob_proto->set_blob_id(blob_work.blob_id);
+ text_blob_proto->set_size(blob_work.size);
+
+ container->addWorkOrderProto(proto, op_index_);
+ }
+ // Done if all split work orders are completed, and no blobs are left to
+ // process.
+ return num_done_split_work_orders_.load(std::memory_order_acquire) == num_split_work_orders_ &&
+ text_blob_queue_.empty();
+ }
+ }
+ return false;
+ } else {
+ // Serial implementation.
+ if (blocking_dependencies_met_ && !work_generated_) {
+ for (const string &file : files) {
+ serialization::WorkOrder *proto = new serialization::WorkOrder;
+ proto->set_work_order_type(serialization::TEXT_SCAN);
+
+ proto->SetExtension(serialization::TextScanWorkOrder::field_terminator, field_terminator_);
+ proto->SetExtension(serialization::TextScanWorkOrder::process_escape_sequences,
+ process_escape_sequences_);
+ proto->SetExtension(serialization::TextScanWorkOrder::insert_destination_index, output_destination_index_);
+ proto->SetExtension(serialization::TextScanWorkOrder::filename, file);
+
+ container->addWorkOrderProto(proto, op_index_);
+ }
+ work_generated_ = true;
+ }
+ return work_generated_;
+ }
+}
+
void TextScanOperator::receiveFeedbackMessage(const WorkOrder::FeedbackMessage &msg) {
switch (msg.type()) {
case kSplitWorkOrderCompletionMessage: {
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3e35844f/relational_operators/TextScanOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/TextScanOperator.hpp b/relational_operators/TextScanOperator.hpp
index a2d4ced..e97227d 100644
--- a/relational_operators/TextScanOperator.hpp
+++ b/relational_operators/TextScanOperator.hpp
@@ -48,6 +48,7 @@ namespace quickstep {
class CatalogRelationSchema;
class InsertDestination;
class StorageManager;
+class WorkOrderProtosContainer;
class WorkOrdersContainer;
/** \addtogroup RelationalOperators
@@ -159,6 +160,8 @@ class TextScanOperator : public RelationalOperator {
const tmb::client_id scheduler_client_id,
tmb::MessageBus *bus) override;
+ bool getAllWorkOrderProtos(WorkOrderProtosContainer *container) override;
+
QueryContext::insert_destination_id getInsertDestinationID() const override {
return output_destination_index_;
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3e35844f/relational_operators/UpdateOperator.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/UpdateOperator.cpp b/relational_operators/UpdateOperator.cpp
index 7585db1..db96fc7 100644
--- a/relational_operators/UpdateOperator.cpp
+++ b/relational_operators/UpdateOperator.cpp
@@ -26,7 +26,9 @@
#include "query_execution/QueryContext.hpp"
#include "query_execution/QueryExecutionMessages.pb.h"
#include "query_execution/QueryExecutionUtil.hpp"
+#include "query_execution/WorkOrderProtosContainer.hpp"
#include "query_execution/WorkOrdersContainer.hpp"
+#include "relational_operators/WorkOrder.pb.h"
#include "storage/InsertDestination.hpp"
#include "storage/StorageBlock.hpp"
#include "storage/StorageBlockInfo.hpp"
@@ -69,6 +71,26 @@ bool UpdateOperator::getAllWorkOrders(
return started_;
}
+bool UpdateOperator::getAllWorkOrderProtos(WorkOrderProtosContainer *container) {
+ if (blocking_dependencies_met_ && !started_) {
+ for (const block_id input_block_id : input_blocks_) {
+ serialization::WorkOrder *proto = new serialization::WorkOrder;
+ proto->set_work_order_type(serialization::UPDATE);
+
+ proto->SetExtension(serialization::UpdateWorkOrder::operator_index, op_index_);
+ proto->SetExtension(serialization::UpdateWorkOrder::relation_id, relation_.getID());
+ proto->SetExtension(serialization::UpdateWorkOrder::insert_destination_index, relocation_destination_index_);
+ proto->SetExtension(serialization::UpdateWorkOrder::predicate_index, predicate_index_);
+ proto->SetExtension(serialization::UpdateWorkOrder::update_group_index, update_group_index_);
+ proto->SetExtension(serialization::UpdateWorkOrder::block_id, input_block_id);
+
+ container->addWorkOrderProto(proto, op_index_);
+ }
+ started_ = true;
+ }
+ return started_;
+}
+
void UpdateWorkOrder::execute() {
MutableBlockReference block(
storage_manager_->getBlockMutable(input_block_id_, relation_));
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/3e35844f/relational_operators/UpdateOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/UpdateOperator.hpp b/relational_operators/UpdateOperator.hpp
index 78f8fe0..20f4c50 100644
--- a/relational_operators/UpdateOperator.hpp
+++ b/relational_operators/UpdateOperator.hpp
@@ -45,6 +45,7 @@ class InsertDestination;
class Predicate;
class Scalar;
class StorageManager;
+class WorkOrderProtosContainer;
class WorkOrdersContainer;
/** \addtogroup RelationalOperators
@@ -95,6 +96,8 @@ class UpdateOperator : public RelationalOperator {
const tmb::client_id scheduler_client_id,
tmb::MessageBus *bus) override;
+ bool getAllWorkOrderProtos(WorkOrderProtosContainer *container) override;
+
QueryContext::insert_destination_id getInsertDestinationID() const override {
return relocation_destination_index_;
}
[14/50] [abbrv] incubator-quickstep git commit: Revert "Change
default aggregate_hashtable_type from LinearOpenAddressing to
SeparateChaining" (#208)
Posted by zu...@apache.org.
Revert "Change default aggregate_hashtable_type from LinearOpenAddressing to SeparateChaining" (#208)
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/a3889a39
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/a3889a39
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/a3889a39
Branch: refs/heads/work-order-serialization
Commit: a3889a392abde3b04633465404d6e889e6c6343d
Parents: 8444e2d
Author: Jignesh Patel <pa...@users.noreply.github.com>
Authored: Wed May 4 14:21:58 2016 -0500
Committer: Jignesh Patel <pa...@users.noreply.github.com>
Committed: Wed May 4 14:21:58 2016 -0500
----------------------------------------------------------------------
query_optimizer/ExecutionGenerator.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/a3889a39/query_optimizer/ExecutionGenerator.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/ExecutionGenerator.cpp b/query_optimizer/ExecutionGenerator.cpp
index 3698701..c34f084 100644
--- a/query_optimizer/ExecutionGenerator.cpp
+++ b/query_optimizer/ExecutionGenerator.cpp
@@ -135,7 +135,7 @@ static const volatile bool join_hashtable_type_dummy
= gflags::RegisterFlagValidator(&FLAGS_join_hashtable_type,
&ValidateHashTableImplTypeString);
-DEFINE_string(aggregate_hashtable_type, "SeparateChaining",
+DEFINE_string(aggregate_hashtable_type, "LinearOpenAddressing",
"HashTable implementation to use for aggregates with GROUP BY "
"(valid options are SeparateChaining or LinearOpenAddressing)");
static const volatile bool aggregate_hashtable_type_dummy
[37/50] [abbrv] incubator-quickstep git commit: Quickstep gen stats
(#225)
Posted by zu...@apache.org.
Quickstep gen stats (#225)
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/30f0981d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/30f0981d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/30f0981d
Branch: refs/heads/work-order-serialization
Commit: 30f0981db21cb53c9edbb7799a90f0de8979dde7
Parents: be35bb5
Author: Rogers Jeffrey Leo John <ro...@gmail.com>
Authored: Thu May 19 20:31:37 2016 -0500
Committer: Zuyu Zhang <zz...@pivotal.io>
Committed: Mon May 30 15:47:52 2016 -0700
----------------------------------------------------------------------
catalog/Catalog.proto | 2 +-
query_optimizer/CMakeLists.txt | 1 +
query_optimizer/ExecutionGenerator.cpp | 10 +++
relational_operators/CMakeLists.txt | 11 +++
.../GenerateNumRowsStatsOperator.cpp | 42 +++++++++++
.../GenerateNumRowsStatsOperator.hpp | 79 ++++++++++++++++++++
6 files changed, 144 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/30f0981d/catalog/Catalog.proto
----------------------------------------------------------------------
diff --git a/catalog/Catalog.proto b/catalog/Catalog.proto
index ce4bc2e..8e44181 100644
--- a/catalog/Catalog.proto
+++ b/catalog/Catalog.proto
@@ -82,7 +82,7 @@ message IndexScheme {
message CatalogRelationStatistics {
optional fixed64 num_tuples = 1;
-
+
message NumDistinctValuesEntry {
required int32 attr_id = 1;
required fixed64 num_distinct_values = 2;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/30f0981d/query_optimizer/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_optimizer/CMakeLists.txt b/query_optimizer/CMakeLists.txt
index aa2873e..1cc38d1 100644
--- a/query_optimizer/CMakeLists.txt
+++ b/query_optimizer/CMakeLists.txt
@@ -111,6 +111,7 @@ target_link_libraries(quickstep_queryoptimizer_ExecutionGenerator
quickstep_relationaloperators_DestroyHashOperator
quickstep_relationaloperators_DropTableOperator
quickstep_relationaloperators_FinalizeAggregationOperator
+ quickstep_relationaloperators_GenerateNumRowsStatsOperator
quickstep_relationaloperators_HashJoinOperator
quickstep_relationaloperators_InsertOperator
quickstep_relationaloperators_NestedLoopsJoinOperator
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/30f0981d/query_optimizer/ExecutionGenerator.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/ExecutionGenerator.cpp b/query_optimizer/ExecutionGenerator.cpp
index c590b6e..612efd9 100644
--- a/query_optimizer/ExecutionGenerator.cpp
+++ b/query_optimizer/ExecutionGenerator.cpp
@@ -91,6 +91,7 @@
#include "relational_operators/DestroyHashOperator.hpp"
#include "relational_operators/DropTableOperator.hpp"
#include "relational_operators/FinalizeAggregationOperator.hpp"
+#include "relational_operators/GenerateNumRowsStatsOperator.hpp"
#include "relational_operators/HashJoinOperator.hpp"
#include "relational_operators/InsertOperator.hpp"
#include "relational_operators/NestedLoopsJoinOperator.hpp"
@@ -947,6 +948,15 @@ void ExecutionGenerator::convertCopyFrom(
execution_plan_->addDirectDependency(save_blocks_operator_index,
scan_operator_index,
false /* is_pipeline_breaker */);
+
+ const QueryPlan::DAGNodeIndex num_rows_operator_index =
+ execution_plan_->addRelationalOperator(new GenerateNumRowsStatsOperator(
+ optimizer_context_->catalog_database()->getRelationByIdMutable(
+ output_relation->getID())));
+ insert_destination_proto->set_relational_op_index(num_rows_operator_index);
+ execution_plan_->addDirectDependency(num_rows_operator_index,
+ scan_operator_index,
+ true /* is_pipeline_breaker */);
}
void ExecutionGenerator::convertCreateIndex(
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/30f0981d/relational_operators/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/relational_operators/CMakeLists.txt b/relational_operators/CMakeLists.txt
index eec5300..e211630 100644
--- a/relational_operators/CMakeLists.txt
+++ b/relational_operators/CMakeLists.txt
@@ -34,6 +34,9 @@ add_library(quickstep_relationaloperators_DropTableOperator DropTableOperator.cp
add_library(quickstep_relationaloperators_FinalizeAggregationOperator
FinalizeAggregationOperator.cpp
FinalizeAggregationOperator.hpp)
+add_library(quickstep_relationaloperators_GenerateNumRowsStatsOperator
+ GenerateNumRowsStatsOperator.cpp
+ GenerateNumRowsStatsOperator.hpp)
add_library(quickstep_relationaloperators_HashJoinOperator HashJoinOperator.cpp HashJoinOperator.hpp)
add_library(quickstep_relationaloperators_InsertOperator InsertOperator.cpp InsertOperator.hpp)
add_library(quickstep_relationaloperators_NestedLoopsJoinOperator
@@ -159,6 +162,13 @@ target_link_libraries(quickstep_relationaloperators_FinalizeAggregationOperator
quickstep_storage_AggregationOperationState
quickstep_utility_Macros
tmb)
+target_link_libraries(quickstep_relationaloperators_GenerateNumRowsStatsOperator
+ glog
+ quickstep_catalog_CatalogRelation
+ quickstep_cli_PrintToScreen
+ quickstep_relationaloperators_RelationalOperator
+ quickstep_utility_Macros
+ tmb)
target_link_libraries(quickstep_relationaloperators_HashJoinOperator
gflags_nothreads-static
glog
@@ -446,6 +456,7 @@ target_link_libraries(quickstep_relationaloperators
quickstep_relationaloperators_DestroyHashOperator
quickstep_relationaloperators_DropTableOperator
quickstep_relationaloperators_FinalizeAggregationOperator
+ quickstep_relationaloperators_GenerateNumRowsStatsOperator
quickstep_relationaloperators_HashJoinOperator
quickstep_relationaloperators_InsertOperator
quickstep_relationaloperators_NestedLoopsJoinOperator
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/30f0981d/relational_operators/GenerateNumRowsStatsOperator.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/GenerateNumRowsStatsOperator.cpp b/relational_operators/GenerateNumRowsStatsOperator.cpp
new file mode 100644
index 0000000..074e1ca
--- /dev/null
+++ b/relational_operators/GenerateNumRowsStatsOperator.cpp
@@ -0,0 +1,42 @@
+/**
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#include "relational_operators/GenerateNumRowsStatsOperator.hpp"
+
+#include <memory>
+
+#include "catalog/CatalogRelation.hpp"
+#include "cli/PrintToScreen.hpp"
+
+#include "tmb/id_typedefs.h"
+
+namespace quickstep {
+
+bool GenerateNumRowsStatsOperator::getAllWorkOrders(
+ WorkOrdersContainer *container,
+ QueryContext *query_context,
+ StorageManager *storage_manager,
+ const tmb::client_id scheduler_client_id,
+ tmb::MessageBus *bus) {
+ std::size_t num_tuples =
+ PrintToScreen::GetNumTuplesInRelation(*relation_, storage_manager);
+ relation_->getStatisticsMutable()->setNumTuples(num_tuples);
+ return true;
+}
+
+} // namespace quickstep
+
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/30f0981d/relational_operators/GenerateNumRowsStatsOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/GenerateNumRowsStatsOperator.hpp b/relational_operators/GenerateNumRowsStatsOperator.hpp
new file mode 100644
index 0000000..8622a63
--- /dev/null
+++ b/relational_operators/GenerateNumRowsStatsOperator.hpp
@@ -0,0 +1,79 @@
+/**
+ * Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ * University of Wisconsin\u2014Madison.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#ifndef QUICKSTEP_RELATIONAL_OPERATORS_GENERATE_NUM_ROWS_STATS_OPERATOR_HPP_
+#define QUICKSTEP_RELATIONAL_OPERATORS_GENERATE_NUM_ROWS_STATS_OPERATOR_HPP_
+
+#include <memory>
+
+#include "catalog/CatalogRelation.hpp"
+#include "relational_operators/RelationalOperator.hpp"
+#include "utility/Macros.hpp"
+
+#include "glog/logging.h"
+
+#include "tmb/id_typedefs.h"
+
+namespace tmb { class MessageBus; }
+
+namespace quickstep {
+
+class CatalogRelation;
+class QueryContext;
+class StorageManager;
+class WorkOrdersContainer;
+
+/** \addtogroup RelationalOperators
+ * @{
+ */
+
+/**
+ * @brief An operator that gets the number of rows after loading a relation.
+ **/
+class GenerateNumRowsStatsOperator : public RelationalOperator {
+ public:
+ /**
+ * @brief Constructor.
+ *
+ * @param relation The relation to get the number of rows from.
+ * This GenNumRowStatsOperator owns relation until
+ * the WorkOrder it produces is successfully executed.
+ **/
+ explicit GenerateNumRowsStatsOperator(CatalogRelation *relation)
+ : relation_(relation) {}
+ ~GenerateNumRowsStatsOperator() override {}
+
+ /**
+ * @note no WorkOrder is generated for this operator.
+ **/
+ bool getAllWorkOrders(WorkOrdersContainer *container,
+ QueryContext *query_context,
+ StorageManager *storage_manager,
+ const tmb::client_id scheduler_client_id,
+ tmb::MessageBus *bus) override;
+
+ private:
+ CatalogRelation *relation_;
+
+ DISALLOW_COPY_AND_ASSIGN(GenerateNumRowsStatsOperator);
+};
+
+/** @} */
+
+} // namespace quickstep
+
+#endif // QUICKSTEP_RELATIONAL_OPERATORS_GENERATE_NUM_ROWS_STATS_OPERATOR_HPP_
[12/50] [abbrv] incubator-quickstep git commit: Support for NUMA
aware preloading (#206)
Posted by zu...@apache.org.
Support for NUMA aware preloading (#206)
* Support for preloading of NUMA-partitioned relations.
- Stored relations which have been NUMA-partitioned, can now be
preloaded. The preloading can maintain the same NUMA placement of
storage blocks.
* Added missing glog library in CMakeLists
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/0f261ea1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/0f261ea1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/0f261ea1
Branch: refs/heads/work-order-serialization
Commit: 0f261ea16a9b8cdb10ca7d5c2c64a906b71ae9a2
Parents: 456b434
Author: Harshad Deshmukh <d....@gmail.com>
Authored: Wed May 4 10:32:47 2016 -0500
Committer: Jignesh Patel <pa...@users.noreply.github.com>
Committed: Wed May 4 10:32:47 2016 -0500
----------------------------------------------------------------------
storage/CMakeLists.txt | 5 +++
storage/PreloaderThread.cpp | 86 +++++++++++++++++++++++++++++++++++-----
storage/PreloaderThread.hpp | 23 +++++++++++
3 files changed, 105 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/0f261ea1/storage/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/storage/CMakeLists.txt b/storage/CMakeLists.txt
index ed23802..dacacfa 100644
--- a/storage/CMakeLists.txt
+++ b/storage/CMakeLists.txt
@@ -741,6 +741,7 @@ target_link_libraries(quickstep_storage_PackedRowStoreValueAccessor
quickstep_utility_BitVector
quickstep_utility_Macros)
target_link_libraries(quickstep_storage_PreloaderThread
+ glog
quickstep_catalog_CatalogDatabase
quickstep_catalog_CatalogRelation
quickstep_catalog_CatalogTypedefs
@@ -750,6 +751,10 @@ target_link_libraries(quickstep_storage_PreloaderThread
quickstep_threading_Thread
quickstep_threading_ThreadUtil
quickstep_utility_Macros)
+if (QUICKSTEP_HAVE_LIBNUMA)
+ target_link_libraries(quickstep_storage_PreloaderThread
+ quickstep_catalog_NUMAPlacementScheme)
+endif()
target_link_libraries(quickstep_storage_SMAIndexSubBlock
glog
quickstep_catalog_CatalogAttribute
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/0f261ea1/storage/PreloaderThread.cpp
----------------------------------------------------------------------
diff --git a/storage/PreloaderThread.cpp b/storage/PreloaderThread.cpp
index d5dc55b..8f600b8 100644
--- a/storage/PreloaderThread.cpp
+++ b/storage/PreloaderThread.cpp
@@ -17,8 +17,13 @@
#include "storage/PreloaderThread.hpp"
+#include <cstddef>
#include <vector>
+#ifdef QUICKSTEP_HAVE_LIBNUMA
+#include <unordered_map>
+#endif
+
#include "catalog/CatalogDatabase.hpp"
#include "catalog/CatalogRelation.hpp"
#include "catalog/CatalogTypedefs.hpp"
@@ -27,6 +32,12 @@
#include "storage/StorageManager.hpp"
#include "threading/ThreadUtil.hpp"
+#ifdef QUICKSTEP_HAVE_LIBNUMA
+#include "catalog/NUMAPlacementScheme.hpp"
+#endif
+
+#include "glog/logging.h"
+
namespace quickstep {
void PreloaderThread::run() {
@@ -38,24 +49,81 @@ void PreloaderThread::run() {
std::size_t blocks_loaded = 0;
for (const CatalogRelation &relation : database_) {
- std::vector<block_id> blocks = relation.getBlocksSnapshot();
- for (block_id current_block_id : blocks) {
+ if (relation.hasPartitionScheme()) {
+ blocks_loaded += preloadNUMAAware(relation, blocks_loaded, num_slots);
+ } else {
+ std::vector<block_id> blocks = relation.getBlocksSnapshot();
+ for (block_id current_block_id : blocks) {
+ try {
+ BlockReference current_block = storage_manager_->getBlock(current_block_id, relation);
+ } catch (...) {
+ LOG(ERROR) << "Error after loading " << blocks_loaded << "blocks\n";
+ throw;
+ }
+ ++blocks_loaded;
+ if (blocks_loaded == num_slots) {
+ // The buffer pool has filled up. But, some database blocks are not loaded.
+ printf(" The database is larger than the buffer pool. Only %lu blocks were loaded ",
+ blocks_loaded);
+ return;
+ }
+ }
+ }
+ }
+ printf(" Loaded %lu blocks ", blocks_loaded);
+}
+
+std::size_t PreloaderThread::preloadNUMAAware(
+ const CatalogRelation &relation,
+ const std::size_t num_previously_loaded_blocks,
+ const std::size_t num_slots) {
+#ifdef QUICKSTEP_HAVE_LIBNUMA
+ std::size_t blocks_loaded = 0;
+ const NUMAPlacementScheme *placement_scheme = relation.getNUMAPlacementSchemePtr();
+ DCHECK(placement_scheme != nullptr);
+ DCHECK(relation.hasPartitionScheme());
+ const PartitionScheme &part_scheme = relation.getPartitionScheme();
+ const PartitionSchemeHeader &part_scheme_header =
+ part_scheme.getPartitionSchemeHeader();
+ const std::size_t num_partitions = part_scheme_header.getNumPartitions();
+ // Key = NUMA node ID, value = number of blocks loaded from that NUMA node.
+ std::unordered_map<numa_node_id, std::size_t> num_blocks_loaded;
+ for (std::size_t part_id = 0; part_id < num_partitions; ++part_id) {
+ const numa_node_id partition_numa_node_id =
+ placement_scheme->getNUMANodeForPartition(part_id);
+ for (block_id curr_block_id : part_scheme.getBlocksInPartition(part_id)) {
try {
- BlockReference current_block = storage_manager_->getBlock(current_block_id, relation);
+ BlockReference current_block = storage_manager_->getBlock(
+ curr_block_id, relation, partition_numa_node_id);
} catch (...) {
- LOG(ERROR) << "Error after loading " << blocks_loaded << "blocks\n";
+ LOG(ERROR) << "Error after loading "
+ << blocks_loaded + num_previously_loaded_blocks
+ << " blocks\n";
throw;
}
++blocks_loaded;
- if (blocks_loaded == num_slots) {
+ num_blocks_loaded[partition_numa_node_id]++;
+ if ((blocks_loaded + num_previously_loaded_blocks) == num_slots) {
// The buffer pool has filled up. But, some database blocks are not loaded.
- printf(" The database is larger than the buffer pool. Only %lu blocks were loaded ",
- blocks_loaded);
- return;
+ printf(
+ " The database is larger than the buffer pool. Only %lu blocks "
+ "were loaded ",
+ blocks_loaded + num_previously_loaded_blocks);
+ return blocks_loaded;
}
}
}
- printf(" Loaded %lu blocks ", blocks_loaded);
+ LOG(INFO) << "For relation: " << relation.getName();
+ for (auto numa_block_loaded_info : num_blocks_loaded) {
+ LOG(INFO) << "NUMA node: " << numa_block_loaded_info.first
+ << " Number of loaded blocks: " << numa_block_loaded_info.second;
+ }
+ return blocks_loaded;
+#else
+ LOG(INFO) << "Relation: " << relation.getName()
+ << " has partition scheme but the system doesn't support NUMA";
+ return 0;
+#endif
}
} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/0f261ea1/storage/PreloaderThread.hpp
----------------------------------------------------------------------
diff --git a/storage/PreloaderThread.hpp b/storage/PreloaderThread.hpp
index ed866b4..f16fd50 100644
--- a/storage/PreloaderThread.hpp
+++ b/storage/PreloaderThread.hpp
@@ -18,12 +18,15 @@
#ifndef QUICKSTEP_STORAGE_PRELOADER_THREAD_HPP_
#define QUICKSTEP_STORAGE_PRELOADER_THREAD_HPP_
+#include <cstddef>
+
#include "threading/Thread.hpp"
#include "utility/Macros.hpp"
namespace quickstep {
class CatalogDatabase;
+class CatalogRelation;
class StorageManager;
/** \addtogroup Storage
@@ -65,6 +68,26 @@ class PreloaderThread : public Thread {
void run() override;
private:
+ /**
+ * @brief Preload a relation which has a partition and a NUMA placement scheme.
+ *
+ * @param relation The relation to be preloaded.
+ * @param num_previously_loaded_blocks The number of blocks already preloaded.
+ * @param num_slots The maximum number of slots in the StorageManager.
+ *
+ * @warning This function may not detect skew on NUMA sockets, i.e. if a given
+ * NUMA socket has large number of blocks, preloading may cause the
+ * memory on that NUMA socket to be full. It is recommended to use
+ * this preloading when we are sure that each NUMA socket has been
+ * allocated sufficient amount of memory so as not to exceed that
+ * socket's memory limit.
+ *
+ * @return The number of blocks loaded during this function call.
+ **/
+ std::size_t preloadNUMAAware(const CatalogRelation &relation,
+ const std::size_t num_previously_loaded_blocks,
+ const std::size_t num_slots);
+
const CatalogDatabase &database_;
StorageManager *storage_manager_;