You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@quickstep.apache.org by ji...@apache.org on 2017/09/20 01:24:05 UTC
[1/8] incubator-quickstep git commit: Added Partition rules for Sort.
[Forced Update!]
Repository: incubator-quickstep
Updated Branches:
refs/heads/fix-iwyu fe4ad718c -> 165bd1fe5 (forced update)
Added Partition rules for Sort.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/4578c63f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/4578c63f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/4578c63f
Branch: refs/heads/fix-iwyu
Commit: 4578c63f06d7e0d2b9c5d60eb7bfeb33cb678bd8
Parents: 2981651
Author: Zuyu Zhang <zu...@cs.wisc.edu>
Authored: Mon Sep 4 19:59:45 2017 -0500
Committer: Zuyu Zhang <zu...@cs.wisc.edu>
Committed: Mon Sep 4 19:59:45 2017 -0500
----------------------------------------------------------------------
query_optimizer/rules/CMakeLists.txt | 1 +
query_optimizer/rules/Partition.cpp | 11 +++++++++++
2 files changed, 12 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4578c63f/query_optimizer/rules/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_optimizer/rules/CMakeLists.txt b/query_optimizer/rules/CMakeLists.txt
index 7abe0d1..73a80d2 100644
--- a/query_optimizer/rules/CMakeLists.txt
+++ b/query_optimizer/rules/CMakeLists.txt
@@ -182,6 +182,7 @@ target_link_libraries(quickstep_queryoptimizer_rules_Partition
quickstep_queryoptimizer_physical_Physical
quickstep_queryoptimizer_physical_PhysicalType
quickstep_queryoptimizer_physical_Selection
+ quickstep_queryoptimizer_physical_Sort
quickstep_queryoptimizer_physical_TableReference
quickstep_queryoptimizer_physical_TopLevelPlan
quickstep_queryoptimizer_rules_BottomUpRule
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4578c63f/query_optimizer/rules/Partition.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/rules/Partition.cpp b/query_optimizer/rules/Partition.cpp
index 5f8b2c6..5f68cd3 100644
--- a/query_optimizer/rules/Partition.cpp
+++ b/query_optimizer/rules/Partition.cpp
@@ -46,6 +46,7 @@
#include "query_optimizer/physical/Physical.hpp"
#include "query_optimizer/physical/PhysicalType.hpp"
#include "query_optimizer/physical/Selection.hpp"
+#include "query_optimizer/physical/Sort.hpp"
#include "query_optimizer/physical/TableReference.hpp"
#include "query_optimizer/physical/TopLevelPlan.hpp"
#include "types/operations/binary_operations/BinaryOperation.hpp"
@@ -566,6 +567,16 @@ P::PhysicalPtr Partition::applyToNode(const P::PhysicalPtr &node) {
return selection->copyWithNewOutputPartitionSchemeHeader(output_partition_scheme_header.release(),
false /* has_repartition */);
}
+ case P::PhysicalType::kSort: {
+ const P::SortPtr sort = static_pointer_cast<const P::Sort>(node);
+ const P::PhysicalPtr input = sort->input();
+ if (P::SomeTableReference::Matches(input) ||
+ !input->getOutputPartitionSchemeHeader()) {
+ break;
+ }
+
+ return sort->copyWithNewChildren({ input->copyWithNewOutputPartitionSchemeHeader(nullptr) });
+ }
default:
break;
}
[5/8] incubator-quickstep git commit: QUICKSTEP-95: Fixed the
exception due to zero tuple estimation for the empty project expression.
Posted by ji...@apache.org.
QUICKSTEP-95: Fixed the exception due to zero tuple estimation for the empty project expression.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/56b34363
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/56b34363
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/56b34363
Branch: refs/heads/fix-iwyu
Commit: 56b3436365aac95d98f04322869b138dcef31d3a
Parents: b1c3356
Author: Zuyu Zhang <zu...@apache.org>
Authored: Mon Jun 19 15:53:52 2017 -0500
Committer: Zuyu Zhang <zu...@cs.wisc.edu>
Committed: Wed Sep 13 22:37:01 2017 -0500
----------------------------------------------------------------------
storage/CMakeLists.txt | 1 +
storage/SplitRowStoreTupleStorageSubBlock.cpp | 9 +++++++++
2 files changed, 10 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/56b34363/storage/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/storage/CMakeLists.txt b/storage/CMakeLists.txt
index 6446a83..92a3292 100644
--- a/storage/CMakeLists.txt
+++ b/storage/CMakeLists.txt
@@ -907,6 +907,7 @@ target_link_libraries(quickstep_storage_SimpleScalarSeparateChainingHashTable
quickstep_utility_Macros
quickstep_utility_PrimeNumber)
target_link_libraries(quickstep_storage_SplitRowStoreTupleStorageSubBlock
+ glog
quickstep_catalog_CatalogRelationSchema
quickstep_expressions_predicate_PredicateCost
quickstep_storage_SplitRowStoreValueAccessor
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/56b34363/storage/SplitRowStoreTupleStorageSubBlock.cpp
----------------------------------------------------------------------
diff --git a/storage/SplitRowStoreTupleStorageSubBlock.cpp b/storage/SplitRowStoreTupleStorageSubBlock.cpp
index 0e5cfe6..5060208 100644
--- a/storage/SplitRowStoreTupleStorageSubBlock.cpp
+++ b/storage/SplitRowStoreTupleStorageSubBlock.cpp
@@ -37,6 +37,8 @@
#include "utility/Macros.hpp"
#include "utility/ScopedBuffer.hpp"
+#include "glog/logging.h"
+
namespace quickstep {
QUICKSTEP_REGISTER_TUPLE_STORE(SplitRowStoreTupleStorageSubBlock, SPLIT_ROW_STORE);
@@ -128,6 +130,13 @@ SplitRowStoreTupleStorageSubBlock::SplitRowStoreTupleStorageSubBlock(
tuple_slot_bytes_ = per_tuple_null_bitmap_bytes_
+ relation.getFixedByteLength()
+ relation.numVariableLengthAttributes() * (sizeof(std::uint32_t) * 2);
+ if (tuple_slot_bytes_ == 0) {
+ LOG(WARNING)
+ << "Estimated zero bytes per tuple for relation \"" << relation.getName()
+ << "\" (relation_id: " << relation.getID()
+ << "). Adjusting to 1 byte.";
+ tuple_slot_bytes_ = 1;
+ }
// Size the occupancy bitmap by calculating the maximum tuples that can fit
// assuming the bare-minimum per tuple storage is used (no variable-length
[2/8] incubator-quickstep git commit: Redirect stdout and stderr in
network mode.
Posted by ji...@apache.org.
Redirect stdout and stderr in network mode.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/b815b388
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/b815b388
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/b815b388
Branch: refs/heads/fix-iwyu
Commit: b815b388d4fb501c4454a1b4e9560a92e4fbb469
Parents: 4578c63
Author: Jianqiao Zhu <ji...@cs.wisc.edu>
Authored: Fri Sep 1 12:07:13 2017 -0500
Committer: Jianqiao Zhu <ji...@cs.wisc.edu>
Committed: Mon Sep 11 16:15:38 2017 -0500
----------------------------------------------------------------------
CMakeLists.txt | 1 +
cli/Flags.cpp | 3 +
cli/Flags.hpp | 2 +
cli/IOInterface.hpp | 11 +-
cli/NetworkCliClientMain.cpp | 1 -
cli/QuickstepCli.cpp | 15 +-
.../tests/ExecutionGeneratorTestRunner.cpp | 4 +-
utility/CMakeLists.txt | 13 ++
utility/ScopedReassignment.hpp | 81 ++++++++
utility/tests/ScopedReassignment_unittest.cpp | 183 +++++++++++++++++++
10 files changed, 304 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/b815b388/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/CMakeLists.txt b/CMakeLists.txt
index ed80fab..e0d020b 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -751,6 +751,7 @@ target_link_libraries(quickstep_cli_shell
quickstep_utility_ExecutionDAGVisualizer
quickstep_utility_Macros
quickstep_utility_PtrVector
+ quickstep_utility_ScopedReassignment
quickstep_utility_SqlError
quickstep_utility_StringUtil)
if (ENABLE_HDFS)
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/b815b388/cli/Flags.cpp
----------------------------------------------------------------------
diff --git a/cli/Flags.cpp b/cli/Flags.cpp
index 362eac3..279c503 100644
--- a/cli/Flags.cpp
+++ b/cli/Flags.cpp
@@ -36,6 +36,9 @@ DEFINE_bool(print_query, false,
"Print each input query statement. This is useful when running a "
"large number of queries in a batch.");
+DEFINE_bool(display_timing, true,
+ "Whether to display execution time of each statement.");
+
DEFINE_bool(initialize_db, false, "If true, initialize a database.");
static bool ValidateNumWorkers(const char *flagname, int value) {
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/b815b388/cli/Flags.hpp
----------------------------------------------------------------------
diff --git a/cli/Flags.hpp b/cli/Flags.hpp
index 1ae37c4..614cca5 100644
--- a/cli/Flags.hpp
+++ b/cli/Flags.hpp
@@ -45,6 +45,8 @@ DECLARE_string(storage_path);
DECLARE_bool(preload_buffer_pool);
+DECLARE_bool(display_timing);
+
/** @} */
} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/b815b388/cli/IOInterface.hpp
----------------------------------------------------------------------
diff --git a/cli/IOInterface.hpp b/cli/IOInterface.hpp
index 815596e..dc0d5b2 100644
--- a/cli/IOInterface.hpp
+++ b/cli/IOInterface.hpp
@@ -28,7 +28,7 @@
namespace quickstep {
/**
- * An individual IO interaction with Quickstep.
+ * @brief An individual IO interaction with Quickstep.
*/
class IOHandle {
public:
@@ -56,18 +56,21 @@ class IOHandle {
};
/**
- * Virtual base defines a generic, file-based interface around IO. One IO interaction (eg a SQL query) will be assigned
- * an IOHandle. On destruction of the IOHandle, the IO interaction has finished.
+ * @brief Virtual base defines a generic, file-based interface around IO.
+ * One IO interaction (eg a SQL query) will be assigned an IOHandle.
+ * On destruction of the IOHandle, the IO interaction has finished.
*/
class IOInterface {
public:
/**
- * @note Destructing the IOInterface should close any outstanding IO state (eg an open port).
+ * @note Destructing the IOInterface should close any outstanding IO state
+ * (e.g. an open port).
*/
virtual ~IOInterface() {}
/**
* @brief Retrieves the next IOHandle. Blocks if no IO ready.
+ *
* @return An IOHandle.
*/
virtual IOHandle* getNextIOHandle() = 0;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/b815b388/cli/NetworkCliClientMain.cpp
----------------------------------------------------------------------
diff --git a/cli/NetworkCliClientMain.cpp b/cli/NetworkCliClientMain.cpp
index 862941c..c55819b 100644
--- a/cli/NetworkCliClientMain.cpp
+++ b/cli/NetworkCliClientMain.cpp
@@ -53,7 +53,6 @@ int main(int argc, char **argv) {
while (!linereader.bufferEmpty()) {
std::string query = linereader.getNextCommand();
if (!query.empty()) {
- std::cout << query << std::endl;
std::cout << qs_client.Query(query) << std::endl;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/b815b388/cli/QuickstepCli.cpp
----------------------------------------------------------------------
diff --git a/cli/QuickstepCli.cpp b/cli/QuickstepCli.cpp
index c55bdd7..e8ca56c 100644
--- a/cli/QuickstepCli.cpp
+++ b/cli/QuickstepCli.cpp
@@ -69,6 +69,7 @@
#include "utility/ExecutionDAGVisualizer.hpp"
#include "utility/Macros.hpp"
#include "utility/PtrVector.hpp"
+#include "utility/ScopedReassignment.hpp"
#include "utility/SqlError.hpp"
#include "utility/StringUtil.hpp"
@@ -105,6 +106,7 @@ using quickstep::PtrVector;
using quickstep::QueryExecutionUtil;
using quickstep::QueryHandle;
using quickstep::QueryProcessor;
+using quickstep::ScopedReassignment;
using quickstep::SqlParserWrapper;
using quickstep::Worker;
using quickstep::WorkerDirectory;
@@ -303,6 +305,9 @@ int main(int argc, char* argv[]) {
for (;;) {
string *command_string = new string();
std::unique_ptr<quickstep::IOHandle> io_handle(io->getNextIOHandle());
+ ScopedReassignment<FILE*> reassign_stdout(&stdout, io_handle->out());
+ ScopedReassignment<FILE*> reassign_stderr(&stderr, io_handle->err());
+
*command_string = io_handle->getCommand();
LOG(INFO) << "Command received: " << *command_string;
if (command_string->size() == 0) {
@@ -397,10 +402,12 @@ int main(int argc, char* argv[]) {
}
query_processor->saveCatalog();
- std::chrono::duration<double, std::milli> time_ms = end - start;
- fprintf(io_handle->out(), "Time: %s ms\n",
- quickstep::DoubleToStringWithSignificantDigits(
- time_ms.count(), 3).c_str());
+ if (quickstep::FLAGS_display_timing) {
+ std::chrono::duration<double, std::milli> time_ms = end - start;
+ fprintf(io_handle->out(), "Time: %s ms\n",
+ quickstep::DoubleToStringWithSignificantDigits(
+ time_ms.count(), 3).c_str());
+ }
if (quickstep::FLAGS_profile_and_report_workorder_perf) {
// TODO(harshad) - Allow user specified file instead of stdout.
foreman.printWorkOrderProfilingResults(query_id, stdout);
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/b815b388/query_optimizer/tests/ExecutionGeneratorTestRunner.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/tests/ExecutionGeneratorTestRunner.cpp b/query_optimizer/tests/ExecutionGeneratorTestRunner.cpp
index 050ef0d..a854589 100644
--- a/query_optimizer/tests/ExecutionGeneratorTestRunner.cpp
+++ b/query_optimizer/tests/ExecutionGeneratorTestRunner.cpp
@@ -63,7 +63,9 @@ void ExecutionGeneratorTestRunner::runTestCase(
sql_parser_.feedNextBuffer(new std::string(input));
// Redirect stderr to output_stream.
- stderr = output_stream.file();
+ if (!FLAGS_logtostderr) {
+ stderr = output_stream.file();
+ }
while (true) {
ParseResult result = sql_parser_.getNextStatement();
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/b815b388/utility/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/utility/CMakeLists.txt b/utility/CMakeLists.txt
index a832404..c5c9dd8 100644
--- a/utility/CMakeLists.txt
+++ b/utility/CMakeLists.txt
@@ -196,6 +196,7 @@ add_library(quickstep_utility_PtrMap ../empty_src.cpp PtrMap.hpp)
add_library(quickstep_utility_PtrVector ../empty_src.cpp PtrVector.hpp)
add_library(quickstep_utility_ScopedBuffer ../empty_src.cpp ScopedBuffer.hpp)
add_library(quickstep_utility_ScopedDeleter ../empty_src.cpp ScopedDeleter.hpp)
+add_library(quickstep_utility_ScopedReassignment ../empty_src.cpp ScopedReassignment.hpp)
add_library(quickstep_utility_ShardedLockManager ../empty_src.cpp ShardedLockManager.hpp)
add_library(quickstep_utility_SortConfiguration SortConfiguration.cpp SortConfiguration.hpp)
add_library(quickstep_utility_SortConfiguration_proto
@@ -313,6 +314,8 @@ target_link_libraries(quickstep_utility_ScopedBuffer
quickstep_utility_Macros)
target_link_libraries(quickstep_utility_ScopedDeleter
quickstep_utility_Macros)
+target_link_libraries(quickstep_utility_ScopedReassignment
+ quickstep_utility_Macros)
target_link_libraries(quickstep_utility_SqlError
glog)
target_link_libraries(quickstep_utility_SortConfiguration
@@ -379,6 +382,7 @@ target_link_libraries(quickstep_utility
quickstep_utility_PtrVector
quickstep_utility_ScopedBuffer
quickstep_utility_ScopedDeleter
+ quickstep_utility_ScopedReassignment
quickstep_utility_ShardedLockManager
quickstep_utility_SortConfiguration
quickstep_utility_SortConfiguration_proto
@@ -462,6 +466,15 @@ target_link_libraries(ScopedDeleter_unittest
${LIBS})
add_test(ScopedDeleter_unittest ScopedDeleter_unittest)
+add_executable(ScopedReassignment_unittest "${CMAKE_CURRENT_SOURCE_DIR}/tests/ScopedReassignment_unittest.cpp")
+target_link_libraries(ScopedReassignment_unittest
+ gtest
+ gtest_main
+ quickstep_utility_Macros
+ quickstep_utility_ScopedReassignment
+ ${LIBS})
+add_test(ScopedReassignment_unittest ScopedReassignment_unittest)
+
add_executable(SqlError_unittest "${CMAKE_CURRENT_SOURCE_DIR}/tests/SqlError_unittest.cpp")
target_link_libraries(SqlError_unittest
glog
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/b815b388/utility/ScopedReassignment.hpp
----------------------------------------------------------------------
diff --git a/utility/ScopedReassignment.hpp b/utility/ScopedReassignment.hpp
new file mode 100644
index 0000000..bd0bcb2
--- /dev/null
+++ b/utility/ScopedReassignment.hpp
@@ -0,0 +1,81 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ **/
+
+#ifndef QUICKSTEP_UTILITY_SCOPED_REASSIGNMENT_HPP_
+#define QUICKSTEP_UTILITY_SCOPED_REASSIGNMENT_HPP_
+
+#include <utility>
+#include <type_traits>
+
+#include "utility/Macros.hpp"
+
+namespace quickstep {
+
+/** \addtogroup Utility
+ * @{
+ */
+
+/**
+ * @brief RAII helper object that assigns a new value to a variable and restores
+ * the old value when the helper object goes out of scope.
+ **/
+template <typename T>
+class ScopedReassignment {
+ public:
+ /**
+ * @brief Constructor.
+ *
+ * @param var The variable.
+ * @param new_value The new value to be assigned to \p var.
+ **/
+ template <typename U>
+ ScopedReassignment(T *var, U &&new_value)
+ : var_(*var),
+ old_value_(std::move(*var)) {
+ *var = std::forward<U>(new_value);
+ }
+
+ /**
+ * @brief Destructor. Restores the old value to \p var_.
+ **/
+ ~ScopedReassignment() {
+ var_ = std::move(old_value_);
+ }
+
+ /**
+ * @brief Get the old value.
+ *
+ * @return A const reference to the old value.
+ */
+ inline const T& old_value() const {
+ return old_value_;
+ }
+
+ private:
+ T &var_;
+ T old_value_;
+
+ DISALLOW_COPY_AND_ASSIGN(ScopedReassignment);
+};
+
+/** @} */
+
+} // namespace quickstep
+
+#endif // QUICKSTEP_UTILITY_SCOPED_REASSIGNMENT_HPP_
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/b815b388/utility/tests/ScopedReassignment_unittest.cpp
----------------------------------------------------------------------
diff --git a/utility/tests/ScopedReassignment_unittest.cpp b/utility/tests/ScopedReassignment_unittest.cpp
new file mode 100644
index 0000000..b7da017
--- /dev/null
+++ b/utility/tests/ScopedReassignment_unittest.cpp
@@ -0,0 +1,183 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ **/
+
+#include "utility/ScopedReassignment.hpp"
+
+#include "utility/Macros.hpp"
+
+#include "gtest/gtest.h"
+
+namespace quickstep {
+
+namespace {
+const int kOldValue = 10;
+const int kNewValue = 20;
+} // namespace
+
+/**
+ * @brief A test class that is movable but not copyable.
+ */
+class NonCopyable {
+ public:
+ explicit NonCopyable(const int value)
+ : value_(value) {}
+
+ explicit NonCopyable(NonCopyable &&other)
+ : value_(other.value_) {}
+
+ NonCopyable& operator=(NonCopyable &&other) {
+ value_ = other.value_;
+ return *this;
+ }
+
+ int value() const {
+ return value_;
+ }
+
+ private:
+ int value_;
+
+ DISALLOW_COPY_AND_ASSIGN(NonCopyable);
+};
+
+/**
+ * @brief A test class that is copyable but not movable.
+ */
+class NonMovable {
+ public:
+ explicit NonMovable(const int value)
+ : value_(value) {}
+
+ explicit NonMovable(const NonMovable &other)
+ : value_(other.value_) {}
+
+ NonMovable& operator=(const NonMovable &other) {
+ value_ = other.value_;
+ return *this;
+ }
+
+ int value() const {
+ return value_;
+ }
+
+ private:
+ int value_;
+};
+
+/**
+ * @brief A test class that is copyable and movable.
+ */
+class CopyableMovable {
+ public:
+ explicit CopyableMovable(const int value)
+ : value_(value) {}
+
+ explicit CopyableMovable(const CopyableMovable &other)
+ : copy_constructed_(true),
+ value_(other.value_) {}
+
+ explicit CopyableMovable(CopyableMovable &&other)
+ : value_(other.value_) {}
+
+ CopyableMovable& operator=(const CopyableMovable &other) {
+ value_ = other.value_;
+ copy_assigned_ = true;
+ return *this;
+ }
+
+ CopyableMovable& operator=(CopyableMovable &&other) {
+ value_ = other.value_;
+ copy_assigned_ = false;
+ return *this;
+ }
+
+ int value() const {
+ return value_;
+ }
+
+ bool copy_constructed() const {
+ return copy_constructed_;
+ }
+
+ bool copy_assigned() const {
+ return copy_assigned_;
+ }
+
+ private:
+ const bool copy_constructed_ = false;
+ int value_;
+ bool copy_assigned_ = false;
+};
+
+TEST(ScopedReassignment, NonCopyableTest) {
+ NonCopyable non_copyable(kOldValue);
+ {
+ NonCopyable other(kNewValue);
+ ScopedReassignment<NonCopyable> reassign(&non_copyable, std::move(other));
+ EXPECT_EQ(kNewValue, non_copyable.value());
+ }
+ EXPECT_EQ(kOldValue, non_copyable.value());
+
+ {
+ ScopedReassignment<NonCopyable> reassign(&non_copyable, NonCopyable(kNewValue));
+ EXPECT_EQ(kNewValue, non_copyable.value());
+ }
+ EXPECT_EQ(kOldValue, non_copyable.value());
+}
+
+TEST(ScopedReassignment, NonMovableTest) {
+ NonMovable non_movable(kOldValue);
+ {
+ NonMovable other(kNewValue);
+ ScopedReassignment<NonMovable> reassign(&non_movable, other);
+ EXPECT_EQ(kNewValue, non_movable.value());
+ }
+ EXPECT_EQ(kOldValue, non_movable.value());
+
+ {
+ ScopedReassignment<NonMovable> reassign(&non_movable, NonMovable(kNewValue));
+ EXPECT_EQ(kNewValue, non_movable.value());
+ }
+ EXPECT_EQ(kOldValue, non_movable.value());
+}
+
+TEST(ScopedReassignment, CopyableMovableTest) {
+ CopyableMovable copyable_movable(kOldValue);
+ {
+ CopyableMovable other(kNewValue);
+ ScopedReassignment<CopyableMovable> reassign(©able_movable, other);
+ EXPECT_EQ(kNewValue, copyable_movable.value());
+ EXPECT_FALSE(reassign.old_value().copy_constructed());
+ EXPECT_TRUE(copyable_movable.copy_assigned());
+ }
+ EXPECT_EQ(kOldValue, copyable_movable.value());
+ EXPECT_FALSE(copyable_movable.copy_assigned());
+
+ {
+ CopyableMovable other(kNewValue);
+ ScopedReassignment<CopyableMovable> reassign(©able_movable, std::move(other));
+ EXPECT_EQ(kNewValue, copyable_movable.value());
+ EXPECT_FALSE(reassign.old_value().copy_constructed());
+ EXPECT_FALSE(copyable_movable.copy_assigned());
+ }
+ EXPECT_EQ(kOldValue, copyable_movable.value());
+ EXPECT_FALSE(copyable_movable.copy_assigned());
+}
+
+} // namespace quickstep
[8/8] incubator-quickstep git commit: Fix iwyu include path
Posted by ji...@apache.org.
Fix iwyu include path
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/165bd1fe
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/165bd1fe
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/165bd1fe
Branch: refs/heads/fix-iwyu
Commit: 165bd1fe53949f585685765fc06976c32c9ac1db
Parents: 475704e
Author: Jianqiao Zhu <ji...@cs.wisc.edu>
Authored: Fri Sep 1 15:07:41 2017 -0500
Committer: Jianqiao Zhu <ji...@cs.wisc.edu>
Committed: Tue Sep 19 20:23:48 2017 -0500
----------------------------------------------------------------------
third_party/src/iwyu/iwyu_helper.py | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/165bd1fe/third_party/src/iwyu/iwyu_helper.py
----------------------------------------------------------------------
diff --git a/third_party/src/iwyu/iwyu_helper.py b/third_party/src/iwyu/iwyu_helper.py
index dff4d55..42bf84c 100755
--- a/third_party/src/iwyu/iwyu_helper.py
+++ b/third_party/src/iwyu/iwyu_helper.py
@@ -22,12 +22,12 @@ QUICKSTEP_INCLUDES = [ '.',
'./build/third_party/gflags/include',
'./build/third_party/protobuf/include',
'./build/third_party/tmb/include',
- './third_party/benchmark/include',
- './third_party/glog/src',
- './third_party/googletest/googletest/include',
- './third_party/protobuf/src',
- './third_party/re2',
- './third_party/tmb/include']
+ './third_party/src/benchmark/include',
+ './third_party/src/glog/src',
+ './third_party/src/googletest/googletest/include',
+ './third_party/src/protobuf/src',
+ './third_party/src/re2',
+ './third_party/src/tmb/include']
QUICKSTEP_DEFINES = [ '-DQUICKSTEP_DEBUG',
'-DQUICKSTEP_ENABLE_VECTOR_COPY_ELISION_SELECTION', ]
CLANG_FLAGS = [ '-std=c++14', '-x', 'c++', ]
[7/8] incubator-quickstep git commit: Printed out the partition info
in QueryPlan.
Posted by ji...@apache.org.
Printed out the partition info in QueryPlan.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/475704ec
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/475704ec
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/475704ec
Branch: refs/heads/fix-iwyu
Commit: 475704ec9510793a70e149b938d02569611c6177
Parents: d85f7a9
Author: Zuyu Zhang <zu...@cs.wisc.edu>
Authored: Mon Sep 11 11:57:10 2017 -0500
Committer: Zuyu Zhang <zu...@cs.wisc.edu>
Committed: Thu Sep 14 13:15:10 2017 -0500
----------------------------------------------------------------------
query_optimizer/CMakeLists.txt | 1 +
query_optimizer/OptimizerTree.hpp | 39 ++++++++++++++++++++
query_optimizer/logical/CreateTable.cpp | 5 +++
query_optimizer/logical/CreateTable.hpp | 20 ++++++----
query_optimizer/physical/CreateTable.cpp | 5 +++
query_optimizer/physical/CreateTable.hpp | 20 ++++++----
.../tests/logical_generator/Create.test | 15 +++++++-
.../tests/physical_generator/Create.test | 26 +++++++++++++
query_optimizer/tests/resolver/Create.test | 17 +++++++--
9 files changed, 128 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/475704ec/query_optimizer/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_optimizer/CMakeLists.txt b/query_optimizer/CMakeLists.txt
index 4ea21b2..5e0db44 100644
--- a/query_optimizer/CMakeLists.txt
+++ b/query_optimizer/CMakeLists.txt
@@ -212,6 +212,7 @@ target_link_libraries(quickstep_queryoptimizer_OptimizerContext
quickstep_queryoptimizer_expressions_ExprId
quickstep_utility_Macros)
target_link_libraries(quickstep_queryoptimizer_OptimizerTree
+ quickstep_catalog_Catalog_proto
quickstep_storage_StorageBlockLayout_proto
quickstep_utility_Macros
quickstep_utility_TreeStringSerializable)
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/475704ec/query_optimizer/OptimizerTree.hpp
----------------------------------------------------------------------
diff --git a/query_optimizer/OptimizerTree.hpp b/query_optimizer/OptimizerTree.hpp
index 62df66d..c54ce20 100644
--- a/query_optimizer/OptimizerTree.hpp
+++ b/query_optimizer/OptimizerTree.hpp
@@ -25,6 +25,7 @@
#include <string>
#include <vector>
+#include "catalog/Catalog.pb.h"
#include "storage/StorageBlockLayout.pb.h"
#include "utility/Macros.hpp"
#include "utility/TreeStringSerializable.hpp"
@@ -283,6 +284,44 @@ OptimizerProtoRepresentation<TreeNodeType>* getOptimizerRepresentationForProto(
return node.release();
}
+template<class TreeNodeType>
+OptimizerProtoRepresentation<TreeNodeType>* getOptimizerRepresentationForProto(
+ const serialization::PartitionSchemeHeader *partition_header) {
+ if (partition_header == nullptr) {
+ return nullptr;
+ }
+
+ auto node = std::make_unique<OptimizerProtoRepresentation<TreeNodeType>>();
+
+ // Add properties based on the partition type.
+ switch (partition_header->partition_type()) {
+ case serialization::PartitionSchemeHeader::HASH: {
+ node->addProperty("partition_type", "hash");
+ break;
+ }
+ case serialization::PartitionSchemeHeader::RANDOM: {
+ node->addProperty("partition_type", "random");
+ break;
+ }
+ case serialization::PartitionSchemeHeader::RANGE: {
+ node->addProperty("partition_type", "range");
+ // TODO(quickstep-team): display the range boundaries.
+ node->addProperty("range_boundaries", "TODO");
+ break;
+ }
+ default:
+ LOG(FATAL) << "Unrecognized partition type in protobuf message.";
+ }
+ // Every case will specify a partition number and a partition attributes.
+ node->addProperty("num_partitions", partition_header->num_partitions());
+
+ for (int i = 0; i < partition_header->partition_attribute_ids_size(); ++i) {
+ node->addProperty("partition_attr_id", partition_header->partition_attribute_ids(i));
+ }
+
+ return node.release();
+}
+
/** @} */
} // namespace optimizer
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/475704ec/query_optimizer/logical/CreateTable.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/logical/CreateTable.cpp b/query_optimizer/logical/CreateTable.cpp
index 111d04b..9977a0f 100644
--- a/query_optimizer/logical/CreateTable.cpp
+++ b/query_optimizer/logical/CreateTable.cpp
@@ -49,6 +49,11 @@ void CreateTable::getFieldStringItems(
non_container_child_field_names->push_back("block_properties");
non_container_child_fields->push_back(block_properties_representation_);
}
+
+ if (partition_scheme_header_proto_representation_) {
+ non_container_child_field_names->push_back("partition_scheme_header");
+ non_container_child_fields->push_back(partition_scheme_header_proto_representation_);
+ }
}
} // namespace logical
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/475704ec/query_optimizer/logical/CreateTable.hpp
----------------------------------------------------------------------
diff --git a/query_optimizer/logical/CreateTable.hpp b/query_optimizer/logical/CreateTable.hpp
index da4325d..b380ac9 100644
--- a/query_optimizer/logical/CreateTable.hpp
+++ b/query_optimizer/logical/CreateTable.hpp
@@ -48,7 +48,7 @@ typedef std::shared_ptr<const CreateTable> CreateTablePtr;
/**
* @brief Represents an operation that creates a new table.
*/
-class CreateTable : public Logical {
+class CreateTable final : public Logical {
public:
LogicalType getLogicalType() const override { return LogicalType::kCreateTable; }
@@ -138,13 +138,17 @@ class CreateTable : public Logical {
block_properties_(block_properties),
block_properties_representation_(
getOptimizerRepresentationForProto<OptimizerTreeBaseNodePtr>(block_properties_.get())),
- partition_scheme_header_proto_(partition_scheme_header_proto) {}
-
- std::string relation_name_;
- std::vector<expressions::AttributeReferencePtr> attributes_;
- std::shared_ptr<const StorageBlockLayoutDescription> block_properties_;
- std::shared_ptr<const OptimizerProtoRepresentation<OptimizerTreeBaseNodePtr> > block_properties_representation_;
- std::shared_ptr<const serialization::PartitionSchemeHeader> partition_scheme_header_proto_;
+ partition_scheme_header_proto_(partition_scheme_header_proto),
+ partition_scheme_header_proto_representation_(
+ getOptimizerRepresentationForProto<OptimizerTreeBaseNodePtr>(partition_scheme_header_proto_.get())) {}
+
+ const std::string relation_name_;
+ const std::vector<expressions::AttributeReferencePtr> attributes_;
+ const std::shared_ptr<const StorageBlockLayoutDescription> block_properties_;
+ const std::shared_ptr<const OptimizerProtoRepresentation<OptimizerTreeBaseNodePtr>> block_properties_representation_;
+ const std::shared_ptr<const serialization::PartitionSchemeHeader> partition_scheme_header_proto_;
+ const std::shared_ptr<const OptimizerProtoRepresentation<OptimizerTreeBaseNodePtr>>
+ partition_scheme_header_proto_representation_;
DISALLOW_COPY_AND_ASSIGN(CreateTable);
};
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/475704ec/query_optimizer/physical/CreateTable.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/physical/CreateTable.cpp b/query_optimizer/physical/CreateTable.cpp
index d42eac3..95e6a9d 100644
--- a/query_optimizer/physical/CreateTable.cpp
+++ b/query_optimizer/physical/CreateTable.cpp
@@ -49,6 +49,11 @@ void CreateTable::getFieldStringItems(
non_container_child_field_names->push_back("block_properties");
non_container_child_fields->push_back(block_properties_representation_);
}
+
+ if (partition_scheme_header_proto_representation_) {
+ non_container_child_field_names->push_back("partition_scheme_header");
+ non_container_child_fields->push_back(partition_scheme_header_proto_representation_);
+ }
}
} // namespace physical
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/475704ec/query_optimizer/physical/CreateTable.hpp
----------------------------------------------------------------------
diff --git a/query_optimizer/physical/CreateTable.hpp b/query_optimizer/physical/CreateTable.hpp
index 05eab0d..6ebdfa2 100644
--- a/query_optimizer/physical/CreateTable.hpp
+++ b/query_optimizer/physical/CreateTable.hpp
@@ -49,7 +49,7 @@ typedef std::shared_ptr<const CreateTable> CreateTablePtr;
/**
* @brief Creates a table.
*/
-class CreateTable : public Physical {
+class CreateTable final : public Physical {
public:
PhysicalType getPhysicalType() const override {
return PhysicalType::kCreateTable;
@@ -145,13 +145,17 @@ class CreateTable : public Physical {
block_properties_(block_properties),
block_properties_representation_(
getOptimizerRepresentationForProto<OptimizerTreeBaseNodePtr>(block_properties_.get())),
- partition_scheme_header_proto_(partition_scheme_header_proto) {}
-
- std::string relation_name_;
- std::vector<expressions::AttributeReferencePtr> attributes_;
- std::shared_ptr<const StorageBlockLayoutDescription> block_properties_;
- std::shared_ptr<const OptimizerProtoRepresentation<OptimizerTreeBaseNodePtr> > block_properties_representation_;
- std::shared_ptr<const serialization::PartitionSchemeHeader> partition_scheme_header_proto_;
+ partition_scheme_header_proto_(partition_scheme_header_proto),
+ partition_scheme_header_proto_representation_(
+ getOptimizerRepresentationForProto<OptimizerTreeBaseNodePtr>(partition_scheme_header_proto_.get())) {}
+
+ const std::string relation_name_;
+ const std::vector<expressions::AttributeReferencePtr> attributes_;
+ const std::shared_ptr<const StorageBlockLayoutDescription> block_properties_;
+ const std::shared_ptr<const OptimizerProtoRepresentation<OptimizerTreeBaseNodePtr>> block_properties_representation_;
+ const std::shared_ptr<const serialization::PartitionSchemeHeader> partition_scheme_header_proto_;
+ const std::shared_ptr<const OptimizerProtoRepresentation<OptimizerTreeBaseNodePtr>>
+ partition_scheme_header_proto_representation_;
DISALLOW_COPY_AND_ASSIGN(CreateTable);
};
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/475704ec/query_optimizer/tests/logical_generator/Create.test
----------------------------------------------------------------------
diff --git a/query_optimizer/tests/logical_generator/Create.test b/query_optimizer/tests/logical_generator/Create.test
index aac49fb..89b4a99 100644
--- a/query_optimizer/tests/logical_generator/Create.test
+++ b/query_optimizer/tests/logical_generator/Create.test
@@ -52,6 +52,10 @@ CREATE TABLE foo (attr INT) PARTITION BY HASH(attr) PARTITIONS 4;
--
TopLevelPlan
+-plan=CreateTable[relation=foo]
+| +-partition_scheme_header=ProtoDescription
+| | +-Property=ProtoProperty[Property=partition_type,Value=hash]
+| | +-Property=ProtoProperty[Property=num_partitions,Value=4]
+| | +-Property=ProtoProperty[Property=partition_attr_id,Value=0]
| +-attributes=
| +-AttributeReference[id=0,name=attr,relation=foo,type=Int]
+-output_attributes=
@@ -59,10 +63,19 @@ TopLevelPlan
==
CREATE TABLE foo (attr1 INT, attr2 LONG, attr3 FLOAT, attr4 DOUBLE, attr5 CHAR(5), attr6 VARCHAR(4))
-PARTITION BY HASH(attr1, attr2, attr3, attr4, attr5, attr6) PARTITIONS 4;
+PARTITION BY HASH(attr2, attr1, attr3, attr4, attr5, attr6) PARTITIONS 4;
--
TopLevelPlan
+-plan=CreateTable[relation=foo]
+| +-partition_scheme_header=ProtoDescription
+| | +-Property=ProtoProperty[Property=partition_type,Value=hash]
+| | +-Property=ProtoProperty[Property=num_partitions,Value=4]
+| | +-Property=ProtoProperty[Property=partition_attr_id,Value=1]
+| | +-Property=ProtoProperty[Property=partition_attr_id,Value=0]
+| | +-Property=ProtoProperty[Property=partition_attr_id,Value=2]
+| | +-Property=ProtoProperty[Property=partition_attr_id,Value=3]
+| | +-Property=ProtoProperty[Property=partition_attr_id,Value=4]
+| | +-Property=ProtoProperty[Property=partition_attr_id,Value=5]
| +-attributes=
| +-AttributeReference[id=0,name=attr1,relation=foo,type=Int]
| +-AttributeReference[id=1,name=attr2,relation=foo,type=Long]
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/475704ec/query_optimizer/tests/physical_generator/Create.test
----------------------------------------------------------------------
diff --git a/query_optimizer/tests/physical_generator/Create.test b/query_optimizer/tests/physical_generator/Create.test
index 161cc00..a2dd5c0 100644
--- a/query_optimizer/tests/physical_generator/Create.test
+++ b/query_optimizer/tests/physical_generator/Create.test
@@ -122,6 +122,10 @@ CREATE TABLE foo (attr INT) PARTITION BY HASH(attr) PARTITIONS 4;
[Optimized Logical Plan]
TopLevelPlan
+-plan=CreateTable[relation=foo]
+| +-partition_scheme_header=ProtoDescription
+| | +-Property=ProtoProperty[Property=partition_type,Value=hash]
+| | +-Property=ProtoProperty[Property=num_partitions,Value=4]
+| | +-Property=ProtoProperty[Property=partition_attr_id,Value=0]
| +-attributes=
| +-AttributeReference[id=0,name=attr,relation=foo,type=Int]
+-output_attributes=
@@ -129,6 +133,10 @@ TopLevelPlan
[Physical Plan]
TopLevelPlan
+-plan=CreateTable[relation=foo]
+| +-partition_scheme_header=ProtoDescription
+| | +-Property=ProtoProperty[Property=partition_type,Value=hash]
+| | +-Property=ProtoProperty[Property=num_partitions,Value=4]
+| | +-Property=ProtoProperty[Property=partition_attr_id,Value=0]
| +-attributes=
| +-AttributeReference[id=0,name=attr,relation=foo,type=Int]
+-output_attributes=
@@ -141,6 +149,15 @@ PARTITION BY HASH(attr1, attr2, attr3, attr4, attr5, attr6) PARTITIONS 4;
[Optimized Logical Plan]
TopLevelPlan
+-plan=CreateTable[relation=foo]
+| +-partition_scheme_header=ProtoDescription
+| | +-Property=ProtoProperty[Property=partition_type,Value=hash]
+| | +-Property=ProtoProperty[Property=num_partitions,Value=4]
+| | +-Property=ProtoProperty[Property=partition_attr_id,Value=0]
+| | +-Property=ProtoProperty[Property=partition_attr_id,Value=1]
+| | +-Property=ProtoProperty[Property=partition_attr_id,Value=2]
+| | +-Property=ProtoProperty[Property=partition_attr_id,Value=3]
+| | +-Property=ProtoProperty[Property=partition_attr_id,Value=4]
+| | +-Property=ProtoProperty[Property=partition_attr_id,Value=5]
| +-attributes=
| +-AttributeReference[id=0,name=attr1,relation=foo,type=Int]
| +-AttributeReference[id=1,name=attr2,relation=foo,type=Long]
@@ -158,6 +175,15 @@ TopLevelPlan
[Physical Plan]
TopLevelPlan
+-plan=CreateTable[relation=foo]
+| +-partition_scheme_header=ProtoDescription
+| | +-Property=ProtoProperty[Property=partition_type,Value=hash]
+| | +-Property=ProtoProperty[Property=num_partitions,Value=4]
+| | +-Property=ProtoProperty[Property=partition_attr_id,Value=0]
+| | +-Property=ProtoProperty[Property=partition_attr_id,Value=1]
+| | +-Property=ProtoProperty[Property=partition_attr_id,Value=2]
+| | +-Property=ProtoProperty[Property=partition_attr_id,Value=3]
+| | +-Property=ProtoProperty[Property=partition_attr_id,Value=4]
+| | +-Property=ProtoProperty[Property=partition_attr_id,Value=5]
| +-attributes=
| +-AttributeReference[id=0,name=attr1,relation=foo,type=Int]
| +-AttributeReference[id=1,name=attr2,relation=foo,type=Long]
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/475704ec/query_optimizer/tests/resolver/Create.test
----------------------------------------------------------------------
diff --git a/query_optimizer/tests/resolver/Create.test b/query_optimizer/tests/resolver/Create.test
index 1372cf4..c216c85 100644
--- a/query_optimizer/tests/resolver/Create.test
+++ b/query_optimizer/tests/resolver/Create.test
@@ -181,9 +181,7 @@ BLOCKPROPERTIES (TYPE compresse...
CREATE TABLE foo (attr INT, attr2 INT) WITH
BLOCKPROPERTIES (TYPE compressed_rowstore, COMPRESS 1);
--
-ERROR: The COMPRESS property must be specified as ALL or a list of attributes. (2 : 1)
-BLOCKPROPERTIES (TYPE compresse...
-^
+[same as above]
==
# All specified COMPRESS columns must exist.
@@ -235,6 +233,10 @@ CREATE TABLE foo (attr INT) PARTITION BY HASH(attr) PARTITIONS 4;
--
TopLevelPlan
+-plan=CreateTable[relation=foo]
+| +-partition_scheme_header=ProtoDescription
+| | +-Property=ProtoProperty[Property=partition_type,Value=hash]
+| | +-Property=ProtoProperty[Property=num_partitions,Value=4]
+| | +-Property=ProtoProperty[Property=partition_attr_id,Value=0]
| +-attributes=
| +-AttributeReference[id=0,name=attr,relation=foo,type=Int]
+-output_attributes=
@@ -246,6 +248,15 @@ PARTITION BY HASH(attr1, attr2, attr3, attr4, attr5, attr6) PARTITIONS 4;
--
TopLevelPlan
+-plan=CreateTable[relation=foo]
+| +-partition_scheme_header=ProtoDescription
+| | +-Property=ProtoProperty[Property=partition_type,Value=hash]
+| | +-Property=ProtoProperty[Property=num_partitions,Value=4]
+| | +-Property=ProtoProperty[Property=partition_attr_id,Value=0]
+| | +-Property=ProtoProperty[Property=partition_attr_id,Value=1]
+| | +-Property=ProtoProperty[Property=partition_attr_id,Value=2]
+| | +-Property=ProtoProperty[Property=partition_attr_id,Value=3]
+| | +-Property=ProtoProperty[Property=partition_attr_id,Value=4]
+| | +-Property=ProtoProperty[Property=partition_attr_id,Value=5]
| +-attributes=
| +-AttributeReference[id=0,name=attr1,relation=foo,type=Int]
| +-AttributeReference[id=1,name=attr2,relation=foo,type=Long]
[3/8] incubator-quickstep git commit: QUICKSTEP-78: Displayed
Partition Info using \d.
Posted by ji...@apache.org.
QUICKSTEP-78: Displayed Partition Info using \d.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/c4f7614d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/c4f7614d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/c4f7614d
Branch: refs/heads/fix-iwyu
Commit: c4f7614dcb1584319986bf8841c1426a89f138dc
Parents: b815b38
Author: Zuyu Zhang <zu...@cs.wisc.edu>
Authored: Mon Sep 11 16:35:45 2017 -0500
Committer: Zuyu Zhang <zu...@cs.wisc.edu>
Committed: Mon Sep 11 17:03:00 2017 -0500
----------------------------------------------------------------------
catalog/CMakeLists.txt | 2 ++
catalog/PartitionScheme.cpp | 14 +++++++++++++
catalog/PartitionScheme.hpp | 12 +++++++++++
catalog/PartitionSchemeHeader.cpp | 38 ++++++++++++++++++++++++++++++++++
catalog/PartitionSchemeHeader.hpp | 6 ++++++
cli/CMakeLists.txt | 1 +
cli/CommandExecutorUtil.cpp | 15 ++++++++++----
cli/tests/command_executor/D.test | 17 +++++++++++++++
8 files changed, 101 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c4f7614d/catalog/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/catalog/CMakeLists.txt b/catalog/CMakeLists.txt
index 1aa2b41..414d736 100644
--- a/catalog/CMakeLists.txt
+++ b/catalog/CMakeLists.txt
@@ -167,6 +167,8 @@ target_link_libraries(quickstep_catalog_PartitionScheme
quickstep_utility_Macros)
target_link_libraries(quickstep_catalog_PartitionSchemeHeader
glog
+ quickstep_catalog_CatalogAttribute
+ quickstep_catalog_CatalogRelationSchema
quickstep_catalog_CatalogTypedefs
quickstep_catalog_Catalog_proto
quickstep_storage_StorageConstants
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c4f7614d/catalog/PartitionScheme.cpp
----------------------------------------------------------------------
diff --git a/catalog/PartitionScheme.cpp b/catalog/PartitionScheme.cpp
index 1d7dce0..f7aeb2c 100644
--- a/catalog/PartitionScheme.cpp
+++ b/catalog/PartitionScheme.cpp
@@ -21,6 +21,8 @@
#include <cstddef>
#include <limits>
+#include <sstream>
+#include <string>
#include <unordered_set>
#include <utility>
#include <vector>
@@ -111,4 +113,16 @@ partition_id PartitionScheme::getPartitionForBlock(const block_id block) const {
return std::numeric_limits<std::size_t>::max();
}
+std::string PartitionScheme::toString(const CatalogRelationSchema &relation_schema) const {
+ std::ostringstream oss;
+ oss << " |";
+ for (std::size_t i = 0; i < blocks_in_partition_.size(); ++i) {
+ SpinSharedMutexSharedLock<false> lock(blocks_in_partition_mutexes_[i]);
+ oss << ' ' << blocks_in_partition_[i].size() << " |";
+ }
+ oss << '\n';
+
+ return header_->toString(relation_schema) + oss.str();
+}
+
} // namespace quickstep
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c4f7614d/catalog/PartitionScheme.hpp
----------------------------------------------------------------------
diff --git a/catalog/PartitionScheme.hpp b/catalog/PartitionScheme.hpp
index b69a33f..739a723 100644
--- a/catalog/PartitionScheme.hpp
+++ b/catalog/PartitionScheme.hpp
@@ -22,6 +22,7 @@
#include <cstddef>
#include <memory>
+#include <string>
#include <unordered_set>
#include <utility>
#include <vector>
@@ -39,6 +40,8 @@
namespace quickstep {
+class CatalogRelationSchema;
+
/** \addtogroup Catalog
* @{
*/
@@ -168,6 +171,15 @@ class PartitionScheme {
**/
partition_id getPartitionForBlock(const block_id block) const;
+ /**
+ * @brief Display the partition scheme as a string.
+ *
+ * @param relation_schema The relation schema that owns this scheme.
+ *
+ * @return the string of the partition scheme.
+ **/
+ std::string toString(const CatalogRelationSchema &relation_schema) const;
+
private:
std::unique_ptr<const PartitionSchemeHeader> header_;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c4f7614d/catalog/PartitionSchemeHeader.cpp
----------------------------------------------------------------------
diff --git a/catalog/PartitionSchemeHeader.cpp b/catalog/PartitionSchemeHeader.cpp
index f7a0971..af69f8f 100644
--- a/catalog/PartitionSchemeHeader.cpp
+++ b/catalog/PartitionSchemeHeader.cpp
@@ -20,11 +20,15 @@
#include "catalog/PartitionSchemeHeader.hpp"
#include <cstddef>
+#include <sstream>
+#include <string>
#include <unordered_set>
#include <utility>
#include <vector>
#include "catalog/Catalog.pb.h"
+#include "catalog/CatalogAttribute.hpp"
+#include "catalog/CatalogRelationSchema.hpp"
#include "catalog/CatalogTypedefs.hpp"
#include "types/Type.hpp"
#include "types/Type.pb.h"
@@ -165,6 +169,40 @@ serialization::PartitionSchemeHeader PartitionSchemeHeader::getProto() const {
return proto;
}
+std::string PartitionSchemeHeader::toString(const CatalogRelationSchema &relation_schema) const {
+ std::ostringstream oss;
+ oss << "PARTITION BY ";
+ switch (partition_type_) {
+ case PartitionType::kHash:
+ oss << "HASH";
+ break;
+ case PartitionType::kRandom:
+ oss << "RANDOM";
+ break;
+ case PartitionType::kRange:
+ oss << "RANGE";
+ break;
+ default:
+ LOG(FATAL) << "Invalid Partition Type.";
+ }
+
+ oss << " ( ";
+ if (!partition_attribute_ids_.empty()) {
+ const CatalogAttribute *attr = relation_schema.getAttributeById(partition_attribute_ids_[0]);
+ DCHECK(attr);
+ oss << attr->getName();
+
+ for (size_t i = 1; i < partition_attribute_ids_.size(); ++i) {
+ attr = relation_schema.getAttributeById(partition_attribute_ids_[i]);
+ DCHECK(attr);
+ oss << ", " << attr->getName();
+ }
+ }
+ oss << " ) PARTITIONS " << num_partitions_ << '\n';
+
+ return oss.str();
+}
+
serialization::PartitionSchemeHeader RangePartitionSchemeHeader::getProto() const {
serialization::PartitionSchemeHeader proto = PartitionSchemeHeader::getProto();
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c4f7614d/catalog/PartitionSchemeHeader.hpp
----------------------------------------------------------------------
diff --git a/catalog/PartitionSchemeHeader.hpp b/catalog/PartitionSchemeHeader.hpp
index 9bbbc0f..d34ca1f 100644
--- a/catalog/PartitionSchemeHeader.hpp
+++ b/catalog/PartitionSchemeHeader.hpp
@@ -23,6 +23,7 @@
#include <cstddef>
#include <memory>
#include <random>
+#include <string>
#include <utility>
#include <vector>
@@ -41,6 +42,7 @@
namespace quickstep {
+class CatalogRelationSchema;
class Type;
/** \addtogroup Catalog
@@ -164,6 +166,10 @@ class PartitionSchemeHeader {
const PartitionAttributeIds partition_attribute_ids_;
private:
+ std::string toString(const CatalogRelationSchema &relation_schema) const;
+
+ friend class PartitionScheme;
+
DISALLOW_COPY_AND_ASSIGN(PartitionSchemeHeader);
};
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c4f7614d/cli/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/cli/CMakeLists.txt b/cli/CMakeLists.txt
index e802e8d..33d10e3 100644
--- a/cli/CMakeLists.txt
+++ b/cli/CMakeLists.txt
@@ -147,6 +147,7 @@ target_link_libraries(quickstep_cli_CommandExecutorUtil
quickstep_catalog_CatalogDatabase
quickstep_catalog_CatalogRelation
quickstep_catalog_IndexScheme
+ quickstep_catalog_PartitionScheme
quickstep_cli_PrintToScreen
quickstep_parser_ParseString
quickstep_storage_StorageBlockLayout_proto
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c4f7614d/cli/CommandExecutorUtil.cpp
----------------------------------------------------------------------
diff --git a/cli/CommandExecutorUtil.cpp b/cli/CommandExecutorUtil.cpp
index d17617f..b3109f9 100644
--- a/cli/CommandExecutorUtil.cpp
+++ b/cli/CommandExecutorUtil.cpp
@@ -22,6 +22,7 @@
#include <algorithm>
#include <cstddef>
#include <iomanip>
+#include <sstream>
#include <string>
#include <vector>
@@ -29,6 +30,7 @@
#include "catalog/CatalogDatabase.hpp"
#include "catalog/CatalogRelation.hpp"
#include "catalog/IndexScheme.hpp"
+#include "catalog/PartitionScheme.hpp"
#include "cli/PrintToScreen.hpp"
#include "parser/ParseString.hpp"
#include "storage/StorageBlockLayout.pb.h"
@@ -146,9 +148,9 @@ string ExecuteDescribeTable(
}
ostringstream oss;
- oss << setw(kInitMaxColumnWidth) << "Table" << " \"" << table_name_val << "\"\n";
- oss << std::left << setw(max_attr_column_width + 1) << " Column" << " |";
- oss << setw(max_type_column_width + 1) << " Type" << '\n';
+ oss << setw(kInitMaxColumnWidth) << "Table" << " \"" << table_name_val << "\"\n"
+ << std::left << setw(max_attr_column_width + 1) << " Column" << " |"
+ << setw(max_type_column_width + 1) << " Type" << '\n';
// Add room for one extra character to allow spacing between the column ending and the vertical bar
oss << PrintToScreen::GenerateHBar({ max_attr_column_width + 1, max_type_column_width + 1 });
@@ -157,7 +159,7 @@ string ExecuteDescribeTable(
oss << ' ' << setw(max_attr_column_width) << attr.getDisplayName() << " | "
<< setw(max_type_column_width) << attr.getType().getName() << '\n';
}
- // TODO(rogers): Add handlers for partitioning information.
+
if (relation->hasIndexScheme()) {
oss << setw(kInitMaxColumnWidth + 2) << " Indexes" << '\n';
for (const auto &index : relation->getIndexScheme()) {
@@ -173,6 +175,11 @@ string ExecuteDescribeTable(
}
}
+ if (relation->hasPartitionScheme()) {
+ oss << setw(kInitMaxColumnWidth + 2) << " Partition Info\n "
+ << relation->getPartitionScheme()->toString(*relation);
+ }
+
return oss.str();
}
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/c4f7614d/cli/tests/command_executor/D.test
----------------------------------------------------------------------
diff --git a/cli/tests/command_executor/D.test b/cli/tests/command_executor/D.test
index 1b35b58..36e9a92 100644
--- a/cli/tests/command_executor/D.test
+++ b/cli/tests/command_executor/D.test
@@ -38,6 +38,11 @@ CREATE TABLE foo4 (col1 INT,
col5 CHAR(5));
CREATE INDEX foo4_index_1 ON foo4 (col1, col2) USING CSBTREE;
CREATE INDEX foo4_index_2 ON foo4 (col3, col4) USING CSBTREE;
+CREATE TABLE foo_hash_part (col1 INT,
+ col2 FLOAT)
+PARTITION BY HASH(col1) PARTITIONS 4;
+INSERT INTO foo_hash_part
+SELECT i, i * 3.0 FROM generate_series(1, 100) AS gs(i);
CREATE TABLE averylongtablenamethatseemstoneverend (col1 INT);
DROP TABLE TEST;
INSERT INTO averylongtablenamethatseemstoneverend VALUES (1);
@@ -102,6 +107,17 @@ INSERT INTO foo3 values(5, 1, 1.0, 1.0, 'XYZZ');
"foo4_index_2" CSB_TREE (col3, col4)
"foo4_index_1" CSB_TREE (col1, col2)
==
+\d foo_hash_part
+--
+ Table "foo_hash_part"
+ Column | Type
++-------+-------+
+ col1 | Int
+ col2 | Float
+ Partition Info
+ PARTITION BY HASH ( col1 ) PARTITIONS 4
+ | 1 | 1 | 1 | 1 |
+==
\d
--
List of relations
@@ -112,6 +128,7 @@ INSERT INTO foo3 values(5, 1, 1.0, 1.0, 'XYZZ');
foo2 | table | 1
foo3 | table | 1
foo4 | table | 0
+ foo_hash_part | table | 4
averylongtablenamethatseemstoneverend | table | 1
==
[6/8] incubator-quickstep git commit: Fixed a bug in partitioned NLJ.
Posted by ji...@apache.org.
Fixed a bug in partitioned NLJ.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/d85f7a9a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/d85f7a9a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/d85f7a9a
Branch: refs/heads/fix-iwyu
Commit: d85f7a9a3b285c60892f40b1ba0a8b32ba629885
Parents: 56b3436
Author: Zuyu Zhang <zu...@cs.wisc.edu>
Authored: Wed Sep 13 21:21:35 2017 -0500
Committer: Zuyu Zhang <zu...@cs.wisc.edu>
Committed: Thu Sep 14 11:32:14 2017 -0500
----------------------------------------------------------------------
.../tests/execution_generator/Partition.test | 37 +++++++++++++++-
.../NestedLoopsJoinOperator.cpp | 44 ++++++++++----------
.../NestedLoopsJoinOperator.hpp | 25 ++---------
3 files changed, 62 insertions(+), 44 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/d85f7a9a/query_optimizer/tests/execution_generator/Partition.test
----------------------------------------------------------------------
diff --git a/query_optimizer/tests/execution_generator/Partition.test b/query_optimizer/tests/execution_generator/Partition.test
index 5d44164..da9b6b8 100644
--- a/query_optimizer/tests/execution_generator/Partition.test
+++ b/query_optimizer/tests/execution_generator/Partition.test
@@ -226,7 +226,42 @@ GROUP BY fact.id;
+-----------+------------------------+
==
-# Partitioned NestedLoopsJoin
+# Partitioned NestedLoopsJoin w/ both stored relations.
+SELECT COUNT(*)
+FROM dim_4_hash_partitions, fact;
+--
++--------------------+
+|COUNT(*) |
++--------------------+
+| 220|
++--------------------+
+==
+
+# Partitioned NestedLoopsJoin w/ one stored relation.
+SELECT COUNT(*)
+FROM dim_4_hash_partitions, fact
+WHERE dim_4_hash_partitions.id > 20;
+--
++--------------------+
+|COUNT(*) |
++--------------------+
+| 20|
++--------------------+
+==
+
+# Partitioned NestedLoopsJoin w/ the other stored relation.
+SELECT COUNT(*)
+FROM dim_4_hash_partitions, fact
+WHERE fact.id > 0;
+--
++--------------------+
+|COUNT(*) |
++--------------------+
+| 220|
++--------------------+
+==
+
+# Partitioned NestedLoopsJoin w/ both non-stored relations.
SELECT dim_4_hash_partitions.id as dim_id, fact.id as fact_id
FROM dim_4_hash_partitions, fact
WHERE dim_4_hash_partitions.id > 20 AND fact.id > 0;
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/d85f7a9a/relational_operators/NestedLoopsJoinOperator.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/NestedLoopsJoinOperator.cpp b/relational_operators/NestedLoopsJoinOperator.cpp
index 121b1c3..658f84b 100644
--- a/relational_operators/NestedLoopsJoinOperator.cpp
+++ b/relational_operators/NestedLoopsJoinOperator.cpp
@@ -63,7 +63,7 @@ bool NestedLoopsJoinOperator::getAllWorkOrders(
for (partition_id part_id = 0; part_id < num_partitions_; ++part_id) {
for (const block_id left_block_id : left_relation_block_ids_[part_id]) {
- for (const block_id right_block_id : right_relation_block_ids_[part_id]) {
+ for (const block_id right_block_id : right_relation_block_ids_) {
container->addNormalWorkOrder(
new NestedLoopsJoinWorkOrder(
query_id_,
@@ -89,7 +89,7 @@ bool NestedLoopsJoinOperator::getAllWorkOrders(
std::vector<block_id>::size_type new_left_blocks
= left_relation_block_ids_[part_id].size() - num_left_workorders_generated_[part_id];
std::vector<block_id>::size_type new_right_blocks
- = right_relation_block_ids_[part_id].size() - num_right_workorders_generated_[part_id];
+ = right_relation_block_ids_.size() - num_right_workorders_generated_[part_id];
std::size_t new_workorders = 0;
if (new_left_blocks > 0 && new_right_blocks > 0) {
@@ -102,7 +102,7 @@ bool NestedLoopsJoinOperator::getAllWorkOrders(
0,
left_relation_block_ids_[part_id].size(),
num_right_workorders_generated_[part_id],
- right_relation_block_ids_[part_id].size());
+ right_relation_block_ids_.size());
// Now generate new_left_blocks * (right).
new_workorders += getAllWorkOrdersHelperBothNotStored(container,
@@ -122,7 +122,7 @@ bool NestedLoopsJoinOperator::getAllWorkOrders(
0,
left_relation_block_ids_[part_id].size(),
num_right_workorders_generated_[part_id],
- right_relation_block_ids_[part_id].size());
+ right_relation_block_ids_.size());
} else if (new_left_blocks > 0 && new_right_blocks == 0) {
// Generate new_left_blocks * right
new_workorders = getAllWorkOrdersHelperBothNotStored(container,
@@ -132,11 +132,11 @@ bool NestedLoopsJoinOperator::getAllWorkOrders(
num_left_workorders_generated_[part_id],
left_relation_block_ids_[part_id].size(),
0,
- right_relation_block_ids_[part_id].size());
+ right_relation_block_ids_.size());
}
if (new_workorders > 0) {
num_left_workorders_generated_[part_id] = left_relation_block_ids_[part_id].size();
- num_right_workorders_generated_[part_id] = right_relation_block_ids_[part_id].size();
+ num_right_workorders_generated_[part_id] = right_relation_block_ids_.size();
}
}
return done_feeding_left_relation_ && done_feeding_right_relation_;
@@ -155,7 +155,7 @@ bool NestedLoopsJoinOperator::getAllWorkOrderProtos(WorkOrderProtosContainer *co
for (partition_id part_id = 0; part_id < num_partitions_; ++part_id) {
for (const block_id left_block_id : left_relation_block_ids_[part_id]) {
- for (const block_id right_block_id : right_relation_block_ids_[part_id]) {
+ for (const block_id right_block_id : right_relation_block_ids_) {
container->addWorkOrderProto(createWorkOrderProto(part_id, left_block_id, right_block_id),
op_index_);
}
@@ -169,7 +169,7 @@ bool NestedLoopsJoinOperator::getAllWorkOrderProtos(WorkOrderProtosContainer *co
const std::vector<block_id>::size_type new_left_blocks
= left_relation_block_ids_[part_id].size() - num_left_workorders_generated_[part_id];
const std::vector<block_id>::size_type new_right_blocks
- = right_relation_block_ids_[part_id].size() - num_right_workorders_generated_[part_id];
+ = right_relation_block_ids_.size() - num_right_workorders_generated_[part_id];
std::size_t new_workorders = 0;
if (new_left_blocks > 0 && new_right_blocks > 0) {
@@ -181,7 +181,7 @@ bool NestedLoopsJoinOperator::getAllWorkOrderProtos(WorkOrderProtosContainer *co
0,
left_relation_block_ids_[part_id].size(),
num_right_workorders_generated_[part_id],
- right_relation_block_ids_[part_id].size());
+ right_relation_block_ids_.size());
// Now generate new_left_blocks * (right).
new_workorders +=
@@ -199,7 +199,7 @@ bool NestedLoopsJoinOperator::getAllWorkOrderProtos(WorkOrderProtosContainer *co
0,
left_relation_block_ids_[part_id].size(),
num_right_workorders_generated_[part_id],
- right_relation_block_ids_[part_id].size());
+ right_relation_block_ids_.size());
} else if (new_left_blocks > 0 && new_right_blocks == 0) {
// Generate new_left_blocks * right
new_workorders =
@@ -208,11 +208,11 @@ bool NestedLoopsJoinOperator::getAllWorkOrderProtos(WorkOrderProtosContainer *co
num_left_workorders_generated_[part_id],
left_relation_block_ids_[part_id].size(),
0,
- right_relation_block_ids_[part_id].size());
+ right_relation_block_ids_.size());
}
if (new_workorders > 0) {
num_left_workorders_generated_[part_id] = left_relation_block_ids_[part_id].size();
- num_right_workorders_generated_[part_id] = right_relation_block_ids_[part_id].size();
+ num_right_workorders_generated_[part_id] = right_relation_block_ids_.size();
}
}
return done_feeding_left_relation_ && done_feeding_right_relation_;
@@ -247,7 +247,7 @@ std::size_t NestedLoopsJoinOperator::getAllWorkOrdersHelperBothNotStored(WorkOrd
right_input_relation_,
part_id,
left_relation_block_ids_[part_id][left_index],
- right_relation_block_ids_[part_id][right_index],
+ right_relation_block_ids_[right_index],
query_context->getPredicate(join_predicate_index_),
query_context->getScalarGroup(selection_index_),
query_context->getInsertDestination(output_destination_index_),
@@ -274,7 +274,7 @@ bool NestedLoopsJoinOperator::getAllWorkOrdersHelperOneStored(WorkOrdersContaine
if (left_relation_is_stored_) {
for (partition_id part_id = 0; part_id < num_partitions_; ++part_id) {
for (std::vector<block_id>::size_type right_index = num_right_workorders_generated_[part_id];
- right_index < right_relation_block_ids_[part_id].size();
+ right_index < right_relation_block_ids_.size();
++right_index) {
for (const block_id left_block_id : left_relation_block_ids_[part_id]) {
container->addNormalWorkOrder(
@@ -284,7 +284,7 @@ bool NestedLoopsJoinOperator::getAllWorkOrdersHelperOneStored(WorkOrdersContaine
right_input_relation_,
part_id,
left_block_id,
- right_relation_block_ids_[part_id][right_index],
+ right_relation_block_ids_[right_index],
join_predicate,
selection,
output_destination,
@@ -292,7 +292,7 @@ bool NestedLoopsJoinOperator::getAllWorkOrdersHelperOneStored(WorkOrdersContaine
op_index_);
}
}
- num_right_workorders_generated_[part_id] = right_relation_block_ids_[part_id].size();
+ num_right_workorders_generated_[part_id] = right_relation_block_ids_.size();
}
return done_feeding_right_relation_;
} else {
@@ -300,7 +300,7 @@ bool NestedLoopsJoinOperator::getAllWorkOrdersHelperOneStored(WorkOrdersContaine
for (std::vector<block_id>::size_type left_index = num_left_workorders_generated_[part_id];
left_index < left_relation_block_ids_[part_id].size();
++left_index) {
- for (const block_id right_block_id : right_relation_block_ids_[part_id]) {
+ for (const block_id right_block_id : right_relation_block_ids_) {
container->addNormalWorkOrder(
new NestedLoopsJoinWorkOrder(query_id_,
left_input_relation_,
@@ -340,7 +340,7 @@ std::size_t NestedLoopsJoinOperator::getAllWorkOrderProtosHelperBothNotStored(
++right_index) {
container->addWorkOrderProto(
createWorkOrderProto(part_id, left_relation_block_ids_[part_id][left_index],
- right_relation_block_ids_[part_id][right_index]),
+ right_relation_block_ids_[right_index]),
op_index_);
}
}
@@ -354,15 +354,15 @@ bool NestedLoopsJoinOperator::getAllWorkOrderProtosHelperOneStored(WorkOrderProt
if (left_relation_is_stored_) {
for (partition_id part_id = 0; part_id < num_partitions_; ++part_id) {
for (std::vector<block_id>::size_type right_index = num_right_workorders_generated_[part_id];
- right_index < right_relation_block_ids_[part_id].size();
+ right_index < right_relation_block_ids_.size();
++right_index) {
for (const block_id left_block_id : left_relation_block_ids_[part_id]) {
container->addWorkOrderProto(
- createWorkOrderProto(part_id, left_block_id, right_relation_block_ids_[part_id][right_index]),
+ createWorkOrderProto(part_id, left_block_id, right_relation_block_ids_[right_index]),
op_index_);
}
}
- num_right_workorders_generated_[part_id] = right_relation_block_ids_[part_id].size();
+ num_right_workorders_generated_[part_id] = right_relation_block_ids_.size();
}
return done_feeding_right_relation_;
} else {
@@ -370,7 +370,7 @@ bool NestedLoopsJoinOperator::getAllWorkOrderProtosHelperOneStored(WorkOrderProt
for (std::vector<block_id>::size_type left_index = num_left_workorders_generated_[part_id];
left_index < left_relation_block_ids_[part_id].size();
++left_index) {
- for (const block_id right_block_id : right_relation_block_ids_[part_id]) {
+ for (const block_id right_block_id : right_relation_block_ids_) {
container->addWorkOrderProto(
createWorkOrderProto(part_id, left_relation_block_ids_[part_id][left_index], right_block_id),
op_index_);
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/d85f7a9a/relational_operators/NestedLoopsJoinOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/NestedLoopsJoinOperator.hpp b/relational_operators/NestedLoopsJoinOperator.hpp
index 5960b81..a379c17 100644
--- a/relational_operators/NestedLoopsJoinOperator.hpp
+++ b/relational_operators/NestedLoopsJoinOperator.hpp
@@ -113,7 +113,8 @@ class NestedLoopsJoinOperator : public RelationalOperator {
left_relation_is_stored_(left_relation_is_stored),
right_relation_is_stored_(right_relation_is_stored),
left_relation_block_ids_(num_partitions),
- right_relation_block_ids_(num_partitions),
+ right_relation_block_ids_(right_relation_is_stored ? right_input_relation_.getBlocksSnapshot()
+ : std::vector<block_id>()),
num_left_workorders_generated_(num_partitions),
num_right_workorders_generated_(num_partitions),
done_feeding_left_relation_(false),
@@ -133,21 +134,6 @@ class NestedLoopsJoinOperator : public RelationalOperator {
left_relation_block_ids_[0] = left_input_relation_.getBlocksSnapshot();
}
}
-
- if (right_relation_is_stored) {
- if (right_input_relation_.hasPartitionScheme()) {
- const PartitionScheme &part_scheme = *right_input_relation_.getPartitionScheme();
- DCHECK_EQ(num_partitions_, part_scheme.getPartitionSchemeHeader().getNumPartitions());
- for (std::size_t part_id = 0; part_id < num_partitions_; ++part_id) {
- right_relation_block_ids_[part_id] = part_scheme.getBlocksInPartition(part_id);
- }
- } else {
- // Broadcast right (smaller) side upon partitioned nlj.
- for (partition_id part_id = 0; part_id < num_partitions_; ++part_id) {
- right_relation_block_ids_[part_id] = right_input_relation_.getBlocksSnapshot();
- }
- }
- }
}
~NestedLoopsJoinOperator() override {}
@@ -183,10 +169,7 @@ class NestedLoopsJoinOperator : public RelationalOperator {
if (input_relation_id == left_input_relation_.getID()) {
left_relation_block_ids_[part_id].push_back(input_block_id);
} else if (input_relation_id == right_input_relation_.getID()) {
- // Broadcast right (smaller) side upon partitioned nlj.
- for (partition_id input_part_id = 0; input_part_id < num_partitions_; ++input_part_id) {
- right_relation_block_ids_[input_part_id].push_back(input_block_id);
- }
+ right_relation_block_ids_.push_back(input_block_id);
} else {
LOG(FATAL) << "The input block sent to the NestedLoopsJoinOperator belongs "
<< "to a different relation than the left and right relations";
@@ -315,7 +298,7 @@ class NestedLoopsJoinOperator : public RelationalOperator {
const bool right_relation_is_stored_;
std::vector<BlocksInPartition> left_relation_block_ids_;
- std::vector<BlocksInPartition> right_relation_block_ids_;
+ std::vector<block_id> right_relation_block_ids_;
// At a given point of time, we have paired num_left_workorders_generated[part_id]
// number of blocks from the left relation with num_right_workorders_generated[part_id]
[4/8] incubator-quickstep git commit: Crash if block not found in the
local storage in the single node.
Posted by ji...@apache.org.
Crash if block not found in the local storage in the single node.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/b1c3356a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/b1c3356a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/b1c3356a
Branch: refs/heads/fix-iwyu
Commit: b1c3356a19efef985ea87ae5e127f4ed2dd40ad7
Parents: c4f7614
Author: Zuyu Zhang <zu...@cs.wisc.edu>
Authored: Tue Sep 12 20:29:39 2017 -0500
Committer: Zuyu Zhang <zu...@cs.wisc.edu>
Committed: Tue Sep 12 20:29:39 2017 -0500
----------------------------------------------------------------------
storage/StorageManager.cpp | 8 ++------
1 file changed, 2 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/b1c3356a/storage/StorageManager.cpp
----------------------------------------------------------------------
diff --git a/storage/StorageManager.cpp b/storage/StorageManager.cpp
index 7eb132c..5924607 100644
--- a/storage/StorageManager.cpp
+++ b/storage/StorageManager.cpp
@@ -683,8 +683,6 @@ StorageManager::BlockHandle StorageManager::loadBlockOrBlob(
loaded_handle.block_memory = block_buffer;
loaded_handle.block_memory_size = num_slots;
} else {
- bool pull_succeeded = false;
-
#ifdef QUICKSTEP_DISTRIBUTED
const string domain_network_address = getPeerDomainNetworkAddress(BlockIdUtil::Domain(block));
DLOG(INFO) << "Pulling Block " << BlockIdUtil::ToString(block) << " from " << domain_network_address;
@@ -693,11 +691,9 @@ StorageManager::BlockHandle StorageManager::loadBlockOrBlob(
while (!client.Pull(block, numa_node, &loaded_handle)) {
LOG(INFO) << "Retry pulling Block " << BlockIdUtil::ToString(block) << " from " << domain_network_address;
}
-
- pull_succeeded = true;
+#else
+ LOG(FATAL) << "Block not found from persistent storage: " << block;
#endif
-
- CHECK(pull_succeeded) << "Failed to pull Block " << BlockIdUtil::ToString(block) << " from remote peers.";
}
#ifdef QUICKSTEP_DISTRIBUTED