You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by mg...@apache.org on 2024/09/09 11:04:13 UTC
(kudu) branch master updated (e0f9aee54 -> 7acd71e9a)
This is an automated email from the ASF dual-hosted git repository.
mgreber pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git
from e0f9aee54 [util] modernize code in bitmap
new 43a7cf535 Table copy supports creating a table with no range partition
new 7acd71e9a [rpc] micro-cleanup on InboundCall and ServicePool
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
src/kudu/rpc/inbound_call.cc | 6 ++-
src/kudu/rpc/inbound_call.h | 4 +-
src/kudu/rpc/rpcz_store.cc | 6 +--
src/kudu/rpc/rpcz_store.h | 2 +-
src/kudu/rpc/service_pool.cc | 26 +++++------
src/kudu/rpc/service_pool.h | 5 +--
src/kudu/tools/kudu-tool-test.cc | 97 ++++++++++++++++++++++++++++++++++++++++
src/kudu/tools/table_scanner.cc | 3 +-
8 files changed, 122 insertions(+), 27 deletions(-)
(kudu) 01/02: Table copy supports creating a table with no range partition
Posted by mg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mgreber pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git
commit 43a7cf5358304aa17b47967540e713e4d6140c0a
Author: xinghuayu007 <14...@qq.com>
AuthorDate: Fri Aug 2 15:44:59 2024 +0800
Table copy supports creating a table with no range partition
Currently, using CLI 'table copy' to create a destination table
when the source table doesn't have any range partition, it creates
a table with an unbounded range partition. That makes the source
table's schema different from the destination table's. That
also causes the tablet schema to be different in the same partition.
The following is an example:
The source table schema:
TABLE test1 (
key INT32 NOT NULL,
int_val INT32 NOT NULL,
string_val STRING NULLABLE,
PRIMARY KEY (key)
)
OWNER root
REPLICAS 1
COMMENT
CLI command:
'kudu table copy $ma test1 $ma --create_table=true'
The destination table schema:
TABLE test1 (
key INT32 NOT NULL,
int_val INT32 NOT NULL,
string_val STRING NULLABLE,
PRIMARY KEY (key)
)
RANGE (key) (
PARTITION UNBOUNDED
)
OWNER root
REPLICAS 1
COMMENT
Semantically it is right, but is better to keep the source table
schema the same with the destination schema.
Change-Id: I94f99f845393ff2460761cc7bd82ecb3f3ead79b
Reviewed-on: http://gerrit.cloudera.org:8080/21634
Reviewed-by: Yingchun Lai <la...@apache.org>
Tested-by: Marton Greber <gr...@gmail.com>
Reviewed-by: Marton Greber <gr...@gmail.com>
---
src/kudu/tools/kudu-tool-test.cc | 97 ++++++++++++++++++++++++++++++++++++++++
src/kudu/tools/table_scanner.cc | 3 +-
2 files changed, 98 insertions(+), 2 deletions(-)
diff --git a/src/kudu/tools/kudu-tool-test.cc b/src/kudu/tools/kudu-tool-test.cc
index 6112f31d9..9dc36d6fc 100644
--- a/src/kudu/tools/kudu-tool-test.cc
+++ b/src/kudu/tools/kudu-tool-test.cc
@@ -1781,6 +1781,103 @@ TEST_P(CreateEmptyPartitionTableTest, TestCreateEmptyPartitionTable) {
}
}
+
+enum class PartitionCombinationMode {
+ kOnlyHashPartition,
+ kOnlyRangePartition,
+ kNoAnyPartition
+};
+
+class CreateNoRangePartitionTable :
+ public ToolTest,
+ public ::testing::WithParamInterface<PartitionCombinationMode> {
+};
+
+INSTANTIATE_TEST_SUITE_P(, CreateNoRangePartitionTable,
+ ::testing::Values(PartitionCombinationMode::kOnlyHashPartition,
+ PartitionCombinationMode::kOnlyRangePartition,
+ PartitionCombinationMode::kNoAnyPartition));
+
+TEST_P(CreateNoRangePartitionTable, TestCreateNoRangePartitionTable) {
+ PartitionCombinationMode create_table_mode = GetParam();
+ const string& kSrcTableName = "test1";
+ const string& kDstTableName = "test2";
+ NO_FATALS(StartExternalMiniCluster());
+ shared_ptr<KuduClient> client;
+ ASSERT_OK(cluster_->CreateClient(nullptr, &client));
+ unique_ptr<KuduTableCreator> table_creator(client->NewTableCreator());
+ KuduSchema schema = KuduSchema::FromSchema(GetSimpleTestSchema());
+ // Create a table in different modes.
+ switch (create_table_mode) {
+ case PartitionCombinationMode::kOnlyHashPartition:
+ ASSERT_OK(table_creator->table_name(kSrcTableName)
+ .schema(&schema)
+ .num_replicas(1)
+ .add_hash_partitions({"key"}, 2)
+ .set_range_partition_columns({})
+ .Create());
+ break;
+ case PartitionCombinationMode::kOnlyRangePartition:
+ ASSERT_OK(table_creator->table_name(kSrcTableName)
+ .schema(&schema)
+ .num_replicas(1)
+ .set_range_partition_columns({"key"})
+ .Create());
+ break;
+ case PartitionCombinationMode::kNoAnyPartition:
+ ASSERT_OK(table_creator->table_name(kSrcTableName)
+ .schema(&schema)
+ .num_replicas(1)
+ .set_range_partition_columns({})
+ .Create());
+ break;
+ default:
+ ASSERT_TRUE(false);
+ }
+
+ // Describe the source table and check the partition schema.
+ const string& master_addr = cluster_->master()->bound_rpc_addr().ToString();
+ vector<string> stdout_before;
+ NO_FATALS(RunActionStdoutLines(Substitute("table describe $0 $1",
+ master_addr,
+ kSrcTableName), &stdout_before));
+ const string& src_table_schema = JoinStrings(stdout_before, "\\");
+ switch (create_table_mode) {
+ case PartitionCombinationMode::kOnlyHashPartition:
+ ASSERT_STR_CONTAINS(src_table_schema, "HASH (key) PARTITIONS");
+ ASSERT_STR_NOT_CONTAINS(src_table_schema, "RANG");
+ break;
+ case PartitionCombinationMode::kOnlyRangePartition:
+ ASSERT_STR_CONTAINS(src_table_schema, "RANGE (key)");
+ ASSERT_STR_NOT_CONTAINS(src_table_schema, "HASH");
+ break;
+ case PartitionCombinationMode::kNoAnyPartition:
+ ASSERT_STR_NOT_CONTAINS(src_table_schema, "HASH");
+ ASSERT_STR_NOT_CONTAINS(src_table_schema, "RANG");
+ break;
+ default:
+ ASSERT_TRUE(false);
+ }
+
+ // Table copy the source table and create the destination table.
+ NO_FATALS(RunTool(
+ Substitute("table copy $0 $1 $2 --dst_table=$3 --create_table=true ",
+ master_addr, kSrcTableName,
+ master_addr, kDstTableName
+ ), nullptr, nullptr));
+ // Describe the destination table and check the partition schema.
+ vector<string> stdout_after;
+ NO_FATALS(RunActionStdoutLines(Substitute("table describe $0 $1",
+ master_addr,
+ kDstTableName), &stdout_after));
+ ASSERT_EQ(stdout_before.size(), stdout_after.size());
+ // Other than the table names, the schemas are the same.
+ // Therefore, compare the content from the second line.
+ for (int i = 1; i < stdout_after.size(); i++) {
+ ASSERT_EQ(stdout_before[i], stdout_after[i]);
+ }
+}
+
TEST_F(ToolTest, TestFsCheck) {
const string kTestDir = GetTestPath("test");
const string kTabletId = "ffffffffffffffffffffffffffffffff";
diff --git a/src/kudu/tools/table_scanner.cc b/src/kudu/tools/table_scanner.cc
index 34e74daec..8fb4fd932 100644
--- a/src/kudu/tools/table_scanner.cc
+++ b/src/kudu/tools/table_scanner.cc
@@ -547,8 +547,7 @@ Status CreateDstTableIfNeeded(const client::sp::shared_ptr<KuduTable>& src_table
table_creator->add_range_partition(lower.release(), upper.release());
}
- if (partition_schema.hash_schema().empty() &&
- partition_schema.range_schema().column_ids.empty()) {
+ if (partition_schema.range_schema().column_ids.empty()) {
// This src table is unpartitioned, just create a table range partitioned on no columns.
table_creator->set_range_partition_columns({});
}
(kudu) 02/02: [rpc] micro-cleanup on InboundCall and ServicePool
Posted by mg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mgreber pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git
commit 7acd71e9a5a33a92ab55122c316c89a8be61b3e1
Author: Alexey Serbin <al...@apache.org>
AuthorDate: Fri Sep 6 11:18:05 2024 -0700
[rpc] micro-cleanup on InboundCall and ServicePool
I was troubleshooting an issue and went over the related code
to clean it up a bit:
* don't crash on errors in ServicePool::Init(), but return the status
to be handled at the upper level
* ServicePool::RejectTooBusy() first sends corresponding response
before logging about the rejected request
* fix const-correctness of a few methods in InboundCall, ServicePool
* remove unnecessary mutex primitive in ServicePool
Change-Id: I255754da8e66263a00aa28e9061102d4ec421105
Reviewed-on: http://gerrit.cloudera.org:8080/21759
Tested-by: Alexey Serbin <al...@apache.org>
Reviewed-by: Marton Greber <gr...@gmail.com>
Reviewed-by: Zoltan Chovan <zc...@cloudera.com>
---
src/kudu/rpc/inbound_call.cc | 6 ++++--
src/kudu/rpc/inbound_call.h | 4 ++--
src/kudu/rpc/rpcz_store.cc | 6 +++---
src/kudu/rpc/rpcz_store.h | 2 +-
src/kudu/rpc/service_pool.cc | 26 ++++++++++++--------------
src/kudu/rpc/service_pool.h | 5 ++---
6 files changed, 24 insertions(+), 25 deletions(-)
diff --git a/src/kudu/rpc/inbound_call.cc b/src/kudu/rpc/inbound_call.cc
index b204eae3e..18ac13582 100644
--- a/src/kudu/rpc/inbound_call.cc
+++ b/src/kudu/rpc/inbound_call.cc
@@ -361,8 +361,10 @@ void InboundCall::DiscardTransfer() {
transfer_.reset();
}
-size_t InboundCall::GetTransferSize() {
- if (!transfer_) return 0;
+size_t InboundCall::GetTransferSize() const {
+ if (PREDICT_FALSE(!transfer_)) {
+ return 0;
+ }
return transfer_->data().size();
}
diff --git a/src/kudu/rpc/inbound_call.h b/src/kudu/rpc/inbound_call.h
index d0dd75e5f..65af6e3be 100644
--- a/src/kudu/rpc/inbound_call.h
+++ b/src/kudu/rpc/inbound_call.h
@@ -174,7 +174,7 @@ class InboundCall {
// Return the method associated with this call. This is set just before
// the call is enqueued onto the service queue, and therefore may be
// 'nullptr' for much of the lifecycle of a call.
- RpcMethodInfo* method_info() {
+ const RpcMethodInfo* method_info() const {
return method_info_.get();
}
@@ -221,7 +221,7 @@ class InboundCall {
// Returns the size of the transfer buffer that backs this call. If the transfer does
// not exist (e.g. GetTransferSize() is called after DiscardTransfer()), returns 0.
- size_t GetTransferSize();
+ size_t GetTransferSize() const;
private:
friend class RpczStore;
diff --git a/src/kudu/rpc/rpcz_store.cc b/src/kudu/rpc/rpcz_store.cc
index 52a91f317..6e46b3492 100644
--- a/src/kudu/rpc/rpcz_store.cc
+++ b/src/kudu/rpc/rpcz_store.cc
@@ -127,7 +127,7 @@ class MethodSampler {
};
MethodSampler* RpczStore::SamplerForCall(InboundCall* call) {
- auto* method_info = call->method_info();
+ const auto* method_info = call->method_info();
if (PREDICT_FALSE(!method_info)) {
return nullptr;
}
@@ -229,9 +229,9 @@ void RpczStore::AddCall(InboundCall* call) {
sampler->SampleCall(call);
}
-void RpczStore::DumpPB(const DumpRpczStoreRequestPB& req,
+void RpczStore::DumpPB(const DumpRpczStoreRequestPB& /* req */,
DumpRpczStoreResponsePB* resp) {
- vector<pair<RpcMethodInfo*, MethodSampler*>> samplers;
+ vector<pair<const RpcMethodInfo*, MethodSampler*>> samplers;
{
shared_lock<rw_spinlock> l(samplers_lock_.get_lock());
for (const auto& [mi, ms] : method_samplers_) {
diff --git a/src/kudu/rpc/rpcz_store.h b/src/kudu/rpc/rpcz_store.h
index 563f51754..099f481dc 100644
--- a/src/kudu/rpc/rpcz_store.h
+++ b/src/kudu/rpc/rpcz_store.h
@@ -65,7 +65,7 @@ class RpczStore final {
percpu_rwlock samplers_lock_;
// Protected by samplers_lock_.
- std::unordered_map<RpcMethodInfo*, std::unique_ptr<MethodSampler>> method_samplers_;
+ std::unordered_map<const RpcMethodInfo*, std::unique_ptr<MethodSampler>> method_samplers_;
DISALLOW_COPY_AND_ASSIGN(RpczStore);
};
diff --git a/src/kudu/rpc/service_pool.cc b/src/kudu/rpc/service_pool.cc
index 1d7a4a07b..ad5fb03cd 100644
--- a/src/kudu/rpc/service_pool.cc
+++ b/src/kudu/rpc/service_pool.cc
@@ -89,7 +89,7 @@ ServicePool::~ServicePool() {
Status ServicePool::Init(int num_threads) {
for (int i = 0; i < num_threads; i++) {
scoped_refptr<kudu::Thread> new_thread;
- CHECK_OK(kudu::Thread::Create(
+ RETURN_NOT_OK(kudu::Thread::Create(
Substitute("service pool $0", service_->service_name()),
"rpc worker",
[this]() { this->RunThread(); }, &new_thread));
@@ -101,11 +101,10 @@ Status ServicePool::Init(int num_threads) {
void ServicePool::Shutdown() {
service_queue_.Shutdown();
- std::lock_guard lock(shutdown_lock_);
- if (closing_) {
+ bool is_shut_down = false;
+ if (!closing_.compare_exchange_strong(is_shut_down, true)) {
return;
}
- closing_ = true;
// TODO(mpercy): Use a proper thread pool implementation.
for (scoped_refptr<kudu::Thread>& thread : threads_) {
CHECK_OK(ThreadJoiner(thread.get()).Join());
@@ -122,21 +121,20 @@ void ServicePool::Shutdown() {
}
void ServicePool::RejectTooBusy(InboundCall* c) {
- string err_msg =
- Substitute("$0 request on $1 from $2 dropped due to backpressure. "
- "The service queue is full; it has $3 items.",
- c->remote_method().method_name(),
- service_->service_name(),
- c->remote_address().ToString(),
- service_queue_.max_size());
rpcs_queue_overflow_->Increment();
- auto* minfo = c->method_info();
- if (minfo) {
+ if (const auto* minfo = c->method_info(); minfo != nullptr) {
minfo->queue_overflow_rejections->Increment();
}
- KLOG_EVERY_N_SECS(WARNING, 1) << err_msg << THROTTLE_MSG;
+ const string err_msg = Substitute(
+ "$0 request on $1 from $2 dropped due to backpressure: "
+ "service queue is full with $3 items",
+ c->remote_method().method_name(),
+ service_->service_name(),
+ c->remote_address().ToString(),
+ service_queue_.max_size());
c->RespondFailure(ErrorStatusPB::ERROR_SERVER_TOO_BUSY,
Status::ServiceUnavailable(err_msg));
+ KLOG_EVERY_N_SECS(WARNING, 1) << err_msg << THROTTLE_MSG;
DLOG(INFO) << err_msg << " Contents of service queue:\n"
<< service_queue_.ToString();
diff --git a/src/kudu/rpc/service_pool.h b/src/kudu/rpc/service_pool.h
index 62f68cef1..daaeb822c 100644
--- a/src/kudu/rpc/service_pool.h
+++ b/src/kudu/rpc/service_pool.h
@@ -16,6 +16,7 @@
// under the License.
#pragma once
+#include <atomic>
#include <cstddef>
#include <functional>
#include <memory>
@@ -27,7 +28,6 @@
#include "kudu/gutil/ref_counted.h"
#include "kudu/rpc/rpc_service.h"
#include "kudu/rpc/service_queue.h"
-#include "kudu/util/mutex.h"
#include "kudu/util/status.h"
namespace kudu {
@@ -99,8 +99,7 @@ class ServicePool : public RpcService {
scoped_refptr<Counter> rpcs_timed_out_in_queue_;
scoped_refptr<Counter> rpcs_queue_overflow_;
- Mutex shutdown_lock_;
- bool closing_;
+ std::atomic<bool> closing_;
std::function<void(void)> too_busy_hook_;