You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by gr...@apache.org on 2020/11/30 00:19:23 UTC
[kudu] branch master updated (42425c0 -> 0e2c288)
This is an automated email from the ASF dual-hosted git repository.
granthenke pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git.
from 42425c0 [tools] add missing top-level entries into TestModeHelp
new f47caaf txns: use HostPorts instead of strings to build TxnSystemClient
new 0e2c288 [thirdparty] Upgrade CMake to 3.19.1
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
CMakeLists.txt | 5 +++++
src/kudu/codegen/CMakeLists.txt | 11 +++--------
src/kudu/integration-tests/auth_token_expire-itest.cc | 6 +-----
.../integration-tests/client-negotiation-failover-itest.cc | 7 +------
src/kudu/integration-tests/master_authz-itest.cc | 9 ++-------
src/kudu/integration-tests/master_hms-itest.cc | 7 +------
src/kudu/integration-tests/txn_status_table-itest.cc | 13 ++-----------
src/kudu/master/txn_manager.cc | 7 +------
src/kudu/transactions/txn_system_client.cc | 9 +++++++--
src/kudu/transactions/txn_system_client.h | 4 +++-
src/kudu/util/env_posix.cc | 5 +++++
thirdparty/vars.sh | 2 +-
12 files changed, 32 insertions(+), 53 deletions(-)
[kudu] 02/02: [thirdparty] Upgrade CMake to 3.19.1
Posted by gr...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
granthenke pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git
commit 0e2c28864c27727c9723db7f319a5d7ebcbeead7
Author: Grant Henke <gr...@apache.org>
AuthorDate: Thu Nov 19 13:59:44 2020 -0600
[thirdparty] Upgrade CMake to 3.19.1
CMake 3.19.1 has some improvements when building on macOS,
especially with Xcode 12+.
This patch also includes some fixes for macOS that would
likely occur when using this new version of CMake and
potentially occurred on older versions as well depending
on the environment.
Change-Id: I125448a3057d0c9da8c82f7e29683ef7844b0aa8
Reviewed-on: http://gerrit.cloudera.org:8080/16785
Tested-by: Grant Henke <gr...@apache.org>
Reviewed-by: Alexey Serbin <as...@cloudera.com>
---
CMakeLists.txt | 5 +++++
src/kudu/codegen/CMakeLists.txt | 11 +++--------
src/kudu/util/env_posix.cc | 5 +++++
thirdparty/vars.sh | 2 +-
4 files changed, 14 insertions(+), 9 deletions(-)
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 5a3b607..aeccb9a 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -24,6 +24,11 @@ cmake_minimum_required(VERSION 3.11.2)
project(Kudu)
+# Print out the CMAKE_OSX_SYSROOT to help debug Apple build issues.
+if (CMAKE_OSX_SYSROOT)
+ message(STATUS "Using CMAKE_OSX_SYSROOT: ${CMAKE_OSX_SYSROOT}")
+endif()
+
# Prevent builds from the top-level source directory. This ensures that build
# output is well isolated from the source tree.
#
diff --git a/src/kudu/codegen/CMakeLists.txt b/src/kudu/codegen/CMakeLists.txt
index 5ce82ac..19ef2f5 100644
--- a/src/kudu/codegen/CMakeLists.txt
+++ b/src/kudu/codegen/CMakeLists.txt
@@ -69,7 +69,7 @@ if (APPLE)
# Clang does not know about by default.
set(PREFIXED_IR_INCLUDES
${PREFIXED_IR_INCLUDES}
- -cxx-isystem "/Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/include/c++/v1")
+ -cxx-isystem "/Library/Developer/CommandLineTools/usr/include/c++/v1")
endif()
# Get preprocessing definitions, which enable directives for glog and gtest.
@@ -90,19 +90,14 @@ set(IR_FLAGS
${PREFIXED_IR_INCLUDES})
# /usr/include doesn't have the necessary headers on macOS 10.14 (Mojave) with
-# Xcode 10[1]. In this case the --sysroot has to be set to the output of "xcrun
+# Xcode 10[1]+. In this case the --sysroot has to be set to the output of "xcrun
# --show-sdk-path" for the thirdparty clang-6.0 to work correctly.
#
# [1] https://developer.apple.com/documentation/xcode_release_notes/xcode_10_release_notes
if (APPLE AND NOT EXISTS /usr/include/libc.h)
- EXECUTE_PROCESS (COMMAND
- xcrun --show-sdk-path
- OUTPUT_STRIP_TRAILING_WHITESPACE
- OUTPUT_VARIABLE SDKPATH)
-
set(IR_FLAGS
${IR_FLAGS}
- --sysroot="${SDKPATH}")
+ --sysroot="${CMAKE_OSX_SYSROOT}")
endif()
# Provide clang with any explicitly defined system prefix path. If there isn't
diff --git a/src/kudu/util/env_posix.cc b/src/kudu/util/env_posix.cc
index 4b1078a..4a83314 100644
--- a/src/kudu/util/env_posix.cc
+++ b/src/kudu/util/env_posix.cc
@@ -248,6 +248,9 @@ int fallocate(int fd, int mode, off_t offset, off_t len) {
return 0;
}
+// Implementations for `preadv` and `pwritev` are available in the MacOSX11+ SDK.
+// We provide simulated implementations for older versions.
+#if defined(__MAC_OS_X_VERSION_MAX_ALLOWED) && __MAC_OS_X_VERSION_MAX_ALLOWED < 110000
// Simulates Linux's preadv API on OS X.
ssize_t preadv(int fd, const struct iovec* iovec, int count, off_t offset) {
ssize_t total_read_bytes = 0;
@@ -285,6 +288,8 @@ ssize_t pwritev(int fd, const struct iovec* iovec, int count, off_t offset) {
}
#endif
+#endif
+
void DoClose(int fd) {
int err;
RETRY_ON_EINTR(err, close(fd));
diff --git a/thirdparty/vars.sh b/thirdparty/vars.sh
index 8233075..04fa883 100644
--- a/thirdparty/vars.sh
+++ b/thirdparty/vars.sh
@@ -60,7 +60,7 @@ PROTOBUF_SOURCE=$TP_SOURCE_DIR/$PROTOBUF_NAME
# Note: CMake gets patched on SLES12SP0. When changing the CMake version, please check if
# cmake-issue-15873-dont-use-select.patch needs to be updated.
-CMAKE_VERSION=3.16.4
+CMAKE_VERSION=3.19.1
CMAKE_NAME=cmake-$CMAKE_VERSION
CMAKE_SOURCE=$TP_SOURCE_DIR/$CMAKE_NAME
[kudu] 01/02: txns: use HostPorts instead of strings to build
TxnSystemClient
Posted by gr...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
granthenke pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git
commit f47caaf3ee4339f5e34b9ad75ba4cbcef686c508
Author: Andrew Wong <aw...@cloudera.com>
AuthorDate: Wed Nov 25 22:25:31 2020 -0800
txns: use HostPorts instead of strings to build TxnSystemClient
Strings are the inputs for the ClientBuilder, but in most tests, we have
HostPorts available, so we currently convert the master addresses to a
vector<string> and pass it to the TxnSystemClient consturctor. Rather
than converting at all the call-sites, this patch converts the
constructor to use HostPorts instead.
Change-Id: I92a09a256e84020756f3073cd7289a2245058e4a
Reviewed-on: http://gerrit.cloudera.org:8080/16787
Tested-by: Kudu Jenkins
Reviewed-by: Alexey Serbin <as...@cloudera.com>
---
src/kudu/integration-tests/auth_token_expire-itest.cc | 6 +-----
.../integration-tests/client-negotiation-failover-itest.cc | 7 +------
src/kudu/integration-tests/master_authz-itest.cc | 9 ++-------
src/kudu/integration-tests/master_hms-itest.cc | 7 +------
src/kudu/integration-tests/txn_status_table-itest.cc | 13 ++-----------
src/kudu/master/txn_manager.cc | 7 +------
src/kudu/transactions/txn_system_client.cc | 9 +++++++--
src/kudu/transactions/txn_system_client.h | 4 +++-
8 files changed, 18 insertions(+), 44 deletions(-)
diff --git a/src/kudu/integration-tests/auth_token_expire-itest.cc b/src/kudu/integration-tests/auth_token_expire-itest.cc
index 4f07bb9..b553cce 100644
--- a/src/kudu/integration-tests/auth_token_expire-itest.cc
+++ b/src/kudu/integration-tests/auth_token_expire-itest.cc
@@ -456,12 +456,8 @@ TEST_F(TokenBasedConnectionITest, ReacquireAuthnToken) {
// of the transaction status table.
TEST_F(TokenBasedConnectionITest, TxnSystemClientReacquireAuthnToken) {
SKIP_IF_SLOW_NOT_ALLOWED();
- vector<string> master_addrs;
- for (const auto& hp : cluster_->master_rpc_addrs()) {
- master_addrs.emplace_back(hp.ToString());
- }
unique_ptr<TxnSystemClient> txn_client;
- ASSERT_OK(TxnSystemClient::Create(master_addrs, &txn_client));
+ ASSERT_OK(TxnSystemClient::Create(cluster_->master_rpc_addrs(), &txn_client));
ASSERT_OK(txn_client->CreateTxnStatusTable(10));
ASSERT_OK(txn_client->OpenTxnStatusTable());
diff --git a/src/kudu/integration-tests/client-negotiation-failover-itest.cc b/src/kudu/integration-tests/client-negotiation-failover-itest.cc
index 7b80bba..ee831b6 100644
--- a/src/kudu/integration-tests/client-negotiation-failover-itest.cc
+++ b/src/kudu/integration-tests/client-negotiation-failover-itest.cc
@@ -36,7 +36,6 @@
#include "kudu/tablet/key_value_test_schema.h"
#include "kudu/transactions/txn_system_client.h"
#include "kudu/util/monotime.h"
-#include "kudu/util/net/net_util.h"
#include "kudu/util/scoped_cleanup.h"
#include "kudu/util/status.h"
#include "kudu/util/test_macros.h"
@@ -207,12 +206,8 @@ TEST_F(ClientFailoverOnNegotiationTimeoutITest, TestTxnSystemClientRetryOnPause)
cluster_opts_.num_tablet_servers = kNumTabletServers;
ASSERT_OK(CreateAndStartCluster());
- vector<string> master_addrs;
- for (const auto& hp : cluster_->master_rpc_addrs()) {
- master_addrs.emplace_back(hp.ToString());
- }
unique_ptr<TxnSystemClient> txn_client;
- ASSERT_OK(TxnSystemClient::Create(master_addrs, &txn_client));
+ ASSERT_OK(TxnSystemClient::Create(cluster_->master_rpc_addrs(), &txn_client));
ASSERT_OK(txn_client->CreateTxnStatusTable(100, kNumTabletServers));
ASSERT_OK(txn_client->OpenTxnStatusTable());
diff --git a/src/kudu/integration-tests/master_authz-itest.cc b/src/kudu/integration-tests/master_authz-itest.cc
index bbce5c1..cf1864c 100644
--- a/src/kudu/integration-tests/master_authz-itest.cc
+++ b/src/kudu/integration-tests/master_authz-itest.cc
@@ -49,7 +49,6 @@
#include "kudu/transactions/txn_system_client.h"
#include "kudu/util/decimal_util.h"
#include "kudu/util/monotime.h"
-#include "kudu/util/net/net_util.h"
#include "kudu/util/slice.h"
#include "kudu/util/status.h"
#include "kudu/util/test_macros.h"
@@ -741,16 +740,12 @@ INSTANTIATE_TEST_CASE_P(AuthzProviders, MasterAuthzITest,
TEST_P(MasterAuthzITest, TestCreateTransactionStatusTable) {
// Create a transaction status table and add a range. Both requests should
// succeed, despite no privileges being granted in Ranger.
- vector<string> master_addrs;
- for (const auto& hp : cluster_->master_rpc_addrs()) {
- master_addrs.emplace_back(hp.ToString());
- }
ASSERT_OK(this->cluster_->kdc()->Kinit(kTestUser));
// Attempting to call system client DDL as a non-admin should result in a
// NotAuthorized error.
{
unique_ptr<TxnSystemClient> non_admin_client;
- ASSERT_OK(TxnSystemClient::Create(master_addrs, &non_admin_client));
+ ASSERT_OK(TxnSystemClient::Create(cluster_->master_rpc_addrs(), &non_admin_client));
Status s = non_admin_client->CreateTxnStatusTable(100);
ASSERT_TRUE(s.IsNotAuthorized()) << s.ToString();
s = non_admin_client->AddTxnStatusTableRange(100, 200);
@@ -759,7 +754,7 @@ TEST_P(MasterAuthzITest, TestCreateTransactionStatusTable) {
// But as service user, we should have no trouble making the calls.
ASSERT_OK(this->cluster_->kdc()->Kinit(kAdminUser));
unique_ptr<TxnSystemClient> txn_sys_client;
- ASSERT_OK(TxnSystemClient::Create(master_addrs, &txn_sys_client));
+ ASSERT_OK(TxnSystemClient::Create(cluster_->master_rpc_addrs(), &txn_sys_client));
ASSERT_OK(txn_sys_client->CreateTxnStatusTable(100));
ASSERT_OK(txn_sys_client->AddTxnStatusTableRange(100, 200));
}
diff --git a/src/kudu/integration-tests/master_hms-itest.cc b/src/kudu/integration-tests/master_hms-itest.cc
index 621fea9..bcb3911 100644
--- a/src/kudu/integration-tests/master_hms-itest.cc
+++ b/src/kudu/integration-tests/master_hms-itest.cc
@@ -45,7 +45,6 @@
#include "kudu/thrift/client.h"
#include "kudu/transactions/txn_system_client.h"
#include "kudu/util/monotime.h"
-#include "kudu/util/net/net_util.h"
#include "kudu/util/status.h"
#include "kudu/util/test_macros.h"
#include "kudu/util/test_util.h"
@@ -677,12 +676,8 @@ TEST_F(MasterHmsTest, TestUppercaseIdentifiers) {
// synchronized to the HMS.
TEST_F(MasterHmsTest, TestTransactionStatusTableDoesntSync) {
// Create a transaction status table.
- vector<string> master_addrs;
- for (const auto& hp : cluster_->master_rpc_addrs()) {
- master_addrs.emplace_back(hp.ToString());
- }
unique_ptr<TxnSystemClient> txn_sys_client;
- ASSERT_OK(TxnSystemClient::Create(master_addrs, &txn_sys_client));
+ ASSERT_OK(TxnSystemClient::Create(cluster_->master_rpc_addrs(), &txn_sys_client));
ASSERT_OK(txn_sys_client->CreateTxnStatusTable(100));
// We shouldn't see the table in the HMS catalog.
diff --git a/src/kudu/integration-tests/txn_status_table-itest.cc b/src/kudu/integration-tests/txn_status_table-itest.cc
index edeb4bb..aa94827 100644
--- a/src/kudu/integration-tests/txn_status_table-itest.cc
+++ b/src/kudu/integration-tests/txn_status_table-itest.cc
@@ -57,7 +57,6 @@
#include "kudu/tserver/tablet_server.h"
#include "kudu/tserver/ts_tablet_manager.h"
#include "kudu/util/monotime.h"
-#include "kudu/util/net/net_util.h"
#include "kudu/util/scoped_cleanup.h"
#include "kudu/util/status.h"
#include "kudu/util/test_macros.h"
@@ -108,11 +107,7 @@ class TxnStatusTableITest : public KuduTest {
ASSERT_OK(cluster_->Start());
// Create the txn system client with which to communicate with the cluster.
- vector<string> master_addrs;
- for (const auto& hp : cluster_->master_rpc_addrs()) {
- master_addrs.emplace_back(hp.ToString());
- }
- ASSERT_OK(TxnSystemClient::Create(master_addrs, &txn_sys_client_));
+ ASSERT_OK(TxnSystemClient::Create(cluster_->master_rpc_addrs(), &txn_sys_client_));
}
// Ensures that all replicas have the right table type set.
@@ -631,11 +626,7 @@ class MultiServerTxnStatusTableITest : public TxnStatusTableITest {
opts.num_tablet_servers = 4;
cluster_.reset(new InternalMiniCluster(env_, std::move(opts)));
ASSERT_OK(cluster_->Start());
- vector<string> master_addrs;
- for (const auto& hp : cluster_->master_rpc_addrs()) {
- master_addrs.emplace_back(hp.ToString());
- }
- ASSERT_OK(TxnSystemClient::Create(master_addrs, &txn_sys_client_));
+ ASSERT_OK(TxnSystemClient::Create(cluster_->master_rpc_addrs(), &txn_sys_client_));
// Create the initial transaction status table partitions and start an
// initial transaction.
diff --git a/src/kudu/master/txn_manager.cc b/src/kudu/master/txn_manager.cc
index 7d6596d..4a016bc 100644
--- a/src/kudu/master/txn_manager.cc
+++ b/src/kudu/master/txn_manager.cc
@@ -247,12 +247,7 @@ Status TxnManager::Init() {
}
vector<HostPort> hostports;
RETURN_NOT_OK(server_->GetMasterHostPorts(&hostports));
- vector<string> master_addrs;
- master_addrs.reserve(hostports.size());
- for (const auto& hp : hostports) {
- master_addrs.emplace_back(hp.ToString());
- }
- RETURN_NOT_OK(TxnSystemClient::Create(master_addrs, &txn_sys_client_));
+ RETURN_NOT_OK(TxnSystemClient::Create(hostports, &txn_sys_client_));
DCHECK(txn_sys_client_);
auto s = txn_sys_client_->CreateTxnStatusTable(
FLAGS_txn_manager_status_table_range_partition_span,
diff --git a/src/kudu/transactions/txn_system_client.cc b/src/kudu/transactions/txn_system_client.cc
index cfefae4..2d999e1 100644
--- a/src/kudu/transactions/txn_system_client.cc
+++ b/src/kudu/transactions/txn_system_client.cc
@@ -40,6 +40,7 @@
#include "kudu/transactions/transactions.pb.h"
#include "kudu/tserver/tserver_admin.pb.h"
#include "kudu/util/async_util.h"
+#include "kudu/util/net/net_util.h"
using kudu::client::KuduClient;
using kudu::client::KuduSchema;
@@ -57,11 +58,15 @@ using std::vector;
namespace kudu {
namespace transactions {
-Status TxnSystemClient::Create(const vector<string>& master_addrs,
+Status TxnSystemClient::Create(const vector<HostPort>& master_addrs,
unique_ptr<TxnSystemClient>* sys_client) {
+ vector<string> master_strings;
+ for (const auto& hp : master_addrs) {
+ master_strings.emplace_back(hp.ToString());
+ }
DCHECK(!master_addrs.empty());
KuduClientBuilder builder;
- builder.master_server_addrs(master_addrs);
+ builder.master_server_addrs(master_strings);
client::sp::shared_ptr<KuduClient> client;
RETURN_NOT_OK(builder.Build(&client));
sys_client->reset(new TxnSystemClient(std::move(client)));
diff --git a/src/kudu/transactions/txn_system_client.h b/src/kudu/transactions/txn_system_client.h
index f5b3a21..acf1d67 100644
--- a/src/kudu/transactions/txn_system_client.h
+++ b/src/kudu/transactions/txn_system_client.h
@@ -32,6 +32,8 @@
#include "kudu/util/status_callback.h"
namespace kudu {
+class HostPort;
+
namespace client {
class KuduClient;
class KuduTable;
@@ -55,7 +57,7 @@ class TxnStatusEntryPB;
// calls to various servers.
class TxnSystemClient {
public:
- static Status Create(const std::vector<std::string>& master_addrs,
+ static Status Create(const std::vector<HostPort>& master_addrs,
std::unique_ptr<TxnSystemClient>* sys_client);
// Creates the transaction status table with a single range partition of the