You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by al...@apache.org on 2020/10/22 23:28:58 UTC
[kudu] branch master updated (14dec6d -> 185d2c9)
This is an automated email from the ASF dual-hosted git repository.
alexey pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git.
from 14dec6d [tool] fix inconsistent definition among tool_action_common header and src
new 303f425 [backup] Fix Spark 2 and Scala 2.11 compilation
new 185d2c9 [common] introduce TxnId wrapper
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../scala/org/apache/kudu/backup/KuduBackup.scala | 3 +-
.../scala/org/apache/kudu/backup/KuduRestore.scala | 3 +-
src/kudu/common/CMakeLists.txt | 2 +
src/kudu/common/txn_id-test.cc | 89 ++++++++++++++++++++++
.../{util/website_util.cc => common/txn_id.cc} | 31 ++++----
src/kudu/common/txn_id.h | 80 +++++++++++++++++++
6 files changed, 191 insertions(+), 17 deletions(-)
create mode 100644 src/kudu/common/txn_id-test.cc
copy src/kudu/{util/website_util.cc => common/txn_id.cc} (61%)
create mode 100644 src/kudu/common/txn_id.h
[kudu] 02/02: [common] introduce TxnId wrapper
Posted by al...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
alexey pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git
commit 185d2c9faa403a6214520ea2d98d8833ab7d36ea
Author: Alexey Serbin <al...@apache.org>
AuthorDate: Tue Oct 20 16:50:22 2020 -0700
[common] introduce TxnId wrapper
TxnId is a wrapper around int64_t with transaction-related
specifics. It's an alternative to boost::optional<int64_t> since
it lightweight and it's possible to use it as an atomic
(i.e. std::atomic<TxnId> works as expected).
This patch also contains appropriate test coverage for the newly
introduced functionality.
Change-Id: I0511637702e946f140dc617ee6bf7e1c3d5289e3
Reviewed-on: http://gerrit.cloudera.org:8080/16624
Tested-by: Kudu Jenkins
Reviewed-by: Andrew Wong <aw...@cloudera.com>
---
src/kudu/common/CMakeLists.txt | 2 +
src/kudu/common/txn_id-test.cc | 89 ++++++++++++++++++++++++++++++++++++++++++
src/kudu/common/txn_id.cc | 48 +++++++++++++++++++++++
src/kudu/common/txn_id.h | 80 +++++++++++++++++++++++++++++++++++++
4 files changed, 219 insertions(+)
diff --git a/src/kudu/common/CMakeLists.txt b/src/kudu/common/CMakeLists.txt
index b9aa87f..e25d07e 100644
--- a/src/kudu/common/CMakeLists.txt
+++ b/src/kudu/common/CMakeLists.txt
@@ -61,6 +61,7 @@ set(COMMON_SRCS
schema.cc
table_util.cc
timestamp.cc
+ txn_id.cc
types.cc
wire_protocol.cc
zp7.cc)
@@ -99,5 +100,6 @@ ADD_KUDU_TEST(row_operations-test)
ADD_KUDU_TEST(scan_spec-test)
ADD_KUDU_TEST(schema-test)
ADD_KUDU_TEST(table_util-test)
+ADD_KUDU_TEST(txn_id-test)
ADD_KUDU_TEST(types-test)
ADD_KUDU_TEST(wire_protocol-test NUM_SHARDS 10)
diff --git a/src/kudu/common/txn_id-test.cc b/src/kudu/common/txn_id-test.cc
new file mode 100644
index 0000000..92e6859
--- /dev/null
+++ b/src/kudu/common/txn_id-test.cc
@@ -0,0 +1,89 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "kudu/common/txn_id.h"
+
+#include <cstdint>
+
+#include <glog/logging.h>
+#include <gtest/gtest.h>
+
+namespace kudu {
+
+// Basic unit test scenario for TxnId.
+TEST(TxnIdTest, Basic) {
+ TxnId id(0);
+ ASSERT_TRUE(id.IsValid());
+ ASSERT_EQ(0, id.value());
+
+ // Implicit convesion: operator int64_t().
+ int64_t id_value = id;
+ ASSERT_EQ(0, id_value);
+
+ // operator!()
+ ASSERT_FALSE(!id);
+
+ ASSERT_EQ("0", id.ToString());
+
+ TxnId invalid_id;
+ ASSERT_GT(id, invalid_id);
+
+ TxnId id_0(0);
+ ASSERT_EQ(id, id_0);
+ ASSERT_TRUE(id == id_0);
+ ASSERT_TRUE(id_0 >= id);
+ ASSERT_TRUE(id_0 <= id);
+
+ TxnId id_1(1);
+ ASSERT_GT(id_1, id);
+ ASSERT_TRUE(id_1 > id);
+ ASSERT_TRUE(id_1 >= id);
+ ASSERT_TRUE(id < id_1);
+ ASSERT_TRUE(id <= id_1);
+}
+
+// Test scenarios specific to invalid transaction identifiers.
+TEST(TxnIdTest, InvalidTxnIdValue) {
+ // Non-initialized txn id has invalid value.
+ TxnId invalid_id;
+
+ // operator!()
+ ASSERT_TRUE(!invalid_id);
+
+#if DCHECK_IS_ON()
+ ASSERT_DEATH({
+ auto v = invalid_id.value();
+ TxnId(v).ToString(); // unreachable
+ },
+ "TxnId contains an invalid value");
+#endif // #if DCHECK_IS_ON() ...
+
+ ASSERT_EQ("InvalidTxnId", invalid_id.ToString());
+
+#if DCHECK_IS_ON()
+ ASSERT_DEATH({
+ TxnId id(-1);
+ },
+ "negative value is not allowed for TxnId");
+ ASSERT_DEATH({
+ TxnId id(-2);
+ },
+ "negative value is not allowed for TxnId");
+#endif // #if DCHECK_IS_ON() ...
+}
+
+} // namespace kudu
diff --git a/src/kudu/common/txn_id.cc b/src/kudu/common/txn_id.cc
new file mode 100644
index 0000000..29abf60
--- /dev/null
+++ b/src/kudu/common/txn_id.cc
@@ -0,0 +1,48 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "kudu/common/txn_id.h"
+
+#include <ostream> // IWYU pragma: keep
+
+#include <glog/logging.h>
+
+#include "kudu/gutil/strings/substitute.h"
+
+namespace kudu {
+
+const TxnId TxnId::kInvalidTxnId;
+
+TxnId::TxnId(int64_t id) noexcept
+ : id_(id) {
+ DCHECK_GE(id, 0) << "negative value is not allowed for TxnId";
+}
+
+int64_t TxnId::value() const {
+ DCHECK(IsValid()) << "TxnId contains an invalid value";
+ return id_;
+}
+
+std::string TxnId::ToString() const {
+ return IsValid() ? strings::Substitute("$0", id_) : "InvalidTxnId";
+}
+
+std::ostream& operator<<(std::ostream& o, const TxnId& id) {
+ return o << id.ToString();
+}
+
+} // namespace kudu
diff --git a/src/kudu/common/txn_id.h b/src/kudu/common/txn_id.h
new file mode 100644
index 0000000..c7cd50e
--- /dev/null
+++ b/src/kudu/common/txn_id.h
@@ -0,0 +1,80 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+#pragma once
+
+#include <cstdint>
+#include <iosfwd>
+#include <string>
+
+namespace kudu {
+
+// A representation of transaction identifier.
+class TxnId {
+ public:
+ // An invalid transaction identifier: the default constructed instances are
+ // initialized to this value.
+ static const TxnId kInvalidTxnId;
+
+ TxnId() noexcept : id_(-1) {}
+ explicit TxnId(int64_t id) noexcept;
+
+ int64_t value() const;
+ bool IsValid() const { return id_ >= 0; }
+ std::string ToString() const;
+
+ // Cast operator: returns TxnId's int64_t value.
+ operator int64_t () const { return value(); }
+ bool operator!() const { return !IsValid(); }
+
+ private:
+ friend bool operator==(const TxnId& lhs, const TxnId& rhs);
+ friend bool operator!=(const TxnId& lhs, const TxnId& rhs);
+ friend bool operator<(const TxnId& lhs, const TxnId& rhs);
+ friend bool operator<=(const TxnId& lhs, const TxnId& rhs);
+ friend bool operator>(const TxnId& lhs, const TxnId& rhs);
+ friend bool operator>=(const TxnId& lhs, const TxnId& rhs);
+
+ const int64_t id_;
+};
+
+std::ostream& operator<<(std::ostream& o, const TxnId& txn_id);
+
+inline bool operator==(const TxnId& lhs, const TxnId& rhs) {
+ return lhs.id_ == rhs.id_;
+}
+
+inline bool operator!=(const TxnId& lhs, const TxnId& rhs) {
+ return !(lhs.id_ == rhs.id_);
+}
+
+inline bool operator<(const TxnId& lhs, const TxnId& rhs) {
+ return lhs.id_ < rhs.id_;
+}
+
+inline bool operator>(const TxnId& lhs, const TxnId& rhs) {
+ return rhs < lhs;
+}
+
+inline bool operator<=(const TxnId& lhs, const TxnId& rhs) {
+ return !(lhs > rhs);
+}
+
+inline bool operator>=(const TxnId& lhs, const TxnId& rhs) {
+ return !(lhs < rhs);
+}
+
+} // namespace kudu
[kudu] 01/02: [backup] Fix Spark 2 and Scala 2.11 compilation
Posted by al...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
alexey pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git
commit 303f4253ee5d5e18d2dbfa73e55091d2e50cdfb6
Author: Grant Henke <gr...@apache.org>
AuthorDate: Thu Oct 22 11:40:02 2020 -0500
[backup] Fix Spark 2 and Scala 2.11 compilation
When compiling with Spark 2 and Scala 2.11, the build fails
due to the use of `java.util.concurrent.ForkJoinPool` which
was added in 41a7c4e. This patch reverts that part of the change.
Change-Id: I75110921a536b6c78d3231251ff68f6f25dba6dc
Reviewed-on: http://gerrit.cloudera.org:8080/16629
Tested-by: Kudu Jenkins
Reviewed-by: Andrew Wong <aw...@cloudera.com>
---
.../kudu-backup/src/main/scala/org/apache/kudu/backup/KuduBackup.scala | 3 +--
.../src/main/scala/org/apache/kudu/backup/KuduRestore.scala | 3 +--
2 files changed, 2 insertions(+), 4 deletions(-)
diff --git a/java/kudu-backup/src/main/scala/org/apache/kudu/backup/KuduBackup.scala b/java/kudu-backup/src/main/scala/org/apache/kudu/backup/KuduBackup.scala
index 4d97452..c02f5de 100644
--- a/java/kudu-backup/src/main/scala/org/apache/kudu/backup/KuduBackup.scala
+++ b/java/kudu-backup/src/main/scala/org/apache/kudu/backup/KuduBackup.scala
@@ -17,8 +17,6 @@
package org.apache.kudu.backup
-import java.util.concurrent.ForkJoinPool
-
import org.apache.kudu.spark.kudu.KuduContext
import org.apache.spark.sql.SaveMode
import org.apache.spark.sql.SparkSession
@@ -28,6 +26,7 @@ import org.slf4j.Logger
import org.slf4j.LoggerFactory
import scala.collection.JavaConverters._
import scala.collection.parallel.ForkJoinTaskSupport
+import scala.concurrent.forkjoin.ForkJoinPool
import scala.util.Failure
import scala.util.Success
import scala.util.Try
diff --git a/java/kudu-backup/src/main/scala/org/apache/kudu/backup/KuduRestore.scala b/java/kudu-backup/src/main/scala/org/apache/kudu/backup/KuduRestore.scala
index f7ce9b4..6549cd6 100644
--- a/java/kudu-backup/src/main/scala/org/apache/kudu/backup/KuduRestore.scala
+++ b/java/kudu-backup/src/main/scala/org/apache/kudu/backup/KuduRestore.scala
@@ -17,8 +17,6 @@
package org.apache.kudu.backup
-import java.util.concurrent.ForkJoinPool
-
import org.apache.kudu.backup.Backup.TableMetadataPB
import org.apache.kudu.client.AlterTableOptions
import org.apache.kudu.client.KuduPartitioner
@@ -34,6 +32,7 @@ import org.slf4j.Logger
import org.slf4j.LoggerFactory
import scala.collection.JavaConverters._
import scala.collection.parallel.ForkJoinTaskSupport
+import scala.concurrent.forkjoin.ForkJoinPool
import scala.util.Failure
import scala.util.Success
import scala.util.Try