You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by wd...@apache.org on 2019/01/29 17:45:09 UTC
[kudu] 02/02: [tools] Describe more table attributes
This is an automated email from the ASF dual-hosted git repository.
wdberkeley pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git
commit 3047db9283432746a3282b180d41f08a97d4aa3f
Author: acelyc111 <40...@qq.com>
AuthorDate: Tue Jan 22 00:51:23 2019 +0800
[tools] Describe more table attributes
This adds more attributes to `kudu table describe`, including column
encoding type, compression type, and default read/write value.
My use case for the extra information is to compare two tables in the
same or different clusters: export the full descriptions of the two
tables into text, and then use any text compare tool to check the
difference.
Change-Id: I4f8cf682f3a2a30f98edbc4aff6951225d8d81e8
Reviewed-on: http://gerrit.cloudera.org:8080/12281
Tested-by: Kudu Jenkins
Reviewed-by: Will Berkeley <wd...@gmail.com>
---
src/kudu/cfile/encoding-test.cc | 1 -
src/kudu/client/schema.cc | 11 +++++-
src/kudu/common/schema-test.cc | 2 +-
src/kudu/common/schema.cc | 24 ++++++++---
src/kudu/common/schema.h | 23 ++++++++---
src/kudu/common/types.h | 1 +
src/kudu/tools/kudu-admin-test.cc | 79 +++++++++++++++++++++++++++++++------
src/kudu/tools/kudu-tool-test.cc | 1 -
src/kudu/tools/tool_action_table.cc | 1 +
src/kudu/util/int128_util.h | 3 +-
10 files changed, 118 insertions(+), 28 deletions(-)
diff --git a/src/kudu/cfile/encoding-test.cc b/src/kudu/cfile/encoding-test.cc
index 9bffc8f..5d6e3b7 100644
--- a/src/kudu/cfile/encoding-test.cc
+++ b/src/kudu/cfile/encoding-test.cc
@@ -51,7 +51,6 @@
#include "kudu/util/group_varint-inl.h"
#include "kudu/util/hexdump.h"
#include "kudu/util/int128.h"
-#include "kudu/util/int128_util.h" // IWYU pragma: keep
#include "kudu/util/memory/arena.h"
#include "kudu/util/random.h"
#include "kudu/util/random_util.h"
diff --git a/src/kudu/client/schema.cc b/src/kudu/client/schema.cc
index 6e7824e..cdc510c 100644
--- a/src/kudu/client/schema.cc
+++ b/src/kudu/client/schema.cc
@@ -24,6 +24,7 @@
#include <utility>
#include <boost/optional/optional.hpp>
+#include <gflags/gflags.h>
#include <glog/logging.h>
#include "kudu/client/schema-internal.h"
@@ -38,10 +39,14 @@
#include "kudu/gutil/macros.h"
#include "kudu/gutil/map-util.h"
#include "kudu/gutil/strings/substitute.h"
-#include "kudu/util/decimal_util.h"
#include "kudu/util/compression/compression.pb.h"
+#include "kudu/util/decimal_util.h"
#include "kudu/util/slice.h"
+DEFINE_bool(show_attributes, false,
+ "Whether to show column attributes, including column encoding type, "
+ "compression type, and default read/write value.");
+
MAKE_ENUM_LIMITS(kudu::client::KuduColumnStorageAttributes::EncodingType,
kudu::client::KuduColumnStorageAttributes::AUTO_ENCODING,
kudu::client::KuduColumnStorageAttributes::RLE);
@@ -747,7 +752,9 @@ void KuduSchema::GetPrimaryKeyColumnIndexes(vector<int>* indexes) const {
}
string KuduSchema::ToString() const {
- return schema_ ? schema_->ToString(Schema::ToStringMode::WITHOUT_COLUMN_IDS)
+ return schema_ ? schema_->ToString(FLAGS_show_attributes ?
+ Schema::ToStringMode::WITH_COLUMN_ATTRIBUTES
+ : Schema::ToStringMode::BASE_INFO)
: "()";
}
diff --git a/src/kudu/common/schema-test.cc b/src/kudu/common/schema-test.cc
index d32e666..198bf00 100644
--- a/src/kudu/common/schema-test.cc
+++ b/src/kudu/common/schema-test.cc
@@ -118,7 +118,7 @@ TEST_F(TestSchema, TestSchemaToStringMode) {
" key INT32 NOT NULL,\n"
" PRIMARY KEY (key)\n"
")",
- schema.ToString(Schema::ToStringMode::WITHOUT_COLUMN_IDS));
+ schema.ToString(Schema::ToStringMode::BASE_INFO));
}
TEST_F(TestSchema, TestCopyAndMove) {
diff --git a/src/kudu/common/schema.cc b/src/kudu/common/schema.cc
index df04e67..46f74a1 100644
--- a/src/kudu/common/schema.cc
+++ b/src/kudu/common/schema.cc
@@ -118,10 +118,11 @@ Status ColumnSchema::ApplyDelta(const ColumnSchemaDelta& col_delta) {
return Status::OK();
}
-string ColumnSchema::ToString() const {
- return Substitute("$0 $1",
+string ColumnSchema::ToString(ToStringMode mode) const {
+ return Substitute("$0 $1$2",
name_,
- TypeToString());
+ TypeToString(),
+ mode == ToStringMode::WITH_ATTRIBUTES ? " " + AttrToString() : "");
}
string ColumnSchema::TypeToString() const {
@@ -133,6 +134,13 @@ string ColumnSchema::TypeToString() const {
is_nullable_ ? "NULLABLE" : "NOT NULL");
}
+string ColumnSchema::AttrToString() const {
+ return Substitute("$0 $1 $2",
+ attributes_.ToString(),
+ has_read_default() ? Stringify(read_default_value()) : "-",
+ has_write_default() ? Stringify(write_default_value()) : "-");
+}
+
size_t ColumnSchema::memory_footprint_excluding_this() const {
// Rough approximation.
return name_.capacity();
@@ -399,14 +407,18 @@ string Schema::ToString(ToStringMode mode) const {
pk_strs.push_back(cols_[i].name());
}
+ auto col_mode = ColumnSchema::ToStringMode::WITHOUT_ATTRIBUTES;
+ if (mode & ToStringMode::WITH_COLUMN_ATTRIBUTES) {
+ col_mode = ColumnSchema::ToStringMode::WITH_ATTRIBUTES;
+ }
vector<string> col_strs;
- if (has_column_ids() && mode != ToStringMode::WITHOUT_COLUMN_IDS) {
+ if (has_column_ids() && (mode & ToStringMode::WITH_COLUMN_IDS)) {
for (int i = 0; i < cols_.size(); ++i) {
- col_strs.push_back(Substitute("$0:$1", col_ids_[i], cols_[i].ToString()));
+ col_strs.push_back(Substitute("$0:$1", col_ids_[i], cols_[i].ToString(col_mode)));
}
} else {
for (const ColumnSchema &col : cols_) {
- col_strs.push_back(col.ToString());
+ col_strs.push_back(col.ToString(col_mode));
}
}
diff --git a/src/kudu/common/schema.h b/src/kudu/common/schema.h
index 4c19969..cd6dbc9 100644
--- a/src/kudu/common/schema.h
+++ b/src/kudu/common/schema.h
@@ -228,14 +228,26 @@ class ColumnSchema {
return name_;
}
+ // Enum to configure how a ColumnSchema is stringified.
+ enum class ToStringMode {
+ // Include encoding type, compression type, and default read/write value.
+ WITH_ATTRIBUTES,
+ // Do not include above attributes.
+ WITHOUT_ATTRIBUTES,
+ };
+
// Return a string identifying this column, including its
// name.
- std::string ToString() const;
+ std::string ToString(ToStringMode mode = ToStringMode::WITHOUT_ATTRIBUTES) const;
// Same as above, but only including the type information.
// For example, "STRING NOT NULL".
std::string TypeToString() const;
+ // Same as above, but only including the attributes information.
+ // For example, "AUTO_ENCODING ZLIB 123 123".
+ std::string AttrToString() const;
+
// Returns true if the column has a read default value
bool has_read_default() const {
return read_default_ != nullptr;
@@ -724,11 +736,12 @@ class Schema {
}
// Enum to configure how a Schema is stringified.
- enum class ToStringMode {
+ enum ToStringMode {
+ BASE_INFO = 0,
// Include column ids if this instance has them.
- WITH_COLUMN_IDS,
- // Do not include column ids.
- WITHOUT_COLUMN_IDS,
+ WITH_COLUMN_IDS = 1 << 0,
+ // Include column attributes.
+ WITH_COLUMN_ATTRIBUTES = 1 << 1,
};
// Stringify this Schema. This is not particularly efficient,
// so should only be used when necessary for output.
diff --git a/src/kudu/common/types.h b/src/kudu/common/types.h
index b3603da..36f9b6f 100644
--- a/src/kudu/common/types.h
+++ b/src/kudu/common/types.h
@@ -37,6 +37,7 @@
#include "kudu/gutil/strings/escaping.h"
#include "kudu/gutil/strings/numbers.h"
#include "kudu/util/int128.h"
+#include "kudu/util/int128_util.h"
#include "kudu/util/slice.h"
// IWYU pragma: no_include "kudu/util/status.h"
diff --git a/src/kudu/tools/kudu-admin-test.cc b/src/kudu/tools/kudu-admin-test.cc
index a870026..a5c8b0c 100644
--- a/src/kudu/tools/kudu-admin-test.cc
+++ b/src/kudu/tools/kudu-admin-test.cc
@@ -38,6 +38,7 @@
#include "kudu/client/client.h"
#include "kudu/client/schema.h"
#include "kudu/client/shared_ptr.h"
+#include "kudu/client/value.h"
#include "kudu/client/write_op.h"
#include "kudu/common/common.pb.h"
#include "kudu/common/partial_row.h"
@@ -66,6 +67,7 @@
#include "kudu/util/net/net_util.h"
#include "kudu/util/net/sockaddr.h"
#include "kudu/util/pb_util.h"
+#include "kudu/util/slice.h"
#include "kudu/util/status.h"
#include "kudu/util/test_macros.h"
#include "kudu/util/test_util.h"
@@ -82,12 +84,14 @@ DECLARE_int32(num_tablet_servers);
using kudu::client::KuduClient;
using kudu::client::KuduClientBuilder;
using kudu::client::KuduColumnSchema;
+using kudu::client::KuduColumnStorageAttributes;
using kudu::client::KuduInsert;
using kudu::client::KuduSchema;
using kudu::client::KuduSchemaBuilder;
using kudu::client::KuduTable;
using kudu::client::KuduTableAlterer;
using kudu::client::KuduTableCreator;
+using kudu::client::KuduValue;
using kudu::client::sp::shared_ptr;
using kudu::cluster::ExternalTabletServer;
using kudu::consensus::COMMITTED_OPID;
@@ -1730,18 +1734,31 @@ TEST_F(AdminCliTest, TestDescribeTable) {
builder.AddColumn("key_hash1")->Type(KuduColumnSchema::INT32)->NotNull();
builder.AddColumn("key_hash2")->Type(KuduColumnSchema::INT32)->NotNull();
builder.AddColumn("key_range")->Type(KuduColumnSchema::INT32)->NotNull();
- builder.AddColumn("int8_val")->Type(KuduColumnSchema::INT8);
- builder.AddColumn("int16_val")->Type(KuduColumnSchema::INT16);
- builder.AddColumn("int32_val")->Type(KuduColumnSchema::INT32);
- builder.AddColumn("int64_val")->Type(KuduColumnSchema::INT64);
+ builder.AddColumn("int8_val")->Type(KuduColumnSchema::INT8)
+ ->Compression(KuduColumnStorageAttributes::CompressionType::NO_COMPRESSION)
+ ->Encoding(KuduColumnStorageAttributes::EncodingType::PLAIN_ENCODING);
+ builder.AddColumn("int16_val")->Type(KuduColumnSchema::INT16)
+ ->Compression(KuduColumnStorageAttributes::CompressionType::SNAPPY)
+ ->Encoding(KuduColumnStorageAttributes::EncodingType::RLE);
+ builder.AddColumn("int32_val")->Type(KuduColumnSchema::INT32)
+ ->Compression(KuduColumnStorageAttributes::CompressionType::LZ4)
+ ->Encoding(KuduColumnStorageAttributes::EncodingType::BIT_SHUFFLE);
+ builder.AddColumn("int64_val")->Type(KuduColumnSchema::INT64)
+ ->Compression(KuduColumnStorageAttributes::CompressionType::ZLIB)
+ ->Default(KuduValue::FromInt(123));
builder.AddColumn("timestamp_val")->Type(KuduColumnSchema::UNIXTIME_MICROS);
- builder.AddColumn("string_val")->Type(KuduColumnSchema::STRING);
- builder.AddColumn("bool_val")->Type(KuduColumnSchema::BOOL);
+ builder.AddColumn("string_val")->Type(KuduColumnSchema::STRING)
+ ->Encoding(KuduColumnStorageAttributes::EncodingType::PREFIX_ENCODING)
+ ->Default(KuduValue::CopyString(Slice("hello")));;
+ builder.AddColumn("bool_val")->Type(KuduColumnSchema::BOOL)
+ ->Default(KuduValue::FromBool(false));
builder.AddColumn("float_val")->Type(KuduColumnSchema::FLOAT);
- builder.AddColumn("double_val")->Type(KuduColumnSchema::DOUBLE);
- builder.AddColumn("binary_val")->Type(KuduColumnSchema::BINARY);
+ builder.AddColumn("double_val")->Type(KuduColumnSchema::DOUBLE)
+ ->Default(KuduValue::FromDouble(123.4));
+ builder.AddColumn("binary_val")->Type(KuduColumnSchema::BINARY)
+ ->Encoding(KuduColumnStorageAttributes::EncodingType::DICT_ENCODING);
builder.AddColumn("decimal_val")->Type(KuduColumnSchema::DECIMAL)
- ->Precision(10)
+ ->Precision(30)
->Scale(4);
builder.SetPrimaryKey({ "key_hash0", "key_hash1", "key_hash2", "key_range" });
ASSERT_OK(builder.Build(&schema));
@@ -1776,7 +1793,7 @@ TEST_F(AdminCliTest, TestDescribeTable) {
"table",
"describe",
cluster_->master()->bound_rpc_addr().ToString(),
- kAnotherTableId
+ kAnotherTableId,
}, &stdout, &stderr);
ASSERT_TRUE(s.ok()) << ToolRunInfo(s, stdout, stderr);
@@ -1797,7 +1814,47 @@ TEST_F(AdminCliTest, TestDescribeTable) {
" float_val FLOAT NULLABLE,\n"
" double_val DOUBLE NULLABLE,\n"
" binary_val BINARY NULLABLE,\n"
- " decimal_val DECIMAL(10, 4) NULLABLE,\n"
+ " decimal_val DECIMAL(30, 4) NULLABLE,\n"
+ " PRIMARY KEY (key_hash0, key_hash1, key_hash2, key_range)\n"
+ ")\n"
+ "HASH (key_hash0) PARTITIONS 2,\n"
+ "HASH (key_hash1, key_hash2) PARTITIONS 3,\n"
+ "RANGE (key_range) (\n"
+ " PARTITION 0 <= VALUES < 1,\n"
+ " PARTITION 2 <= VALUES < 3\n"
+ ")\n"
+ "REPLICAS 1");
+
+ // Test the describe output with `-show_attributes=true`.
+ stdout.clear();
+ stderr.clear();
+ s = RunKuduTool({
+ "table",
+ "describe",
+ cluster_->master()->bound_rpc_addr().ToString(),
+ kAnotherTableId,
+ "-show_attributes=true"
+ }, &stdout, &stderr);
+ ASSERT_TRUE(s.ok()) << ToolRunInfo(s, stdout, stderr);
+
+ ASSERT_STR_CONTAINS(
+ stdout,
+ "(\n"
+ " key_hash0 INT32 NOT NULL AUTO_ENCODING DEFAULT_COMPRESSION - -,\n"
+ " key_hash1 INT32 NOT NULL AUTO_ENCODING DEFAULT_COMPRESSION - -,\n"
+ " key_hash2 INT32 NOT NULL AUTO_ENCODING DEFAULT_COMPRESSION - -,\n"
+ " key_range INT32 NOT NULL AUTO_ENCODING DEFAULT_COMPRESSION - -,\n"
+ " int8_val INT8 NULLABLE PLAIN_ENCODING NO_COMPRESSION - -,\n"
+ " int16_val INT16 NULLABLE RLE SNAPPY - -,\n"
+ " int32_val INT32 NULLABLE BIT_SHUFFLE LZ4 - -,\n"
+ " int64_val INT64 NULLABLE AUTO_ENCODING ZLIB 123 123,\n"
+ " timestamp_val UNIXTIME_MICROS NULLABLE AUTO_ENCODING DEFAULT_COMPRESSION - -,\n"
+ " string_val STRING NULLABLE PREFIX_ENCODING DEFAULT_COMPRESSION \"hello\" \"hello\",\n"
+ " bool_val BOOL NULLABLE AUTO_ENCODING DEFAULT_COMPRESSION false false,\n"
+ " float_val FLOAT NULLABLE AUTO_ENCODING DEFAULT_COMPRESSION - -,\n"
+ " double_val DOUBLE NULLABLE AUTO_ENCODING DEFAULT_COMPRESSION 123.4 123.4,\n"
+ " binary_val BINARY NULLABLE DICT_ENCODING DEFAULT_COMPRESSION - -,\n"
+ " decimal_val DECIMAL(30, 4) NULLABLE AUTO_ENCODING DEFAULT_COMPRESSION - -,\n"
" PRIMARY KEY (key_hash0, key_hash1, key_hash2, key_range)\n"
")\n"
"HASH (key_hash0) PARTITIONS 2,\n"
diff --git a/src/kudu/tools/kudu-tool-test.cc b/src/kudu/tools/kudu-tool-test.cc
index 1c0633f..9838620 100644
--- a/src/kudu/tools/kudu-tool-test.cc
+++ b/src/kudu/tools/kudu-tool-test.cc
@@ -112,7 +112,6 @@
#include "kudu/tserver/tserver_admin.proxy.h"
#include "kudu/util/async_util.h"
#include "kudu/util/env.h"
-#include "kudu/util/int128_util.h" // IWYU pragma: keep
#include "kudu/util/metrics.h"
#include "kudu/util/monotime.h"
#include "kudu/util/net/net_util.h"
diff --git a/src/kudu/tools/tool_action_table.cc b/src/kudu/tools/tool_action_table.cc
index a0c3f06..4710c12 100644
--- a/src/kudu/tools/tool_action_table.cc
+++ b/src/kudu/tools/tool_action_table.cc
@@ -418,6 +418,7 @@ unique_ptr<Mode> BuildTableMode() {
.Description("Describe a table")
.AddRequiredParameter({ kMasterAddressesArg, kMasterAddressesArgDesc })
.AddRequiredParameter({ kTableNameArg, "Name of the table to describe" })
+ .AddOptionalParameter("show_attributes")
.Build();
unique_ptr<Action> list_tables =
diff --git a/src/kudu/util/int128_util.h b/src/kudu/util/int128_util.h
index 2d01de7..5965da1 100644
--- a/src/kudu/util/int128_util.h
+++ b/src/kudu/util/int128_util.h
@@ -15,6 +15,8 @@
// specific language governing permissions and limitations
// under the License.
+#pragma once
+
#include "kudu/util/int128.h"
#include <iostream>
@@ -36,4 +38,3 @@ inline std::ostream& operator<<(std::ostream& os, const unsigned __int128& val)
}
} // namespace std
-