You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by da...@apache.org on 2018/04/02 21:41:43 UTC
[1/3] kudu git commit: [examples] Add licenses to Python examples
Repository: kudu
Updated Branches:
refs/heads/master ba955733d -> fff4185e0
[examples] Add licenses to Python examples
This also adds a README to the graphite example warning that it's
not a real integration, just an example.
Change-Id: I8825bfac0416fc19fdb99c972eb104898ba78e7d
Reviewed-on: http://gerrit.cloudera.org:8080/9858
Tested-by: Kudu Jenkins
Reviewed-by: Adar Dembo <ad...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/357ef6c9
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/357ef6c9
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/357ef6c9
Branch: refs/heads/master
Commit: 357ef6c9f736a77ae1cec786088adff4d8ac5091
Parents: ba95573
Author: Will Berkeley <wd...@apache.org>
Authored: Thu Mar 29 14:34:26 2018 -0700
Committer: Will Berkeley <wd...@gmail.com>
Committed: Mon Apr 2 04:26:44 2018 +0000
----------------------------------------------------------------------
examples/python/dstat-kudu/README.md | 14 ++++++++++++
examples/python/dstat-kudu/kudu_dstat.py | 17 ++++++++++++++
examples/python/graphite-kudu/README.adoc | 24 ++++++++++++++++++++
examples/python/graphite-kudu/kudu/__init__.py | 17 ++++++++++++++
.../python/graphite-kudu/kudu/kudu_graphite.py | 19 +++++++++++++++-
examples/python/graphite-kudu/setup.py | 18 ++++++++++++++-
python/setup.cfg | 2 +-
7 files changed, 108 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kudu/blob/357ef6c9/examples/python/dstat-kudu/README.md
----------------------------------------------------------------------
diff --git a/examples/python/dstat-kudu/README.md b/examples/python/dstat-kudu/README.md
index a4f443c..f2ba031 100644
--- a/examples/python/dstat-kudu/README.md
+++ b/examples/python/dstat-kudu/README.md
@@ -1,3 +1,17 @@
+<!---
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+
# Kudu + dstat + Impala
This is an example program that shows how to use the Kudu API in Python to load data into
http://git-wip-us.apache.org/repos/asf/kudu/blob/357ef6c9/examples/python/dstat-kudu/kudu_dstat.py
----------------------------------------------------------------------
diff --git a/examples/python/dstat-kudu/kudu_dstat.py b/examples/python/dstat-kudu/kudu_dstat.py
index f1f5bb2..4bc3485 100644
--- a/examples/python/dstat-kudu/kudu_dstat.py
+++ b/examples/python/dstat-kudu/kudu_dstat.py
@@ -1,3 +1,20 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
import kudu
import os
import subprocess
http://git-wip-us.apache.org/repos/asf/kudu/blob/357ef6c9/examples/python/graphite-kudu/README.adoc
----------------------------------------------------------------------
diff --git a/examples/python/graphite-kudu/README.adoc b/examples/python/graphite-kudu/README.adoc
new file mode 100644
index 0000000..eede0d4
--- /dev/null
+++ b/examples/python/graphite-kudu/README.adoc
@@ -0,0 +1,24 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+= Kudu Python client Graphite example
+:author: Kudu Team
+:homepage: https://kudu.apache.org/
+
+This uses the Python client to implement a plugin for using graphite-web with
+Kudu as a backend. Be aware it's meant just as an example of how to use the
+Python client. It is not a well-tested integration.
http://git-wip-us.apache.org/repos/asf/kudu/blob/357ef6c9/examples/python/graphite-kudu/kudu/__init__.py
----------------------------------------------------------------------
diff --git a/examples/python/graphite-kudu/kudu/__init__.py b/examples/python/graphite-kudu/kudu/__init__.py
index 34da70a..c53d0d3 100644
--- a/examples/python/graphite-kudu/kudu/__init__.py
+++ b/examples/python/graphite-kudu/kudu/__init__.py
@@ -1,3 +1,20 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
from pkgutil import extend_path
__path__ = extend_path(__path__, __name__)
http://git-wip-us.apache.org/repos/asf/kudu/blob/357ef6c9/examples/python/graphite-kudu/kudu/kudu_graphite.py
----------------------------------------------------------------------
diff --git a/examples/python/graphite-kudu/kudu/kudu_graphite.py b/examples/python/graphite-kudu/kudu/kudu_graphite.py
index 86a142e..1765303 100644
--- a/examples/python/graphite-kudu/kudu/kudu_graphite.py
+++ b/examples/python/graphite-kudu/kudu/kudu_graphite.py
@@ -1,5 +1,22 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
import sys, os
-
+
import re
import time
import math
http://git-wip-us.apache.org/repos/asf/kudu/blob/357ef6c9/examples/python/graphite-kudu/setup.py
----------------------------------------------------------------------
diff --git a/examples/python/graphite-kudu/setup.py b/examples/python/graphite-kudu/setup.py
index 386edeb..e02ac96 100644
--- a/examples/python/graphite-kudu/setup.py
+++ b/examples/python/graphite-kudu/setup.py
@@ -1,4 +1,20 @@
-# coding: utf-8
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
from setuptools import setup
setup(
http://git-wip-us.apache.org/repos/asf/kudu/blob/357ef6c9/python/setup.cfg
----------------------------------------------------------------------
diff --git a/python/setup.cfg b/python/setup.cfg
index 9af7e6f..b7e4789 100644
--- a/python/setup.cfg
+++ b/python/setup.cfg
@@ -1,2 +1,2 @@
[aliases]
-test=pytest
\ No newline at end of file
+test=pytest
[2/3] kudu git commit: KUDU-2384. Don't collect thread stacks when
running under debugger
Posted by da...@apache.org.
KUDU-2384. Don't collect thread stacks when running under debugger
This changes the stack watchdog and the 'collect all stacks' utility
code to not run when it appears that the process is running under a
tracer such as gdb or strace. This makes debugging Kudu processes
significantly easier because you don't need to know the magical
incantation 'handle SIGUSR2 nostop noprint'.
I tested manually by running a master with
--diagnostics-log-stack-traces-interval-ms=10 and attaching to it with
gdb. With this patch, I didn't see any signals sent.
This patch also changes the Env EIO fault injection to never inject
failures on /proc/. This is to avoid a TSAN warning since
KernelStackWatchdog would otherwise access the env_inject_eio_globs flag
concurrent to tests modifying it.
Change-Id: Ib17ffb9514e9b440b8d97843374d2a60913a398b
Reviewed-on: http://gerrit.cloudera.org:8080/9835
Reviewed-by: Adar Dembo <ad...@cloudera.com>
Tested-by: Kudu Jenkins
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/4db748f3
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/4db748f3
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/4db748f3
Branch: refs/heads/master
Commit: 4db748f3021711a67017fe51f30097350f343101
Parents: 357ef6c
Author: Todd Lipcon <to...@apache.org>
Authored: Tue Mar 27 16:58:37 2018 -0700
Committer: Todd Lipcon <to...@apache.org>
Committed: Mon Apr 2 05:04:29 2018 +0000
----------------------------------------------------------------------
src/kudu/util/debug-util.cc | 5 +++++
src/kudu/util/debug-util.h | 18 ++++++++++++---
src/kudu/util/env_posix.cc | 17 +++++++++++---
src/kudu/util/kernel_stack_watchdog.cc | 7 ++++++
src/kudu/util/os-util.cc | 35 +++++++++++++++++++++++++++++
src/kudu/util/os-util.h | 7 +++++-
6 files changed, 82 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kudu/blob/4db748f3/src/kudu/util/debug-util.cc
----------------------------------------------------------------------
diff --git a/src/kudu/util/debug-util.cc b/src/kudu/util/debug-util.cc
index 07703ea..93a9543 100644
--- a/src/kudu/util/debug-util.cc
+++ b/src/kudu/util/debug-util.cc
@@ -67,6 +67,7 @@
#include "kudu/util/errno.h"
#include "kudu/util/faststring.h"
#include "kudu/util/monotime.h"
+#include "kudu/util/os-util.h"
#include "kudu/util/scoped_cleanup.h"
#include "kudu/util/thread.h"
@@ -724,6 +725,10 @@ bool StackTrace::LessThan(const StackTrace& s) const {
}
Status StackTraceSnapshot::SnapshotAllStacks() {
+ if (IsBeingDebugged()) {
+ return Status::Incomplete("not collecting stack trace since debugger or strace is attached");
+ }
+
vector<pid_t> tids;
RETURN_NOT_OK_PREPEND(ListThreads(&tids), "could not list threads");
http://git-wip-us.apache.org/repos/asf/kudu/blob/4db748f3/src/kudu/util/debug-util.h
----------------------------------------------------------------------
diff --git a/src/kudu/util/debug-util.h b/src/kudu/util/debug-util.h
index 44fccc3..e8c94ea 100644
--- a/src/kudu/util/debug-util.h
+++ b/src/kudu/util/debug-util.h
@@ -68,10 +68,20 @@ Status SetStackTraceSignal(int signum);
// thread. It requires that the target thread has not blocked POSIX signals. If
// it has, an error message will be returned.
//
-// This function is thread-safe but coarsely synchronized: only one "dumper" thread
-// may be active at a time.
+// This function is thread-safe.
+//
+// NOTE: if Kudu is running inside a debugger, this can be annoying to a developer since
+// it internally uses signals that will cause the debugger to stop. Consider checking
+// 'IsBeingDebugged()' from os-util.h before using this function for non-critical use
+// cases.
std::string DumpThreadStack(int64_t tid);
+// Capture the thread stack of another thread
+//
+// NOTE: if Kudu is running inside a debugger, this can be annoying to a developer since
+// it internally uses signals that will cause the debugger to stop. Consider checking
+// 'IsBeingDebugged()' from os-util.h before using this function for non-critical use
+// cases.
Status GetThreadStack(int64_t tid, StackTrace* stack);
// Return the current stack trace, stringified.
@@ -230,7 +240,9 @@ class StackTraceSnapshot {
capture_thread_names_ = c;
}
- // Snapshot the stack traces of all threads in the process.
+ // Snapshot the stack traces of all threads in the process. This may return a bad
+ // Status in the case that stack traces aren't supported on the platform, or if
+ // the process is running inside a debugger.
//
// NOTE: this may take some time and should not be called in a latency-sensitive
// context.
http://git-wip-us.apache.org/repos/asf/kudu/blob/4db748f3/src/kudu/util/env_posix.cc
----------------------------------------------------------------------
diff --git a/src/kudu/util/env_posix.cc b/src/kudu/util/env_posix.cc
index 9ba583b..d180cb3 100644
--- a/src/kudu/util/env_posix.cc
+++ b/src/kudu/util/env_posix.cc
@@ -47,6 +47,7 @@
#include "kudu/gutil/stringprintf.h"
#include "kudu/gutil/strings/split.h"
#include "kudu/gutil/strings/substitute.h"
+#include "kudu/gutil/strings/util.h"
#include "kudu/util/array_view.h"
#include "kudu/util/debug/trace_event.h"
#include "kudu/util/env.h"
@@ -132,10 +133,20 @@ typedef struct xfs_flock64 {
#define MAYBE_RETURN_EIO(filename_expr, error_expr) do { \
const string& f_ = (filename_expr); \
MAYBE_RETURN_FAILURE(FLAGS_env_inject_eio, \
- StringMatchesGlob(f_, FLAGS_env_inject_eio_globs) ? (error_expr) : Status::OK()) \
+ ShouldInject(f_, FLAGS_env_inject_eio_globs) ? (error_expr) : Status::OK()) \
} while (0);
-bool StringMatchesGlob(const string& candidate, const string& glob_patterns) {
+bool ShouldInject(const string& candidate, const string& glob_patterns) {
+ // Never inject on /proc/ file accesses regardless of the configured flag,
+ // since it's not possible for /proc to "go bad".
+ //
+ // NB: it's important that this is done here _before_ consulting glob_patterns
+ // since some background threads read /proc/ after gflags have already been
+ // destructed.
+ if (HasPrefixString(candidate, "/proc/")) {
+ return false;
+ }
+
vector<string> globs = strings::Split(glob_patterns, ",", strings::SkipEmpty());
for (const auto& glob : globs) {
if (fnmatch(glob.c_str(), candidate.c_str(), 0) == 0) {
@@ -1339,7 +1350,7 @@ class PosixEnv : public Env {
virtual Status LockFile(const std::string& fname, FileLock** lock) OVERRIDE {
TRACE_EVENT1("io", "PosixEnv::LockFile", "path", fname);
MAYBE_RETURN_EIO(fname, IOError(Env::kInjectedFailureStatusMsg, EIO));
- if (StringMatchesGlob(fname, FLAGS_env_inject_lock_failure_globs)) {
+ if (ShouldInject(fname, FLAGS_env_inject_lock_failure_globs)) {
return IOError("lock " + fname, EAGAIN);
}
ThreadRestrictions::AssertIOAllowed();
http://git-wip-us.apache.org/repos/asf/kudu/blob/4db748f3/src/kudu/util/kernel_stack_watchdog.cc
----------------------------------------------------------------------
diff --git a/src/kudu/util/kernel_stack_watchdog.cc b/src/kudu/util/kernel_stack_watchdog.cc
index 2b890dc..27a259c 100644
--- a/src/kudu/util/kernel_stack_watchdog.cc
+++ b/src/kudu/util/kernel_stack_watchdog.cc
@@ -38,6 +38,7 @@
#include "kudu/util/fault_injection.h"
#include "kudu/util/flag_tags.h"
#include "kudu/util/monotime.h"
+#include "kudu/util/os-util.h"
#include "kudu/util/status.h"
#include "kudu/util/thread.h"
@@ -135,6 +136,12 @@ void KernelStackWatchdog::RunThread() {
break;
}
+ // Don't send signals while the debugger is running, since it makes it hard to
+ // use.
+ if (IsBeingDebugged()) {
+ continue;
+ }
+
// Prevent threads from deleting their TLS objects between the snapshot loop and the sending of
// signals. This makes it safe for us to access their TLS.
//
http://git-wip-us.apache.org/repos/asf/kudu/blob/4db748f3/src/kudu/util/os-util.cc
----------------------------------------------------------------------
diff --git a/src/kudu/util/os-util.cc b/src/kudu/util/os-util.cc
index d29eaff..d4ab2c7 100644
--- a/src/kudu/util/os-util.cc
+++ b/src/kudu/util/os-util.cc
@@ -33,11 +33,17 @@
#include <cstddef>
#include <fstream>
#include <string>
+#include <utility>
#include <vector>
#include "kudu/gutil/strings/numbers.h"
#include "kudu/gutil/strings/split.h"
+#include "kudu/gutil/strings/stringpiece.h"
#include "kudu/gutil/strings/substitute.h"
+#include "kudu/gutil/strings/util.h"
+#include "kudu/util/env.h"
+#include "kudu/util/faststring.h"
+#include "kudu/util/logging.h"
using std::ifstream;
using std::istreambuf_iterator;
@@ -142,5 +148,34 @@ void DisableCoreDumps() {
}
}
+bool IsBeingDebugged() {
+#ifndef __linux__
+ return false;
+#else
+ // Look for the TracerPid line in /proc/self/status.
+ // If this is non-zero, we are being ptraced, which is indicative of gdb or strace
+ // being attached.
+ faststring buf;
+ Status s = ReadFileToString(Env::Default(), "/proc/self/status", &buf);
+ if (!s.ok()) {
+ KLOG_FIRST_N(WARNING, 1) << "could not read /proc/self/status: " << s.ToString();
+ return false;
+ }
+ StringPiece buf_sp(reinterpret_cast<const char*>(buf.data()), buf.size());
+ vector<StringPiece> lines = Split(buf_sp, "\n");
+ for (const auto& l : lines) {
+ if (!HasPrefixString(l, "TracerPid:")) continue;
+ std::pair<StringPiece, StringPiece> key_val = Split(l, "\t");
+ int64_t tracer_pid = -1;
+ if (!safe_strto64(key_val.second.data(), key_val.second.size(), &tracer_pid)) {
+ KLOG_FIRST_N(WARNING, 1) << "Invalid line in /proc/self/status: " << l;
+ return false;
+ }
+ return tracer_pid != 0;
+ }
+ KLOG_FIRST_N(WARNING, 1) << "Could not find TracerPid line in /proc/self/status";
+ return false;
+#endif // __linux__
+}
} // namespace kudu
http://git-wip-us.apache.org/repos/asf/kudu/blob/4db748f3/src/kudu/util/os-util.h
----------------------------------------------------------------------
diff --git a/src/kudu/util/os-util.h b/src/kudu/util/os-util.h
index 04ba8f8..c79d579 100644
--- a/src/kudu/util/os-util.h
+++ b/src/kudu/util/os-util.h
@@ -27,7 +27,7 @@
#include <cstdint>
#include <string>
-#include <type_traits>
+#include <type_traits> // IWYU pragma: keep
#include "kudu/util/status.h"
@@ -71,6 +71,11 @@ Status GetThreadStats(int64_t tid, ThreadStats* stats);
// want to generate a core dump from an "expected" crash.
void DisableCoreDumps();
+// Return true if this process appears to be running under a debugger or strace.
+//
+// This may return false on unsupported (non-Linux) platforms.
+bool IsBeingDebugged();
+
} // namespace kudu
#endif /* KUDU_UTIL_OS_UTIL_H */
[3/3] kudu git commit: KUDU-2191 (6/n): Fixups to the C++ HMS client
Posted by da...@apache.org.
KUDU-2191 (6/n): Fixups to the C++ HMS client
These are changes necessary for subsequent patches in the series, but
they are easier to review in isolation.
Change-Id: I86a8d984e67fe6b4c004725828b4296476c2389e
Reviewed-on: http://gerrit.cloudera.org:8080/9861
Reviewed-by: Adar Dembo <ad...@cloudera.com>
Tested-by: Kudu Jenkins
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/fff4185e
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/fff4185e
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/fff4185e
Branch: refs/heads/master
Commit: fff4185e0f88954c67841a13b0924db4eab4c761
Parents: 4db748f
Author: Dan Burkert <da...@apache.org>
Authored: Wed Mar 28 16:35:45 2018 -0700
Committer: Dan Burkert <da...@apache.org>
Committed: Mon Apr 2 21:39:49 2018 +0000
----------------------------------------------------------------------
.../hive/metastore/TestKuduMetastorePlugin.java | 3 +-
src/kudu/hms/hms_client-test.cc | 15 +++++--
src/kudu/hms/hms_client.cc | 44 ++++++++++++++++++--
src/kudu/hms/hms_client.h | 18 +++++++-
src/kudu/hms/mini_hms.cc | 10 ++++-
5 files changed, 79 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kudu/blob/fff4185e/java/kudu-hive/src/test/java/org/apache/kudu/hive/metastore/TestKuduMetastorePlugin.java
----------------------------------------------------------------------
diff --git a/java/kudu-hive/src/test/java/org/apache/kudu/hive/metastore/TestKuduMetastorePlugin.java b/java/kudu-hive/src/test/java/org/apache/kudu/hive/metastore/TestKuduMetastorePlugin.java
index 8eb6927..1abcd6a 100644
--- a/java/kudu-hive/src/test/java/org/apache/kudu/hive/metastore/TestKuduMetastorePlugin.java
+++ b/java/kudu-hive/src/test/java/org/apache/kudu/hive/metastore/TestKuduMetastorePlugin.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
import org.apache.hadoop.hive.metastore.MetaStoreUtils;
import org.apache.hadoop.hive.metastore.MockPartitionExpressionForMetastore;
import org.apache.hadoop.hive.metastore.PartitionExpressionProxy;
+import org.apache.hadoop.hive.metastore.TableType;
import org.apache.hadoop.hive.metastore.api.EnvironmentContext;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.SerDeInfo;
@@ -77,7 +78,7 @@ public class TestKuduMetastorePlugin {
Table table = new Table();
table.setDbName("default");
table.setTableName(name);
- table.setTableType("MANAGED_TABLE");
+ table.setTableType(TableType.MANAGED_TABLE.toString());
table.putToParameters(hive_metastoreConstants.META_TABLE_STORAGE,
KuduMetastorePlugin.KUDU_STORAGE_HANDLER);
table.putToParameters(KuduMetastorePlugin.KUDU_TABLE_ID_KEY,
http://git-wip-us.apache.org/repos/asf/kudu/blob/fff4185e/src/kudu/hms/hms_client-test.cc
----------------------------------------------------------------------
diff --git a/src/kudu/hms/hms_client-test.cc b/src/kudu/hms/hms_client-test.cc
index 87601eb..a4a6ffe 100644
--- a/src/kudu/hms/hms_client-test.cc
+++ b/src/kudu/hms/hms_client-test.cc
@@ -17,11 +17,16 @@
#include "kudu/hms/hms_client.h"
+#include <algorithm> // IWYU pragma: keep
#include <cstdint>
+#include <map> // IWYU pragma: keep
+#include <memory> // IWYU pragma: keep
#include <string>
+#include <type_traits> // IWYU pragma: keep
#include <utility>
#include <vector>
+#include <boost/none_t.hpp> // IWYU pragma: keep
#include <boost/optional/optional.hpp>
#include <glog/stl_logging.h> // IWYU pragma: keep
#include <gtest/gtest.h>
@@ -59,7 +64,7 @@ class HmsClientTest : public KuduTest,
hive::Table table;
table.dbName = database_name;
table.tableName = table_name;
- table.tableType = "MANAGED_TABLE";
+ table.tableType = HmsClient::kManagedTable;
table.__set_parameters({
make_pair(HmsClient::kKuduTableIdKey, table_id),
@@ -122,7 +127,9 @@ TEST_P(HmsClientTest, TestHmsOperations) {
ASSERT_OK(hms.Start());
HmsClient client(hms.address(), hms_client_opts);
+ ASSERT_FALSE(client.IsConnected());
ASSERT_OK(client.Start());
+ ASSERT_TRUE(client.IsConnected());
// Create a database.
string database_name = "my_db";
@@ -161,7 +168,7 @@ TEST_P(HmsClientTest, TestHmsOperations) {
EXPECT_EQ(table_id, my_table.parameters[HmsClient::kKuduTableIdKey]);
EXPECT_EQ(HmsClient::kKuduStorageHandler,
my_table.parameters[hive::g_hive_metastore_constants.META_TABLE_STORAGE]);
- EXPECT_EQ("MANAGED_TABLE", my_table.tableType);
+ EXPECT_EQ(HmsClient::kManagedTable, my_table.tableType);
string new_table_name = "my_altered_table";
@@ -186,7 +193,7 @@ TEST_P(HmsClientTest, TestHmsOperations) {
EXPECT_EQ(table_id, renamed_table.parameters[HmsClient::kKuduTableIdKey]);
EXPECT_EQ(HmsClient::kKuduStorageHandler,
renamed_table.parameters[hive::g_hive_metastore_constants.META_TABLE_STORAGE]);
- EXPECT_EQ("MANAGED_TABLE", renamed_table.tableType);
+ EXPECT_EQ(HmsClient::kManagedTable, renamed_table.tableType);
// Create a table with an uppercase name.
string uppercase_table_name = "my_UPPERCASE_Table";
@@ -279,7 +286,7 @@ TEST_P(HmsClientTest, TestLargeObjects) {
hive::Table table;
table.dbName = database_name;
table.tableName = table_name;
- table.tableType = "MANAGED_TABLE";
+ table.tableType = HmsClient::kManagedTable;
hive::FieldSchema partition_key;
partition_key.name = "c1";
partition_key.type = "int";
http://git-wip-us.apache.org/repos/asf/kudu/blob/fff4185e/src/kudu/hms/hms_client.cc
----------------------------------------------------------------------
diff --git a/src/kudu/hms/hms_client.cc b/src/kudu/hms/hms_client.cc
index 6e2f4c6..1e889ad 100644
--- a/src/kudu/hms/hms_client.cc
+++ b/src/kudu/hms/hms_client.cc
@@ -25,6 +25,7 @@
#include <utility>
#include <vector>
+#include <boost/algorithm/string/predicate.hpp>
#include <glog/logging.h>
#include <thrift/Thrift.h>
#include <thrift/protocol/TBinaryProtocol.h>
@@ -65,6 +66,7 @@ using std::make_shared;
using std::shared_ptr;
using std::string;
using std::vector;
+using strings::Substitute;
namespace kudu {
namespace hms {
@@ -110,11 +112,17 @@ const char* const HmsClient::kKuduStorageHandler = "org.apache.kudu.hive.KuduSto
const char* const HmsClient::kTransactionalEventListeners =
"hive.metastore.transactional.event.listeners";
+const char* const HmsClient::kDisallowIncompatibleColTypeChanges =
+ "hive.metastore.disallow.incompatible.col.type.changes";
const char* const HmsClient::kDbNotificationListener =
"org.apache.hive.hcatalog.listener.DbNotificationListener";
const char* const HmsClient::kKuduMetastorePlugin =
"org.apache.kudu.hive.metastore.KuduMetastorePlugin";
+const char* const HmsClient::kManagedTable = "MANAGED_TABLE";
+
+const uint16_t HmsClient::kDefaultHmsPort = 9083;
+
const int kSlowExecutionWarningThresholdMs = 1000;
namespace {
@@ -166,7 +174,8 @@ Status HmsClient::Start() {
// the required event listeners.
string event_listener_config;
HMS_RET_NOT_OK(client_.get_config_value(event_listener_config, kTransactionalEventListeners, ""),
- "failed to get Hive MetaStore transactional event listener configuration");
+ Substitute("failed to get Hive MetaStore $0 configuration",
+ kTransactionalEventListeners));
// Parse the set of listeners from the configuration string.
vector<string> listeners = strings::Split(event_listener_config, ",", strings::SkipWhitespace());
@@ -177,12 +186,35 @@ Status HmsClient::Start() {
for (const auto& required_listener : { kDbNotificationListener, kKuduMetastorePlugin }) {
if (std::find(listeners.begin(), listeners.end(), required_listener) == listeners.end()) {
return Status::IllegalState(
- strings::Substitute("Hive Metastore configuration is missing required "
- "transactional event listener ($0): $1",
- kTransactionalEventListeners, required_listener));
+ Substitute("Hive Metastore configuration is missing required "
+ "transactional event listener ($0): $1",
+ kTransactionalEventListeners, required_listener));
}
}
+ // Also check that the HMS is configured to allow changing the type of
+ // columns, which is required to support dropping columns. File-based Hive
+ // tables handle columns by offset, and removing a column simply shifts all
+ // remaining columns down by one. The actual data is not changed, but instead
+ // reassigned to the next column. If the HMS is configured to check column
+ // type changes it will validate that the existing data matches the shifted
+ // column layout. This is overly strict for Kudu, since we handle DDL
+ // operations properly.
+ //
+ // See org.apache.hadoop.hive.metastore.MetaStoreUtils.throwExceptionIfIncompatibleColTypeChange.
+ string disallow_incompatible_column_type_changes;
+ HMS_RET_NOT_OK(client_.get_config_value(disallow_incompatible_column_type_changes,
+ kDisallowIncompatibleColTypeChanges,
+ "false"),
+ Substitute("failed to get Hive MetaStore $0 configuration",
+ kDisallowIncompatibleColTypeChanges));
+
+ if (boost::iequals(disallow_incompatible_column_type_changes, "true")) {
+ return Status::IllegalState(Substitute(
+ "Hive Metastore configuration is invalid: $0 must be set to false",
+ kDisallowIncompatibleColTypeChanges));
+ }
+
return Status::OK();
}
@@ -193,6 +225,10 @@ Status HmsClient::Stop() {
return Status::OK();
}
+bool HmsClient::IsConnected() {
+ return client_.getInputProtocol()->getTransport()->isOpen();
+}
+
Status HmsClient::CreateDatabase(const hive::Database& database) {
SCOPED_LOG_SLOW_EXECUTION(WARNING, kSlowExecutionWarningThresholdMs, "create HMS database");
HMS_RET_NOT_OK(client_.create_database(database), "failed to create Hive MetaStore database");
http://git-wip-us.apache.org/repos/asf/kudu/blob/fff4185e/src/kudu/hms/hms_client.h
----------------------------------------------------------------------
diff --git a/src/kudu/hms/hms_client.h b/src/kudu/hms/hms_client.h
index 21f18f9..cafa417 100644
--- a/src/kudu/hms/hms_client.h
+++ b/src/kudu/hms/hms_client.h
@@ -23,11 +23,18 @@
#include "kudu/gutil/port.h"
#include "kudu/hms/ThriftHiveMetastore.h"
-#include "kudu/hms/hive_metastore_types.h"
#include "kudu/util/monotime.h"
#include "kudu/util/slice.h"
#include "kudu/util/status.h"
+namespace hive {
+class Database;
+class EnvironmentContext;
+class NotificationEvent; // IWYU pragma: keep
+class Partition; // IWYU pragma: keep
+class Table;
+}
+
namespace kudu {
class HostPort;
@@ -85,9 +92,15 @@ class HmsClient {
static const char* const kKuduStorageHandler;
static const char* const kTransactionalEventListeners;
+ static const char* const kDisallowIncompatibleColTypeChanges;
static const char* const kDbNotificationListener;
static const char* const kKuduMetastorePlugin;
+ // See org.apache.hadoop.hive.metastore.TableType.
+ static const char* const kManagedTable;
+
+ static const uint16_t kDefaultHmsPort;
+
// Create an HmsClient connection to the proided HMS Thrift RPC address.
//
// The individual timeouts may be set to enforce per-operation
@@ -109,6 +122,9 @@ class HmsClient {
// This is optional; if not called the destructor will stop the client.
Status Stop() WARN_UNUSED_RESULT;
+ // Returns 'true' if the client is connected to the remote server.
+ bool IsConnected() WARN_UNUSED_RESULT;
+
// Creates a new database in the HMS.
//
// If a database already exists by the same name an AlreadyPresent status is
http://git-wip-us.apache.org/repos/asf/kudu/blob/fff4185e/src/kudu/hms/mini_hms.cc
----------------------------------------------------------------------
diff --git a/src/kudu/hms/mini_hms.cc b/src/kudu/hms/mini_hms.cc
index 078341d..f571b6c 100644
--- a/src/kudu/hms/mini_hms.cc
+++ b/src/kudu/hms/mini_hms.cc
@@ -181,6 +181,9 @@ Status MiniHms::CreateHiveSite(const string& tmp_dir) const {
// - hive.metastore.kerberos.keytab.file
// - hive.metastore.kerberos.principal
// Configures the HMS to use Kerberos for its Thrift RPC interface.
+ //
+ // - hive.metastore.disallow.incompatible.col.type.changes
+ // Configures the HMS to allow altering and dropping columns.
static const string kFileTemplate = R"(
<configuration>
<property>
@@ -208,7 +211,7 @@ Status MiniHms::CreateHiveSite(const string& tmp_dir) const {
<property>
<name>javax.jdo.option.ConnectionURL</name>
- <value>jdbc:derby:memory:$1/metadb;create=true</value>
+ <value>jdbc:derby:$1/metadb;create=true</value>
</property>
<property>
@@ -235,6 +238,11 @@ Status MiniHms::CreateHiveSite(const string& tmp_dir) const {
<name>hadoop.rpc.protection</name>
<value>$5</value>
</property>
+
+ <property>
+ <name>hive.metastore.disallow.incompatible.col.type.changes</name>
+ <value>false</value>
+ </property>
</configuration>
)";