You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by mj...@apache.org on 2017/07/11 15:36:21 UTC
[1/3] incubator-impala git commit: IMPALA-5643: Add total number of
threads created per group to /threadz
Repository: incubator-impala
Updated Branches:
refs/heads/master 7761692fb -> 36cca141e
IMPALA-5643: Add total number of threads created per group to /threadz
Change-Id: I8b532220b6940aa3d6b88a0ebaa247f6914fef53
Reviewed-on: http://gerrit.cloudera.org:8080/7390
Reviewed-by: Henry Robinson <he...@cloudera.com>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/dbdab4a3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/dbdab4a3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/dbdab4a3
Branch: refs/heads/master
Commit: dbdab4a3385c636a39dce106f8a6854ccc546014
Parents: 7761692
Author: Lars Volker <lv...@cloudera.com>
Authored: Wed Jun 28 19:04:32 2017 -0700
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Tue Jul 11 05:49:27 2017 +0000
----------------------------------------------------------------------
be/src/util/thread.cc | 20 ++++++++++++++------
www/threadz.tmpl | 2 +-
2 files changed, 15 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/dbdab4a3/be/src/util/thread.cc
----------------------------------------------------------------------
diff --git a/be/src/util/thread.cc b/be/src/util/thread.cc
index b3c7d17..39a344f 100644
--- a/be/src/util/thread.cc
+++ b/be/src/util/thread.cc
@@ -163,7 +163,10 @@ class ThreadMgr {
// A ThreadCategory is a set of threads that are logically related.
// TODO: unordered_map is incompatible with boost::thread::id, but would be more
// efficient here.
- typedef map<const thread::id, ThreadDescriptor> ThreadCategory;
+ struct ThreadCategory {
+ int64_t num_threads_created;
+ map<const thread::id, ThreadDescriptor> threads_by_id;
+ };
// All thread categorys, keyed on the category name.
typedef map<string, ThreadCategory> ThreadCategoryMap;
@@ -197,7 +200,9 @@ Status ThreadMgr::StartInstrumentation(MetricGroup* metrics) {
void ThreadMgr::AddThread(const thread::id& thread, const string& name,
const string& category, int64_t tid) {
lock_guard<mutex> l(lock_);
- thread_categories_[category][thread] = ThreadDescriptor(category, name, tid);
+ ThreadCategory& thread_category = thread_categories_[category];
+ thread_category.threads_by_id[thread] = ThreadDescriptor(category, name, tid);
+ ++thread_category.num_threads_created;
if (metrics_enabled_) {
current_num_threads_metric_->Increment(1L);
total_threads_metric_->Increment(1L);
@@ -208,7 +213,7 @@ void ThreadMgr::RemoveThread(const thread::id& boost_id, const string& category)
lock_guard<mutex> l(lock_);
ThreadCategoryMap::iterator category_it = thread_categories_.find(category);
DCHECK(category_it != thread_categories_.end());
- category_it->second.erase(boost_id);
+ category_it->second.threads_by_id.erase(boost_id);
if (metrics_enabled_) current_num_threads_metric_->Increment(-1L);
}
@@ -222,7 +227,10 @@ void ThreadMgr::GetThreadOverview(Document* document) {
for (const ThreadCategoryMap::value_type& category: thread_categories_) {
Value val(kObjectType);
val.AddMember("name", category.first.c_str(), document->GetAllocator());
- val.AddMember("size", static_cast<uint64_t>(category.second.size()),
+ val.AddMember("size", static_cast<uint64_t>(category.second.threads_by_id.size()),
+ document->GetAllocator());
+ val.AddMember("num_created",
+ static_cast<uint64_t>(category.second.num_threads_created),
document->GetAllocator());
// TODO: URLEncode() name?
lst.PushBack(val, document->GetAllocator());
@@ -245,7 +253,7 @@ void ThreadMgr::ThreadGroupUrlCallback(const Webserver::ArgumentMap& args,
categories_to_print.push_back(&category->second);
Value val(kObjectType);
val.AddMember("category", category->first.c_str(), document->GetAllocator());
- val.AddMember("size", static_cast<uint64_t>(category->second.size()),
+ val.AddMember("size", static_cast<uint64_t>(category->second.threads_by_id.size()),
document->GetAllocator());
document->AddMember("thread-group", val, document->GetAllocator());
} else {
@@ -256,7 +264,7 @@ void ThreadMgr::ThreadGroupUrlCallback(const Webserver::ArgumentMap& args,
Value lst(kArrayType);
for (const ThreadCategory* category: categories_to_print) {
- for (const ThreadCategory::value_type& thread: *category) {
+ for (const auto& thread : category->threads_by_id) {
Value val(kObjectType);
val.AddMember("name", thread.second.name().c_str(), document->GetAllocator());
val.AddMember("id", thread.second.thread_id(), document->GetAllocator());
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/dbdab4a3/www/threadz.tmpl
----------------------------------------------------------------------
diff --git a/www/threadz.tmpl b/www/threadz.tmpl
index 9a9e3c1..cc5ec38 100644
--- a/www/threadz.tmpl
+++ b/www/threadz.tmpl
@@ -30,7 +30,7 @@ under the License.
{{#thread-groups}}
<a href='/thread-group?group={{name}}'>
- <h3>{{name}} : {{size}}</h3>
+ <h3>{{name}} : (running: {{size}}, total created: {{num_created}})</h3>
</a>
{{/thread-groups}}
[3/3] incubator-impala git commit: IMPALA-5507: Add clear description
to help information of KEYVAL option
Posted by mj...@apache.org.
IMPALA-5507: Add clear description to help information of KEYVAL option
Help information of KEYVAL option in impala-shell is not clear enough.
I fix this issue by adding clear description to help information of
KEYVAL option.
Change-Id: I68cfc16838c6c0e7813f03dd4296f9eb54ec4c63
Reviewed-on: http://gerrit.cloudera.org:8080/7179
Reviewed-by: Jim Apple <jb...@apache.org>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/36cca141
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/36cca141
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/36cca141
Branch: refs/heads/master
Commit: 36cca141e0bff77b35e2890114dac41c3088e307
Parents: bd3b95e
Author: davidxdh <xu...@zte.com.cn>
Authored: Tue Jul 11 09:03:44 2017 +0800
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Tue Jul 11 11:05:14 2017 +0000
----------------------------------------------------------------------
shell/option_parser.py | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/36cca141/shell/option_parser.py
----------------------------------------------------------------------
diff --git a/shell/option_parser.py b/shell/option_parser.py
index 6092439..f0f3573 100755
--- a/shell/option_parser.py
+++ b/shell/option_parser.py
@@ -85,7 +85,8 @@ def get_option_parser(defaults):
help="Execute a query without the shell")
parser.add_option("-f", "--query_file", dest="query_file",
help="Execute the queries in the query file, delimited by ;."
- " Queries may be read from stdin if the argument to -f is -.")
+ " If the argument to -f is \"-\", then queries are read from"
+ " stdin and terminated with ctrl-d.")
parser.add_option("-k", "--kerberos", dest="use_kerberos",
action="store_true", help="Connect to a kerberized impalad")
parser.add_option("-o", "--output_file", dest="output_file",
@@ -157,7 +158,10 @@ def get_option_parser(defaults):
parser.add_option("--ldap_password_cmd",
help="Shell command to run to retrieve the LDAP password")
parser.add_option("--var", dest="keyval", action="append",
- help="Define variable(s) to be used within the Impala session.")
+ help="Define variable(s) to be used within the Impala session."
+ " It must follow the pattern \"KEY=VALUE\","
+ " KEY starts with an alphabetic character and"
+ " contains alphanumeric characters or underscores.")
# add default values to the help text
for option in parser.option_list:
[2/3] incubator-impala git commit: IMPALA-5514: Throw an error when
--ldap_password_cmd is used without LDAP auth
Posted by mj...@apache.org.
IMPALA-5514: Throw an error when --ldap_password_cmd is used without LDAP auth
When only with ldap_password_cmd option, impala-shell runs successfully.
I solved this problem by throwing an error when --ldap_password_cmd is
used without LDAP auth, that is, ldap_password_cmd option will only
take effect if ldap option presents.
Change-Id: I3711d8a0eca2fa8612e2943fa9121945db6b012e
Reviewed-on: http://gerrit.cloudera.org:8080/7188
Reviewed-by: Bharath Vissapragada <bh...@cloudera.com>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/bd3b95e3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/bd3b95e3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/bd3b95e3
Branch: refs/heads/master
Commit: bd3b95e3a9ac0df329d523e3ef4dac9812d39399
Parents: dbdab4a
Author: davidxdh <xu...@zte.com.cn>
Authored: Tue Jul 11 09:56:34 2017 +0800
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Tue Jul 11 08:43:11 2017 +0000
----------------------------------------------------------------------
shell/impala_shell.py | 5 +++++
1 file changed, 5 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/bd3b95e3/shell/impala_shell.py
----------------------------------------------------------------------
diff --git a/shell/impala_shell.py b/shell/impala_shell.py
index cbfa8a5..d71b9ad 100755
--- a/shell/impala_shell.py
+++ b/shell/impala_shell.py
@@ -1332,6 +1332,11 @@ if __name__ == "__main__":
"connections. Enable SSL or set --auth_creds_ok_in_clear")
sys.exit(1)
+ if not options.use_ldap and options.ldap_password_cmd:
+ print_to_stderr("Option --ldap_password_cmd requires using LDAP authentication " +
+ "mechanism (-l)")
+ sys.exit(1)
+
if options.use_kerberos:
print_to_stderr("Starting Impala Shell using Kerberos authentication")
print_to_stderr("Using service name '%s'" % options.kerberos_service_name)