You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ta...@apache.org on 2018/07/11 22:54:08 UTC
[1/2] impala git commit: IMPALA-7275: Create table authorization
error should not show table name
Repository: impala
Updated Branches:
refs/heads/master 36ce54bd5 -> 192771511
IMPALA-7275: Create table authorization error should not show table name
The authorization correctly checks database level authorization but
the exception includes the table name. This fix adds a check for
create and modifies the error string to remove the table name. Having
an error with CREATE on a table is not correct since to create a table
a user needs CREATE on the parent database.
Example:
SQL: CREATE TABLE db.tbl(col1 INT)
Before:
User 'user1' does not have privileges to execute 'CREATE' on: db.tbl
After:
User 'user1' does not have privileges to execute 'CREATE' on: db
Testsing: Ran all FE tests
Change-Id: I6711a744541abd5ff26574974ba8517b6e51c453
Reviewed-on: http://gerrit.cloudera.org:8080/10919
Reviewed-by: Fredy Wijaya <fw...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/0ae9d58c
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/0ae9d58c
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/0ae9d58c
Branch: refs/heads/master
Commit: 0ae9d58c0f760dace23e31780bfe67cfc025b155
Parents: 36ce54b
Author: Adam Holley <gi...@holleyism.com>
Authored: Tue Jul 10 22:58:17 2018 -0500
Committer: Impala Public Jenkins <im...@cloudera.com>
Committed: Wed Jul 11 22:40:18 2018 +0000
----------------------------------------------------------------------
.../authorization/AuthorizationChecker.java | 16 ++++-
.../impala/analysis/AuthorizationStmtTest.java | 74 ++++++++++----------
2 files changed, 50 insertions(+), 40 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/0ae9d58c/fe/src/main/java/org/apache/impala/authorization/AuthorizationChecker.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/authorization/AuthorizationChecker.java b/fe/src/main/java/org/apache/impala/authorization/AuthorizationChecker.java
index d1d906c..82dcbc8 100644
--- a/fe/src/main/java/org/apache/impala/authorization/AuthorizationChecker.java
+++ b/fe/src/main/java/org/apache/impala/authorization/AuthorizationChecker.java
@@ -19,6 +19,7 @@ package org.apache.impala.authorization;
import java.util.Collections;
import java.util.EnumSet;
+import java.util.Iterator;
import java.util.List;
import java.util.Set;
@@ -113,10 +114,19 @@ public class AuthorizationChecker {
"User '%s' does not have privileges to access: %s",
user.getName(), privilegeRequest.getName()));
} else if (privilege == Privilege.REFRESH) {
+ throw new AuthorizationException(String.format(
+ "User '%s' does not have privileges to execute " +
+ "'INVALIDATE METADATA/REFRESH' on: %s", user.getName(),
+ privilegeRequest.getName()));
+ } else if (privilege == Privilege.CREATE &&
+ privilegeRequest.getAuthorizeable() instanceof AuthorizeableTable) {
+ // Creating a table requires CREATE on a database and we shouldn't
+ // expose the table name.
+ AuthorizeableTable authorizeableTable =
+ (AuthorizeableTable) privilegeRequest.getAuthorizeable();
throw new AuthorizationException(String.format(
- "User '%s' does not have privileges to execute " +
- "'INVALIDATE METADATA/REFRESH' on: %s", user.getName(),
- privilegeRequest.getName()));
+ "User '%s' does not have privileges to execute '%s' on: %s",
+ user.getName(), privilege, authorizeableTable.getDbName()));
} else {
throw new AuthorizationException(String.format(
"User '%s' does not have privileges to execute '%s' on: %s",
http://git-wip-us.apache.org/repos/asf/impala/blob/0ae9d58c/fe/src/test/java/org/apache/impala/analysis/AuthorizationStmtTest.java
----------------------------------------------------------------------
diff --git a/fe/src/test/java/org/apache/impala/analysis/AuthorizationStmtTest.java b/fe/src/test/java/org/apache/impala/analysis/AuthorizationStmtTest.java
index 4f88463..ee440ce 100644
--- a/fe/src/test/java/org/apache/impala/analysis/AuthorizationStmtTest.java
+++ b/fe/src/test/java/org/apache/impala/analysis/AuthorizationStmtTest.java
@@ -1239,9 +1239,9 @@ public class AuthorizationStmtTest extends FrontendTestBase {
.ok(onServer(TPrivilegeLevel.CREATE))
.ok(onDatabase("functional", TPrivilegeLevel.ALL))
.ok(onDatabase("functional", TPrivilegeLevel.CREATE))
- .error(createError("functional.new_table"), onServer(allExcept(
+ .error(createError("functional"), onServer(allExcept(
TPrivilegeLevel.ALL, TPrivilegeLevel.CREATE)))
- .error(createError("functional.new_table"), onDatabase("functional", allExcept(
+ .error(createError("functional"), onDatabase("functional", allExcept(
TPrivilegeLevel.ALL, TPrivilegeLevel.CREATE)));
}
@@ -1257,9 +1257,9 @@ public class AuthorizationStmtTest extends FrontendTestBase {
.error(accessError("functional.alltypes"))
.error(accessError("functional.alltypes"), onServer(allExcept(
join(viewMetadataPrivileges(), TPrivilegeLevel.CREATE))))
- .error(createError("functional.new_table"), onDatabase("functional", allExcept(
+ .error(createError("functional"), onDatabase("functional", allExcept(
TPrivilegeLevel.ALL, TPrivilegeLevel.CREATE, TPrivilegeLevel.SELECT)))
- .error(createError("functional.new_table"), onDatabase("functional", allExcept(
+ .error(createError("functional"), onDatabase("functional", allExcept(
TPrivilegeLevel.ALL, TPrivilegeLevel.CREATE)), onTable(
"functional", "alltypes", viewMetadataPrivileges()))
.error(accessError("functional.alltypes"), onDatabase("functional",
@@ -1270,10 +1270,10 @@ public class AuthorizationStmtTest extends FrontendTestBase {
for (AuthzTest test : new AuthzTest[]{
authorize("create table functional.alltypes(i int)"),
authorize("create table if not exists functional.alltypes(i int)")}) {
- test.error(createError("functional.alltypes"))
- .error(createError("functional.alltypes"), onServer(allExcept(
+ test.error(createError("functional"))
+ .error(createError("functional"), onServer(allExcept(
TPrivilegeLevel.ALL, TPrivilegeLevel.CREATE)))
- .error(createError("functional.alltypes"), onDatabase("functional", allExcept(
+ .error(createError("functional"), onDatabase("functional", allExcept(
TPrivilegeLevel.ALL, TPrivilegeLevel.CREATE)));
}
@@ -1296,17 +1296,17 @@ public class AuthorizationStmtTest extends FrontendTestBase {
.ok(onDatabase("functional"), onDatabase("functional", TPrivilegeLevel.CREATE,
TPrivilegeLevel.INSERT), onColumn("functional", "alltypes", "int_col",
TPrivilegeLevel.ALL))
- .error(createError("functional.new_table"))
- .error(createError("functional.new_table"), onServer(allExcept(
+ .error(createError("functional"))
+ .error(createError("functional"), onServer(allExcept(
TPrivilegeLevel.ALL, TPrivilegeLevel.CREATE, TPrivilegeLevel.INSERT,
TPrivilegeLevel.SELECT)))
- .error(createError("functional.new_table"), onDatabase("functional", allExcept(
+ .error(createError("functional"), onDatabase("functional", allExcept(
TPrivilegeLevel.ALL, TPrivilegeLevel.CREATE, TPrivilegeLevel.INSERT,
TPrivilegeLevel.SELECT)))
- .error(createError("functional.new_table"), onDatabase("functional", allExcept(
+ .error(createError("functional"), onDatabase("functional", allExcept(
TPrivilegeLevel.ALL, TPrivilegeLevel.CREATE, TPrivilegeLevel.INSERT)),
onTable("functional", "alltypes", TPrivilegeLevel.SELECT))
- .error(selectError("functional.alltypes"), onDatabase("functional",
+ .error(selectError("functional"), onDatabase("functional",
TPrivilegeLevel.CREATE, TPrivilegeLevel.INSERT), onTable("functional",
"alltypes", allExcept(TPrivilegeLevel.ALL, TPrivilegeLevel.SELECT)));
}
@@ -1317,9 +1317,9 @@ public class AuthorizationStmtTest extends FrontendTestBase {
.ok(onServer(TPrivilegeLevel.ALL))
.ok(onServer(TPrivilegeLevel.CREATE),
onUri("hdfs://localhost:20500/test-warehouse/new_table", TPrivilegeLevel.ALL))
- .error(createError("functional.new_table"), onServer(allExcept(
+ .error(createError("functional"), onServer(allExcept(
TPrivilegeLevel.ALL, TPrivilegeLevel.CREATE)))
- .error(createError("functional.new_table"), onDatabase("functional", allExcept(
+ .error(createError("functional"), onDatabase("functional", allExcept(
TPrivilegeLevel.ALL, TPrivilegeLevel.CREATE)), onUri(
"hdfs://localhost:20500/test-warehouse/new_table", TPrivilegeLevel.ALL))
.error(accessError("hdfs://localhost:20500/test-warehouse/new_table"),
@@ -1337,9 +1337,9 @@ public class AuthorizationStmtTest extends FrontendTestBase {
onServer(TPrivilegeLevel.CREATE),
onUri("hdfs://localhost:20500/test-warehouse/upper_case/test",
TPrivilegeLevel.ALL))
- .error(createError("functional.new_table"), onServer(allExcept(
+ .error(createError("functional"), onServer(allExcept(
TPrivilegeLevel.ALL, TPrivilegeLevel.CREATE)))
- .error(createError("functional.new_table"), onDatabase("functional", allExcept(
+ .error(createError("functional"), onDatabase("functional", allExcept(
TPrivilegeLevel.ALL, TPrivilegeLevel.CREATE)), onUri(
"hdfs://localhost:20500/test-warehouse/UPPER_CASE/test", TPrivilegeLevel.ALL))
.error(accessError("hdfs://localhost:20500/test-warehouse/UPPER_CASE/test"),
@@ -1354,7 +1354,7 @@ public class AuthorizationStmtTest extends FrontendTestBase {
.error(accessError(
"hdfs://localhost:20500/test-warehouse/schemas/alltypestiny.parquet"),
onServer(allExcept(TPrivilegeLevel.ALL, TPrivilegeLevel.CREATE)))
- .error(createError("functional.new_table"), onDatabase("functional", allExcept(
+ .error(createError("functional"), onDatabase("functional", allExcept(
TPrivilegeLevel.ALL, TPrivilegeLevel.CREATE)), onUri(
"hdfs://localhost:20500/test-warehouse/schemas/alltypestiny.parquet",
TPrivilegeLevel.ALL))
@@ -1370,7 +1370,7 @@ public class AuthorizationStmtTest extends FrontendTestBase {
authorize("create external table functional.kudu_tbl stored as kudu " +
"tblproperties ('kudu.master_addresses'='127.0.0.1', 'kudu.table_name'='tbl')")
.ok(onServer(TPrivilegeLevel.ALL))
- .error(createError("functional.kudu_tbl"))
+ .error(createError("functional"))
.error(accessError("server1"), onServer(allExcept(TPrivilegeLevel.ALL)))
.error(accessError("server1"), onDatabase("functional", TPrivilegeLevel.ALL));
@@ -1380,10 +1380,10 @@ public class AuthorizationStmtTest extends FrontendTestBase {
.ok(onServer(TPrivilegeLevel.ALL))
.ok(onDatabase("functional", TPrivilegeLevel.ALL))
.ok(onDatabase("functional", TPrivilegeLevel.CREATE))
- .error(createError("functional.kudu_tbl"))
- .error(createError("functional.kudu_tbl"), onServer(allExcept(
+ .error(createError("functional"))
+ .error(createError("functional"), onServer(allExcept(
TPrivilegeLevel.ALL, TPrivilegeLevel.CREATE)))
- .error(createError("functional.kudu_tbl"), onDatabase("functional", allExcept(
+ .error(createError("functional"), onDatabase("functional", allExcept(
TPrivilegeLevel.ALL, TPrivilegeLevel.CREATE)));
// IMPALA-6451: CTAS for Kudu tables on non-external tables and without
@@ -1398,19 +1398,19 @@ public class AuthorizationStmtTest extends FrontendTestBase {
.ok(onServer(TPrivilegeLevel.ALL))
.ok(onDatabase("functional", TPrivilegeLevel.CREATE, TPrivilegeLevel.INSERT,
TPrivilegeLevel.SELECT))
- .error(createError("functional.kudu_tbl"))
- .error(createError("functional.kudu_tbl"), onServer(allExcept(TPrivilegeLevel.ALL,
+ .error(createError("functional"))
+ .error(createError("functional"), onServer(allExcept(TPrivilegeLevel.ALL,
TPrivilegeLevel.CREATE, TPrivilegeLevel.INSERT, TPrivilegeLevel.SELECT)))
- .error(createError("functional.kudu_tbl"), onDatabase("functional", allExcept(
+ .error(createError("functional"), onDatabase("functional", allExcept(
TPrivilegeLevel.ALL, TPrivilegeLevel.CREATE, TPrivilegeLevel.INSERT,
TPrivilegeLevel.SELECT)));
// Database does not exist.
authorize("create table nodb.new_table(i int)")
- .error(createError("nodb.new_table"))
- .error(createError("nodb.new_table"), onServer(allExcept(
+ .error(createError("nodb"))
+ .error(createError("nodb"), onServer(allExcept(
TPrivilegeLevel.ALL, TPrivilegeLevel.CREATE)))
- .error(createError("nodb.new_table"), onDatabase("functional", allExcept(
+ .error(createError("nodb"), onDatabase("functional", allExcept(
TPrivilegeLevel.ALL, TPrivilegeLevel.CREATE)));
}
@@ -1432,7 +1432,7 @@ public class AuthorizationStmtTest extends FrontendTestBase {
.error(selectError("functional.alltypes"))
.error(selectError("functional.alltypes"), onServer(allExcept(
TPrivilegeLevel.ALL, TPrivilegeLevel.CREATE, TPrivilegeLevel.SELECT)))
- .error(createError("functional.new_view"), onDatabase("functional", allExcept(
+ .error(createError("functional"), onDatabase("functional", allExcept(
TPrivilegeLevel.ALL, TPrivilegeLevel.CREATE)))
.error(selectError("functional.alltypes"), onDatabase("functional", allExcept(
TPrivilegeLevel.ALL, TPrivilegeLevel.SELECT)))
@@ -1446,10 +1446,10 @@ public class AuthorizationStmtTest extends FrontendTestBase {
.ok(onServer(TPrivilegeLevel.CREATE))
.ok(onDatabase("functional", TPrivilegeLevel.ALL))
.ok(onDatabase("functional", TPrivilegeLevel.CREATE))
- .error(createError("functional.new_view"))
- .error(createError("functional.new_view"), onServer(allExcept(
+ .error(createError("functional"))
+ .error(createError("functional"), onServer(allExcept(
TPrivilegeLevel.ALL, TPrivilegeLevel.CREATE, TPrivilegeLevel.SELECT)))
- .error(createError("functional.new_view"), onDatabase("functional", allExcept(
+ .error(createError("functional"), onDatabase("functional", allExcept(
TPrivilegeLevel.ALL, TPrivilegeLevel.CREATE)));
// View already exists.
@@ -1461,7 +1461,7 @@ public class AuthorizationStmtTest extends FrontendTestBase {
test.error(selectError("functional.alltypes"))
.error(selectError("functional.alltypes"), onServer(allExcept(
TPrivilegeLevel.ALL, TPrivilegeLevel.CREATE, TPrivilegeLevel.SELECT)))
- .error(createError("functional.alltypes_view"), onDatabase("functional", allExcept(
+ .error(createError("functional"), onDatabase("functional", allExcept(
TPrivilegeLevel.ALL, TPrivilegeLevel.CREATE)))
.error(selectError("functional.alltypes"), onDatabase("functional", allExcept(
TPrivilegeLevel.ALL, TPrivilegeLevel.SELECT)))
@@ -1474,10 +1474,10 @@ public class AuthorizationStmtTest extends FrontendTestBase {
// Database does not exist.
authorize("create view nodb.new_view as select 1")
- .error(createError("nodb.new_view"))
- .error(createError("nodb.new_view"), onServer(allExcept(
+ .error(createError("nodb"))
+ .error(createError("nodb"), onServer(allExcept(
TPrivilegeLevel.ALL, TPrivilegeLevel.CREATE)))
- .error(createError("nodb.new_view"), onDatabase("functional", allExcept(
+ .error(createError("nodb"), onDatabase("functional", allExcept(
TPrivilegeLevel.ALL, TPrivilegeLevel.CREATE)));
}
@@ -1682,7 +1682,7 @@ public class AuthorizationStmtTest extends FrontendTestBase {
.error(alterError("functional.alltypes"), onDatabase("functional",
TPrivilegeLevel.CREATE), onTable("functional", "alltypes", allExcept(
TPrivilegeLevel.ALL, TPrivilegeLevel.ALTER)))
- .error(createError("functional.new_table"), onDatabase("functional",
+ .error(createError("functional"), onDatabase("functional",
allExcept(TPrivilegeLevel.ALL, TPrivilegeLevel.CREATE)),
onTable("functional", "alltypes", TPrivilegeLevel.ALTER));
@@ -1804,7 +1804,7 @@ public class AuthorizationStmtTest extends FrontendTestBase {
.error(alterError("functional.alltypes_view"), onDatabase("functional",
TPrivilegeLevel.CREATE), onTable("functional", "alltypes_view", allExcept(
TPrivilegeLevel.ALL, TPrivilegeLevel.ALTER)))
- .error(createError("functional.new_view"), onDatabase("functional",
+ .error(createError("functional"), onDatabase("functional",
allExcept(TPrivilegeLevel.ALL, TPrivilegeLevel.CREATE)),
onTable("functional", "alltypes_view", TPrivilegeLevel.ALTER));
[2/2] impala git commit: IMPALA-7239: Disable smaps parsing by default
Posted by ta...@apache.org.
IMPALA-7239: Disable smaps parsing by default
Accessing smaps has proven to be too expensive to enable on all
operating systems. Let's move the metrics behind a hidden flag
so that they can be enabled for development if needed but
won't affect production workloads.
Change-Id: I235eddde8fe925866e0581b235752354a3f36d5b
Reviewed-on: http://gerrit.cloudera.org:8080/10884
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/19277151
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/19277151
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/19277151
Branch: refs/heads/master
Commit: 192771511b62a05d1183ecf974f4ff9eb174617e
Parents: 0ae9d58
Author: Tim Armstrong <ta...@cloudera.com>
Authored: Fri Jul 6 08:53:05 2018 -0700
Committer: Impala Public Jenkins <im...@cloudera.com>
Committed: Wed Jul 11 22:44:27 2018 +0000
----------------------------------------------------------------------
be/src/common/init.cc | 2 +-
be/src/util/memory-metrics.cc | 18 +++++---
be/src/util/process-state-info.cc | 79 ++++++++++++++++++----------------
be/src/util/process-state-info.h | 13 +++++-
4 files changed, 66 insertions(+), 46 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/19277151/be/src/common/init.cc
----------------------------------------------------------------------
diff --git a/be/src/common/init.cc b/be/src/common/init.cc
index b57d2af..30c194e 100644
--- a/be/src/common/init.cc
+++ b/be/src/common/init.cc
@@ -280,7 +280,7 @@ void impala::InitCommonRuntime(int argc, char** argv, bool init_jvm,
}
Status impala::StartMemoryMaintenanceThread() {
- DCHECK(AggregateMemoryMetrics::NUM_MAPS != nullptr) << "Mem metrics not registered.";
+ DCHECK(AggregateMemoryMetrics::TOTAL_USED != nullptr) << "Mem metrics not registered.";
return Thread::Create("common", "memory-maintenance-thread",
&MemoryMaintenanceThread, &memory_maintenance_thread);
}
http://git-wip-us.apache.org/repos/asf/impala/blob/19277151/be/src/util/memory-metrics.cc
----------------------------------------------------------------------
diff --git a/be/src/util/memory-metrics.cc b/be/src/util/memory-metrics.cc
index d37fb3a..390bd7c 100644
--- a/be/src/util/memory-metrics.cc
+++ b/be/src/util/memory-metrics.cc
@@ -25,6 +25,7 @@
#include "runtime/mem-tracker.h"
#include "util/jni-util.h"
#include "util/mem-info.h"
+#include "util/process-state-info.h"
#include "util/time.h"
using boost::algorithm::to_lower;
@@ -32,6 +33,10 @@ using namespace impala;
using namespace strings;
DECLARE_bool(mmap_buffers);
+DEFINE_bool_hidden(enable_extended_memory_metrics, false,
+ "(Experimental) enable extended memory metrics, including those that can be "
+ "expensive to compute. This was introduced as a workaround for poor /proc/*/smaps "
+ "performance in certain Linux kernel versions - see IMPALA-7239.");
SumGauge* AggregateMemoryMetrics::TOTAL_USED = nullptr;
IntGauge* AggregateMemoryMetrics::NUM_MAPS = nullptr;
@@ -116,15 +121,15 @@ Status impala::RegisterMemoryMetrics(MetricGroup* metrics, bool register_jvm_met
RETURN_IF_ERROR(JvmMetric::InitMetrics(metrics->GetOrCreateChildGroup("jvm")));
}
- if (MemInfo::HaveSmaps()) {
+ if (FLAGS_enable_extended_memory_metrics && MemInfo::HaveSmaps()) {
AggregateMemoryMetrics::NUM_MAPS =
aggregate_metrics->AddGauge("memory.num-maps", 0U);
- AggregateMemoryMetrics::MAPPED_BYTES =
- aggregate_metrics->AddGauge("memory.mapped-bytes", 0U);
- AggregateMemoryMetrics::RSS = aggregate_metrics->AddGauge("memory.rss", 0U);
AggregateMemoryMetrics::ANON_HUGE_PAGE_BYTES =
aggregate_metrics->AddGauge("memory.anon-huge-page-bytes", 0U);
}
+ AggregateMemoryMetrics::MAPPED_BYTES =
+ aggregate_metrics->AddGauge("memory.mapped-bytes", 0U);
+ AggregateMemoryMetrics::RSS = aggregate_metrics->AddGauge("memory.rss", 0U);
ThpConfig thp_config = MemInfo::ParseThpConfig();
AggregateMemoryMetrics::THP_ENABLED =
aggregate_metrics->AddProperty("memory.thp.enabled", thp_config.enabled);
@@ -141,10 +146,11 @@ void AggregateMemoryMetrics::Refresh() {
// Only call ParseSmaps() if the metrics were created.
MappedMemInfo map_info = MemInfo::ParseSmaps();
NUM_MAPS->SetValue(map_info.num_maps);
- MAPPED_BYTES->SetValue(map_info.size_kb * 1024);
- RSS->SetValue(map_info.rss_kb * 1024);
ANON_HUGE_PAGE_BYTES->SetValue(map_info.anon_huge_pages_kb * 1024);
}
+ ProcessStateInfo proc_state(false);
+ MAPPED_BYTES->SetValue(proc_state.GetVmSize());
+ RSS->SetValue(proc_state.GetRss());
ThpConfig thp_config = MemInfo::ParseThpConfig();
THP_ENABLED->SetValue(thp_config.enabled);
http://git-wip-us.apache.org/repos/asf/impala/blob/19277151/be/src/util/process-state-info.cc
----------------------------------------------------------------------
diff --git a/be/src/util/process-state-info.cc b/be/src/util/process-state-info.cc
index 7b8ae91..1ef8dd1 100644
--- a/be/src/util/process-state-info.cc
+++ b/be/src/util/process-state-info.cc
@@ -165,51 +165,27 @@ void ProcessStateInfo::ReadProcFileDescriptorCount() {
process_state_map_["fd/count"] = to_string(fd_count);
}
-ProcessStateInfo::ProcessStateInfo() {
- ReadProcIO();
- ReadProcCgroup();
- ReadProcSched();
+ProcessStateInfo::ProcessStateInfo(bool get_extended_metrics)
+ : have_extended_metrics_(get_extended_metrics) {
ReadProcStatus();
- ReadProcFileDescriptorCount();
+ if (get_extended_metrics) {
+ ReadProcIO();
+ ReadProcCgroup();
+ ReadProcSched();
+ ReadProcFileDescriptorCount();
+ }
}
string ProcessStateInfo::DebugString() const {
stringstream stream;
stream << "Process State: " << endl
- << " I/O: " << endl
- << " Read: "
- << PrettyPrinter::Print(GetBytes("io/read_bytes"), TUnit::BYTES) << endl
- << " Write: "
- << PrettyPrinter::Print(GetBytes("io/write_bytes"), TUnit::BYTES) << endl
- << " Read I/O: " << GetInt64("io/syscr") << endl
- << " Write I/O: " << GetInt64("io/syscw") << endl
- << " CGroups: " << endl
- << " Hierarchy: " << GetString("cgroup/hierarchy_id") << endl
- << " Subsystems: " << GetString("cgroup/subsystems") <<endl
- << " Control Group: " << GetString("cgroup/control_group") << endl
- << " Schedule: " << endl
- << " Sum Execute Time: " << GetString("sched/se.sum_exec_runtime") << endl
- << " Max Wait Time: " << GetString("sched/se.statistics.wait_max") << endl
- << " Sum Wait Time: " << GetString("sched/se.statistics.wait_sum") << endl
- << " Wait Count: " << GetInt64("sched/se.statistics.wait_count") << endl
- << " Sum I/O Wait Time: "
- << GetString("sched/se.statistics.iowait_sum") << endl
- << " I/O Wait Count: "
- << GetInt64("sched/se.statistics.iowait_count") << endl
- << " Wakeup Count with cpu migration: "
- << GetInt64("sched/se.statistics.nr_wakeups_migrate") << endl
- << " Switches: " << GetInt64("sched/nr_switches") << endl
- << " Voluntary Switches: " << GetInt("sched/nr_voluntary_switches") << endl
- << " Involuntary Switches: "
- << GetInt("sched/nr_involuntary_switches") << endl
- << " Process Priority: " << GetInt("sched/prio") << endl
<< " Status: " << endl
<< " Process ID: " << getpid() << endl
<< " Thread Number: " << GetInt("status/Threads") << endl
<< " VM Peak: "
<< PrettyPrinter::Print(GetBytes("status/VmPeak"), TUnit::BYTES) << endl
<< " VM Size: "
- << PrettyPrinter::Print(GetBytes("status/VmSize"), TUnit::BYTES) << endl
+ << PrettyPrinter::Print(GetVmSize(), TUnit::BYTES) << endl
<< " VM Lock: "
<< PrettyPrinter::Print(GetBytes("status/VmLck"), TUnit::BYTES) << endl
<< " VM Pin: "
@@ -217,7 +193,7 @@ string ProcessStateInfo::DebugString() const {
<< " VM HWM: "
<< PrettyPrinter::Print(GetBytes("status/VmHWM"), TUnit::BYTES) << endl
<< " VM RSS: "
- << PrettyPrinter::Print(GetBytes("status/VmRSS"), TUnit::BYTES) << endl
+ << PrettyPrinter::Print(GetRss(), TUnit::BYTES) << endl
<< " VM Data: "
<< PrettyPrinter::Print(GetBytes("status/VmData"), TUnit::BYTES) << endl
<< " VM Stk: "
@@ -231,9 +207,38 @@ string ProcessStateInfo::DebugString() const {
<< " VM Swap: "
<< PrettyPrinter::Print(GetBytes("status/VmSwap"), TUnit::BYTES) << endl
<< " Cpus Allowed List: " << GetString("status/Cpus_allowed_list") << endl
- << " Mems Allowed List: " << GetString("status/Mems_allowed_list") << endl
- << " File Descriptors: " << endl
- << " Number of File Descriptors: " << GetInt("fd/count") << endl;
+ << " Mems Allowed List: " << GetString("status/Mems_allowed_list") << endl;
+ if (have_extended_metrics_) {
+ stream << " I/O: " << endl
+ << " Read: "
+ << PrettyPrinter::Print(GetBytes("io/read_bytes"), TUnit::BYTES) << endl
+ << " Write: "
+ << PrettyPrinter::Print(GetBytes("io/write_bytes"), TUnit::BYTES) << endl
+ << " Read I/O: " << GetInt64("io/syscr") << endl
+ << " Write I/O: " << GetInt64("io/syscw") << endl
+ << " CGroups: " << endl
+ << " Hierarchy: " << GetString("cgroup/hierarchy_id") << endl
+ << " Subsystems: " << GetString("cgroup/subsystems") <<endl
+ << " Control Group: " << GetString("cgroup/control_group") << endl
+ << " Schedule: " << endl
+ << " Sum Execute Time: " << GetString("sched/se.sum_exec_runtime") << endl
+ << " Max Wait Time: " << GetString("sched/se.statistics.wait_max") << endl
+ << " Sum Wait Time: " << GetString("sched/se.statistics.wait_sum") << endl
+ << " Wait Count: " << GetInt64("sched/se.statistics.wait_count") << endl
+ << " Sum I/O Wait Time: "
+ << GetString("sched/se.statistics.iowait_sum") << endl
+ << " I/O Wait Count: "
+ << GetInt64("sched/se.statistics.iowait_count") << endl
+ << " Wakeup Count with cpu migration: "
+ << GetInt64("sched/se.statistics.nr_wakeups_migrate") << endl
+ << " Switches: " << GetInt64("sched/nr_switches") << endl
+ << " Voluntary Switches: " << GetInt("sched/nr_voluntary_switches") << endl
+ << " Involuntary Switches: "
+ << GetInt("sched/nr_involuntary_switches") << endl
+ << " Process Priority: " << GetInt("sched/prio") << endl
+ << " File Descriptors: " << endl
+ << " Number of File Descriptors: " << GetInt("fd/count") << endl;
+ }
stream << endl;
return stream.str();
}
http://git-wip-us.apache.org/repos/asf/impala/blob/19277151/be/src/util/process-state-info.h
----------------------------------------------------------------------
diff --git a/be/src/util/process-state-info.h b/be/src/util/process-state-info.h
index 63b6559..0b7158b 100644
--- a/be/src/util/process-state-info.h
+++ b/be/src/util/process-state-info.h
@@ -81,8 +81,9 @@ namespace impala {
class ProcessStateInfo {
public:
/// Read the current process state info when constructed. There is no need
- /// to be thread safe.
- ProcessStateInfo();
+ /// to be thread safe. If 'get_extended_metrics' is false, only the contents
+ /// of /status are collected. It it is true, all metrics are collected.
+ ProcessStateInfo(bool get_extended_metrics=true);
std::string DebugString() const;
@@ -92,10 +93,18 @@ class ProcessStateInfo {
/// Original data's unit is B or KB.
int64_t GetBytes(const std::string& state_key) const;
+
+ /// Return the virtual memory size in bytes.
+ int64_t GetVmSize() const { return GetBytes("status/VmSize"); }
+
+ //// Return the process RSS in bytes.
+ int64_t GetRss() const { return GetBytes("status/VmRSS"); }
private:
typedef std::map<std::string, std::string> ProcessStateMap;
ProcessStateMap process_state_map_;
+ const bool have_extended_metrics_;
+
/// Read I/O info from /proc/self/io.
void ReadProcIO();