You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by jr...@apache.org on 2017/11/07 22:57:03 UTC
[1/2] incubator-impala git commit: IMPALA-6151: add query-level
fragment/backend counters
Repository: incubator-impala
Updated Branches:
refs/heads/master 216642e28 -> 40ec6d008
IMPALA-6151: add query-level fragment/backend counters
This adds NumBackends, NumFragments and NumFragmentInstances
counters to the query profile to make it easier to manually or
programmatically analyse the query.
Also add a num-queries-registered metric to track the number
of queries that have been executed but are not yet unregistered.
Testing:
Ran "select count(*) from alltypessmall" and checked profile:
Backend startup latencies: Count: 3, min / max: 1ms / 1ms, 25th %-ile: 1ms, 50th %-ile: 1ms, 75th %-ile: 1ms, 90th %-ile: 1ms, 95th %-ile: 1ms, 99.9th %-ile: 1ms
Per Node Peak Memory Usage: tarmstrong-box:22000(1.10 MB) tarmstrong-box:22001(1.02 MB) tarmstrong-box:22002(1.02 MB)
- FiltersReceived: 0 (0)
- FinalizationTimer: 0.000ns
- NumBackends: 3 (3)
- NumFragmentInstances: 4 (4)
- NumFragments: 2 (2)
Ran some query tests (both beeswax and HS2) and manually checked the
num-queries-registered metric on the /metrics page when the queries
were running and after they finished. Added the metric to
test_metrics_are_zero() to make sure that there are no accounting
errors.
Change-Id: I3df350414733e98d1ec28adc1c98f45bb0c4e3e9
Reviewed-on: http://gerrit.cloudera.org:8080/8461
Reviewed-by: Tim Armstrong <ta...@cloudera.com>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/bf9c2f52
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/bf9c2f52
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/bf9c2f52
Branch: refs/heads/master
Commit: bf9c2f521fc920ac78ba88a186f100de354c60e1
Parents: 216642e
Author: Tim Armstrong <ta...@cloudera.com>
Authored: Fri Nov 3 11:12:05 2017 -0700
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Tue Nov 7 21:44:34 2017 +0000
----------------------------------------------------------------------
be/src/runtime/coordinator.cc | 12 ++++++++++++
be/src/service/impala-server.cc | 4 +++-
be/src/util/impalad-metrics.cc | 10 ++++++----
be/src/util/impalad-metrics.h | 5 +++++
common/thrift/metrics.json | 10 ++++++++++
tests/verifiers/metric_verifier.py | 1 +
6 files changed, 37 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/bf9c2f52/be/src/runtime/coordinator.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/coordinator.cc b/be/src/runtime/coordinator.cc
index 94b0bdb..7fe6fb7 100644
--- a/be/src/runtime/coordinator.cc
+++ b/be/src/runtime/coordinator.cc
@@ -187,6 +187,7 @@ void Coordinator::InitFragmentStats() {
vector<const TPlanFragment*> fragments;
schedule_.GetTPlanFragments(&fragments);
const TPlanFragment* coord_fragment = schedule_.GetCoordFragment();
+ int64_t total_num_finstances = 0;
for (const TPlanFragment* fragment: fragments) {
string root_profile_name =
@@ -197,6 +198,7 @@ void Coordinator::InitFragmentStats() {
Substitute("Averaged Fragment $0", fragment->display_name);
int num_instances =
schedule_.GetFragmentExecParams(fragment->idx).instance_exec_params.size();
+ total_num_finstances += num_instances;
// TODO: special-case the coordinator fragment?
FragmentStats* fragment_stats = obj_pool()->Add(
new FragmentStats(
@@ -205,6 +207,12 @@ void Coordinator::InitFragmentStats() {
query_profile_->AddChild(fragment_stats->avg_profile(), true);
query_profile_->AddChild(fragment_stats->root_profile());
}
+ RuntimeProfile::Counter* num_fragments =
+ ADD_COUNTER(query_profile_, "NumFragments", TUnit::UNIT);
+ num_fragments->Set(static_cast<int64_t>(fragments.size()));
+ RuntimeProfile::Counter* num_finstances =
+ ADD_COUNTER(query_profile_, "NumFragmentInstances", TUnit::UNIT);
+ num_finstances->Set(total_num_finstances);
}
void Coordinator::InitBackendStates() {
@@ -212,6 +220,10 @@ void Coordinator::InitBackendStates() {
DCHECK_GT(num_backends, 0);
backend_states_.resize(num_backends);
+ RuntimeProfile::Counter* num_backends_counter =
+ ADD_COUNTER(query_profile_, "NumBackends", TUnit::UNIT);
+ num_backends_counter->Set(num_backends);
+
// create BackendStates
bool has_coord_fragment = schedule_.GetCoordFragment() != nullptr;
const TNetworkAddress& coord_address = ExecEnv::GetInstance()->backend_address();
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/bf9c2f52/be/src/service/impala-server.cc
----------------------------------------------------------------------
diff --git a/be/src/service/impala-server.cc b/be/src/service/impala-server.cc
index bc8613b..4b225b4 100644
--- a/be/src/service/impala-server.cc
+++ b/be/src/service/impala-server.cc
@@ -869,7 +869,6 @@ Status ImpalaServer::ExecuteInternal(
RETURN_IF_ERROR(RegisterQuery(session_state, *request_state));
*registered_request_state = true;
-
#ifndef NDEBUG
// Inject a sleep to simulate metadata loading pauses for tables. This
// is only used for testing.
@@ -950,6 +949,8 @@ Status ImpalaServer::RegisterQuery(shared_ptr<SessionState> session_state,
}
client_request_state_map_.insert(make_pair(query_id, request_state));
}
+ // Metric is decremented in UnregisterQuery().
+ ImpaladMetrics::NUM_QUERIES_REGISTERED->Increment(1L);
return Status::OK();
}
@@ -1049,6 +1050,7 @@ Status ImpalaServer::UnregisterQuery(const TUniqueId& query_id, bool check_infli
}
}
ArchiveQuery(*request_state);
+ ImpaladMetrics::NUM_QUERIES_REGISTERED->Increment(-1L);
return Status::OK();
}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/bf9c2f52/be/src/util/impalad-metrics.cc
----------------------------------------------------------------------
diff --git a/be/src/util/impalad-metrics.cc b/be/src/util/impalad-metrics.cc
index 6d214cd..1325f2e 100644
--- a/be/src/util/impalad-metrics.cc
+++ b/be/src/util/impalad-metrics.cc
@@ -28,10 +28,10 @@ namespace impala {
// be separated by '-'.
const char* ImpaladMetricKeys::IMPALA_SERVER_VERSION =
"impala-server.version";
-const char* ImpaladMetricKeys::IMPALA_SERVER_READY =
- "impala-server.ready";
-const char* ImpaladMetricKeys::IMPALA_SERVER_NUM_QUERIES =
- "impala-server.num-queries";
+const char* ImpaladMetricKeys::IMPALA_SERVER_READY = "impala-server.ready";
+const char* ImpaladMetricKeys::IMPALA_SERVER_NUM_QUERIES = "impala-server.num-queries";
+const char* ImpaladMetricKeys::NUM_QUERIES_REGISTERED =
+ "impala-server.num-queries-registered";
const char* ImpaladMetricKeys::IMPALA_SERVER_NUM_FRAGMENTS =
"impala-server.num-fragments";
const char* ImpaladMetricKeys::IMPALA_SERVER_NUM_FRAGMENTS_IN_FLIGHT =
@@ -138,6 +138,7 @@ IntGauge* ImpaladMetrics::IO_MGR_CACHED_FILE_HANDLES_MISS_COUNT = NULL;
IntGauge* ImpaladMetrics::IO_MGR_TOTAL_BYTES = NULL;
IntGauge* ImpaladMetrics::MEM_POOL_TOTAL_BYTES = NULL;
IntGauge* ImpaladMetrics::NUM_FILES_OPEN_FOR_INSERT = NULL;
+IntGauge* ImpaladMetrics::NUM_QUERIES_REGISTERED = NULL;
IntGauge* ImpaladMetrics::RESULTSET_CACHE_TOTAL_NUM_ROWS = NULL;
IntGauge* ImpaladMetrics::RESULTSET_CACHE_TOTAL_BYTES = NULL;
@@ -163,6 +164,7 @@ void ImpaladMetrics::CreateMetrics(MetricGroup* m) {
IMPALA_SERVER_NUM_QUERIES = m->AddCounter<int64_t>(
ImpaladMetricKeys::IMPALA_SERVER_NUM_QUERIES, 0);
+ NUM_QUERIES_REGISTERED = m->AddGauge<int64_t>(ImpaladMetricKeys::NUM_QUERIES_REGISTERED, 0);
NUM_QUERIES_EXPIRED = m->AddCounter<int64_t>(
ImpaladMetricKeys::NUM_QUERIES_EXPIRED, 0);
NUM_QUERIES_SPILLED = m->AddCounter<int64_t>(
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/bf9c2f52/be/src/util/impalad-metrics.h
----------------------------------------------------------------------
diff --git a/be/src/util/impalad-metrics.h b/be/src/util/impalad-metrics.h
index 9d1cfc2..65ddf1e 100644
--- a/be/src/util/impalad-metrics.h
+++ b/be/src/util/impalad-metrics.h
@@ -127,6 +127,10 @@ class ImpaladMetricKeys {
/// Number of queries expired due to inactivity
static const char* NUM_QUERIES_EXPIRED;
+ /// Number of queries currently registered on this server, i.e. that have been
+ /// registered but are not yet unregistered.
+ static const char* NUM_QUERIES_REGISTERED;
+
/// Number of queries that spilled.
static const char* NUM_QUERIES_SPILLED;
@@ -188,6 +192,7 @@ class ImpaladMetrics {
static IntGauge* IO_MGR_TOTAL_BYTES;
static IntGauge* MEM_POOL_TOTAL_BYTES;
static IntGauge* NUM_FILES_OPEN_FOR_INSERT;
+ static IntGauge* NUM_QUERIES_REGISTERED;
static IntGauge* RESULTSET_CACHE_TOTAL_NUM_ROWS;
static IntGauge* RESULTSET_CACHE_TOTAL_BYTES;
// Properties
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/bf9c2f52/common/thrift/metrics.json
----------------------------------------------------------------------
diff --git a/common/thrift/metrics.json b/common/thrift/metrics.json
index 4e67eae..dafe986 100644
--- a/common/thrift/metrics.json
+++ b/common/thrift/metrics.json
@@ -470,6 +470,16 @@
"key": "impala-server.num-queries"
},
{
+ "description": "The total number of queries registered on this Impala server instance. Includes queries that are in flight and waiting to be closed",
+ "contexts": [
+ "IMPALAD"
+ ],
+ "label": "Queries Registered",
+ "units": "UNIT",
+ "kind": "GAUGE",
+ "key": "impala-server.num-queries-registered"
+ },
+ {
"description": "Number of queries expired due to inactivity.",
"contexts": [
"IMPALAD"
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/bf9c2f52/tests/verifiers/metric_verifier.py
----------------------------------------------------------------------
diff --git a/tests/verifiers/metric_verifier.py b/tests/verifiers/metric_verifier.py
index 1e61494..0c1777b 100644
--- a/tests/verifiers/metric_verifier.py
+++ b/tests/verifiers/metric_verifier.py
@@ -19,6 +19,7 @@
# List of metrics that should be equal to zero when there are no outstanding queries.
METRIC_LIST = [
+ "impala-server.num-queries-registered",
# TODO (IMPALA-3377): Re-enable
# "impala-server.backends.client-cache.clients-in-use", disabled as a
# work-around due to IMPALA-3327.
[2/2] incubator-impala git commit: IMPALA-5473: [DOCS] Document TLS
min version & cipher options
Posted by jr...@apache.org.
IMPALA-5473: [DOCS] Document TLS min version & cipher options
Under the doc JIRA IMPALA-6065.
Change-Id: Ia1705262f8c01e38c616541d1c48f5d0cad5498e
Reviewed-on: http://gerrit.cloudera.org:8080/8401
Reviewed-by: Michael Brown <mi...@cloudera.com>
Reviewed-by: Sailesh Mukil <sa...@cloudera.com>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/40ec6d00
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/40ec6d00
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/40ec6d00
Branch: refs/heads/master
Commit: 40ec6d0080638efaf3260672ab54ea4674896c5e
Parents: bf9c2f5
Author: John Russell <jr...@cloudera.com>
Authored: Thu Oct 26 13:53:46 2017 -0700
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Tue Nov 7 22:29:30 2017 +0000
----------------------------------------------------------------------
docs/topics/impala_ssl.xml | 71 +++++++++++++++++++++++++++++++++++++++++
1 file changed, 71 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/40ec6d00/docs/topics/impala_ssl.xml
----------------------------------------------------------------------
diff --git a/docs/topics/impala_ssl.xml b/docs/topics/impala_ssl.xml
index 726e040..9feb758 100644
--- a/docs/topics/impala_ssl.xml
+++ b/docs/topics/impala_ssl.xml
@@ -132,4 +132,75 @@ under the License.
</conbody>
</concept>
+ <concept id="tls_min_version" rev="2.10.0 IMPALA-5743">
+
+ <title>Specifying TLS/SSL Minimum Allowed Version and Ciphers</title>
+
+ <conbody>
+
+ <p>
+ Depending on your cluster configuration and the security practices in your
+ organization, you might need to restrict the allowed versions of TLS/SSL
+ used by Impala. Older TLS/SSL versions might have vulnerabilities or lack
+ certain features. In <keyword keyref="impala210_full"/>, you can use startup
+ options for the <cmdname>impalad</cmdname>, <cmdname>catalogd</cmdname>,
+ and <cmdname>statestored</cmdname> daemons to specify a minimum allowed
+ version of TLS/SSL.
+ </p>
+
+ <p>
+ Specify one of the following values for the <codeph>--ssl_minimum_version</codeph>
+ configuration setting:
+ </p>
+
+ <ul>
+ <li>
+ <p>
+ <codeph>tlsv1</codeph>: Allow any TLS version of 1.0 or higher.
+ This setting is the default when TLS/SSL is enabled.
+ </p>
+ </li>
+ <li>
+ <p>
+ <codeph>tlsv1.1</codeph>: Allow any TLS version of 1.1 or higher.
+ </p>
+ </li>
+ <li>
+ <p>
+ <codeph>tlsv1.2</codeph>: Allow any TLS version of 1.2 or higher.
+ </p>
+ </li>
+ </ul>
+ <note>
+ <p>
+ As of <keyword keyref="impala210_full"/>, TLSv1.2 may not work for Impala on RHEL 6
+ or CentOS 6, even if OpenSSL 1.0.1 is available. The daemons fail to start, with a
+ socket error stating the TLS version is not supported. The underlying cause is related to
+ <xref href="https://bugzilla.redhat.com/show_bug.cgi?id=1497859" scope="external" format="html">Red Hat issue 1497859</xref>.
+ The issue applies if you build on a RHEL 6 or CentOS 6 system with OpenSSL 1.0.0, and
+ run on a RHEL 6 or CentOS 6 system with OpenSSL 1.0.1.
+ </p>
+ </note>
+
+ <p>
+ Along with specifying the version, you can also specify the allowed set of TLS ciphers
+ by using the <codeph>--ssl_cipher_list</codeph> configuration setting. The argument to
+ this option is a list of keywords, separated by colons, commas, or spaces, and
+ optionally including other notation. For example:
+ </p>
+
+<codeblock>
+--ssl_cipher_list="RC4-SHA,RC4-MD5"
+</codeblock>
+
+ <p>
+ By default, the cipher list is empty, and Impala uses the default cipher list for
+ the underlying platform. See the output of <cmdname>man ciphers</cmdname> for the full
+ set of keywords and notation allowed in the argument string.
+ </p>
+
+ </conbody>
+
+ </concept>
+
</concept>