You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by he...@apache.org on 2017/05/17 16:16:59 UTC
[1/5] incubator-impala git commit: IMPALA-4764: Add Hedged read
metrics
Repository: incubator-impala
Updated Branches:
refs/heads/master a2caacf87 -> 8afb59045
IMPALA-4764: Add Hedged read metrics
The following Hadoop metrics have been added to the /metrics page:
hedgedReadOps - the number of hedged reads that have occurred
hedgedReadOpsWin - the number of times the hedged read returned
faster than the original read
The metrics will be updated only when --use_hdfs_pread is set to
'true'.
This change depends on the following new commit to HDFS:
https://github.com/apache/hadoop/commit/8c81a16a1fa40d3a3796b1c3633328f1340883ca
Testing: Not adding tests since it requires some custom hadoop
configuration. Tested manually by setting the configurations and
verifying that the metrics work.
Change-Id: Id4a5d396abb3373d352ad2df8c2272db018114da
Reviewed-on: http://gerrit.cloudera.org:8080/6886
Reviewed-by: Matthew Jacobs <mj...@cloudera.com>
Reviewed-by: Lars Volker <lv...@cloudera.com>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/c826af68
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/c826af68
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/c826af68
Branch: refs/heads/master
Commit: c826af6831719cfe5e967a56099493a103c68a90
Parents: a2caacf
Author: Sailesh Mukil <sa...@cloudera.com>
Authored: Fri May 12 15:18:13 2017 -0700
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Mon May 15 22:57:32 2017 +0000
----------------------------------------------------------------------
be/src/runtime/disk-io-mgr-scan-range.cc | 14 ++++++++++++++
be/src/util/impalad-metrics.cc | 11 +++++++++++
be/src/util/impalad-metrics.h | 12 +++++++++++-
common/thrift/metrics.json | 20 ++++++++++++++++++++
4 files changed, 56 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/c826af68/be/src/runtime/disk-io-mgr-scan-range.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/disk-io-mgr-scan-range.cc b/be/src/runtime/disk-io-mgr-scan-range.cc
index 5111b70..9ccd1da 100644
--- a/be/src/runtime/disk-io-mgr-scan-range.cc
+++ b/be/src/runtime/disk-io-mgr-scan-range.cc
@@ -345,6 +345,20 @@ void DiskIoMgr::ScanRange::Close() {
}
hdfsFileFreeReadStatistics(stats);
}
+
+ if (FLAGS_use_hdfs_pread) {
+ // Update Hedged Read Metrics.
+ // We call it only if the --use_hdfs_pread flag is set, to avoid having the
+ // libhdfs client malloc and free a hdfsHedgedReadMetrics object unnecessarily
+ // otherwise.
+ struct hdfsHedgedReadMetrics* hedged_metrics;
+ success = hdfsGetHedgedReadMetrics(fs_, &hedged_metrics);
+ if (success == 0) {
+ ImpaladMetrics::HEDGED_READ_OPS->set_value(hedged_metrics->hedgedReadOps);
+ ImpaladMetrics::HEDGED_READ_OPS_WIN->set_value(hedged_metrics->hedgedReadOpsWin);
+ }
+ hdfsFreeHedgedReadMetrics(hedged_metrics);
+ }
}
if (external_buffer_tag_ == ExternalBufferTag::CACHED_BUFFER) {
hadoopRzBufferFree(hdfs_file_->file(), cached_buffer_);
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/c826af68/be/src/util/impalad-metrics.cc
----------------------------------------------------------------------
diff --git a/be/src/util/impalad-metrics.cc b/be/src/util/impalad-metrics.cc
index 271f8d7..7155a8d 100644
--- a/be/src/util/impalad-metrics.cc
+++ b/be/src/util/impalad-metrics.cc
@@ -100,6 +100,10 @@ const char* ImpaladMetricKeys::QUERY_DURATIONS =
"impala-server.query-durations-ms";
const char* ImpaladMetricKeys::DDL_DURATIONS =
"impala-server.ddl-durations-ms";
+const char* ImpaladMetricKeys::HEDGED_READ_OPS =
+ "impala-server.hedged-read-ops";
+const char* ImpaladMetricKeys::HEDGED_READ_OPS_WIN =
+ "impala-server.hedged-read-ops-win";
// These are created by impala-server during startup.
// =======
@@ -118,6 +122,8 @@ IntCounter* ImpaladMetrics::IO_MGR_LOCAL_BYTES_READ = NULL;
IntCounter* ImpaladMetrics::IO_MGR_SHORT_CIRCUIT_BYTES_READ = NULL;
IntCounter* ImpaladMetrics::IO_MGR_CACHED_BYTES_READ = NULL;
IntCounter* ImpaladMetrics::IO_MGR_BYTES_WRITTEN = NULL;
+IntCounter* ImpaladMetrics::HEDGED_READ_OPS = NULL;
+IntCounter* ImpaladMetrics::HEDGED_READ_OPS_WIN = NULL;
// Gauges
IntGauge* ImpaladMetrics::CATALOG_NUM_DBS = NULL;
@@ -243,6 +249,11 @@ void ImpaladMetrics::CreateMetrics(MetricGroup* m) {
MetricDefs::Get(ImpaladMetricKeys::QUERY_DURATIONS), FIVE_HOURS_IN_MS, 3));
DDL_DURATIONS = m->RegisterMetric(new HistogramMetric(
MetricDefs::Get(ImpaladMetricKeys::DDL_DURATIONS), FIVE_HOURS_IN_MS, 3));
+
+ // Initialize Hedged read metrics
+ HEDGED_READ_OPS = m->AddCounter<int64_t>(ImpaladMetricKeys::HEDGED_READ_OPS, 0);
+ HEDGED_READ_OPS_WIN = m->AddCounter<int64_t>(ImpaladMetricKeys::HEDGED_READ_OPS_WIN, 0);
+
}
}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/c826af68/be/src/util/impalad-metrics.h
----------------------------------------------------------------------
diff --git a/be/src/util/impalad-metrics.h b/be/src/util/impalad-metrics.h
index cc3162a..2dafc6a 100644
--- a/be/src/util/impalad-metrics.h
+++ b/be/src/util/impalad-metrics.h
@@ -139,9 +139,17 @@ class ImpaladMetricKeys {
/// Total bytes consumed for rows cached to support HS2 FETCH_FIRST.
static const char* RESULTSET_CACHE_TOTAL_BYTES;
- // Distribution of execution times for queries and DDL statements, in ms.
+ /// Distribution of execution times for queries and DDL statements, in ms.
static const char* QUERY_DURATIONS;
static const char* DDL_DURATIONS;
+
+ /// Total number of attempted hedged reads operations.
+ static const char* HEDGED_READ_OPS;
+
+ /// Total number of hedged reads operations that won
+ /// (i.e. returned faster than original read).
+ static const char* HEDGED_READ_OPS_WIN;
+
};
/// Global impalad-wide metrics. This is useful for objects that want to update metrics
@@ -165,6 +173,8 @@ class ImpaladMetrics {
static IntCounter* IO_MGR_CACHED_BYTES_READ;
static IntCounter* IO_MGR_SHORT_CIRCUIT_BYTES_READ;
static IntCounter* IO_MGR_BYTES_WRITTEN;
+ static IntCounter* HEDGED_READ_OPS;
+ static IntCounter* HEDGED_READ_OPS_WIN;
// Gauges
static IntGauge* CATALOG_NUM_DBS;
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/c826af68/common/thrift/metrics.json
----------------------------------------------------------------------
diff --git a/common/thrift/metrics.json b/common/thrift/metrics.json
index 49d13ae..cef9b02 100644
--- a/common/thrift/metrics.json
+++ b/common/thrift/metrics.json
@@ -550,6 +550,26 @@
"key": "impala-server.scan-ranges.total"
},
{
+ "description": "The total number of hedged reads attempted over the life of the process",
+ "contexts": [
+ "IMPALAD"
+ ],
+ "label": "Hedged Read Operations",
+ "units": "UNIT",
+ "kind": "COUNTER",
+ "key": "impala-server.hedged-read-ops"
+ },
+ {
+ "description": "The total number of times hedged reads were faster than regular read operations",
+ "contexts": [
+ "IMPALAD"
+ ],
+ "label": "Hedged Read Operations Won",
+ "units": "UNIT",
+ "kind": "COUNTER",
+ "key": "impala-server.hedged-read-ops-win"
+ },
+ {
"description": "The local start time of the Impala Server.",
"contexts": [
"IMPALAD"
[4/5] incubator-impala git commit: IMPALA-5174: Bump gflags to
2.2.0-p1
Posted by he...@apache.org.
IMPALA-5174: Bump gflags to 2.2.0-p1
This gflags patch adds DEFINE_int32_hidden() etc. macros, which suppress
flags from appearing in /varz, --help and other flag enumerations.
Our toolchain glog is statically linked against gflags, and therefore
had to be rebuilt, however its version number did not change. You will
likely need to do the following:
rm -rf ${IMPALA_TOOLCHAIN_DIR}/glog-0.3.4-p2/
before running bin/bootstrap_toolchain.py, otherwise building Impala may
fail with a linking error.
Change-Id: Ibc09a750879a8eae8b3549b9438241cb7c4448ed
Reviewed-on: http://gerrit.cloudera.org:8080/6889
Reviewed-by: Lars Volker <lv...@cloudera.com>
Reviewed-by: Matthew Jacobs <mj...@cloudera.com>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/d1910a39
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/d1910a39
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/d1910a39
Branch: refs/heads/master
Commit: d1910a39fcc50ce211b95c3552c0c90b4bc37bbd
Parents: b73c445
Author: Henry Robinson <he...@cloudera.com>
Authored: Mon May 15 11:17:05 2017 -0700
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Wed May 17 09:11:05 2017 +0000
----------------------------------------------------------------------
bin/impala-config.sh | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/d1910a39/bin/impala-config.sh
----------------------------------------------------------------------
diff --git a/bin/impala-config.sh b/bin/impala-config.sh
index a6ae9ca..467451b 100755
--- a/bin/impala-config.sh
+++ b/bin/impala-config.sh
@@ -72,7 +72,7 @@ fi
# moving to a different build of the toolchain, e.g. when a version is bumped or a
# compile option is changed. The build id can be found in the output of the toolchain
# build jobs, it is constructed from the build number and toolchain git hash prefix.
-export IMPALA_TOOLCHAIN_BUILD_ID=381-5fbf43fa0f
+export IMPALA_TOOLCHAIN_BUILD_ID=386-57eed04522
# Versions of toolchain dependencies.
# -----------------------------------
export IMPALA_AVRO_VERSION=1.7.4-p4
@@ -85,7 +85,7 @@ export IMPALA_CRCUTIL_VERSION=440ba7babeff77ffad992df3a10c767f184e946e
export IMPALA_CYRUS_SASL_VERSION=2.1.23
export IMPALA_FLATBUFFERS_VERSION=1.6.0
export IMPALA_GCC_VERSION=4.9.2
-export IMPALA_GFLAGS_VERSION=2.2.0
+export IMPALA_GFLAGS_VERSION=2.2.0-p1
export IMPALA_GLOG_VERSION=0.3.4-p2
export IMPALA_GPERFTOOLS_VERSION=2.5
export IMPALA_GTEST_VERSION=1.6.0
[2/5] incubator-impala git commit: IMPALA-5127: Add history_max option
Posted by he...@apache.org.
IMPALA-5127: Add history_max option
Allow users to keep a longer history of queries if desired. I
personally find it useful to keep a long history of queries to
reference and want to bump this up to a very large value, but
keep the default reasonable. Also change the config loader
to not freak out over unknown parameters so as not to break
for users that end up with new options set running on older
shells.
Testing: Created .impalarc as follows, now getting more history saved.
Put broken things in .impalarc and make sure they are logged as
warnings.
[impala]
history_max=1000
Change-Id: Iaf65bbecb8fd7f1105aac62b6745d6125a603d7f
Reviewed-on: http://gerrit.cloudera.org:8080/6335
Reviewed-by: Michael Brown <mi...@cloudera.com>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/5905083b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/5905083b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/5905083b
Branch: refs/heads/master
Commit: 5905083bfec82dd179fe94c1a109445f69d1faea
Parents: c826af6
Author: Zach Amsden <za...@cloudera.com>
Authored: Thu Mar 9 22:24:16 2017 +0000
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Tue May 16 02:35:22 2017 +0000
----------------------------------------------------------------------
docs/topics/impala_shell_options.xml | 17 +++++++++++++++
shell/impala_shell.py | 8 ++++++--
shell/impala_shell_config_defaults.py | 33 +++++++++++++++---------------
shell/option_parser.py | 6 ++++++
4 files changed, 46 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/5905083b/docs/topics/impala_shell_options.xml
----------------------------------------------------------------------
diff --git a/docs/topics/impala_shell_options.xml b/docs/topics/impala_shell_options.xml
index b2ce883..6e370fe 100644
--- a/docs/topics/impala_shell_options.xml
+++ b/docs/topics/impala_shell_options.xml
@@ -198,6 +198,23 @@ under the License.
<row>
<entry>
<p>
+ N/A
+ </p>
+ </entry>
+ <entry rev="2.9.0 IMPALA-5127">
+ <p>
+ history_max=1000
+ </p>
+ </entry>
+ <entry>
+ <p>
+ Sets the maximum number of queries to store in the history file.
+ </p>
+ </entry>
+ </row>
+ <row>
+ <entry>
+ <p>
-i <varname>hostname</varname> or
--impalad=<varname>hostname</varname>[:<varname>portnum</varname>]
</p>
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/5905083b/shell/impala_shell.py
----------------------------------------------------------------------
diff --git a/shell/impala_shell.py b/shell/impala_shell.py
index 630a4e1..8af2b57 100755
--- a/shell/impala_shell.py
+++ b/shell/impala_shell.py
@@ -46,7 +46,6 @@ from thrift.Thrift import TException
VERSION_FORMAT = "Impala Shell v%(version)s (%(git_hash)s) built on %(build_date)s"
VERSION_STRING = "build version not available"
-HISTORY_LENGTH = 100
# Tarball / packaging build makes impala_build_version available
try:
@@ -181,7 +180,12 @@ class ImpalaShell(cmd.Cmd):
self.interactive = True
try:
self.readline = __import__('readline')
- self.readline.set_history_length(HISTORY_LENGTH)
+ try:
+ self.readline.set_history_length(int(options.history_max))
+ except ValueError:
+ print_to_stderr("WARNING: history_max option malformed %s\n"
+ % options.history_max)
+ self.readline.set_history_length(1000)
except ImportError:
self._disable_readline()
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/5905083b/shell/impala_shell_config_defaults.py
----------------------------------------------------------------------
diff --git a/shell/impala_shell_config_defaults.py b/shell/impala_shell_config_defaults.py
index 4ae02dd..c50ad87 100644
--- a/shell/impala_shell_config_defaults.py
+++ b/shell/impala_shell_config_defaults.py
@@ -24,26 +24,27 @@ import os
import socket
impala_shell_defaults = {
+ 'ca_cert': None,
+ 'config_file': os.path.expanduser("~/.impalarc"),
+ 'default_db': None,
+ 'history_max': 1000,
+ 'ignore_query_failure': False,
'impalad': socket.getfqdn() + ':21000',
- 'query': None,
- 'query_file': None,
- 'use_kerberos': False,
+ 'kerberos_service_name': 'impala',
+ 'output_delimiter': '\\t',
'output_file': None,
- 'write_delimited': False,
'print_header': False,
- 'output_delimiter': '\\t',
- 'kerberos_service_name': 'impala',
- 'verbose': True,
- 'show_profiles': False,
- 'version': False,
- 'ignore_query_failure': False,
+ 'print_progress' : False,
+ 'print_summary' : False,
+ 'query': None,
+ 'query_file': None,
'refresh_after_connect': False,
- 'default_db': None,
+ 'show_profiles': False,
+ 'ssl': False,
+ 'use_kerberos': False,
'use_ldap': False,
'user': getpass.getuser(),
- 'ssl': False,
- 'ca_cert': None,
- 'config_file': os.path.expanduser("~/.impalarc"),
- 'print_progress' : False,
- 'print_summary' : False
+ 'verbose': True,
+ 'version': False,
+ 'write_delimited': False,
}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/5905083b/shell/option_parser.py
----------------------------------------------------------------------
diff --git a/shell/option_parser.py b/shell/option_parser.py
index c455f57..0a30ed4 100755
--- a/shell/option_parser.py
+++ b/shell/option_parser.py
@@ -26,6 +26,7 @@
#
import ConfigParser
+import sys
from impala_shell_config_defaults import impala_shell_defaults
from optparse import OptionParser
@@ -44,6 +45,11 @@ def get_config_from_file(config_filename):
loaded_options = config.items(section_title);
for i, (option, value) in enumerate(loaded_options):
+ if option not in impala_shell_defaults:
+ print >> sys.stderr, "WARNING: Unable to read configuration file correctly. " \
+ "Check formatting: '%s'\n" % option;
+ continue
+
if impala_shell_defaults[option] in [True, False]:
# validate the option if it can only be a boolean value
# the only choice for these options is true or false
[5/5] incubator-impala git commit: IMPALA-5187,
IMPALA-5208: Bump Breakpad Version, undo IMPALA-3794
Posted by he...@apache.org.
IMPALA-5187, IMPALA-5208: Bump Breakpad Version, undo IMPALA-3794
This change switches to a new Breakpad version, which includes fixes for
Breakpad bugs #681 and #728. The toolchain change was reviewed here:
https://gerrit.cloudera.org/6866
The change also undoes the workaround introduced in IMPALA-3794.
In addition to running test_breakpad.py in a loop for a while, I tested
Then I verified that the test fails with the old toolchain version
(88e5b2) and works with the new one (ffe3e4).
To test #728 I added a sleep() call before SendContinueSignalToChild()
and then killed the parent process, manually observing that the child
would die, too.
Change-Id: Ic541ccd565f2bb51f68c085747fc47ae8c905d19
Reviewed-on: http://gerrit.cloudera.org:8080/6883
Reviewed-by: Lars Volker <lv...@cloudera.com>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/8afb5904
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/8afb5904
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/8afb5904
Branch: refs/heads/master
Commit: 8afb59045eb5c5a69f11ac955e16d08e98277b40
Parents: d1910a3
Author: Lars Volker <lv...@cloudera.com>
Authored: Sat May 13 13:37:32 2017 +0200
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Wed May 17 15:19:12 2017 +0000
----------------------------------------------------------------------
bin/impala-config.sh | 4 ++--
tests/custom_cluster/test_breakpad.py | 15 +++++----------
2 files changed, 7 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/8afb5904/bin/impala-config.sh
----------------------------------------------------------------------
diff --git a/bin/impala-config.sh b/bin/impala-config.sh
index 467451b..86a115e 100755
--- a/bin/impala-config.sh
+++ b/bin/impala-config.sh
@@ -72,13 +72,13 @@ fi
# moving to a different build of the toolchain, e.g. when a version is bumped or a
# compile option is changed. The build id can be found in the output of the toolchain
# build jobs, it is constructed from the build number and toolchain git hash prefix.
-export IMPALA_TOOLCHAIN_BUILD_ID=386-57eed04522
+export IMPALA_TOOLCHAIN_BUILD_ID=390-07ebc4fbe0
# Versions of toolchain dependencies.
# -----------------------------------
export IMPALA_AVRO_VERSION=1.7.4-p4
export IMPALA_BINUTILS_VERSION=2.26.1
export IMPALA_BOOST_VERSION=1.57.0-p1
-export IMPALA_BREAKPAD_VERSION=88e5b2c8806bac3f2c80d2fe80094be5bd371601-p2
+export IMPALA_BREAKPAD_VERSION=ffe3e478657dc7126fca6329dfcedc49f4c726d9-p1
export IMPALA_BZIP2_VERSION=1.0.6-p2
export IMPALA_CMAKE_VERSION=3.2.3-p1
export IMPALA_CRCUTIL_VERSION=440ba7babeff77ffad992df3a10c767f184e946e
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/8afb5904/tests/custom_cluster/test_breakpad.py
----------------------------------------------------------------------
diff --git a/tests/custom_cluster/test_breakpad.py b/tests/custom_cluster/test_breakpad.py
index bf480e9..5479a77 100644
--- a/tests/custom_cluster/test_breakpad.py
+++ b/tests/custom_cluster/test_breakpad.py
@@ -143,14 +143,9 @@ class TestBreakpad(CustomClusterTestSuite):
def count_all_minidumps(self, base_dir=None):
return sum((self.count_minidumps(daemon, base_dir) for daemon in DAEMONS))
- def assert_num_minidumps_for_all_daemons(self, base_dir=None):
+ def assert_num_minidumps_for_all_daemons(self, cluster_size, base_dir=None):
self.assert_num_logfile_entries(1)
- # IMPALA-3794 / Breakpad-681: Weak minidump ID generation can lead to name conflicts,
- # so that one process overwrites the minidump of others. See IMPALA-3794 for more
- # information.
- # TODO: Change this here and elsewhere in this file to expect 'cluster_size' minidumps
- # once Breakpad-681 has been fixed.
- assert self.count_minidumps('impalad', base_dir) >= 1
+ assert self.count_minidumps('impalad', base_dir) == cluster_size
assert self.count_minidumps('statestored', base_dir) == 1
assert self.count_minidumps('catalogd', base_dir) == 1
@@ -169,7 +164,7 @@ class TestBreakpad(CustomClusterTestSuite):
assert self.count_all_minidumps() == 0
cluster_size = self.get_num_processes('impalad')
self.kill_cluster(SIGSEGV)
- self.assert_num_minidumps_for_all_daemons()
+ self.assert_num_minidumps_for_all_daemons(cluster_size)
@pytest.mark.execute_serially
def test_sigusr1_writes_minidump(self):
@@ -188,7 +183,7 @@ class TestBreakpad(CustomClusterTestSuite):
self.execute_query_expect_success(client, "SELECT COUNT(*) FROM functional.alltypes")
# Kill the cluster. Sending SIGKILL will not trigger minidumps to be written.
self.kill_cluster(SIGKILL)
- self.assert_num_minidumps_for_all_daemons()
+ self.assert_num_minidumps_for_all_daemons(cluster_size)
@pytest.mark.execute_serially
def test_minidump_relative_path(self):
@@ -203,7 +198,7 @@ class TestBreakpad(CustomClusterTestSuite):
assert self.count_all_minidumps(minidump_base_dir) == 0
cluster_size = self.get_num_processes('impalad')
self.kill_cluster(SIGSEGV)
- self.assert_num_minidumps_for_all_daemons(minidump_base_dir)
+ self.assert_num_minidumps_for_all_daemons(cluster_size, minidump_base_dir)
shutil.rmtree(minidump_base_dir)
@pytest.mark.execute_serially
[3/5] incubator-impala git commit: IMPALA-5246: Verify that a UDF
test completely finishes before moving on
Posted by he...@apache.org.
IMPALA-5246: Verify that a UDF test completely finishes before moving on
A memory intensive UDF test takes a while to completely finish and for
the memory in Impala to be completely freed. This caused a problem in
ASAN builds (and potentially in normal builds) because we would start
the next test right away, before the memory is freed.
We fix the issue by checking that all fragments finish executing before
starting the next test.
Testing:
- Ran a private ASAN build which passed.
Change-Id: I0555b5327945c522f70f449caa1214ee0bfd84fe
Reviewed-on: http://gerrit.cloudera.org:8080/6893
Reviewed-by: Alex Behm <al...@cloudera.com>
Reviewed-by: Michael Ho <kw...@cloudera.com>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/b73c445e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/b73c445e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/b73c445e
Branch: refs/heads/master
Commit: b73c445eba48a4637c5642fa1866e237617a322d
Parents: 5905083
Author: Taras Bobrovytsky <tb...@cloudera.com>
Authored: Thu May 11 13:54:13 2017 -0700
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Wed May 17 04:31:08 2017 +0000
----------------------------------------------------------------------
tests/query_test/test_udfs.py | 9 +++++++++
1 file changed, 9 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b73c445e/tests/query_test/test_udfs.py
----------------------------------------------------------------------
diff --git a/tests/query_test/test_udfs.py b/tests/query_test/test_udfs.py
index ec24c9f..e8e4ead 100644
--- a/tests/query_test/test_udfs.py
+++ b/tests/query_test/test_udfs.py
@@ -30,6 +30,7 @@ from tests.common.test_dimensions import (
create_uncompressed_text_dimension)
from tests.util.calculation_util import get_random_id
from tests.util.filesystem_utils import get_fs_path, IS_S3
+from tests.verifiers.metric_verifier import MetricVerifier
class TestUdfBase(ImpalaTestSuite):
"""
@@ -341,6 +342,14 @@ class TestUdfExecution(TestUdfBase):
except ImpalaBeeswaxException, e:
self._check_exception(e)
+ # It takes a long time for Impala to free up memory after this test, especially if
+ # ASAN is enabled. Verify that all fragments finish executing before moving on to the
+ # next test to make sure that the next test is not affected.
+ for impalad in ImpalaCluster().impalads:
+ verifier = MetricVerifier(impalad.service)
+ verifier.wait_for_metric("impala-server.num-fragments-in-flight", 0)
+ verifier.verify_num_unused_buffers()
+
def test_udf_constant_folding(self, vector, unique_database):
"""Test that constant folding of UDFs is handled correctly. Uses count_rows(),
which returns a unique value every time it is evaluated in the same thread."""