You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ta...@apache.org on 2016/07/14 19:04:58 UTC
[01/18] incubator-impala git commit: IMPALA-3839: Fix race condition
in impala_cluster.py
Repository: incubator-impala
Updated Branches:
refs/heads/master 476f687b4 -> baf8fe202
IMPALA-3839: Fix race condition in impala_cluster.py
Accesses to properties of Process objects from psutil hit the operating
system, at least under Linux. Therefore any access can throw
NoSuchProcess if the underlying process has already terminated. The fix
extends the try/except block to all such accesses.
To ensure that this change does not introduce a regression I ran
test_breakpad.py, which exercises this code. I also ran
test_breakpad.py:test_sigusr1_writes_minidump() in a loop for about 40
iterations with the changes from https://gerrit.cloudera.org/#/c/3312/6,
which made me discover this issue in the first place.
Change-Id: I3c0d5e43f8c58b4922ab5db78236915fcc8b588d
Reviewed-on: http://gerrit.cloudera.org:8080/3592
Tested-by: Internal Jenkins
Reviewed-by: Lars Volker <lv...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/db4294c7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/db4294c7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/db4294c7
Branch: refs/heads/master
Commit: db4294c76139209b663f880d7aa7d03f6bcd1b3d
Parents: c6ce32b
Author: Lars Volker <lv...@cloudera.com>
Authored: Fri Jul 8 00:32:40 2016 +0200
Committer: Taras Bobrovytsky <ta...@apache.org>
Committed: Thu Jul 14 19:04:43 2016 +0000
----------------------------------------------------------------------
tests/common/impala_cluster.py | 26 +++++++++++++-------------
1 file changed, 13 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/db4294c7/tests/common/impala_cluster.py
----------------------------------------------------------------------
diff --git a/tests/common/impala_cluster.py b/tests/common/impala_cluster.py
index 4a762c0..2a8b56c 100644
--- a/tests/common/impala_cluster.py
+++ b/tests/common/impala_cluster.py
@@ -96,23 +96,23 @@ class ImpalaCluster(object):
for pid in psutil.get_pid_list():
try:
process = psutil.Process(pid)
+ try:
+ if process.username != getuser():
+ continue
+ except KeyError, e:
+ if "uid not found" in str(e):
+ continue
+ raise
+ if process.name == 'impalad' and len(process.cmdline) >= 1:
+ impalads.append(ImpaladProcess(process.cmdline))
+ elif process.name == 'statestored' and len(process.cmdline) >= 1:
+ statestored.append(StateStoreProcess(process.cmdline))
+ elif process.name == 'catalogd' and len(process.cmdline) >= 1:
+ catalogd = CatalogdProcess(process.cmdline)
except psutil.NoSuchProcess, e:
# A process from get_pid_list() no longer exists, continue.
LOG.info(e)
continue
- try:
- if process.username != getuser():
- continue
- except KeyError, e:
- if "uid not found" in str(e):
- continue
- raise
- if process.name == 'impalad' and len(process.cmdline) >= 1:
- impalads.append(ImpaladProcess(process.cmdline))
- elif process.name == 'statestored' and len(process.cmdline) >= 1:
- statestored.append(StateStoreProcess(process.cmdline))
- elif process.name == 'catalogd' and len(process.cmdline) >=1:
- catalogd = CatalogdProcess(process.cmdline)
return impalads, statestored, catalogd
# Represents a process running on a machine and common actions that can be performed
[14/18] incubator-impala git commit: IMPALA-3778: Fix ASF packaging
build
Posted by ta...@apache.org.
IMPALA-3778: Fix ASF packaging build
The tarballs in IMPALA_HOME/infra/python/deps and the thirdparty
directory have been removed in the ASF repository. All Python
dependencies and CDH components must now be downloaded as part of every
build. This caused the ASF packaging build to fail. Before this patch,
we used the system pip to download the Python dependencies, which caused
flakiness and inconsistency on different operating systems. This patch
fixes the problem by using our own script (which requires Python 2.6+ to
be installed on the system), to download all the files in
requirements.txt.
Also replaced all whl and zip Python packages with tar.gz to make it
consistent with the ASF build.
Change-Id: Ibe5a743096cda2059bd330805d324983f6730e19
Reviewed-on: http://gerrit.cloudera.org:8080/3647
Reviewed-by: Jim Apple <jb...@cloudera.com>
Tested-by: Taras Bobrovytsky <tb...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/baf8fe20
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/baf8fe20
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/baf8fe20
Branch: refs/heads/master
Commit: baf8fe202c1e212bbed3c73a6a63017eceb4a180
Parents: 7441032
Author: Taras Bobrovytsky <tb...@cloudera.com>
Authored: Sat Jul 9 02:07:17 2016 +0000
Committer: Taras Bobrovytsky <ta...@apache.org>
Committed: Thu Jul 14 19:04:45 2016 +0000
----------------------------------------------------------------------
bin/impala-config.sh | 48 ++++++-----
infra/python/deps/download_requirements | 28 ++----
infra/python/deps/find_py26.py | 41 +++++++++
infra/python/deps/pip_download.py | 124 ++++++++++++++++++++-------
infra/python/deps/requirements.txt | 2 +
5 files changed, 170 insertions(+), 73 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/baf8fe20/bin/impala-config.sh
----------------------------------------------------------------------
diff --git a/bin/impala-config.sh b/bin/impala-config.sh
index 9db640a..481d2cb 100755
--- a/bin/impala-config.sh
+++ b/bin/impala-config.sh
@@ -66,9 +66,14 @@ fi
# If enabled, debug symbols are added to cross-compiled IR.
: ${ENABLE_IMPALA_IR_DEBUG_INFO=false}
+if [ -d $IMPALA_HOME/thirdparty ]; then
+ NO_THIRDPARTY=false
+else
+ NO_THIRDPARTY=true
+fi
# If true, download and use the CDH components from S3 instead of the ones
# in $IMPALA_HOME/thirdparty.
-: ${DOWNLOAD_CDH_COMPONENTS=false}
+: ${DOWNLOAD_CDH_COMPONENTS=$NO_THIRDPARTY}
export IMPALA_TOOLCHAIN
export SKIP_TOOLCHAIN_BOOTSTRAP
@@ -426,26 +431,27 @@ alias gerrit-verify-merge="${IMPALA_AUX_TEST_HOME}/jenkins/gerrit-verify-merge.s
# A marker in the environment to prove that we really did source this file
export IMPALA_CONFIG_SOURCED=1
-echo "IMPALA_HOME = $IMPALA_HOME"
-echo "HADOOP_HOME = $HADOOP_HOME"
-echo "HADOOP_CONF_DIR = $HADOOP_CONF_DIR"
-echo "MINI_DFS_BASE_DATA_DIR = $MINI_DFS_BASE_DATA_DIR"
-echo "HIVE_HOME = $HIVE_HOME"
-echo "HIVE_CONF_DIR = $HIVE_CONF_DIR"
-echo "HBASE_HOME = $HBASE_HOME"
-echo "HBASE_CONF_DIR = $HBASE_CONF_DIR"
-echo "MINIKDC_HOME = $MINIKDC_HOME"
-echo "THRIFT_HOME = $THRIFT_HOME"
-echo "HADOOP_LZO = $HADOOP_LZO"
-echo "IMPALA_LZO = $IMPALA_LZO"
-echo "CLASSPATH = $CLASSPATH"
-echo "LIBHDFS_OPTS = $LIBHDFS_OPTS"
-echo "PYTHONPATH = $PYTHONPATH"
-echo "JAVA_HOME = $JAVA_HOME"
-echo "LD_LIBRARY_PATH = $LD_LIBRARY_PATH"
-echo "LD_PRELOAD = $LD_PRELOAD"
-echo "POSTGRES_JDBC_DRIVER = $POSTGRES_JDBC_DRIVER"
-echo "IMPALA_TOOLCHAIN = $IMPALA_TOOLCHAIN"
+echo "IMPALA_HOME = $IMPALA_HOME"
+echo "HADOOP_HOME = $HADOOP_HOME"
+echo "HADOOP_CONF_DIR = $HADOOP_CONF_DIR"
+echo "MINI_DFS_BASE_DATA_DIR = $MINI_DFS_BASE_DATA_DIR"
+echo "HIVE_HOME = $HIVE_HOME"
+echo "HIVE_CONF_DIR = $HIVE_CONF_DIR"
+echo "HBASE_HOME = $HBASE_HOME"
+echo "HBASE_CONF_DIR = $HBASE_CONF_DIR"
+echo "MINIKDC_HOME = $MINIKDC_HOME"
+echo "THRIFT_HOME = $THRIFT_HOME"
+echo "HADOOP_LZO = $HADOOP_LZO"
+echo "IMPALA_LZO = $IMPALA_LZO"
+echo "CLASSPATH = $CLASSPATH"
+echo "LIBHDFS_OPTS = $LIBHDFS_OPTS"
+echo "PYTHONPATH = $PYTHONPATH"
+echo "JAVA_HOME = $JAVA_HOME"
+echo "LD_LIBRARY_PATH = $LD_LIBRARY_PATH"
+echo "LD_PRELOAD = $LD_PRELOAD"
+echo "POSTGRES_JDBC_DRIVER = $POSTGRES_JDBC_DRIVER"
+echo "IMPALA_TOOLCHAIN = $IMPALA_TOOLCHAIN"
+echo "DOWNLOAD_CDH_COMPONENTS = $DOWNLOAD_CDH_COMPONENTS"
# Kerberos things. If the cluster exists and is kerberized, source
# the required environment. This is required for any hadoop tool to
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/baf8fe20/infra/python/deps/download_requirements
----------------------------------------------------------------------
diff --git a/infra/python/deps/download_requirements b/infra/python/deps/download_requirements
index eee7a1b..e054653 100755
--- a/infra/python/deps/download_requirements
+++ b/infra/python/deps/download_requirements
@@ -2,31 +2,15 @@
set -euo pipefail
-# Prefer the virtualenv pip and python since this is what will actually be used during the
-# installation and it may be a different version than the system default.
-VIRTUAL_ENV_PIP="$IMPALA_HOME"/infra/python/env/bin/pip
-if [[ -e "$VIRTUAL_ENV_PIP" ]]; then
- PIP="$VIRTUAL_ENV_PIP"
- # Assume python is also available in the virtualenv.
- PYTHON="$IMPALA_HOME"/infra/python/env/bin/python
-else
- PIP=pip
- PYTHON=python
-fi
-
DIR=$(dirname "$0")
-# Download but don't install all packages listed in requirements.txt.
-# Don't use pip to download Impyla because its setup.py requires a newer version
-# of setuptools than is available on some systems we support (e.g. CentOS 6).
-"$PIP" -q install --download "$DIR" -r <(grep -v "impyla" "$DIR"/requirements.txt)
-# Directly download required packages not listed in requirements.txt.
+pushd $DIR
+PY26=$(./find_py26.py)
+# Directly download packages listed in requirements.txt, but don't install them.
+$PY26 pip_download.py
# For virtualenv, other scripts rely on the .tar.gz package (not a .whl package).
+$PY26 pip_download.py virtualenv 13.1.0
# kudu-python is downloaded separately because pip install attempts to execute a
# setup.py subcommand for kudu-python that can fail even if the download succeeds.
-pushd $DIR
-$PYTHON ./pip_download.py virtualenv 13.1.0
-$PYTHON ./pip_download.py kudu-python 0.1.1
-IMPYLA_VERSION=$(grep impyla ./requirements.txt | grep -o '[0-9.]*')
-$PYTHON ./pip_download.py impyla ${IMPYLA_VERSION}
+$PY26 pip_download.py kudu-python 0.1.1
popd
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/baf8fe20/infra/python/deps/find_py26.py
----------------------------------------------------------------------
diff --git a/infra/python/deps/find_py26.py b/infra/python/deps/find_py26.py
new file mode 100755
index 0000000..2ba5d23
--- /dev/null
+++ b/infra/python/deps/find_py26.py
@@ -0,0 +1,41 @@
+#!/usr/bin/python
+# Copyright (c) 2015 Cloudera, Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script finds Python 2.6 or higher on the system and outputs the
+# system command to stdout. The script exits with a nonzero exit code if
+# Python 2.6+ is not present.
+
+import os
+import subprocess
+import sys
+import textwrap
+
+def detect_python_cmd():
+ '''Returns the system command that provides python 2.6 or greater.'''
+ paths = os.getenv("PATH").split(os.path.pathsep)
+ for cmd in ("python", "python27", "python2.7", "python-27", "python-2.7", "python26",
+ "python2.6", "python-26", "python-2.6"):
+ for path in paths:
+ cmd_path = os.path.join(path, cmd)
+ if not os.path.exists(cmd_path) or not os.access(cmd_path, os.X_OK):
+ continue
+ exit = subprocess.call([cmd_path, "-c", textwrap.dedent("""
+ import sys
+ sys.exit(int(sys.version_info[:2] < (2, 6)))""")])
+ if exit == 0:
+ return cmd_path
+ raise Exception("Could not find minimum required python version 2.6")
+
+print detect_python_cmd()
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/baf8fe20/infra/python/deps/pip_download.py
----------------------------------------------------------------------
diff --git a/infra/python/deps/pip_download.py b/infra/python/deps/pip_download.py
index b1e1fa7..a3268b6 100755
--- a/infra/python/deps/pip_download.py
+++ b/infra/python/deps/pip_download.py
@@ -1,42 +1,106 @@
#!/usr/bin/python
+# Copyright (c) 2015 Cloudera, Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
# Implement the basic 'pip download' functionality in a way that gives us more control
# over which archive type is downloaded and what post-download steps are executed.
-import hashlib
+# This script requires Python 2.6+.
+
import json
import os.path
-from urllib import urlopen, URLopener
import sys
+from hashlib import md5
+from time import sleep
+from urllib import urlopen, URLopener
-pkg_name = sys.argv[1]
-pkg_version = sys.argv[2]
-pkg_type = 'sdist' # Don't download wheel archives for now
-pkg_info = json.loads(urlopen('https://pypi.python.org/pypi/%s/json' % pkg_name).read())
+NUM_TRIES = 3
def check_md5sum(filename, expected_md5):
- expected_md5 = pkg['md5_digest']
- actual_md5 = hashlib.md5(open(filename).read()).hexdigest()
+ actual_md5 = md5(open(filename).read()).hexdigest()
return actual_md5 == expected_md5
-found = False
-downloader = URLopener()
-for pkg in pkg_info['releases'][pkg_version]:
- if pkg['packagetype'] == pkg_type:
- filename = pkg['filename']
- expected_md5 = pkg['md5_digest']
- print "Downloading %s from %s " % (filename, pkg['url'])
- if os.path.isfile(filename) and check_md5sum(filename, expected_md5):
- print "File with matching md5sum already exists, skipping download."
- found = True
- break
- downloader.retrieve(pkg['url'], filename)
- actual_md5 = hashlib.md5(open(filename).read()).hexdigest()
- if not check_md5sum(filename, expected_md5):
- print "MD5 mismatch in file %s." % filename
- sys.exit(1)
- found = True
- break
-
-if not found:
- print "Could not find archive to download for %s %s %s" % (pkg_name, pkg_version,
- pkg_type)
+def retry(func):
+ '''Retry decorator.'''
+
+ def wrapper(*args, **kwargs):
+ for _ in xrange(NUM_TRIES):
+ try:
+ result = func(*args, **kwargs)
+ if result: return result
+ except Exception as e:
+ print e
+ sleep(5)
+ print "Download failed after several attempts."
+ sys.exit(1)
+
+ return wrapper
+
+@retry
+def download_package(pkg_name, pkg_version):
+ '''Download the required package. Sometimes the download can be flaky, so we use the
+ retry decorator.'''
+ pkg_type = 'sdist' # Don't download wheel archives for now
+ pkg_info = json.loads(urlopen('https://pypi.python.org/pypi/%s/json' % pkg_name).read())
+
+ downloader = URLopener()
+ for pkg in pkg_info['releases'][pkg_version]:
+ if pkg['packagetype'] == pkg_type:
+ filename = pkg['filename']
+ expected_md5 = pkg['md5_digest']
+ if os.path.isfile(filename) and check_md5sum(filename, expected_md5):
+ print "File with matching md5sum already exists, skipping download."
+ return True
+ print "Downloading %s from %s " % (filename, pkg['url'])
+ downloader.retrieve(pkg['url'], filename)
+ actual_md5 = md5(open(filename).read()).hexdigest()
+ if check_md5sum(filename, expected_md5):
+ return True
+ else:
+ print "MD5 mismatch in file %s." % filename
+ return False
+ print "Could not find archive to download for %s %s %s" % (
+ pkg_name, pkg_version, pkg_type)
sys.exit(1)
+
+def main():
+ if len(sys.argv) > 1:
+ _, pkg_name, pkg_version = sys.argv
+ download_package(pkg_name, pkg_version)
+ else:
+ # If the package name and version are not specified in the command line arguments,
+ # download the packages that in requirements.txt.
+ f = open("requirements.txt", 'r')
+ try:
+ # requirements.txt follows the standard pip grammar.
+ for line in f:
+ # A hash symbol ("#") represents a comment that should be ignored.
+ hash_index = line.find('#')
+ if hash_index != -1:
+ line = line[:hash_index]
+ # A semi colon (";") specifies some additional condition for when the package
+ # should be installed (for example a specific OS). We can ignore this and download
+ # the package anyways because the installation script(bootstrap_virtualenv.py) can
+ # take it into account.
+ semi_colon_index = line.find(';')
+ if semi_colon_index != -1:
+ line = line[:semi_colon_index]
+ l = line.strip()
+ if len(l) > 0:
+ pkg_name, pkg_version = l.split('==')
+ download_package(pkg_name.strip(), pkg_version.strip())
+ finally:
+ f.close()
+
+if __name__ == '__main__':
+ main()
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/baf8fe20/infra/python/deps/requirements.txt
----------------------------------------------------------------------
diff --git a/infra/python/deps/requirements.txt b/infra/python/deps/requirements.txt
index b3ebfb1..77e4be9 100644
--- a/infra/python/deps/requirements.txt
+++ b/infra/python/deps/requirements.txt
@@ -19,6 +19,7 @@ cm-api == 10.0.0
readline == 6.2.4.1; sys_platform == 'darwin'
Fabric == 1.10.2
paramiko == 1.15.2
+ ecdsa == 0.13
pycrypto == 2.6.1
Flask == 0.10.1
Jinja2 == 2.8
@@ -73,3 +74,4 @@ texttable == 0.8.3
# For dev purposes, not used in scripting. Version 1.2.1 is the latest that supports 2.6.
ipython == 1.2.1
+ apipkg == 1.4
[13/18] incubator-impala git commit: IMPALA-3018: Don't return NULL
on zero length allocations.
Posted by ta...@apache.org.
IMPALA-3018: Don't return NULL on zero length allocations.
FunctionContext::Allocate() and FunctionContext::AllocateLocal()
used to return NULL for zero length allocations. This makes
it hard to distinguish between allocation failures and zero
length allocations. Such confusion may lead to DCHECK failure
in the macro RETURN_IF_NULL() in debug builds or access to NULL
pointers in non-debug builds.
This change fixes the problem above by returning NULL only if
there is allocation failure. Zero-length allocations will always
return a dummy non-NULL pointer.
Change-Id: Id8c3211f4d9417f44b8018ccc58ae182682693da
Reviewed-on: http://gerrit.cloudera.org:8080/3601
Reviewed-by: Michael Ho <kw...@cloudera.com>
Tested-by: Internal Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/f129dfd2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/f129dfd2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/f129dfd2
Branch: refs/heads/master
Commit: f129dfd2022d268a5658bac9da279df79dd6c857
Parents: 900f148
Author: Michael Ho <kw...@cloudera.com>
Authored: Fri Jul 8 14:39:44 2016 -0700
Committer: Taras Bobrovytsky <ta...@apache.org>
Committed: Thu Jul 14 19:04:45 2016 +0000
----------------------------------------------------------------------
be/src/exprs/aggregate-functions-ir.cc | 1 +
be/src/runtime/free-pool.h | 8 +++----
be/src/udf/udf-test.cc | 5 ++--
be/src/udf/udf.cc | 2 --
.../queries/QueryTest/aggregation.test | 24 +++++++++++++++++---
5 files changed, 28 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f129dfd2/be/src/exprs/aggregate-functions-ir.cc
----------------------------------------------------------------------
diff --git a/be/src/exprs/aggregate-functions-ir.cc b/be/src/exprs/aggregate-functions-ir.cc
index 79d7ac7..853661c 100644
--- a/be/src/exprs/aggregate-functions-ir.cc
+++ b/be/src/exprs/aggregate-functions-ir.cc
@@ -1388,6 +1388,7 @@ void AggregateFunctions::LastValUpdate(FunctionContext* ctx, const StringVal& sr
} else {
new_ptr = ctx->Reallocate(dst->ptr, src.len);
}
+ // Note that a zero-length string is not the same as StringVal::null().
RETURN_IF_NULL(ctx, new_ptr);
dst->ptr = new_ptr;
memcpy(dst->ptr, src.ptr, src.len);
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f129dfd2/be/src/runtime/free-pool.h
----------------------------------------------------------------------
diff --git a/be/src/runtime/free-pool.h b/be/src/runtime/free-pool.h
index 90df749..8762f1d 100644
--- a/be/src/runtime/free-pool.h
+++ b/be/src/runtime/free-pool.h
@@ -62,12 +62,10 @@ class FreePool {
return NULL;
}
#endif
- ++net_allocations_;
- if (FLAGS_disable_mem_pools) return reinterpret_cast<uint8_t*>(malloc(size));
-
/// Return a non-NULL dummy pointer. NULL is reserved for failures.
if (UNLIKELY(size == 0)) return mem_pool_->EmptyAllocPtr();
-
+ ++net_allocations_;
+ if (FLAGS_disable_mem_pools) return reinterpret_cast<uint8_t*>(malloc(size));
int free_list_idx = Bits::Log2Ceiling64(size);
DCHECK_LT(free_list_idx, NUM_LISTS);
FreeListNode* allocation = lists_[free_list_idx].next;
@@ -92,12 +90,12 @@ class FreePool {
}
void Free(uint8_t* ptr) {
+ if (UNLIKELY(ptr == NULL || ptr == mem_pool_->EmptyAllocPtr())) return;
--net_allocations_;
if (FLAGS_disable_mem_pools) {
free(ptr);
return;
}
- if (UNLIKELY(ptr == NULL || ptr == mem_pool_->EmptyAllocPtr())) return;
FreeListNode* node = reinterpret_cast<FreeListNode*>(ptr - sizeof(FreeListNode));
FreeListNode* list = node->list;
#ifndef NDEBUG
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f129dfd2/be/src/udf/udf-test.cc
----------------------------------------------------------------------
diff --git a/be/src/udf/udf-test.cc b/be/src/udf/udf-test.cc
index 4857a06..86f02c2 100644
--- a/be/src/udf/udf-test.cc
+++ b/be/src/udf/udf-test.cc
@@ -123,8 +123,9 @@ IntVal ValidateFail(FunctionContext* context) {
}
IntVal ValidateMem(FunctionContext* context) {
- EXPECT_TRUE(context->Allocate(0) == NULL);
- uint8_t* buffer = context->Allocate(10);
+ uint8_t* buffer = context->Allocate(0);
+ EXPECT_TRUE(buffer != NULL);
+ buffer = context->Reallocate(buffer, 10);
EXPECT_TRUE(buffer != NULL);
memset(buffer, 0, 10);
context->Free(buffer);
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f129dfd2/be/src/udf/udf.cc
----------------------------------------------------------------------
diff --git a/be/src/udf/udf.cc b/be/src/udf/udf.cc
index ff81c9c..c1fb5be 100644
--- a/be/src/udf/udf.cc
+++ b/be/src/udf/udf.cc
@@ -286,7 +286,6 @@ inline bool FunctionContextImpl::CheckAllocResult(const char* fn_name,
uint8_t* FunctionContext::Allocate(int byte_size) noexcept {
assert(!impl_->closed_);
- if (byte_size == 0) return NULL;
uint8_t* buffer = impl_->pool_->Allocate(byte_size);
if (UNLIKELY(!impl_->CheckAllocResult("FunctionContext::Allocate",
buffer, byte_size))) {
@@ -418,7 +417,6 @@ void FunctionContext::SetFunctionState(FunctionStateScope scope, void* ptr) {
uint8_t* FunctionContextImpl::AllocateLocal(int64_t byte_size) noexcept {
assert(!closed_);
- if (byte_size == 0) return NULL;
uint8_t* buffer = pool_->Allocate(byte_size);
if (UNLIKELY(!CheckAllocResult("FunctionContextImpl::AllocateLocal",
buffer, byte_size))) {
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f129dfd2/testdata/workloads/functional-query/queries/QueryTest/aggregation.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/aggregation.test b/testdata/workloads/functional-query/queries/QueryTest/aggregation.test
index 5389b04..740f777 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/aggregation.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/aggregation.test
@@ -154,13 +154,31 @@ from alltypesagg where day is not null
bigint, string, string, string, string
====
---- QUERY
-# Test for IMPALA-3018. Verify update() function of min() handles
+# Test for IMPALA-3018. Verify update() functions of min() and max() handle
# zero-length string correctly.
-select min(str) from (values ('aaa' as str), ('')) as tmp
+select max(str), min(str) from (values ('aaa' as str), (''), ('123')) as tmp
+---- RESULTS
+'aaa',''
+---- TYPES
+string,string
+====
+---- QUERY
+# Test for IMPALA-3018. Verify update() function of last_value() handles
+# zero-length string correctly.
+select last_value(b) over (partition by a order by d) from functional.nulltable;
---- RESULTS
''
---- TYPES
-String
+string
+====
+---- QUERY
+# Test for IMPALA-3018. Verify update() function of first_value() handles
+# zero-length string correctly.
+select first_value(b) over (partition by a order by d) from functional.nulltable;
+---- RESULTS
+''
+---- TYPES
+string
====
---- QUERY
# grouping by different data types, with NULLs
[12/18] incubator-impala git commit: IMPALA-3225: Add script to push
from gerrit to ASF
Posted by ta...@apache.org.
IMPALA-3225: Add script to push from gerrit to ASF
Adds a script (copied and modified from Kudu) that attempts
to push any changes to ASF branches from gerrit branches
that have new changes.
Change-Id: I15de939acc4b08a8f01511fffcb6ab1743d09c01
Reviewed-on: http://gerrit.cloudera.org:8080/3468
Reviewed-by: Matthew Jacobs <mj...@cloudera.com>
Tested-by: Internal Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/761c39c5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/761c39c5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/761c39c5
Branch: refs/heads/master
Commit: 761c39c54510f08b10c9dc567335243d61d6b2d9
Parents: d75f327
Author: Matthew Jacobs <mj...@cloudera.com>
Authored: Thu Jun 23 10:45:45 2016 -0700
Committer: Taras Bobrovytsky <ta...@apache.org>
Committed: Thu Jul 14 19:04:44 2016 +0000
----------------------------------------------------------------------
bin/push_to_asf.py | 295 ++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 295 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/761c39c5/bin/push_to_asf.py
----------------------------------------------------------------------
diff --git a/bin/push_to_asf.py b/bin/push_to_asf.py
new file mode 100755
index 0000000..cbadad8
--- /dev/null
+++ b/bin/push_to_asf.py
@@ -0,0 +1,295 @@
+#!/usr/bin/env python
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# This script fetches branches from the Gerrit repository and
+# allows ASF committers to propagate commits from gerrit into the
+# official ASF repository.
+#
+# Current ASF policy is that this mirroring cannot be automatic
+# and should be driven by a committer who inspects and signs off
+# on the commits being made into the ASF. Additionally, the ASF
+# prefers that in most cases, the committer according to source
+# control should be the same person to push the commit to a git
+# repository.
+#
+# This script provides the committer the opportunity to review the
+# changes to be pushed, warns them if they are pushing code for
+# which they weren't the committer, and performs the actual push.
+#
+# TODO: Improve console output: replace 'print' with format strings
+# and use sys.stderr/sys.stdout.
+
+import logging
+import optparse
+import os
+import re
+import subprocess
+import sys
+
+APACHE_REPO = "https://git-wip-us.apache.org/repos/asf/incubator-impala.git"
+GERRIT_URL_RE = re.compile(r"ssh://.+@gerrit.cloudera.org:29418/Impala")
+
+# Parsed options, filled in by main().
+OPTIONS = None
+
+class Colors(object):
+ """ ANSI color codes. """
+
+ def __on_tty(x):
+ if not os.isatty(sys.stdout.fileno()):
+ return ""
+ return x
+
+ RED = __on_tty("\x1b[31m")
+ GREEN = __on_tty("\x1b[32m")
+ YELLOW = __on_tty("\x1b[33m")
+ RESET = __on_tty("\x1b[m")
+
+
+def check_output(*popenargs, **kwargs):
+ r"""Run command with arguments and return its output as a byte string.
+ Backported from Python 2.7 as it's implemented as pure python on stdlib.
+ >>> check_output(['/usr/bin/python', '--version'])
+ Python 2.6.2
+ """
+ process = subprocess.Popen(stdout=subprocess.PIPE, *popenargs, **kwargs)
+ output, unused_err = process.communicate()
+ retcode = process.poll()
+ if retcode:
+ cmd = kwargs.get("args")
+ if cmd is None:
+ cmd = popenargs[0]
+ error = subprocess.CalledProcessError(retcode, cmd)
+ error.output = output
+ raise error
+ return output
+
+
+def confirm_prompt(prompt):
+ """
+ Issue the given prompt, and ask the user to confirm yes/no. Returns true
+ if the user confirms.
+ """
+ while True:
+ print prompt, "[Y/n]:",
+
+ if not os.isatty(sys.stdout.fileno()):
+ print "Not running interactively. Assuming 'N'."
+ return False
+
+ r = raw_input().strip().lower()
+ if r in ['y', 'yes', '']:
+ return True
+ elif r in ['n', 'no']:
+ return False
+
+
+def get_my_email():
+ """ Return the email address in the user's git config. """
+ return check_output(['git', 'config', '--get', 'user.email']).strip()
+
+
+def check_apache_remote():
+ """
+ Checks that there is a remote named 'apache' set up correctly.
+ Otherwise, exits with an error message.
+ """
+ try:
+ url = check_output(\
+ ['git', 'config', '--local', '--get', 'remote.apache.url']).strip()
+ except subprocess.CalledProcessError:
+ print >>sys.stderr, "No remote named 'apache'. Please set one up, for example with: "
+ print >>sys.stderr, " git remote add apache", APACHE_REPO
+ sys.exit(1)
+ if url != APACHE_REPO:
+ print >>sys.stderr, "Unexpected URL for remote 'apache'."
+ print >>sys.stderr, " Got: ", url
+ print >>sys.stderr, " Expected:", APACHE_REPO
+ sys.exit(1)
+
+
+def check_gerrit_remote():
+ """
+ Checks that there is a remote named 'gerrit' set up correctly.
+ Otherwise, exits with an error message.
+ """
+ try:
+ url = check_output(\
+ ['git', 'config', '--local', '--get', 'remote.gerrit.url']).strip()
+ except subprocess.CalledProcessError:
+ print >>sys.stderr, "No remote named 'gerrit'. Please set one up following "
+ print >>sys.stderr, "the contributor guide."
+ sys.exit(1)
+ if not GERRIT_URL_RE.match(url):
+ print >>sys.stderr, "Unexpected URL for remote 'gerrit'."
+ print >>sys.stderr, " Got: ", url
+ print >>sys.stderr, " Expected to find host '%s' in the URL" % GERRIT_HOST
+ sys.exit(1)
+
+
+def fetch(remote):
+ """Run git fetch for the given remote, including some logging."""
+ logging.info("Fetching from remote '%s'..." % remote)
+ subprocess.check_call(['git', 'fetch', remote])
+ logging.info("done")
+
+
+def get_branches(remote):
+ """ Fetch a dictionary mapping branch name to SHA1 hash from the given remote. """
+ out = check_output(["git", "ls-remote", remote, "refs/heads/*"])
+ ret = {}
+ for l in out.splitlines():
+ sha, ref = l.split("\t")
+ branch = ref.replace("refs/heads/", "", 1)
+ ret[branch] = sha
+ return ret
+
+
+def rev_parse(rev):
+ """Run git rev-parse, returning the sha1, or None if not found"""
+ try:
+ return check_output(['git', 'rev-parse', rev], stderr=subprocess.STDOUT).strip()
+ except subprocess.CalledProcessError:
+ return None
+
+
+def rev_list(arg):
+ """Run git rev-list, returning an array of SHA1 commit hashes."""
+ return check_output(['git', 'rev-list', arg]).splitlines()
+
+
+def describe_commit(rev):
+ """ Return a one-line description of a commit. """
+ return subprocess.check_output(
+ ['git', 'log', '--color', '-n1', '--oneline', rev]).strip()
+
+
+def is_fast_forward(ancestor, child):
+ """
+ Return True if 'child' is a descendent of 'ancestor' and thus
+ could be fast-forward merged.
+ """
+ try:
+ merge_base = check_output(['git', 'merge-base', ancestor, child]).strip()
+ except:
+ # If either of the commits is unknown, count this as a non-fast-forward.
+ return False
+ return merge_base == rev_parse(ancestor)
+
+
+def get_committer_email(rev):
+ """ Return the email address of the committer of the given revision. """
+ return check_output(['git', 'log', '-n1', '--pretty=format:%ce', rev]).strip()
+
+
+def do_update(branch, gerrit_sha, apache_sha):
+ """
+ Displays and performs a proposed update of the Apache repository
+ for branch 'branch' from 'apache_sha' to 'gerrit_sha'.
+ """
+ # First, verify that the update is fast-forward. If it's not, then something
+ # must have gotten committed to Apache outside of gerrit, and we'd need some
+ # manual intervention.
+ if not is_fast_forward(apache_sha, gerrit_sha):
+ print >>sys.stderr, "Cannot update branch '%s' from gerrit:" % branch
+ print >>sys.stderr, "Apache revision %s is not an ancestor of gerrit revision %s" % (
+ apache_sha[:8], gerrit_sha[:8])
+ print >>sys.stderr,\
+ "Something must have been committed to Apache and bypassed gerrit."
+ print >>sys.stderr, "Manual intervention is required."
+ sys.exit(1)
+
+ # List the commits that are going to be pushed to the ASF, so that the committer
+ # can verify and "sign off".
+ commits = rev_list("%s..%s" % (apache_sha, gerrit_sha))
+ commits.reverse() # Display from oldest to newest.
+ print "-" * 60
+ print Colors.GREEN + ("%d commit(s) need to be pushed from Gerrit to ASF:" %\
+ len(commits)) + Colors.RESET
+ push_sha = None
+ for sha in commits:
+ oneline = describe_commit(sha)
+ print " ", oneline
+ committer = get_committer_email(sha)
+ if committer != get_my_email():
+ print Colors.RED + " !!! Committed by someone else (%s) !!!" %\
+ committer, Colors.RESET
+ if not confirm_prompt(Colors.RED +\
+ " !!! Are you sure you want to push on behalf of another committer?" +\
+ Colors.RESET):
+ # Even if they don't want to push this commit, we could still push any
+ # earlier commits that the user _did_ author.
+ if push_sha is not None:
+ print "... will still update to prior commit %s..." % push_sha
+ break
+ push_sha = sha
+ if push_sha is None:
+ print "Nothing to push"
+ return
+
+ # Everything has been confirmed. Do the actual push
+ cmd = ['git', 'push', 'apache']
+ if OPTIONS.dry_run:
+ cmd.append('--dry-run')
+ cmd.append('%s:refs/heads/%s' % (push_sha, branch))
+ print Colors.GREEN + "Running: " + Colors.RESET + " ".join(cmd)
+ subprocess.check_call(cmd)
+ print Colors.GREEN + "Successfully updated %s to %s" % (branch, gerrit_sha) +\
+ Colors.RESET
+ print
+
+
+def main():
+ global OPTIONS
+ p = optparse.OptionParser(
+ epilog=("See the top of the source code for more information on the purpose of "
+ "this script."))
+ p.add_option("-n", "--dry-run", action="store_true",
+ help="Perform git pushes with --dry-run")
+ OPTIONS, args = p.parse_args()
+ if args:
+ p.error("no arguments expected")
+ sys.exit(1)
+
+ # Pre-flight checks.
+ check_apache_remote()
+ check_gerrit_remote()
+
+ # Ensure we have the latest state of gerrit.
+ fetch('gerrit')
+
+ # Check the current state of branches on Apache.
+ # For each branch, we try to update it if the revisions don't match.
+ apache_branches = get_branches('apache')
+ for branch, apache_sha in sorted(apache_branches.iteritems()):
+ gerrit_sha = rev_parse("remotes/gerrit/" + branch)
+ print "Branch '%s':\t" % branch,
+ if gerrit_sha is None:
+ print Colors.YELLOW, "found on Apache but not in gerrit", Colors.RESET
+ continue
+ if gerrit_sha == apache_sha:
+ print Colors.GREEN, "up to date", Colors.RESET
+ continue
+ print Colors.YELLOW, "needs update", Colors.RESET
+ do_update(branch, gerrit_sha, apache_sha)
+
+
+if __name__ == "__main__":
+ logging.basicConfig(level=logging.INFO)
+ main()
[08/18] incubator-impala git commit: IMPALA-3757: Add missing lock in
RuntimeProfile::ComputeTimeInProfile
Posted by ta...@apache.org.
IMPALA-3757: Add missing lock in RuntimeProfile::ComputeTimeInProfile
We've seen a number of issues that appear to be a race
around the RuntimeProfile handling on the coordinator. We
don't have a consistent repro and haven't been able to find
the root cause of the crashes/hangs, but we have seen a
number of stacks from crashes that include the following
frames:
...
RuntimeProfile::ComputeTimeInProfile
RuntimeProfile::UpdateAverage
Coordinator::UpdateAverageProfile
Coordinator::UpdateFragmentExecStatus
...
This patch fixes a case where we iterate over children_
without holding the lock. I can't prove that this causes the
hangs but we certainly should fix this regardless.
Testing: Passed several exhaustive test runs successfully.
Change-Id: Ibb40ebfa2e58ecaa8668d30ad92003fc75483517
Reviewed-on: http://gerrit.cloudera.org:8080/3397
Reviewed-by: Matthew Jacobs <mj...@cloudera.com>
Tested-by: Internal Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/d75f327a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/d75f327a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/d75f327a
Branch: refs/heads/master
Commit: d75f327adda7377300e448cdda12d321da837efe
Parents: 59cdec2
Author: Matthew Jacobs <mj...@cloudera.com>
Authored: Thu Jun 16 16:31:48 2016 -0700
Committer: Taras Bobrovytsky <ta...@apache.org>
Committed: Thu Jul 14 19:04:44 2016 +0000
----------------------------------------------------------------------
be/src/util/runtime-profile.cc | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/d75f327a/be/src/util/runtime-profile.cc
----------------------------------------------------------------------
diff --git a/be/src/util/runtime-profile.cc b/be/src/util/runtime-profile.cc
index 714180a..ce5e934 100644
--- a/be/src/util/runtime-profile.cc
+++ b/be/src/util/runtime-profile.cc
@@ -363,8 +363,11 @@ void RuntimeProfile::ComputeTimeInProfile(int64_t total) {
local_time_percent_ = ::min(1.0, local_time_percent_) * 100;
// Recurse on children
- for (int i = 0; i < children_.size(); ++i) {
- children_[i].first->ComputeTimeInProfile(total);
+ {
+ lock_guard<SpinLock> l(children_lock_);
+ for (int i = 0; i < children_.size(); ++i) {
+ children_[i].first->ComputeTimeInProfile(total);
+ }
}
}
[02/18] incubator-impala git commit: IMPALA-3687: Prefer Avro field
name during schema reconciliation
Posted by ta...@apache.org.
IMPALA-3687: Prefer Avro field name during schema reconciliation
Since it is possible to create an Avro table with both column
definitions and an Avro schema, Impala attempts to reconcile
inconsistencies in the two schema definitions, generally preferring the
Avro schema. The only exception to this rule was with
CHAR/VARCHAR/STRING columns, where the column definition was preferred
in order to support tables with CHAR/VARCHAR columns although Avro only
supports STRING. This exception is confusing because the name for such a
column will be taken from the column definition (and not from the Avro
schema).
This patch prefers name, comment from Avro schema definition and
uses column type from column definition for CHAR/VARCHAR/STRING
columns.
Change-Id: Ia3e43b2885853c2b4f207a45a873c9d7f31379cd
Reviewed-on: http://gerrit.cloudera.org:8080/3331
Reviewed-by: Huaisi Xu <hx...@cloudera.com>
Tested-by: Internal Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/c6ce32b3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/c6ce32b3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/c6ce32b3
Branch: refs/heads/master
Commit: c6ce32b3b6ae9b16f9b7921d22205c851745f410
Parents: 476f687
Author: Huaisi Xu <hx...@cloudera.com>
Authored: Mon Jun 6 18:12:47 2016 -0700
Committer: Taras Bobrovytsky <ta...@apache.org>
Committed: Thu Jul 14 19:04:43 2016 +0000
----------------------------------------------------------------------
.../impala/util/AvroSchemaConverter.java | 5 +
.../cloudera/impala/util/AvroSchemaUtils.java | 27 +++--
.../queries/QueryTest/avro-schema-changes.test | 2 +-
.../QueryTest/avro-schema-resolution.test | 110 ++++++++++++++++++-
tests/query_test/test_avro_schema_resolution.py | 4 +-
5 files changed, 131 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/c6ce32b3/fe/src/main/java/com/cloudera/impala/util/AvroSchemaConverter.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/com/cloudera/impala/util/AvroSchemaConverter.java b/fe/src/main/java/com/cloudera/impala/util/AvroSchemaConverter.java
index 9a0077c..dce35da 100644
--- a/fe/src/main/java/com/cloudera/impala/util/AvroSchemaConverter.java
+++ b/fe/src/main/java/com/cloudera/impala/util/AvroSchemaConverter.java
@@ -40,6 +40,11 @@ import com.google.common.collect.Lists;
* Error behavior: These functions throw an UnsupportedOperationException when failing
* to generate an Impala-compatible Avro schema, e.g., because of an unknown type or a
* type not supported by Impala.
+ *
+ * Behavior for TIMESTAMP:
+ * A TIMESTAMP column definition maps to an Avro STRING and is created as a STRING column,
+ * because Avro has no binary TIMESTAMP representation. As a result, no Avro table may
+ * have a TIMESTAMP column.
*/
public class AvroSchemaConverter {
// Arbitrarily chosen schema name and record prefix. Note that
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/c6ce32b3/fe/src/main/java/com/cloudera/impala/util/AvroSchemaUtils.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/com/cloudera/impala/util/AvroSchemaUtils.java b/fe/src/main/java/com/cloudera/impala/util/AvroSchemaUtils.java
index 558b569..ede6ebb 100644
--- a/fe/src/main/java/com/cloudera/impala/util/AvroSchemaUtils.java
+++ b/fe/src/main/java/com/cloudera/impala/util/AvroSchemaUtils.java
@@ -109,14 +109,10 @@ public class AvroSchemaUtils {
* resolution policy:
*
* Mismatched number of columns -> Prefer Avro columns.
- * Mismatched name/type -> Prefer Avro column, except:
- * A CHAR/VARCHAR column definition maps to an Avro STRING, and is preserved
- * as a CHAR/VARCHAR in the reconciled schema.
- *
- * Behavior for TIMESTAMP:
- * A TIMESTAMP column definition maps to an Avro STRING and is presented as a STRING
- * in the reconciled schema, because Avro has no binary TIMESTAMP representation.
- * As a result, no Avro table may have a TIMESTAMP column.
+ * Always prefer Avro schema except for column type CHAR/VARCHAR/STRING:
+ * A CHAR/VARCHAR/STRING column definition maps to an Avro STRING. The reconciled
+ * column will preserve the type in the column definition but use the column name
+ * and comment from the Avro schema.
*/
public static List<ColumnDef> reconcileSchemas(
List<ColumnDef> colDefs, List<ColumnDef> avroCols, StringBuilder warning) {
@@ -135,12 +131,21 @@ public class AvroSchemaUtils {
Preconditions.checkNotNull(colDef.getType());
Preconditions.checkNotNull(avroCol.getType());
- // A CHAR/VARCHAR column definition maps to an Avro STRING, and is preserved
- // as a CHAR/VARCHAR in the reconciled schema.
+ // A CHAR/VARCHAR/STRING column definition maps to an Avro STRING, and is preserved
+ // as a CHAR/VARCHAR/STRING in the reconciled schema. Column name and comment
+ // are taken from the Avro schema.
if ((colDef.getType().isStringType() && avroCol.getType().isStringType())) {
Preconditions.checkState(
avroCol.getType().getPrimitiveType() == PrimitiveType.STRING);
- result.add(colDef);
+ ColumnDef reconciledColDef = new ColumnDef(
+ avroCol.getColName(), colDef.getTypeDef(), avroCol.getComment());
+ try {
+ reconciledColDef.analyze();
+ } catch (AnalysisException e) {
+ Preconditions.checkNotNull(
+ null, "reconciledColDef.analyze() should never throw.");
+ }
+ result.add(reconciledColDef);
} else {
result.add(avroCol);
}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/c6ce32b3/testdata/workloads/functional-query/queries/QueryTest/avro-schema-changes.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/avro-schema-changes.test b/testdata/workloads/functional-query/queries/QueryTest/avro-schema-changes.test
index 7fc0bc8..e4b87c5 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/avro-schema-changes.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/avro-schema-changes.test
@@ -1,6 +1,6 @@
====
---- QUERY
-# Create a table with default fileformat and later change it to avro using
+# Create a table with default fileformat and later change it to Avro using
# alter sql. The query runs with stale metadata and a warning should be raised.
# Invalidating metadata should cause the Avro schema to be properly set upon the
# next metadata load.
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/c6ce32b3/testdata/workloads/functional-query/queries/QueryTest/avro-schema-resolution.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/avro-schema-resolution.test b/testdata/workloads/functional-query/queries/QueryTest/avro-schema-resolution.test
index d5d8942..c640fa6 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/avro-schema-resolution.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/avro-schema-resolution.test
@@ -1,7 +1,7 @@
====
---- QUERY
# see testdata/avro_schema_resolution
-select * from schema_resolution_test
+select * from functional_avro_snap.schema_resolution_test
---- TYPES
boolean, int, long, float, double, string, string, string
---- RESULTS
@@ -15,9 +15,9 @@ false,2,2,2,2,'serialized string','','NULL'
# IMPALA-1947: A TIMESTAMP from the column definitions results in a STRING column
# backed by a stored Avro STRING during table loading.
# See testdata/avro_schema_resolution
-select * from no_avro_schema where year = 2009 order by id limit 1
+select * from functional_avro_snap.no_avro_schema where year = 2009 order by id limit 1
union all
-select * from no_avro_schema where year = 2010 order by id limit 1
+select * from functional_avro_snap.no_avro_schema where year = 2010 order by id limit 1
---- TYPES
int, boolean, int, int, int, bigint, float, double, string, string, string, int, int
---- RESULTS: VERIFY_IS_EQUAL_SORTED
@@ -74,3 +74,107 @@ int, string
4,'4'
4,'4'
====
+---- QUERY
+# IMPALA-3687: Create a table with mismatched column definitions and Avro schema.
+CREATE EXTERNAL TABLE alltypesagg_mismatch_column_name_comment (
+ id INT COMMENT 'int commnet',
+ bool_col BOOLEAN COMMENT 'bool commnet',
+ tinyint_col float COMMENT 'tinyint comment',
+ smallint_col double COMMENT 'smallint comment',
+ int_col INT COMMENT 'int comment',
+ bigint_col BIGINT COMMENT 'bigint comment',
+ float_col FLOAT COMMENT 'float comment',
+ double_col DOUBLE COMMENT 'double comment',
+ date_string_col STRING COMMENT 'string comment',
+ char_col char(2) COMMENT 'char comment',
+ varchar_col varchar(5) COMMENT 'varchar comment'
+)
+STORED AS AVRO
+TBLPROPERTIES ('avro.schema.literal'= '{
+"fields": [
+{"type": ["int", "null"], "name": "id_mismatch", "doc": "int comment mismatch"},
+{"type": ["boolean", "null"], "name": "bool_col_mismatch", "doc": "bool comment mismatch"},
+{"type": ["int", "null"], "name": "tinyint_col_mismatch", "doc": "tinyint comment mismatch"},
+{"type": ["int", "null"], "name": "smallint_col_mismatch", "doc": "smallint comment mismatch"},
+{"type": ["int", "null"], "name": "int_col_mismatch", "doc": "int comment mismatch"},
+{"type": ["long", "null"], "name": "bigint_col_mismatch", "doc": "bigint comment mismatch"},
+{"type": ["float", "null"], "name": "float_col_mismatch", "doc": "float comment mismatch"},
+{"type": ["double", "null"], "name": "double_col_mismatch", "doc": "double comment mismatch"},
+{"type": ["string", "null"], "name": "date_string_col_mismatch", "doc": "string comment mismatch"},
+{"type": ["string", "null"], "name": "char_col_mismatch", "doc": "char comment mismatch"},
+{"type": ["string", "null"], "name": "varchar_col_mismatch", "doc": "varchar comment mismatch"}],
+"type": "record", "name": "a"}')
+====
+---- QUERY
+# We favor Avro schema definition over column definitions except for CHAR/VARCHAR, for
+# which we use the type from the column definition.
+DESCRIBE alltypesagg_mismatch_column_name_comment
+---- RESULTS
+'id_mismatch','int','int comment mismatch'
+'bool_col_mismatch','boolean','bool comment mismatch'
+'tinyint_col_mismatch','int','tinyint comment mismatch'
+'smallint_col_mismatch','int','smallint comment mismatch'
+'int_col_mismatch','int','int comment mismatch'
+'bigint_col_mismatch','bigint','bigint comment mismatch'
+'float_col_mismatch','float','float comment mismatch'
+'double_col_mismatch','double','double comment mismatch'
+'date_string_col_mismatch','string','string comment mismatch'
+'char_col_mismatch','char(2)','char comment mismatch'
+'varchar_col_mismatch','varchar(5)','varchar comment mismatch'
+---- TYPES
+string, string, string
+====
+---- QUERY
+# IMPALA-3687: Create a table with only Avro schema and later alter it to be a mismatched
+# schema.
+CREATE EXTERNAL TABLE alltypesagg_alter_avro_name_comment
+STORED AS AVRO
+TBLPROPERTIES ('avro.schema.literal'= '{
+"fields": [
+{"type": ["int", "null"], "name": "id", "doc": "int comment"},
+{"type": ["boolean", "null"], "name": "bool_col", "doc": "bool comment"},
+{"type": ["int", "null"], "name": "tinyint_col", "doc": "tinyint comment"},
+{"type": ["int", "null"], "name": "smallint_col", "doc": "smallint comment"},
+{"type": ["int", "null"], "name": "int_col", "doc": "int comment"},
+{"type": ["long", "null"], "name": "bigint_col", "doc": "bigint comment"},
+{"type": ["float", "null"], "name": "float_col", "doc": "float comment"},
+{"type": ["double", "null"], "name": "double_col", "doc": "double comment"},
+{"type": ["string", "null"], "name": "date_string_col", "doc": "string comment"},
+{"type": ["string", "null"], "name": "char_col", "doc": "char comment"},
+{"type": ["string", "null"], "name": "varchar_col", "doc": "varchar comment"}],
+"type": "record", "name": "a"}');
+
+ALTER TABLE alltypesagg_alter_avro_name_comment SET TBLPROPERTIES ('avro.schema.literal'= '{
+"fields": [
+{"type": ["int", "null"], "name": "id_mismatch", "doc": "int comment mismatch"},
+{"type": ["boolean", "null"], "name": "bool_col_mismatch", "doc": "bool comment mismatch"},
+{"type": ["int", "null"], "name": "tinyint_col_mismatch", "doc": "tinyint comment mismatch"},
+{"type": ["int", "null"], "name": "smallint_col_mismatch", "doc": "smallint comment mismatch"},
+{"type": ["int", "null"], "name": "int_col_mismatch", "doc": "int comment mismatch"},
+{"type": ["long", "null"], "name": "bigint_col_mismatch", "doc": "bigint comment mismatch"},
+{"type": ["float", "null"], "name": "float_col_mismatch", "doc": "float comment mismatch"},
+{"type": ["double", "null"], "name": "double_col_mismatch", "doc": "double comment mismatch"},
+{"type": ["string", "null"], "name": "date_string_col_mismatch", "doc": "string comment mismatch"},
+{"type": ["string", "null"], "name": "char_col_mismatch", "doc": "char comment mismatch"},
+{"type": ["string", "null"], "name": "varchar_col_mismatch", "doc": "varchar comment mismatch"}],
+"type": "record", "name": "a"}');
+
+REFRESH alltypesagg_alter_avro_name_comment;
+====
+---- QUERY
+DESCRIBE alltypesagg_alter_avro_name_comment
+---- RESULTS
+'id_mismatch','int','int comment mismatch'
+'bool_col_mismatch','boolean','bool comment mismatch'
+'tinyint_col_mismatch','int','tinyint comment mismatch'
+'smallint_col_mismatch','int','smallint comment mismatch'
+'int_col_mismatch','int','int comment mismatch'
+'bigint_col_mismatch','bigint','bigint comment mismatch'
+'float_col_mismatch','float','float comment mismatch'
+'double_col_mismatch','double','double comment mismatch'
+'date_string_col_mismatch','string','string comment mismatch'
+'char_col_mismatch','string','char comment mismatch'
+'varchar_col_mismatch','string','varchar comment mismatch'
+---- TYPES
+string, string, string
+====
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/c6ce32b3/tests/query_test/test_avro_schema_resolution.py
----------------------------------------------------------------------
diff --git a/tests/query_test/test_avro_schema_resolution.py b/tests/query_test/test_avro_schema_resolution.py
index 5754c83..527181c 100644
--- a/tests/query_test/test_avro_schema_resolution.py
+++ b/tests/query_test/test_avro_schema_resolution.py
@@ -20,8 +20,8 @@ class TestAvroSchemaResolution(ImpalaTestSuite):
v.get_value('table_format').file_format == 'avro' and\
v.get_value('table_format').compression_codec == 'snap')
- def test_avro_schema_resolution(self, vector):
- self.run_test_case('QueryTest/avro-schema-resolution', vector)
+ def test_avro_schema_resolution(self, vector, unique_database):
+ self.run_test_case('QueryTest/avro-schema-resolution', vector, unique_database)
def test_avro_c_lib_unicode_nulls(self, vector):
"""Test for IMPALA-1136 and IMPALA-2161 and unicode characters in the
[05/18] incubator-impala git commit: IMPALA-3206: Enable codegen for
AVRO_DECIMAL
Posted by ta...@apache.org.
IMPALA-3206: Enable codegen for AVRO_DECIMAL
This change adds the missing switch statement in
CodegenReadScalar() for AVRO_DECIMAL so that we will
also codegen if an avro table contains AVRO_DECIMAL.
With this change, the following query improves by 37.5%,
going from 8s to 5s:
select count(distinct l_linenumber), avg(l_extendedprice), max(l_discount), min(l_tax) from tpch15_avro.lineitem;
This change also un-inlines BitUtil::ByteSwap() as the
third argument 'len' is not compilation constant for
all call sites.
Change-Id: I51adf0c1ba76e055f31ccb0034a0d23ea2afb30e
Reviewed-on: http://gerrit.cloudera.org:8080/3489
Reviewed-by: Michael Ho <kw...@cloudera.com>
Tested-by: Internal Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/df2fc08d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/df2fc08d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/df2fc08d
Branch: refs/heads/master
Commit: df2fc08d22a3be8c9b6a2f2e6a4d9d59c2b6ca44
Parents: 948a6c3
Author: Michael Ho <kw...@cloudera.com>
Authored: Wed Jun 8 15:18:30 2016 -0700
Committer: Taras Bobrovytsky <ta...@apache.org>
Committed: Thu Jul 14 19:04:44 2016 +0000
----------------------------------------------------------------------
be/src/codegen/gen_ir_descriptions.py | 1 +
be/src/exec/hdfs-avro-scanner.cc | 4 +-
be/src/util/CMakeLists.txt | 1 +
be/src/util/bit-util-test.cc | 2 +-
be/src/util/bit-util.cc | 124 ++++++++++++++++++
be/src/util/bit-util.h | 2 +-
be/src/util/bit-util.inline.h | 130 -------------------
be/src/util/decimal-util.h | 2 +-
.../queries/QueryTest/decimal_avro.test | 15 +++
9 files changed, 147 insertions(+), 134 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/df2fc08d/be/src/codegen/gen_ir_descriptions.py
----------------------------------------------------------------------
diff --git a/be/src/codegen/gen_ir_descriptions.py b/be/src/codegen/gen_ir_descriptions.py
index 5cc992c..280ab96 100755
--- a/be/src/codegen/gen_ir_descriptions.py
+++ b/be/src/codegen/gen_ir_descriptions.py
@@ -106,6 +106,7 @@ ir_functions = [
["READ_AVRO_STRING", "ReadAvroString"],
["READ_AVRO_VARCHAR", "ReadAvroVarchar"],
["READ_AVRO_CHAR", "ReadAvroChar"],
+ ["READ_AVRO_DECIMAL", "ReadAvroDecimal"],
["HDFS_SCANNER_WRITE_ALIGNED_TUPLES", "WriteAlignedTuples"],
["HDFS_SCANNER_GET_CONJUNCT_CTX", "GetConjunctCtx"],
["STRING_TO_BOOL", "IrStringToBool"],
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/df2fc08d/be/src/exec/hdfs-avro-scanner.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/hdfs-avro-scanner.cc b/be/src/exec/hdfs-avro-scanner.cc
index fdb0a96..09d3955 100644
--- a/be/src/exec/hdfs-avro-scanner.cc
+++ b/be/src/exec/hdfs-avro-scanner.cc
@@ -957,7 +957,9 @@ Status HdfsAvroScanner::CodegenReadScalar(const AvroSchemaElement& element,
read_field_fn = codegen->GetFunction(IRFunction::READ_AVRO_STRING, false);
}
break;
- // TODO: Add AVRO_DECIMAL here.
+ case AVRO_DECIMAL:
+ read_field_fn = codegen->GetFunction(IRFunction::READ_AVRO_DECIMAL, false);
+ break;
default:
return Status(Substitute(
"Failed to codegen MaterializeTuple() due to unsupported type: $0",
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/df2fc08d/be/src/util/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/be/src/util/CMakeLists.txt b/be/src/util/CMakeLists.txt
index 57bf572..2bf5e49 100644
--- a/be/src/util/CMakeLists.txt
+++ b/be/src/util/CMakeLists.txt
@@ -31,6 +31,7 @@ add_library(Util
avro-util.cc
benchmark.cc
bitmap.cc
+ bit-util.cc
bloom-filter.cc
cgroups-mgr.cc
codec.cc
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/df2fc08d/be/src/util/bit-util-test.cc
----------------------------------------------------------------------
diff --git a/be/src/util/bit-util-test.cc b/be/src/util/bit-util-test.cc
index e2bdd97..9947fca 100644
--- a/be/src/util/bit-util-test.cc
+++ b/be/src/util/bit-util-test.cc
@@ -21,7 +21,7 @@
#include <gtest/gtest.h>
#include "gutil/bits.h"
-#include "util/bit-util.inline.h"
+#include "util/bit-util.h"
#include "util/cpu-info.h"
#include "common/names.h"
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/df2fc08d/be/src/util/bit-util.cc
----------------------------------------------------------------------
diff --git a/be/src/util/bit-util.cc b/be/src/util/bit-util.cc
new file mode 100644
index 0000000..6853764
--- /dev/null
+++ b/be/src/util/bit-util.cc
@@ -0,0 +1,124 @@
+// Copyright 2016 Cloudera Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "util/bit-util.h"
+
+namespace impala {
+
+void BitUtil::ByteSwap(void* dest, const void* source, int len) {
+ uint8_t* dst = reinterpret_cast<uint8_t*>(dest);
+ const uint8_t* src = reinterpret_cast<const uint8_t*>(source);
+ switch (len) {
+ case 1:
+ *reinterpret_cast<uint8_t*>(dst) = *reinterpret_cast<const uint8_t*>(src);
+ return;
+ case 2:
+ *reinterpret_cast<uint16_t*>(dst) =
+ ByteSwap(*reinterpret_cast<const uint16_t*>(src));
+ return;
+ case 3:
+ *reinterpret_cast<uint16_t*>(dst + 1) =
+ ByteSwap(*reinterpret_cast<const uint16_t*>(src));
+ *reinterpret_cast<uint8_t*>(dst) = *reinterpret_cast<const uint8_t*>(src + 2);
+ return;
+ case 4:
+ *reinterpret_cast<uint32_t*>(dst) =
+ ByteSwap(*reinterpret_cast<const uint32_t*>(src));
+ return;
+ case 5:
+ *reinterpret_cast<uint32_t*>(dst + 1) =
+ ByteSwap(*reinterpret_cast<const uint32_t*>(src));
+ *reinterpret_cast<uint8_t*>(dst) = *reinterpret_cast<const uint8_t*>(src + 4);
+ return;
+ case 6:
+ *reinterpret_cast<uint32_t*>(dst + 2) =
+ ByteSwap(*reinterpret_cast<const uint32_t*>(src));
+ *reinterpret_cast<uint16_t*>(dst) =
+ ByteSwap(*reinterpret_cast<const uint16_t*>(src + 4));
+ return;
+ case 7:
+ *reinterpret_cast<uint32_t*>(dst + 3) =
+ ByteSwap(*reinterpret_cast<const uint32_t*>(src));
+ *reinterpret_cast<uint16_t*>(dst + 1) =
+ ByteSwap(*reinterpret_cast<const uint16_t*>(src + 4));
+ *reinterpret_cast<uint8_t*>(dst) = *reinterpret_cast<const uint8_t*>(src + 6);
+ return;
+ case 8:
+ *reinterpret_cast<uint64_t*>(dst) =
+ ByteSwap(*reinterpret_cast<const uint64_t*>(src));
+ return;
+ case 9:
+ *reinterpret_cast<uint64_t*>(dst + 1) =
+ ByteSwap(*reinterpret_cast<const uint64_t*>(src));
+ *reinterpret_cast<uint8_t*>(dst) = *reinterpret_cast<const uint8_t*>(src + 8);
+ return;
+ case 10:
+ *reinterpret_cast<uint64_t*>(dst + 2) =
+ ByteSwap(*reinterpret_cast<const uint64_t*>(src));
+ *reinterpret_cast<uint16_t*>(dst) =
+ ByteSwap(*reinterpret_cast<const uint16_t*>(src + 8));
+ return;
+ case 11:
+ *reinterpret_cast<uint64_t*>(dst + 3) =
+ ByteSwap(*reinterpret_cast<const uint64_t*>(src));
+ *reinterpret_cast<uint16_t*>(dst + 1) =
+ ByteSwap(*reinterpret_cast<const uint16_t*>(src + 8));
+ *reinterpret_cast<uint8_t*>(dst) = *reinterpret_cast<const uint8_t*>(src + 10);
+ return;
+ case 12:
+ *reinterpret_cast<uint64_t*>(dst + 4) =
+ ByteSwap(*reinterpret_cast<const uint64_t*>(src));
+ *reinterpret_cast<uint32_t*>(dst) =
+ ByteSwap(*reinterpret_cast<const uint32_t*>(src + 8));
+ return;
+ case 13:
+ *reinterpret_cast<uint64_t*>(dst + 5) =
+ ByteSwap(*reinterpret_cast<const uint64_t*>(src));
+ *reinterpret_cast<uint32_t*>(dst + 1) =
+ ByteSwap(*reinterpret_cast<const uint32_t*>(src + 8));
+ *reinterpret_cast<uint8_t*>(dst) = *reinterpret_cast<const uint8_t*>(src + 12);
+ return;
+ case 14:
+ *reinterpret_cast<uint64_t*>(dst + 6) =
+ ByteSwap(*reinterpret_cast<const uint64_t*>(src));
+ *reinterpret_cast<uint32_t*>(dst + 2) =
+ ByteSwap(*reinterpret_cast<const uint32_t*>(src + 8));
+ *reinterpret_cast<uint16_t*>(dst) =
+ ByteSwap(*reinterpret_cast<const uint16_t*>(src + 12));
+ return;
+ case 15:
+ *reinterpret_cast<uint64_t*>(dst + 7) =
+ ByteSwap(*reinterpret_cast<const uint64_t*>(src));
+ *reinterpret_cast<uint32_t*>(dst + 3) =
+ ByteSwap(*reinterpret_cast<const uint32_t*>(src + 8));
+ *reinterpret_cast<uint16_t*>(dst + 1) =
+ ByteSwap(*reinterpret_cast<const uint16_t*>(src + 12));
+ *reinterpret_cast<uint8_t*>(dst) = *reinterpret_cast<const uint8_t*>(src + 14);
+ return;
+ case 16:
+ *reinterpret_cast<uint64_t*>(dst + 8) =
+ ByteSwap(*reinterpret_cast<const uint64_t*>(src));
+ *reinterpret_cast<uint64_t*>(dst) =
+ ByteSwap(*reinterpret_cast<const uint64_t*>(src + 8));
+ return;
+ default:
+ // Revert to slow loop-based swap.
+ for (int i = 0; i < len; ++i) {
+ dst[i] = src[len - i - 1];
+ }
+ return;
+ }
+}
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/df2fc08d/be/src/util/bit-util.h
----------------------------------------------------------------------
diff --git a/be/src/util/bit-util.h b/be/src/util/bit-util.h
index 14553ae..a38fcba 100644
--- a/be/src/util/bit-util.h
+++ b/be/src/util/bit-util.h
@@ -167,7 +167,7 @@ class BitUtil {
/// Write the swapped bytes into dest; source and dest must not overlap.
/// This function is optimized for len <= 16. It reverts to a slow loop-based
/// swap for len > 16.
- static inline void ByteSwap(void* dest, const void* source, int len);
+ static void ByteSwap(void* dest, const void* source, int len);
/// Converts to big endian format (if not already in big endian) from the
/// machine's native endian format.
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/df2fc08d/be/src/util/bit-util.inline.h
----------------------------------------------------------------------
diff --git a/be/src/util/bit-util.inline.h b/be/src/util/bit-util.inline.h
deleted file mode 100644
index 33b44c6..0000000
--- a/be/src/util/bit-util.inline.h
+++ /dev/null
@@ -1,130 +0,0 @@
-// Copyright 2016 Cloudera Inc.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-
-#ifndef IMPALA_BIT_UTIL_INLINE_H
-#define IMPALA_BIT_UTIL_INLINE_H
-
-#include "util/bit-util.h"
-
-namespace impala {
-
-inline void BitUtil::ByteSwap(void* dest, const void* source, int len) {
- uint8_t* dst = reinterpret_cast<uint8_t*>(dest);
- const uint8_t* src = reinterpret_cast<const uint8_t*>(source);
- switch (len) {
- case 1:
- *reinterpret_cast<uint8_t*>(dst) = *reinterpret_cast<const uint8_t*>(src);
- return;
- case 2:
- *reinterpret_cast<uint16_t*>(dst) =
- ByteSwap(*reinterpret_cast<const uint16_t*>(src));
- return;
- case 3:
- *reinterpret_cast<uint16_t*>(dst + 1) =
- ByteSwap(*reinterpret_cast<const uint16_t*>(src));
- *reinterpret_cast<uint8_t*>(dst) = *reinterpret_cast<const uint8_t*>(src + 2);
- return;
- case 4:
- *reinterpret_cast<uint32_t*>(dst) =
- ByteSwap(*reinterpret_cast<const uint32_t*>(src));
- return;
- case 5:
- *reinterpret_cast<uint32_t*>(dst + 1) =
- ByteSwap(*reinterpret_cast<const uint32_t*>(src));
- *reinterpret_cast<uint8_t*>(dst) = *reinterpret_cast<const uint8_t*>(src + 4);
- return;
- case 6:
- *reinterpret_cast<uint32_t*>(dst + 2) =
- ByteSwap(*reinterpret_cast<const uint32_t*>(src));
- *reinterpret_cast<uint16_t*>(dst) =
- ByteSwap(*reinterpret_cast<const uint16_t*>(src + 4));
- return;
- case 7:
- *reinterpret_cast<uint32_t*>(dst + 3) =
- ByteSwap(*reinterpret_cast<const uint32_t*>(src));
- *reinterpret_cast<uint16_t*>(dst + 1) =
- ByteSwap(*reinterpret_cast<const uint16_t*>(src + 4));
- *reinterpret_cast<uint8_t*>(dst) = *reinterpret_cast<const uint8_t*>(src + 6);
- return;
- case 8:
- *reinterpret_cast<uint64_t*>(dst) =
- ByteSwap(*reinterpret_cast<const uint64_t*>(src));
- return;
- case 9:
- *reinterpret_cast<uint64_t*>(dst + 1) =
- ByteSwap(*reinterpret_cast<const uint64_t*>(src));
- *reinterpret_cast<uint8_t*>(dst) = *reinterpret_cast<const uint8_t*>(src + 8);
- return;
- case 10:
- *reinterpret_cast<uint64_t*>(dst + 2) =
- ByteSwap(*reinterpret_cast<const uint64_t*>(src));
- *reinterpret_cast<uint16_t*>(dst) =
- ByteSwap(*reinterpret_cast<const uint16_t*>(src + 8));
- return;
- case 11:
- *reinterpret_cast<uint64_t*>(dst + 3) =
- ByteSwap(*reinterpret_cast<const uint64_t*>(src));
- *reinterpret_cast<uint16_t*>(dst + 1) =
- ByteSwap(*reinterpret_cast<const uint16_t*>(src + 8));
- *reinterpret_cast<uint8_t*>(dst) = *reinterpret_cast<const uint8_t*>(src + 10);
- return;
- case 12:
- *reinterpret_cast<uint64_t*>(dst + 4) =
- ByteSwap(*reinterpret_cast<const uint64_t*>(src));
- *reinterpret_cast<uint32_t*>(dst) =
- ByteSwap(*reinterpret_cast<const uint32_t*>(src + 8));
- return;
- case 13:
- *reinterpret_cast<uint64_t*>(dst + 5) =
- ByteSwap(*reinterpret_cast<const uint64_t*>(src));
- *reinterpret_cast<uint32_t*>(dst + 1) =
- ByteSwap(*reinterpret_cast<const uint32_t*>(src + 8));
- *reinterpret_cast<uint8_t*>(dst) = *reinterpret_cast<const uint8_t*>(src + 12);
- return;
- case 14:
- *reinterpret_cast<uint64_t*>(dst + 6) =
- ByteSwap(*reinterpret_cast<const uint64_t*>(src));
- *reinterpret_cast<uint32_t*>(dst + 2) =
- ByteSwap(*reinterpret_cast<const uint32_t*>(src + 8));
- *reinterpret_cast<uint16_t*>(dst) =
- ByteSwap(*reinterpret_cast<const uint16_t*>(src + 12));
- return;
- case 15:
- *reinterpret_cast<uint64_t*>(dst + 7) =
- ByteSwap(*reinterpret_cast<const uint64_t*>(src));
- *reinterpret_cast<uint32_t*>(dst + 3) =
- ByteSwap(*reinterpret_cast<const uint32_t*>(src + 8));
- *reinterpret_cast<uint16_t*>(dst + 1) =
- ByteSwap(*reinterpret_cast<const uint16_t*>(src + 12));
- *reinterpret_cast<uint8_t*>(dst) = *reinterpret_cast<const uint8_t*>(src + 14);
- return;
- case 16:
- *reinterpret_cast<uint64_t*>(dst + 8) =
- ByteSwap(*reinterpret_cast<const uint64_t*>(src));
- *reinterpret_cast<uint64_t*>(dst) =
- ByteSwap(*reinterpret_cast<const uint64_t*>(src + 8));
- return;
- default:
- // Revert to slow loop-based swap.
- for (int i = 0; i < len; ++i) {
- dst[i] = src[len - i - 1];
- }
- return;
- }
-}
-
-}
-
-#endif
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/df2fc08d/be/src/util/decimal-util.h
----------------------------------------------------------------------
diff --git a/be/src/util/decimal-util.h b/be/src/util/decimal-util.h
index 2ce69bc..79b7130 100644
--- a/be/src/util/decimal-util.h
+++ b/be/src/util/decimal-util.h
@@ -22,7 +22,7 @@
#include "runtime/multi-precision.h"
#include "runtime/types.h"
-#include "util/bit-util.inline.h"
+#include "util/bit-util.h"
namespace impala {
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/df2fc08d/testdata/workloads/functional-query/queries/QueryTest/decimal_avro.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/decimal_avro.test b/testdata/workloads/functional-query/queries/QueryTest/decimal_avro.test
index 826dd2f..73fd27d 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/decimal_avro.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/decimal_avro.test
@@ -38,3 +38,18 @@ select count(*) from avro_decimal_tbl
---- TYPES
BIGINT
====
+---- QUERY
+select l_shipmode, count(distinct l_quantity), avg(l_extendedprice), max(l_discount), ndv(l_tax)
+from tpch_avro_snap.lineitem
+group by l_shipmode
+---- RESULTS
+'SHIP',50,38267.37,0.10,9
+'REG AIR',50,38268.41,0.10,9
+'TRUCK',50,38209.82,0.10,9
+'RAIL',50,38269.81,0.10,9
+'FOB',50,38246.23,0.10,9
+'MAIL',50,38224.29,0.10,9
+'AIR',50,38299.98,0.10,9
+----TYPES
+STRING,DECIMAL,DECIMAL,DECIMAL,DECIMAL
+====
[18/18] incubator-impala git commit: IMPALA-3606: Fix Java NPE when
trying to add an existing partition
Posted by ta...@apache.org.
IMPALA-3606: Fix Java NPE when trying to add an existing partition
A NPE is also encountered when trying to drop a partition which does
not exist.
Currently, catalogd sends a response to impalad that contains a dummy
TCatalogObject of type "TABLE" but contains nulls for all other fields.
This causes a null pointer exception when impalad tries to access those
null fields. The exception never gets exposed to the user as it was
caught and ignored. In this case, the fix is that catalogd responds with
an empty TCatalogUpdateResult that contains no dummy TCatalogObject objects.
No testing was added since the NPE never gets exposed to the user.
Change-Id: Ic829e8580c14ed88adf7a0d1571381526b18a206
Reviewed-on: http://gerrit.cloudera.org:8080/3324
Reviewed-by: Dimitris Tsirogiannis <dt...@cloudera.com>
Tested-by: Internal Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/144cc68c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/144cc68c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/144cc68c
Branch: refs/heads/master
Commit: 144cc68cc12e713824f5b9c7269dda4b70385557
Parents: 26266fd
Author: Bikramjeet Vig <bi...@cloudera.com>
Authored: Mon Jun 6 18:00:48 2016 -0700
Committer: Taras Bobrovytsky <ta...@apache.org>
Committed: Thu Jul 14 19:04:45 2016 +0000
----------------------------------------------------------------------
.../impala/service/CatalogOpExecutor.java | 19 ++++++++++++-------
1 file changed, 12 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/144cc68c/fe/src/main/java/com/cloudera/impala/service/CatalogOpExecutor.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/com/cloudera/impala/service/CatalogOpExecutor.java b/fe/src/main/java/com/cloudera/impala/service/CatalogOpExecutor.java
index 602f65a..a63427a 100644
--- a/fe/src/main/java/com/cloudera/impala/service/CatalogOpExecutor.java
+++ b/fe/src/main/java/com/cloudera/impala/service/CatalogOpExecutor.java
@@ -366,12 +366,14 @@ public class CatalogOpExecutor {
// Create and add HdfsPartition object to the corresponding HdfsTable and load
// its block metadata. Get the new table object with an updated catalog
// version. If the partition already exists in Hive and "IfNotExists" is true,
- // then null is returned.
+ // then return without populating the response object.
Table refreshedTable = alterTableAddPartition(tbl,
addPartParams.getPartition_spec(), addPartParams.isIf_not_exists(),
addPartParams.getLocation(), addPartParams.getCache_op());
- if (refreshedTable != null) refreshedTable.setCatalogVersion(newCatalogVersion);
- addTableToCatalogUpdate(refreshedTable, response.result);
+ if (refreshedTable != null) {
+ refreshedTable.setCatalogVersion(newCatalogVersion);
+ addTableToCatalogUpdate(refreshedTable, response.result);
+ }
return;
case DROP_COLUMN:
TAlterTableDropColParams dropColParams = params.getDrop_col_params();
@@ -389,13 +391,16 @@ public class CatalogOpExecutor {
params.getDrop_partition_params();
// Drop the partition from the corresponding table. Get the table object
// with an updated catalog version. If the partition does not exist and
- // "IfExists" is true, null is returned. If "purge" option is specified
- // partition data is purged by skipping Trash, if configured.
+ // "IfExists" is true, then return without populating the response object.
+ // If "purge" option is specified partition data is purged by skipping
+ // Trash, if configured.
refreshedTable = alterTableDropPartition(tbl,
dropPartParams.getPartition_spec(),
dropPartParams.isIf_exists(), dropPartParams.isPurge());
- if (refreshedTable != null) refreshedTable.setCatalogVersion(newCatalogVersion);
- addTableToCatalogUpdate(refreshedTable, response.result);
+ if (refreshedTable != null) {
+ refreshedTable.setCatalogVersion(newCatalogVersion);
+ addTableToCatalogUpdate(refreshedTable, response.result);
+ }
return;
case RENAME_TABLE:
case RENAME_VIEW:
[04/18] incubator-impala git commit: IMPALA-3442: Replace '> >' with
'>>' in template decls
Posted by ta...@apache.org.
IMPALA-3442: Replace '> >' with '>>' in template decls
Change-Id: I9b27dfbda791ec6730c329c91bed015887a5e27c
Reviewed-on: http://gerrit.cloudera.org:8080/3562
Reviewed-by: Henry Robinson <he...@cloudera.com>
Tested-by: Henry Robinson <he...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/55b2a639
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/55b2a639
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/55b2a639
Branch: refs/heads/master
Commit: 55b2a639e81e295a99927d5063adaf896985b29a
Parents: df2fc08
Author: Henry Robinson <he...@cloudera.com>
Authored: Mon May 2 21:03:17 2016 -0700
Committer: Taras Bobrovytsky <ta...@apache.org>
Committed: Thu Jul 14 19:04:44 2016 +0000
----------------------------------------------------------------------
be/src/benchmarks/string-benchmark.cc | 2 +-
be/src/codegen/llvm-codegen.h | 2 +-
be/src/exec/analytic-eval-node.cc | 4 ++--
be/src/exec/analytic-eval-node.h | 4 ++--
be/src/exec/hdfs-parquet-table-writer.cc | 2 +-
be/src/exec/hdfs-scan-node.cc | 4 ++--
be/src/exec/hdfs-scan-node.h | 6 +++---
be/src/exec/hdfs-table-sink.h | 2 +-
be/src/exec/incr-stats-util.cc | 4 ++--
be/src/exec/incr-stats-util.h | 2 +-
be/src/exec/topn-node.cc | 2 +-
be/src/exec/topn-node.h | 2 +-
be/src/exec/union-node.cc | 4 ++--
be/src/exec/union-node.h | 4 ++--
be/src/experiments/data-provider.cc | 4 ++--
be/src/exprs/aggregate-functions-test.cc | 2 +-
be/src/exprs/expr-test.cc | 6 +++---
be/src/resourcebroker/resource-broker.h | 8 ++++----
be/src/rpc/thrift-server.h | 2 +-
be/src/runtime/buffered-block-mgr-test.cc | 10 +++++-----
be/src/runtime/buffered-block-mgr.h | 2 +-
be/src/runtime/buffered-tuple-stream.h | 4 ++--
be/src/runtime/client-cache.h | 4 ++--
be/src/runtime/coordinator.h | 2 +-
be/src/runtime/data-stream-mgr.h | 4 ++--
be/src/runtime/data-stream-recvr.cc | 2 +-
be/src/runtime/disk-io-mgr.h | 2 +-
be/src/runtime/free-pool-test.cc | 3 +--
be/src/runtime/mem-tracker.h | 2 +-
be/src/runtime/multi-precision.h | 2 +-
be/src/runtime/plan-fragment-executor.h | 2 +-
be/src/runtime/row-batch-serialize-test.cc | 8 ++++----
be/src/runtime/sorter.cc | 2 +-
be/src/runtime/test-env.h | 2 +-
be/src/runtime/tmp-file-mgr-test.cc | 2 +-
be/src/scheduling/query-schedule.h | 2 +-
be/src/scheduling/simple-scheduler.cc | 4 ++--
be/src/scheduling/simple-scheduler.h | 2 +-
be/src/service/fragment-mgr.h | 2 +-
be/src/service/hs2-util-test.cc | 2 +-
be/src/service/impala-beeswax-server.cc | 2 +-
be/src/service/impala-server.cc | 4 ++--
be/src/service/impala-server.h | 12 ++++++------
be/src/service/query-exec-state.h | 2 +-
be/src/service/session-expiry-test.cc | 4 ++--
be/src/statestore/statestore.h | 6 +++---
be/src/testutil/impalad-query-executor.h | 2 +-
be/src/transport/TSaslServerTransport.cpp | 2 +-
be/src/transport/TSaslServerTransport.h | 2 +-
be/src/udf/uda-test-harness-impl.h | 4 ++--
be/src/util/blocking-queue-test.cc | 2 +-
be/src/util/collection-metrics.h | 4 ++--
be/src/util/debug-util.cc | 2 +-
be/src/util/internal-queue-test.cc | 2 +-
be/src/util/metrics-test.cc | 2 +-
be/src/util/perf-counters.h | 2 +-
be/src/util/runtime-profile.h | 4 ++--
be/src/util/summary-util.cc | 6 +++---
be/src/util/table-printer.h | 2 +-
be/src/util/thread-pool.h | 4 ++--
be/src/util/url-coding.cc | 2 +-
.../com/cloudera/impala/catalog/HdfsPartitionTest.java | 2 +-
62 files changed, 102 insertions(+), 103 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/55b2a639/be/src/benchmarks/string-benchmark.cc
----------------------------------------------------------------------
diff --git a/be/src/benchmarks/string-benchmark.cc b/be/src/benchmarks/string-benchmark.cc
index 170f3dc..3970609 100644
--- a/be/src/benchmarks/string-benchmark.cc
+++ b/be/src/benchmarks/string-benchmark.cc
@@ -92,7 +92,7 @@ struct CompactStringValue {
struct TestData {
vector<StringValue> normal_strings;
- vector<CompactStringValue<8> > compact_strings;
+ vector<CompactStringValue<8>> compact_strings;
vector<int> random_order;
uint32_t hash_normal;
uint32_t hash_compact;
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/55b2a639/be/src/codegen/llvm-codegen.h
----------------------------------------------------------------------
diff --git a/be/src/codegen/llvm-codegen.h b/be/src/codegen/llvm-codegen.h
index 332e599..e84ea74 100644
--- a/be/src/codegen/llvm-codegen.h
+++ b/be/src/codegen/llvm-codegen.h
@@ -575,7 +575,7 @@ class LlvmCodeGen {
std::set<std::string> linked_modules_;
/// The vector of functions to automatically JIT compile after FinalizeModule().
- std::vector<std::pair<llvm::Function*, void**> > fns_to_jit_compile_;
+ std::vector<std::pair<llvm::Function*, void**>> fns_to_jit_compile_;
/// Debug strings that will be outputted by jitted code. This is a copy of all
/// strings passed to CodegenDebugTrace.
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/55b2a639/be/src/exec/analytic-eval-node.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/analytic-eval-node.cc b/be/src/exec/analytic-eval-node.cc
index d9fa6c3..830fd18 100644
--- a/be/src/exec/analytic-eval-node.cc
+++ b/be/src/exec/analytic-eval-node.cc
@@ -306,7 +306,7 @@ string AnalyticEvalNode::DebugStateString(bool detailed = false) const {
<< " last_result_idx=" << last_result_idx_;
if (detailed) {
ss << " result_tuples idx: [";
- for (list<pair<int64_t, Tuple*> >::const_iterator it = result_tuples_.begin();
+ for (list<pair<int64_t, Tuple*>>::const_iterator it = result_tuples_.begin();
it != result_tuples_.end(); ++it) {
ss << it->first;
if (*it != result_tuples_.back()) ss << ", ";
@@ -314,7 +314,7 @@ string AnalyticEvalNode::DebugStateString(bool detailed = false) const {
ss << "]";
if (fn_scope_ == ROWS && window_.__isset.window_start) {
ss << " window_tuples idx: [";
- for (list<pair<int64_t, Tuple*> >::const_iterator it = window_tuples_.begin();
+ for (list<pair<int64_t, Tuple*>>::const_iterator it = window_tuples_.begin();
it != window_tuples_.end(); ++it) {
ss << it->first;
if (*it != window_tuples_.back()) ss << ", ";
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/55b2a639/be/src/exec/analytic-eval-node.h
----------------------------------------------------------------------
diff --git a/be/src/exec/analytic-eval-node.h b/be/src/exec/analytic-eval-node.h
index 6f17dbd..720c065 100644
--- a/be/src/exec/analytic-eval-node.h
+++ b/be/src/exec/analytic-eval-node.h
@@ -268,7 +268,7 @@ class AnalyticEvalNode : public ExecNode {
/// output row and result_tuples_.size() may be one less than the row batch size, in
/// which case we will process another input row batch (inserting one result tuple per
/// input row) before returning a row batch.
- std::list<std::pair<int64_t, Tuple*> > result_tuples_;
+ std::list<std::pair<int64_t, Tuple*>> result_tuples_;
/// Index in input_stream_ of the most recently added result tuple.
int64_t last_result_idx_;
@@ -278,7 +278,7 @@ class AnalyticEvalNode : public ExecNode {
/// or FOLLOWING. Tuples in this list are deep copied and owned by
/// curr_window_tuple_pool_.
/// TODO: Remove and use BufferedTupleStream (needs support for multiple readers).
- std::list<std::pair<int64_t, Tuple*> > window_tuples_;
+ std::list<std::pair<int64_t, Tuple*>> window_tuples_;
/// The index of the last row from input_stream_ associated with output row containing
/// resources in prev_tuple_pool_. -1 when the pool is empty. Resources from
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/55b2a639/be/src/exec/hdfs-parquet-table-writer.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/hdfs-parquet-table-writer.cc b/be/src/exec/hdfs-parquet-table-writer.cc
index 87e42fb..ce1bbb1 100644
--- a/be/src/exec/hdfs-parquet-table-writer.cc
+++ b/be/src/exec/hdfs-parquet-table-writer.cc
@@ -298,7 +298,7 @@ class HdfsParquetTableWriter::ColumnWriter :
static const int DICTIONARY_DATA_PAGE_SIZE_CHECK_PERIOD = 100;
// Encoder for dictionary encoding for different columns. Only one is set.
- scoped_ptr<DictEncoder<T> > dict_encoder_;
+ scoped_ptr<DictEncoder<T>> dict_encoder_;
// The number of values added since we last checked the dictionary.
int num_values_since_dict_size_check_;
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/55b2a639/be/src/exec/hdfs-scan-node.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/hdfs-scan-node.cc b/be/src/exec/hdfs-scan-node.cc
index 8e808cd..19a071f 100644
--- a/be/src/exec/hdfs-scan-node.cc
+++ b/be/src/exec/hdfs-scan-node.cc
@@ -141,9 +141,9 @@ Status HdfsScanNode::Init(const TPlanNode& tnode, RuntimeState* state) {
RETURN_IF_ERROR(ExecNode::Init(tnode, state));
// Add collection item conjuncts
- const map<TTupleId, vector<TExpr> >& collection_conjuncts =
+ const map<TTupleId, vector<TExpr>>& collection_conjuncts =
tnode.hdfs_scan_node.collection_conjuncts;
- map<TTupleId, vector<TExpr> >::const_iterator iter = collection_conjuncts.begin();
+ map<TTupleId, vector<TExpr>>::const_iterator iter = collection_conjuncts.begin();
for (; iter != collection_conjuncts.end(); ++iter) {
DCHECK(conjuncts_map_[iter->first].empty());
RETURN_IF_ERROR(
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/55b2a639/be/src/exec/hdfs-scan-node.h
----------------------------------------------------------------------
diff --git a/be/src/exec/hdfs-scan-node.h b/be/src/exec/hdfs-scan-node.h
index f0eb8e3..c611a45 100644
--- a/be/src/exec/hdfs-scan-node.h
+++ b/be/src/exec/hdfs-scan-node.h
@@ -164,7 +164,7 @@ class HdfsScanNode : public ScanNode {
DiskIoRequestContext* reader_context() { return reader_context_; }
- typedef std::map<TupleId, std::vector<ExprContext*> > ConjunctsMap;
+ typedef std::map<TupleId, std::vector<ExprContext*>> ConjunctsMap;
const ConjunctsMap& conjuncts_map() const { return conjuncts_map_; }
RuntimeProfile::HighWaterMarkCounter* max_compressed_text_file_length() {
@@ -284,7 +284,7 @@ class HdfsScanNode : public ScanNode {
}
/// map from volume id to <number of split, per volume split lengths>
- typedef boost::unordered_map<int32_t, std::pair<int, int64_t> > PerVolumnStats;
+ typedef boost::unordered_map<int32_t, std::pair<int, int64_t>> PerVolumnStats;
/// Update the per volume stats with the given scan range params list
static void UpdateHdfsSplitStats(
@@ -352,7 +352,7 @@ class HdfsScanNode : public ScanNode {
FileDescMap file_descs_;
/// File format => file descriptors.
- typedef std::map<THdfsFileFormat::type, std::vector<HdfsFileDesc*> > FileFormatsMap;
+ typedef std::map<THdfsFileFormat::type, std::vector<HdfsFileDesc*>> FileFormatsMap;
FileFormatsMap per_type_files_;
/// Conjuncts for each materialized tuple (top-level row batch tuples and collection
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/55b2a639/be/src/exec/hdfs-table-sink.h
----------------------------------------------------------------------
diff --git a/be/src/exec/hdfs-table-sink.h b/be/src/exec/hdfs-table-sink.h
index 99dcc89..c5bdcc4 100644
--- a/be/src/exec/hdfs-table-sink.h
+++ b/be/src/exec/hdfs-table-sink.h
@@ -179,7 +179,7 @@ class HdfsTableSink : public DataSink {
/// dynamic_partition_key_exprs_ generated by GetHashTblKey().
/// Maps to an OutputPartition, which are owned by the object pool and
/// a vector of rows to insert into this partition from the current row batch.
- typedef std::pair<OutputPartition*, std::vector<int32_t> > PartitionPair;
+ typedef std::pair<OutputPartition*, std::vector<int32_t>> PartitionPair;
typedef boost::unordered_map<std::string, PartitionPair> PartitionMap;
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/55b2a639/be/src/exec/incr-stats-util.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/incr-stats-util.cc b/be/src/exec/incr-stats-util.cc
index d83144f..3f1d026 100644
--- a/be/src/exec/incr-stats-util.cc
+++ b/be/src/exec/incr-stats-util.cc
@@ -188,7 +188,7 @@ namespace impala {
void FinalizePartitionedColumnStats(const TTableSchema& col_stats_schema,
const vector<TPartitionStats>& existing_part_stats,
- const vector<vector<string> >& expected_partitions, const TRowSet& rowset,
+ const vector<vector<string>>& expected_partitions, const TRowSet& rowset,
int32_t num_partition_cols, TAlterTableUpdateStatsParams* params) {
// The rowset should have the following schema: for every column in the source table,
// five columns are produced, one row per partition.
@@ -197,7 +197,7 @@ void FinalizePartitionedColumnStats(const TTableSchema& col_stats_schema,
const int num_cols =
(col_stats_schema.columns.size() - num_partition_cols) / COLUMNS_PER_STAT;
- unordered_set<vector<string> > seen_partitions;
+ unordered_set<vector<string>> seen_partitions;
vector<PerColumnStats> stats(num_cols);
if (rowset.rows.size() > 0) {
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/55b2a639/be/src/exec/incr-stats-util.h
----------------------------------------------------------------------
diff --git a/be/src/exec/incr-stats-util.h b/be/src/exec/incr-stats-util.h
index 9861e10..194f7b2 100644
--- a/be/src/exec/incr-stats-util.h
+++ b/be/src/exec/incr-stats-util.h
@@ -27,7 +27,7 @@ namespace impala {
void FinalizePartitionedColumnStats(
const apache::hive::service::cli::thrift::TTableSchema& col_stats_schema,
const std::vector<TPartitionStats>& existing_part_stats,
- const std::vector<std::vector<std::string> >& expected_partitions,
+ const std::vector<std::vector<std::string>>& expected_partitions,
const apache::hive::service::cli::thrift::TRowSet& rowset,
int32_t num_partition_cols, TAlterTableUpdateStatsParams* params);
}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/55b2a639/be/src/exec/topn-node.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/topn-node.cc b/be/src/exec/topn-node.cc
index a2b8e06..e638e36 100644
--- a/be/src/exec/topn-node.cc
+++ b/be/src/exec/topn-node.cc
@@ -113,7 +113,7 @@ Status TopNNode::Prepare(RuntimeState* state) {
}
AddCodegenExecOption(codegen_enabled, codegen_status);
priority_queue_.reset(new priority_queue<Tuple*, vector<Tuple*>,
- ComparatorWrapper<TupleRowComparator> >(*tuple_row_less_than_));
+ ComparatorWrapper<TupleRowComparator>>(*tuple_row_less_than_));
materialized_tuple_desc_ = row_descriptor_.tuple_descriptors()[0];
insert_batch_timer_ = ADD_TIMER(runtime_profile(), "InsertBatchTime");
return Status::OK();
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/55b2a639/be/src/exec/topn-node.h
----------------------------------------------------------------------
diff --git a/be/src/exec/topn-node.h b/be/src/exec/topn-node.h
index 29c89e0..3a521e2 100644
--- a/be/src/exec/topn-node.h
+++ b/be/src/exec/topn-node.h
@@ -113,7 +113,7 @@ class TopNNode : public ExecNode {
/// the order of the queue is the opposite of what the ORDER BY clause specifies, such
/// that the top of the queue is the last sorted element.
boost::scoped_ptr<std::priority_queue<Tuple*, std::vector<Tuple*>,
- ComparatorWrapper<TupleRowComparator> > > priority_queue_;
+ ComparatorWrapper<TupleRowComparator>>> priority_queue_;
/// END: Members that must be Reset()
/////////////////////////////////////////
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/55b2a639/be/src/exec/union-node.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/union-node.cc b/be/src/exec/union-node.cc
index 2e4cbf5..17ae0de 100644
--- a/be/src/exec/union-node.cc
+++ b/be/src/exec/union-node.cc
@@ -42,14 +42,14 @@ Status UnionNode::Init(const TPlanNode& tnode, RuntimeState* state) {
RETURN_IF_ERROR(ExecNode::Init(tnode, state));
DCHECK(tnode.__isset.union_node);
// Create const_expr_ctx_lists_ from thrift exprs.
- const vector<vector<TExpr> >& const_texpr_lists = tnode.union_node.const_expr_lists;
+ const vector<vector<TExpr>>& const_texpr_lists = tnode.union_node.const_expr_lists;
for (int i = 0; i < const_texpr_lists.size(); ++i) {
vector<ExprContext*> ctxs;
RETURN_IF_ERROR(Expr::CreateExprTrees(pool_, const_texpr_lists[i], &ctxs));
const_result_expr_ctx_lists_.push_back(ctxs);
}
// Create result_expr_ctx_lists_ from thrift exprs.
- const vector<vector<TExpr> >& result_texpr_lists = tnode.union_node.result_expr_lists;
+ const vector<vector<TExpr>>& result_texpr_lists = tnode.union_node.result_expr_lists;
for (int i = 0; i < result_texpr_lists.size(); ++i) {
vector<ExprContext*> ctxs;
RETURN_IF_ERROR(Expr::CreateExprTrees(pool_, result_texpr_lists[i], &ctxs));
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/55b2a639/be/src/exec/union-node.h
----------------------------------------------------------------------
diff --git a/be/src/exec/union-node.h b/be/src/exec/union-node.h
index b874739..9f24d03 100644
--- a/be/src/exec/union-node.h
+++ b/be/src/exec/union-node.h
@@ -51,10 +51,10 @@ class UnionNode : public ExecNode {
const TupleDescriptor* tuple_desc_;
/// Const exprs materialized by this node. These exprs don't refer to any children.
- std::vector<std::vector<ExprContext*> > const_result_expr_ctx_lists_;
+ std::vector<std::vector<ExprContext*>> const_result_expr_ctx_lists_;
/// Exprs materialized by this node. The i-th result expr list refers to the i-th child.
- std::vector<std::vector<ExprContext*> > result_expr_ctx_lists_;
+ std::vector<std::vector<ExprContext*>> result_expr_ctx_lists_;
/////////////////////////////////////////
/// BEGIN: Members that must be Reset()
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/55b2a639/be/src/experiments/data-provider.cc
----------------------------------------------------------------------
diff --git a/be/src/experiments/data-provider.cc b/be/src/experiments/data-provider.cc
index fa98b80..f50a2e1 100644
--- a/be/src/experiments/data-provider.cc
+++ b/be/src/experiments/data-provider.cc
@@ -47,7 +47,7 @@ void DataProvider::SetSeed(int seed) {
void RandString(MemPool* pool, StringValue* result,
const StringValue& min, const StringValue& max, double r,
- variate_generator<minstd_rand&, uniform_real<> >& rand) {
+ variate_generator<minstd_rand&, uniform_real<>>& rand) {
int min_len = min.len;
int max_len = max.len;
int len = r * (max_len - min_len) + min_len;
@@ -69,7 +69,7 @@ void* DataProvider::NextBatch(int* rows_returned) {
COUNTER_ADD(bytes_generated_, num_rows * row_size_);
uniform_real<> dist(0,1);
- variate_generator<minstd_rand&, uniform_real<> > rand_double(rand_generator_, dist);
+ variate_generator<minstd_rand&, uniform_real<>> rand_double(rand_generator_, dist);
char* data = data_.get();
for (int i = 0, row_idx = rows_returned_; i < num_rows; ++i, ++row_idx) {
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/55b2a639/be/src/exprs/aggregate-functions-test.cc
----------------------------------------------------------------------
diff --git a/be/src/exprs/aggregate-functions-test.cc b/be/src/exprs/aggregate-functions-test.cc
index 4c1bed3..1265e01 100644
--- a/be/src/exprs/aggregate-functions-test.cc
+++ b/be/src/exprs/aggregate-functions-test.cc
@@ -50,7 +50,7 @@ bool CheckHistogramDistribution(const StringVal& actual,
vector<string> str_vals;
split(str_vals, result, is_any_of(","));
- accumulator_set<int, stats<tag::variance> > acc;
+ accumulator_set<int, stats<tag::variance>> acc;
int prev_val = -1;
for (string& s: str_vals) {
trim(s);
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/55b2a639/be/src/exprs/expr-test.cc
----------------------------------------------------------------------
diff --git a/be/src/exprs/expr-test.cc b/be/src/exprs/expr-test.cc
index 4147093..ce8149e 100644
--- a/be/src/exprs/expr-test.cc
+++ b/be/src/exprs/expr-test.cc
@@ -4795,7 +4795,7 @@ TEST_F(ExprTest, ConditionalFunctionIsNotFalse) {
// - expected_offsets: mapping of byte sizes to a set valid offsets
// exprs that have the same byte size can end up in a number of locations
void ValidateLayout(const vector<Expr*>& exprs, int expected_byte_size,
- int expected_var_begin, const map<int, set<int> >& expected_offsets) {
+ int expected_var_begin, const map<int, set<int>>& expected_offsets) {
vector<int> offsets;
set<int> offsets_found;
@@ -4809,7 +4809,7 @@ void ValidateLayout(const vector<Expr*>& exprs, int expected_byte_size,
// Walk the computed offsets and make sure the resulting sets match expected_offsets
for (int i = 0; i < exprs.size(); ++i) {
int expr_byte_size = exprs[i]->type().GetByteSize();
- map<int, set<int> >::const_iterator iter = expected_offsets.find(expr_byte_size);
+ map<int, set<int>>::const_iterator iter = expected_offsets.find(expr_byte_size);
EXPECT_TRUE(iter != expected_offsets.end());
const set<int>& possible_offsets = iter->second;
@@ -4827,7 +4827,7 @@ TEST_F(ExprTest, ResultsLayoutTest) {
ObjectPool pool;
vector<Expr*> exprs;
- map<int, set<int> > expected_offsets;
+ map<int, set<int>> expected_offsets;
// Test empty exprs
ValidateLayout(exprs, 0, -1, expected_offsets);
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/55b2a639/be/src/resourcebroker/resource-broker.h
----------------------------------------------------------------------
diff --git a/be/src/resourcebroker/resource-broker.h b/be/src/resourcebroker/resource-broker.h
index 0857c1e..e4c5a25 100644
--- a/be/src/resourcebroker/resource-broker.h
+++ b/be/src/resourcebroker/resource-broker.h
@@ -262,7 +262,7 @@ class ResourceBroker {
boost::scoped_ptr<ThriftServer> llama_callback_server_;
/// Cache of Llama client connections.
- boost::scoped_ptr<ClientCache<llama::LlamaAMServiceClient> > llama_client_cache_;
+ boost::scoped_ptr<ClientCache<llama::LlamaAMServiceClient>> llama_client_cache_;
/// Lock to ensure that only a single registration with Llama is sent, e.g.,
/// when multiple concurrent requests realize that Llama has restarted.
@@ -352,7 +352,7 @@ class ResourceBroker {
/// Map from reservation IDs to pending expansion IDs. All pending request IDs have a
/// PendingRequest in pending_requests_.
- typedef boost::unordered_map<llama::TUniqueId, boost::unordered_set<llama::TUniqueId> >
+ typedef boost::unordered_map<llama::TUniqueId, boost::unordered_set<llama::TUniqueId>>
PendingExpansionIdsMap;
PendingExpansionIdsMap pending_expansion_ids_;
@@ -390,13 +390,13 @@ class ResourceBroker {
/// Map from reservation ID to all satisfied requests - reservation and expansion -
/// associated with that reservation. Used only for bookkeeping so that Impala can report
/// on the current resource usage.
- typedef boost::unordered_map<llama::TUniqueId, std::vector<AllocatedRequest> >
+ typedef boost::unordered_map<llama::TUniqueId, std::vector<AllocatedRequest>>
AllocatedRequestMap;
AllocatedRequestMap allocated_requests_;
/// Protects query_resource_mgrs_
boost::mutex query_resource_mgrs_lock_;
- typedef boost::unordered_map<TUniqueId, std::pair<int32_t, QueryResourceMgr*> >
+ typedef boost::unordered_map<TUniqueId, std::pair<int32_t, QueryResourceMgr*>>
QueryResourceMgrsMap;
/// Map from query ID to a (ref_count, QueryResourceMgr*) pair, i.e. one QueryResourceMgr
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/55b2a639/be/src/rpc/thrift-server.h
----------------------------------------------------------------------
diff --git a/be/src/rpc/thrift-server.h b/be/src/rpc/thrift-server.h
index 1fd1121..bd51070 100644
--- a/be/src/rpc/thrift-server.h
+++ b/be/src/rpc/thrift-server.h
@@ -204,7 +204,7 @@ class ThriftServer {
/// Map of active connection context to a shared_ptr containing that context; when an
/// item is removed from the map, it is automatically freed.
- typedef boost::unordered_map<ConnectionContext*, boost::shared_ptr<ConnectionContext> >
+ typedef boost::unordered_map<ConnectionContext*, boost::shared_ptr<ConnectionContext>>
ConnectionContextSet;
ConnectionContextSet connection_contexts_;
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/55b2a639/be/src/runtime/buffered-block-mgr-test.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/buffered-block-mgr-test.cc b/be/src/runtime/buffered-block-mgr-test.cc
index 04630fb..90a63d6 100644
--- a/be/src/runtime/buffered-block-mgr-test.cc
+++ b/be/src/runtime/buffered-block-mgr-test.cc
@@ -222,7 +222,7 @@ class BufferedBlockMgrTest : public ::testing::Test {
}
}
- void DeleteBlocks(const vector<pair<BufferedBlockMgr::Block*, int32_t> >& blocks) {
+ void DeleteBlocks(const vector<pair<BufferedBlockMgr::Block*, int32_t>>& blocks) {
for (int i = 0; i < blocks.size(); ++i) {
blocks[i].first->Delete();
}
@@ -385,9 +385,9 @@ class BufferedBlockMgrTest : public ::testing::Test {
const int iters_before_close = num_iterations - 5000;
bool close_called = false;
unordered_map<BufferedBlockMgr::Block*, int> pinned_block_map;
- vector<pair<BufferedBlockMgr::Block*, int32_t> > pinned_blocks;
+ vector<pair<BufferedBlockMgr::Block*, int32_t>> pinned_blocks;
unordered_map<BufferedBlockMgr::Block*, int> unpinned_block_map;
- vector<pair<BufferedBlockMgr::Block*, int32_t> > unpinned_blocks;
+ vector<pair<BufferedBlockMgr::Block*, int32_t>> unpinned_blocks;
typedef enum { Pin, New, Unpin, Delete, Close } ApiFunction;
ApiFunction api_function;
@@ -990,7 +990,7 @@ TEST_F(BufferedBlockMgrTest, DISABLED_WriteErrorBlacklist) {
client_tracker_.get(), &block_mgrs, &clients);
// Allocate files for all 2x2 combinations by unpinning blocks.
- vector<vector<BufferedBlockMgr::Block*> > blocks;
+ vector<vector<BufferedBlockMgr::Block*>> blocks;
vector<BufferedBlockMgr::Block*> all_blocks;
for (int i = 0; i < NUM_BLOCK_MGRS; ++i) {
vector<BufferedBlockMgr::Block*> mgr_blocks;
@@ -1073,7 +1073,7 @@ TEST_F(BufferedBlockMgrTest, AllocationErrorHandling) {
false, client_tracker_.get(), &block_mgrs, &clients);
// Allocate files for all 2x2 combinations by unpinning blocks.
- vector<vector<BufferedBlockMgr::Block*> > blocks;
+ vector<vector<BufferedBlockMgr::Block*>> blocks;
for (int i = 0; i < num_block_mgrs; ++i) {
vector<BufferedBlockMgr::Block*> mgr_blocks;
LOG(INFO) << "Iter " << i;
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/55b2a639/be/src/runtime/buffered-block-mgr.h
----------------------------------------------------------------------
diff --git a/be/src/runtime/buffered-block-mgr.h b/be/src/runtime/buffered-block-mgr.h
index 78fdb37..416751e 100644
--- a/be/src/runtime/buffered-block-mgr.h
+++ b/be/src/runtime/buffered-block-mgr.h
@@ -637,7 +637,7 @@ class BufferedBlockMgr {
/// map contains only weak ptrs. BufferedBlockMgrs that are handed out are shared ptrs.
/// When all the shared ptrs are no longer referenced, the BufferedBlockMgr
/// d'tor will be called at which point the weak ptr will be removed from the map.
- typedef boost::unordered_map<TUniqueId, boost::weak_ptr<BufferedBlockMgr> >
+ typedef boost::unordered_map<TUniqueId, boost::weak_ptr<BufferedBlockMgr>>
BlockMgrsMap;
static BlockMgrsMap query_to_block_mgrs_;
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/55b2a639/be/src/runtime/buffered-tuple-stream.h
----------------------------------------------------------------------
diff --git a/be/src/runtime/buffered-tuple-stream.h b/be/src/runtime/buffered-tuple-stream.h
index 23c09a9..9238b50 100644
--- a/be/src/runtime/buffered-tuple-stream.h
+++ b/be/src/runtime/buffered-tuple-stream.h
@@ -358,11 +358,11 @@ class BufferedTupleStream {
/// Vectors of all the strings slots that have their varlen data stored in stream
/// grouped by tuple_idx.
- std::vector<std::pair<int, std::vector<SlotDescriptor*> > > inlined_string_slots_;
+ std::vector<std::pair<int, std::vector<SlotDescriptor*>>> inlined_string_slots_;
/// Vectors of all the collection slots that have their varlen data stored in the
/// stream, grouped by tuple_idx.
- std::vector<std::pair<int, std::vector<SlotDescriptor*> > > inlined_coll_slots_;
+ std::vector<std::pair<int, std::vector<SlotDescriptor*>>> inlined_coll_slots_;
/// Block manager and client used to allocate, pin and release blocks. Not owned.
BufferedBlockMgr* block_mgr_;
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/55b2a639/be/src/runtime/client-cache.h
----------------------------------------------------------------------
diff --git a/be/src/runtime/client-cache.h b/be/src/runtime/client-cache.h
index ff38286..350bdd6 100644
--- a/be/src/runtime/client-cache.h
+++ b/be/src/runtime/client-cache.h
@@ -144,7 +144,7 @@ class ClientCacheHelper {
/// client_map_ for that host. The value type is wrapped in a shared_ptr so that the copy
/// c'tor for PerHostCache is not required.
typedef boost::unordered_map<
- TNetworkAddress, boost::shared_ptr<PerHostCache> > PerHostCacheMap;
+ TNetworkAddress, boost::shared_ptr<PerHostCache>> PerHostCacheMap;
PerHostCacheMap per_host_caches_;
/// Protects client_map_.
@@ -154,7 +154,7 @@ class ClientCacheHelper {
/// all the clients are actually stored, and client instances are owned by this class and
/// persist for exactly as long as they are present in this map.
/// We use a map (vs. unordered_map) so we get iterator consistency across operations.
- typedef std::map<ClientKey, boost::shared_ptr<ThriftClientImpl> > ClientMap;
+ typedef std::map<ClientKey, boost::shared_ptr<ThriftClientImpl>> ClientMap;
ClientMap client_map_;
/// Number of attempts to make to open a connection. 0 means retry indefinitely.
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/55b2a639/be/src/runtime/coordinator.h
----------------------------------------------------------------------
diff --git a/be/src/runtime/coordinator.h b/be/src/runtime/coordinator.h
index b4eab0e..9994e1e 100644
--- a/be/src/runtime/coordinator.h
+++ b/be/src/runtime/coordinator.h
@@ -531,7 +531,7 @@ class Coordinator {
/// permissions of the path. The PermissionCache argument is also used to cache the
/// output across repeated calls, to avoid repeatedly calling hdfsGetPathInfo() on the
/// same path.
- typedef boost::unordered_map<std::string, std::pair<bool, short> > PermissionCache;
+ typedef boost::unordered_map<std::string, std::pair<bool, short>> PermissionCache;
void PopulatePathPermissionCache(hdfsFS fs, const std::string& path_str,
PermissionCache* permissions_cache);
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/55b2a639/be/src/runtime/data-stream-mgr.h
----------------------------------------------------------------------
diff --git a/be/src/runtime/data-stream-mgr.h b/be/src/runtime/data-stream-mgr.h
index 21f8336..2b2b253 100644
--- a/be/src/runtime/data-stream-mgr.h
+++ b/be/src/runtime/data-stream-mgr.h
@@ -122,7 +122,7 @@ class DataStreamMgr {
/// we don't want to create a map<pair<TUniqueId, PlanNodeId>, DataStreamRecvr*>,
/// because that requires a bunch of copying of ids for lookup
typedef boost::unordered_multimap<uint32_t,
- boost::shared_ptr<DataStreamRecvr> > RecvrMap;
+ boost::shared_ptr<DataStreamRecvr>> RecvrMap;
RecvrMap receiver_map_;
/// (Fragment instance id, Plan node id) pair that uniquely identifies a stream.
@@ -182,7 +182,7 @@ class DataStreamMgr {
inline uint32_t GetHashValue(const TUniqueId& fragment_instance_id, PlanNodeId node_id);
/// The coordination primitive used to signal the arrival of a waited-for receiver
- typedef Promise<boost::shared_ptr<DataStreamRecvr> > RendezvousPromise;
+ typedef Promise<boost::shared_ptr<DataStreamRecvr>> RendezvousPromise;
/// A reference-counted promise-wrapper used to coordinate between senders and
/// receivers. The ref_count field tracks the number of senders waiting for the arrival
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/55b2a639/be/src/runtime/data-stream-recvr.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/data-stream-recvr.cc b/be/src/runtime/data-stream-recvr.cc
index 83decea..f5db5c8 100644
--- a/be/src/runtime/data-stream-recvr.cc
+++ b/be/src/runtime/data-stream-recvr.cc
@@ -87,7 +87,7 @@ class DataStreamRecvr::SenderQueue {
// queue of (batch length, batch) pairs. The SenderQueue block owns memory to
// these batches. They are handed off to the caller via GetBatch.
- typedef list<pair<int, RowBatch*> > RowBatchQueue;
+ typedef list<pair<int, RowBatch*>> RowBatchQueue;
RowBatchQueue batch_queue_;
// The batch that was most recently returned via GetBatch(), i.e. the current batch
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/55b2a639/be/src/runtime/disk-io-mgr.h
----------------------------------------------------------------------
diff --git a/be/src/runtime/disk-io-mgr.h b/be/src/runtime/disk-io-mgr.h
index 79902c5..9eb663c 100644
--- a/be/src/runtime/disk-io-mgr.h
+++ b/be/src/runtime/disk-io-mgr.h
@@ -733,7 +733,7 @@ class DiskIoMgr {
/// free_buffers_[10] => list of free buffers with size 1 MB
/// free_buffers_[13] => list of free buffers with size 8 MB
/// free_buffers_[n] => list of free buffers with size 2^n * 1024 B
- std::vector<std::list<char*> > free_buffers_;
+ std::vector<std::list<char*>> free_buffers_;
/// List of free buffer desc objects that can be handed out to clients
std::list<BufferDescriptor*> free_buffer_descs_;
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/55b2a639/be/src/runtime/free-pool-test.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/free-pool-test.cc b/be/src/runtime/free-pool-test.cc
index c8ff93d..9e3c218 100644
--- a/be/src/runtime/free-pool-test.cc
+++ b/be/src/runtime/free-pool-test.cc
@@ -112,7 +112,7 @@ TEST(FreePoolTest, Loop) {
MemPool mem_pool(&tracker);
FreePool pool(&mem_pool);
- map<int64_t, pair<uint8_t*, uint8_t*> > primed_allocations;
+ map<int64_t, pair<uint8_t*, uint8_t*>> primed_allocations;
vector<int64_t> allocation_sizes;
int64_t expected_pool_size = 0;
@@ -193,4 +193,3 @@ int main(int argc, char **argv) {
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}
-
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/55b2a639/be/src/runtime/mem-tracker.h
----------------------------------------------------------------------
diff --git a/be/src/runtime/mem-tracker.h b/be/src/runtime/mem-tracker.h
index 73c9300..7700b06 100644
--- a/be/src/runtime/mem-tracker.h
+++ b/be/src/runtime/mem-tracker.h
@@ -408,7 +408,7 @@ class MemTracker {
/// contains only weak ptrs. MemTrackers that are handed out via GetQueryMemTracker()
/// are shared ptrs. When all the shared ptrs are no longer referenced, the MemTracker
/// d'tor will be called at which point the weak ptr will be removed from the map.
- typedef boost::unordered_map<TUniqueId, boost::weak_ptr<MemTracker> >
+ typedef boost::unordered_map<TUniqueId, boost::weak_ptr<MemTracker>>
RequestTrackersMap;
static RequestTrackersMap request_to_mem_trackers_;
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/55b2a639/be/src/runtime/multi-precision.h
----------------------------------------------------------------------
diff --git a/be/src/runtime/multi-precision.h b/be/src/runtime/multi-precision.h
index 0a82077..2e4b803 100644
--- a/be/src/runtime/multi-precision.h
+++ b/be/src/runtime/multi-precision.h
@@ -54,7 +54,7 @@ typedef __int128_t int128_t;
typedef boost::multiprecision::number<
boost::multiprecision::cpp_int_backend<256, 256,
boost::multiprecision::signed_magnitude,
- boost::multiprecision::unchecked, void> > int256_t;
+ boost::multiprecision::unchecked, void>> int256_t;
/// There is no implicit assignment from int128_t to int256_t (or in general, the boost
/// multi precision types and __int128_t).
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/55b2a639/be/src/runtime/plan-fragment-executor.h
----------------------------------------------------------------------
diff --git a/be/src/runtime/plan-fragment-executor.h b/be/src/runtime/plan-fragment-executor.h
index 082442e..9511a3e 100644
--- a/be/src/runtime/plan-fragment-executor.h
+++ b/be/src/runtime/plan-fragment-executor.h
@@ -242,7 +242,7 @@ class PlanFragmentExecutor {
ObjectPool* obj_pool() { return runtime_state_->obj_pool(); }
/// typedef for TPlanFragmentInstanceCtx.per_node_scan_ranges
- typedef std::map<TPlanNodeId, std::vector<TScanRangeParams> > PerNodeScanRanges;
+ typedef std::map<TPlanNodeId, std::vector<TScanRangeParams>> PerNodeScanRanges;
/// Main loop of profile reporting thread.
/// Exits when notified on done_cv_.
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/55b2a639/be/src/runtime/row-batch-serialize-test.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/row-batch-serialize-test.cc b/be/src/runtime/row-batch-serialize-test.cc
index 1f28440..a00efa4 100644
--- a/be/src/runtime/row-batch-serialize-test.cc
+++ b/be/src/runtime/row-batch-serialize-test.cc
@@ -244,7 +244,7 @@ class RowBatchSerializeTest : public testing::Test {
// Create a row batch from preconstructed tuples. Each tuple instance for tuple i
// is consecutively repeated repeats[i] times. The tuple instances are used in the
// order provided, starting at the beginning once all are used.
- void AddTuplesToRowBatch(int num_rows, const vector<vector<Tuple*> >& tuples,
+ void AddTuplesToRowBatch(int num_rows, const vector<vector<Tuple*>>& tuples,
const vector<int>& repeats, RowBatch* batch) {
int tuples_per_row = batch->row_desc().tuple_descriptors().size();
ASSERT_EQ(tuples_per_row, tuples.size());
@@ -268,7 +268,7 @@ class RowBatchSerializeTest : public testing::Test {
// Helper to build a row batch with only one tuple per row.
void AddTuplesToRowBatch(int num_rows, const vector<Tuple*>& tuples, int repeats,
RowBatch* batch) {
- vector<vector<Tuple*> > tmp_tuples(1, tuples);
+ vector<vector<Tuple*>> tmp_tuples(1, tuples);
vector<int> tmp_repeats(1, repeats);
AddTuplesToRowBatch(num_rows, tmp_tuples, tmp_repeats, batch);
}
@@ -435,7 +435,7 @@ void RowBatchSerializeTest::TestDupCorrectness(bool full_dedup) {
// All string dups are consecutive
repeats.push_back(num_rows / distinct_string_tuples + 1);
RowBatch* batch = pool_.Add(new RowBatch(row_desc, num_rows, tracker_.get()));
- vector<vector<Tuple*> > distinct_tuples(2);
+ vector<vector<Tuple*>> distinct_tuples(2);
CreateTuples(*row_desc.tuple_descriptors()[0], batch->tuple_data_pool(),
distinct_int_tuples, 0, 10, &distinct_tuples[0]);
CreateTuples(*row_desc.tuple_descriptors()[1], batch->tuple_data_pool(),
@@ -571,7 +571,7 @@ TEST_F(RowBatchSerializeTest, DedupPathologicalFull) {
string huge_string;
LOG(INFO) << "Try to resize to " << huge_string_size;
huge_string.resize(huge_string_size, 'z');
- vector<vector<Tuple*> > tuples(num_tuples);
+ vector<vector<Tuple*>> tuples(num_tuples);
vector<int> repeats(num_tuples, 1); // Don't repeat tuples adjacently
int64_t total_byte_size = 0;
RowBatch* batch = pool_.Add(new RowBatch(row_desc, num_rows, tracker_.get()));
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/55b2a639/be/src/runtime/sorter.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/sorter.cc b/be/src/runtime/sorter.cc
index 627f0e1..91e9a23 100644
--- a/be/src/runtime/sorter.cc
+++ b/be/src/runtime/sorter.cc
@@ -1534,7 +1534,7 @@ Status Sorter::CreateMerger(int max_num_runs) {
merger_.reset(
new SortedRunMerger(compare_less_than_, output_row_desc_, profile_, true));
- vector<function<Status (RowBatch**)> > merge_runs;
+ vector<function<Status (RowBatch**)>> merge_runs;
merge_runs.reserve(max_num_runs);
for (int i = 0; i < max_num_runs; ++i) {
Run* run = sorted_runs_.front();
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/55b2a639/be/src/runtime/test-env.h
----------------------------------------------------------------------
diff --git a/be/src/runtime/test-env.h b/be/src/runtime/test-env.h
index 5950261..900bb3e 100644
--- a/be/src/runtime/test-env.h
+++ b/be/src/runtime/test-env.h
@@ -74,7 +74,7 @@ class TestEnv {
boost::scoped_ptr<TmpFileMgr> tmp_file_mgr_;
/// Per-query states with associated block managers.
- vector<boost::shared_ptr<RuntimeState> > query_states_;
+ vector<boost::shared_ptr<RuntimeState>> query_states_;
};
}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/55b2a639/be/src/runtime/tmp-file-mgr-test.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/tmp-file-mgr-test.cc b/be/src/runtime/tmp-file-mgr-test.cc
index f6f2406..7276e97 100644
--- a/be/src/runtime/tmp-file-mgr-test.cc
+++ b/be/src/runtime/tmp-file-mgr-test.cc
@@ -49,7 +49,7 @@ class TmpFileMgrTest : public ::testing::Test {
"tmp-file-mgr.active-scratch-dirs");
EXPECT_EQ(active.size(), active_metric->value());
SetMetric<string>* active_set_metric =
- metrics_->FindMetricForTesting<SetMetric<string> >(
+ metrics_->FindMetricForTesting<SetMetric<string>>(
"tmp-file-mgr.active-scratch-dirs.list");
set<string> active_set = active_set_metric->value();
EXPECT_EQ(active.size(), active_set.size());
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/55b2a639/be/src/scheduling/query-schedule.h
----------------------------------------------------------------------
diff --git a/be/src/scheduling/query-schedule.h b/be/src/scheduling/query-schedule.h
index 20f7dfe..4c225c7 100644
--- a/be/src/scheduling/query-schedule.h
+++ b/be/src/scheduling/query-schedule.h
@@ -35,7 +35,7 @@ namespace impala {
class Coordinator;
/// map from scan node id to a list of scan ranges
-typedef std::map<TPlanNodeId, std::vector<TScanRangeParams> > PerNodeScanRanges;
+typedef std::map<TPlanNodeId, std::vector<TScanRangeParams>> PerNodeScanRanges;
/// map from an impalad host address to the per-node assigned scan ranges;
/// records scan range assignment for a single fragment
typedef boost::unordered_map<TNetworkAddress, PerNodeScanRanges>
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/55b2a639/be/src/scheduling/simple-scheduler.cc
----------------------------------------------------------------------
diff --git a/be/src/scheduling/simple-scheduler.cc b/be/src/scheduling/simple-scheduler.cc
index a14509c..1e46c67 100644
--- a/be/src/scheduling/simple-scheduler.cc
+++ b/be/src/scheduling/simple-scheduler.cc
@@ -422,7 +422,7 @@ void SimpleScheduler::GetAllKnownBackends(BackendList* backends) {
Status SimpleScheduler::ComputeScanRangeAssignment(const TQueryExecRequest& exec_request,
QuerySchedule* schedule) {
- map<TPlanNodeId, vector<TScanRangeLocations> >::const_iterator entry;
+ map<TPlanNodeId, vector<TScanRangeLocations>>::const_iterator entry;
for (entry = exec_request.per_node_scan_ranges.begin();
entry != exec_request.per_node_scan_ranges.end(); ++entry) {
const TPlanNodeId node_id = entry->first;
@@ -815,7 +815,7 @@ void SimpleScheduler::FindNodes(const TPlan& plan,
void SimpleScheduler::GetScanHosts(TPlanNodeId scan_id,
const TQueryExecRequest& exec_request, const FragmentExecParams& params,
vector<TNetworkAddress>* scan_hosts) {
- map<TPlanNodeId, vector<TScanRangeLocations> >::const_iterator entry =
+ map<TPlanNodeId, vector<TScanRangeLocations>>::const_iterator entry =
exec_request.per_node_scan_ranges.find(scan_id);
if (entry == exec_request.per_node_scan_ranges.end() || entry->second.empty()) {
// this scan node doesn't have any scan ranges; run it on the coordinator
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/55b2a639/be/src/scheduling/simple-scheduler.h
----------------------------------------------------------------------
diff --git a/be/src/scheduling/simple-scheduler.h b/be/src/scheduling/simple-scheduler.h
index 1d07711..5cff7a5 100644
--- a/be/src/scheduling/simple-scheduler.h
+++ b/be/src/scheduling/simple-scheduler.h
@@ -94,7 +94,7 @@ class SimpleScheduler : public Scheduler {
/// Map from a datanode's IP address to a list of backend addresses running on that
/// node.
- typedef boost::unordered_map<std::string, std::list<TBackendDescriptor> > BackendMap;
+ typedef boost::unordered_map<std::string, std::list<TBackendDescriptor>> BackendMap;
BackendMap backend_map_;
/// Map from a datanode's hostname to its IP address to support both hostname based
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/55b2a639/be/src/service/fragment-mgr.h
----------------------------------------------------------------------
diff --git a/be/src/service/fragment-mgr.h b/be/src/service/fragment-mgr.h
index f63d92f..68eac15 100644
--- a/be/src/service/fragment-mgr.h
+++ b/be/src/service/fragment-mgr.h
@@ -71,7 +71,7 @@ class FragmentMgr {
/// Map from fragment instance id to exec state; FragmentExecState is owned by us and
/// referenced as a shared_ptr to allow asynchronous calls to CancelPlanFragment()
- typedef boost::unordered_map<TUniqueId, boost::shared_ptr<FragmentExecState> >
+ typedef boost::unordered_map<TUniqueId, boost::shared_ptr<FragmentExecState>>
FragmentExecStateMap;
FragmentExecStateMap fragment_exec_state_map_;
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/55b2a639/be/src/service/hs2-util-test.cc
----------------------------------------------------------------------
diff --git a/be/src/service/hs2-util-test.cc b/be/src/service/hs2-util-test.cc
index c0b7b56..0d04164 100644
--- a/be/src/service/hs2-util-test.cc
+++ b/be/src/service/hs2-util-test.cc
@@ -111,7 +111,7 @@ TEST(StitchNullsTest, StitchWithOffset) {
TEST(PrintTColumnValueTest, TestAllTypes) {
using namespace apache::hive::service::cli::thrift;
- vector<pair<apache::hive::service::cli::thrift::TColumnValue, string> > values;
+ vector<pair<apache::hive::service::cli::thrift::TColumnValue, string>> values;
values.resize(9);
values[0].first.__set_boolVal(TBoolValue());
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/55b2a639/be/src/service/impala-beeswax-server.cc
----------------------------------------------------------------------
diff --git a/be/src/service/impala-beeswax-server.cc b/be/src/service/impala-beeswax-server.cc
index f26ace6..988386d 100644
--- a/be/src/service/impala-beeswax-server.cc
+++ b/be/src/service/impala-beeswax-server.cc
@@ -165,7 +165,7 @@ class ImpalaServer::AsciiQueryResultSet : public ImpalaServer::QueryResultSet {
vector<string>* result_set_;
// Set to result_set_ if result_set_ is owned.
- scoped_ptr<vector<string> > owned_result_set_;
+ scoped_ptr<vector<string>> owned_result_set_;
};
void ImpalaServer::query(QueryHandle& query_handle, const Query& query) {
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/55b2a639/be/src/service/impala-server.cc
----------------------------------------------------------------------
diff --git a/be/src/service/impala-server.cc b/be/src/service/impala-server.cc
index ec58ccc..e092f4b 100644
--- a/be/src/service/impala-server.cc
+++ b/be/src/service/impala-server.cc
@@ -1525,7 +1525,7 @@ void ImpalaServer::MembershipCallback(
// Maps from query id (to be cancelled) to a list of failed Impalads that are
// the cause of the cancellation.
- map<TUniqueId, vector<TNetworkAddress> > queries_to_cancel;
+ map<TUniqueId, vector<TNetworkAddress>> queries_to_cancel;
{
// Build a list of queries that are running on failed hosts (as evidenced by their
// absence from the membership list).
@@ -1562,7 +1562,7 @@ void ImpalaServer::MembershipCallback(
// Since we are the only producer for this pool, we know that this cannot block
// indefinitely since the queue is large enough to accept all new cancellation
// requests.
- map<TUniqueId, vector<TNetworkAddress> >::iterator cancellation_entry;
+ map<TUniqueId, vector<TNetworkAddress>>::iterator cancellation_entry;
for (cancellation_entry = queries_to_cancel.begin();
cancellation_entry != queries_to_cancel.end();
++cancellation_entry) {
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/55b2a639/be/src/service/impala-server.h
----------------------------------------------------------------------
diff --git a/be/src/service/impala-server.h b/be/src/service/impala-server.h
index 6a07ccd..9069d75 100644
--- a/be/src/service/impala-server.h
+++ b/be/src/service/impala-server.h
@@ -677,7 +677,7 @@ class ImpalaServer : public ImpalaServiceIf, public ImpalaHiveServer2ServiceIf,
/// Thread pool to process cancellation requests that come from failed Impala demons to
/// avoid blocking the statestore callback.
- boost::scoped_ptr<ThreadPool<CancellationWork> > cancellation_thread_pool_;
+ boost::scoped_ptr<ThreadPool<CancellationWork>> cancellation_thread_pool_;
/// Thread that runs ExpireSessions. It will wake up periodically to check for sessions
/// which are idle for more their timeout values.
@@ -695,7 +695,7 @@ class ImpalaServer : public ImpalaServiceIf, public ImpalaHiveServer2ServiceIf,
/// map from query id to exec state; QueryExecState is owned by us and referenced
/// as a shared_ptr to allow asynchronous deletion
- typedef boost::unordered_map<TUniqueId, boost::shared_ptr<QueryExecState> >
+ typedef boost::unordered_map<TUniqueId, boost::shared_ptr<QueryExecState>>
QueryExecStateMap;
QueryExecStateMap query_exec_state_map_;
boost::mutex query_exec_state_map_lock_; // protects query_exec_state_map_
@@ -827,7 +827,7 @@ class ImpalaServer : public ImpalaServiceIf, public ImpalaHiveServer2ServiceIf,
boost::mutex session_state_map_lock_;
/// A map from session identifier to a structure containing per-session information
- typedef boost::unordered_map<TUniqueId, boost::shared_ptr<SessionState> >
+ typedef boost::unordered_map<TUniqueId, boost::shared_ptr<SessionState>>
SessionStateMap;
SessionStateMap session_state_map_;
@@ -838,7 +838,7 @@ class ImpalaServer : public ImpalaServiceIf, public ImpalaHiveServer2ServiceIf,
/// when the connection ends. HS2 allows for multiplexing several sessions across a
/// single connection. If a session has already been closed (only possible via HS2) it is
/// not removed from this map to avoid the cost of looking it up.
- typedef boost::unordered_map<TUniqueId, std::vector<TUniqueId> >
+ typedef boost::unordered_map<TUniqueId, std::vector<TUniqueId>>
ConnectionToSessionMap;
ConnectionToSessionMap connection_to_sessions_map_;
@@ -863,7 +863,7 @@ class ImpalaServer : public ImpalaServiceIf, public ImpalaHiveServer2ServiceIf,
boost::mutex query_locations_lock_;
/// A map from backend to the list of queries currently running there.
- typedef boost::unordered_map<TNetworkAddress, boost::unordered_set<TUniqueId> >
+ typedef boost::unordered_map<TNetworkAddress, boost::unordered_set<TUniqueId>>
QueryLocations;
QueryLocations query_locations_;
@@ -918,7 +918,7 @@ class ImpalaServer : public ImpalaServiceIf, public ImpalaHiveServer2ServiceIf,
/// Map of short usernames of authorized proxy users to the set of user(s) they are
/// allowed to delegate to. Populated by parsing the --authorized_proxy_users_config
/// flag.
- typedef boost::unordered_map<std::string, boost::unordered_set<std::string> >
+ typedef boost::unordered_map<std::string, boost::unordered_set<std::string>>
ProxyUserMap;
ProxyUserMap authorized_proxy_user_config_;
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/55b2a639/be/src/service/query-exec-state.h
----------------------------------------------------------------------
diff --git a/be/src/service/query-exec-state.h b/be/src/service/query-exec-state.h
index 0b62a28..c2459cf 100644
--- a/be/src/service/query-exec-state.h
+++ b/be/src/service/query-exec-state.h
@@ -241,7 +241,7 @@ class ImpalaServer::QueryExecState {
/// Result set used for requests that return results and are not QUERY
/// statements. For example, EXPLAIN, LOAD, and SHOW use this.
- boost::scoped_ptr<std::vector<TResultRow> > request_result_set_;
+ boost::scoped_ptr<std::vector<TResultRow>> request_result_set_;
/// Cache of the first result_cache_max_size_ query results to allow clients to restart
/// fetching from the beginning of the result set. This cache is appended to in
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/55b2a639/be/src/service/session-expiry-test.cc
----------------------------------------------------------------------
diff --git a/be/src/service/session-expiry-test.cc b/be/src/service/session-expiry-test.cc
index b76edab..48cb75e 100644
--- a/be/src/service/session-expiry-test.cc
+++ b/be/src/service/session-expiry-test.cc
@@ -56,8 +56,8 @@ TEST(SessionTest, TestExpiry) {
EXPECT_EQ(beeswax_session_metric->value(), 0L);
{
- scoped_ptr<ThriftClient<ImpalaServiceClient> > beeswax_clients[5];
- scoped_ptr<ThriftClient<ImpalaHiveServer2ServiceClient> > hs2_clients[5];
+ scoped_ptr<ThriftClient<ImpalaServiceClient>> beeswax_clients[5];
+ scoped_ptr<ThriftClient<ImpalaHiveServer2ServiceClient>> hs2_clients[5];
// Create five Beeswax clients and five HS2 clients (each HS2 gets one session each)
for (int i = 0; i < 5; ++i) {
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/55b2a639/be/src/statestore/statestore.h
----------------------------------------------------------------------
diff --git a/be/src/statestore/statestore.h b/be/src/statestore/statestore.h
index 3ca0498..2061e38 100644
--- a/be/src/statestore/statestore.h
+++ b/be/src/statestore/statestore.h
@@ -350,7 +350,7 @@ class Statestore {
/// Subscribers are held in shared_ptrs so that RegisterSubscriber() may overwrite their
/// entry in this map while UpdateSubscriber() tries to update an existing registration
/// without risk of use-after-free.
- typedef boost::unordered_map<SubscriberId, boost::shared_ptr<Subscriber> >
+ typedef boost::unordered_map<SubscriberId, boost::shared_ptr<Subscriber>>
SubscriberMap;
SubscriberMap subscribers_;
@@ -396,13 +396,13 @@ class Statestore {
/// Cache of subscriber clients used for UpdateState() RPCs. Only one client per
/// subscriber should be used, but the cache helps with the client lifecycle on failure.
- boost::scoped_ptr<ClientCache<StatestoreSubscriberClient> > update_state_client_cache_;
+ boost::scoped_ptr<ClientCache<StatestoreSubscriberClient>> update_state_client_cache_;
/// Cache of subscriber clients used for Heartbeat() RPCs. Separate from
/// update_state_client_cache_ because we enable TCP-level timeouts for these calls,
/// whereas they are not safe for UpdateState() RPCs which can take an unbounded amount
/// of time.
- boost::scoped_ptr<ClientCache<StatestoreSubscriberClient> > heartbeat_client_cache_;
+ boost::scoped_ptr<ClientCache<StatestoreSubscriberClient>> heartbeat_client_cache_;
/// Thrift API implementation which proxies requests onto this Statestore
boost::shared_ptr<StatestoreServiceIf> thrift_iface_;
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/55b2a639/be/src/testutil/impalad-query-executor.h
----------------------------------------------------------------------
diff --git a/be/src/testutil/impalad-query-executor.h b/be/src/testutil/impalad-query-executor.h
index 0fbaf6b..f0448f8 100644
--- a/be/src/testutil/impalad-query-executor.h
+++ b/be/src/testutil/impalad-query-executor.h
@@ -87,7 +87,7 @@ class ImpaladQueryExecutor {
private:
/// fe service-related
- boost::scoped_ptr<ThriftClient<ImpalaServiceClient> > client_;
+ boost::scoped_ptr<ThriftClient<ImpalaServiceClient>> client_;
/// Execution options
std::vector<std::string> exec_options_;
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/55b2a639/be/src/transport/TSaslServerTransport.cpp
----------------------------------------------------------------------
diff --git a/be/src/transport/TSaslServerTransport.cpp b/be/src/transport/TSaslServerTransport.cpp
index 280db85..1dfa99e 100644
--- a/be/src/transport/TSaslServerTransport.cpp
+++ b/be/src/transport/TSaslServerTransport.cpp
@@ -135,7 +135,7 @@ shared_ptr<TTransport> TSaslServerTransport::Factory::getTransport(
// ensure that when ret_transport is eventually deleted, its corresponding map entry is
// removed. That is likely to be error prone given the locking involved; for now we go
// with the simple solution.
- map<shared_ptr<TTransport>, shared_ptr<TBufferedTransport> >::iterator trans_map =
+ map<shared_ptr<TTransport>, shared_ptr<TBufferedTransport>>::iterator trans_map =
transportMap_.find(trans);
VLOG_EVERY_N(2, 100) << "getTransport(): transportMap_ size is: "
<< transportMap_.size();
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/55b2a639/be/src/transport/TSaslServerTransport.h
----------------------------------------------------------------------
diff --git a/be/src/transport/TSaslServerTransport.h b/be/src/transport/TSaslServerTransport.h
index 0db5eca..31041b3 100644
--- a/be/src/transport/TSaslServerTransport.h
+++ b/be/src/transport/TSaslServerTransport.h
@@ -174,7 +174,7 @@ class TSaslServerTransport : public TSaslTransport {
/* Map from a transport to its Sasl Transport (wrapped by a TBufferedTransport). */
std::map<boost::shared_ptr<TTransport>,
- boost::shared_ptr<TBufferedTransport> > transportMap_;
+ boost::shared_ptr<TBufferedTransport>> transportMap_;
/* Lock to synchronize the transport map. */
boost::mutex transportMap_mutex_;
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/55b2a639/be/src/udf/uda-test-harness-impl.h
----------------------------------------------------------------------
diff --git a/be/src/udf/uda-test-harness-impl.h b/be/src/udf/uda-test-harness-impl.h
index e08b9b0..8be05ca 100644
--- a/be/src/udf/uda-test-harness-impl.h
+++ b/be/src/udf/uda-test-harness-impl.h
@@ -181,7 +181,7 @@ RESULT UdaTestHarnessBase<RESULT, INTERMEDIATE>::ExecuteSingleNode(
template<typename RESULT, typename INTERMEDIATE>
RESULT UdaTestHarnessBase<RESULT, INTERMEDIATE>::ExecuteOneLevel(int num_nodes,
ScopedFunctionContext* result_context) {
- std::vector<boost::shared_ptr<ScopedFunctionContext> > contexts;
+ std::vector<boost::shared_ptr<ScopedFunctionContext>> contexts;
std::vector<INTERMEDIATE> intermediates;
contexts.resize(num_nodes);
@@ -237,7 +237,7 @@ RESULT UdaTestHarnessBase<RESULT, INTERMEDIATE>::ExecuteOneLevel(int num_nodes,
template<typename RESULT, typename INTERMEDIATE>
RESULT UdaTestHarnessBase<RESULT, INTERMEDIATE>::ExecuteTwoLevel(
int num1, int num2, ScopedFunctionContext* result_context) {
- std::vector<boost::shared_ptr<ScopedFunctionContext> > level1_contexts, level2_contexts;
+ std::vector<boost::shared_ptr<ScopedFunctionContext>> level1_contexts, level2_contexts;
std::vector<INTERMEDIATE> level1_intermediates, level2_intermediates;
level1_contexts.resize(num1);
level2_contexts.resize(num2);
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/55b2a639/be/src/util/blocking-queue-test.cc
----------------------------------------------------------------------
diff --git a/be/src/util/blocking-queue-test.cc b/be/src/util/blocking-queue-test.cc
index 1e3240b..ba6a9ad 100644
--- a/be/src/util/blocking-queue-test.cc
+++ b/be/src/util/blocking-queue-test.cc
@@ -127,7 +127,7 @@ class MultiThreadTest {
}
private:
- typedef vector<shared_ptr<thread> > ThreadVector;
+ typedef vector<shared_ptr<thread>> ThreadVector;
int iterations_;
int nthreads_;
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/55b2a639/be/src/util/collection-metrics.h
----------------------------------------------------------------------
diff --git a/be/src/util/collection-metrics.h b/be/src/util/collection-metrics.h
index 276f4b9..f836765 100644
--- a/be/src/util/collection-metrics.h
+++ b/be/src/util/collection-metrics.h
@@ -96,7 +96,7 @@ class SetMetric : public Metric {
virtual std::string ToHumanReadable() {
std::stringstream out;
- PrettyPrinter::PrintStringList<std::set<T> >(
+ PrettyPrinter::PrintStringList<std::set<T>>(
value_, TUnit::NONE, &out);
return out.str();
}
@@ -273,7 +273,7 @@ class StatsMetric : public Metric {
boost::accumulators::tag::count,
boost::accumulators::tag::min,
boost::accumulators::tag::max,
- boost::accumulators::tag::variance> > Accumulator;
+ boost::accumulators::tag::variance>> Accumulator;
Accumulator acc_;
};
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/55b2a639/be/src/util/debug-util.cc
----------------------------------------------------------------------
diff --git a/be/src/util/debug-util.cc b/be/src/util/debug-util.cc
index c342965..c3f8c94 100644
--- a/be/src/util/debug-util.cc
+++ b/be/src/util/debug-util.cc
@@ -124,7 +124,7 @@ bool ParseId(const string& s, TUniqueId* id) {
if (separator == NULL) {
// Legacy compatibility branch
char_separator<char> sep(" ");
- tokenizer< char_separator<char> > tokens(s, sep);
+ tokenizer< char_separator<char>> tokens(s, sep);
int i = 0;
for (const string& token: tokens) {
StringParser::ParseResult parse_result = StringParser::PARSE_SUCCESS;
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/55b2a639/be/src/util/internal-queue-test.cc
----------------------------------------------------------------------
diff --git a/be/src/util/internal-queue-test.cc b/be/src/util/internal-queue-test.cc
index 30bb1e8..2142de2 100644
--- a/be/src/util/internal-queue-test.cc
+++ b/be/src/util/internal-queue-test.cc
@@ -240,7 +240,7 @@ TEST(InternalQueue, TestMultiProducerMultiConsumer) {
const int num_per_consumer = nodes.size() / NUM_CONSUMERS;
const int num_per_producer = nodes.size() / num_producers;
- vector<vector<int> > results;
+ vector<vector<int>> results;
results.resize(NUM_CONSUMERS);
int expected_delta = -1;
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/55b2a639/be/src/util/metrics-test.cc
----------------------------------------------------------------------
diff --git a/be/src/util/metrics-test.cc b/be/src/util/metrics-test.cc
index 7baca5f..ce80f56 100644
--- a/be/src/util/metrics-test.cc
+++ b/be/src/util/metrics-test.cc
@@ -205,7 +205,7 @@ TEST_F(MetricsTest, MemMetric) {
// Allocate 100MB to increase the number of bytes used. TCMalloc may also give up some
// bytes during this allocation, so this allocation is deliberately large to ensure that
// the bytes used metric goes up net.
- scoped_ptr<vector<uint64_t> > chunk(new vector<uint64_t>(100 * 1024 * 1024));
+ scoped_ptr<vector<uint64_t>> chunk(new vector<uint64_t>(100 * 1024 * 1024));
EXPECT_GT(bytes_in_use->value(), cur_in_use);
UIntGauge* total_bytes_reserved =
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/55b2a639/be/src/util/perf-counters.h
----------------------------------------------------------------------
diff --git a/be/src/util/perf-counters.h b/be/src/util/perf-counters.h
index 4597ecf..3eb0d51 100644
--- a/be/src/util/perf-counters.h
+++ b/be/src/util/perf-counters.h
@@ -129,7 +129,7 @@ class PerfCounters {
std::vector<CounterData> counters_;
std::vector<std::string> counter_names_;
std::vector<std::string> snapshot_names_;
- std::vector<std::vector<int64_t> > snapshots_;
+ std::vector<std::vector<int64_t>> snapshots_;
/// System perf counters can be grouped together. The OS will update all grouped counters
/// at the same time. This is useful to better correlate counter values.
int group_fd_;
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/55b2a639/be/src/util/runtime-profile.h
----------------------------------------------------------------------
diff --git a/be/src/util/runtime-profile.h b/be/src/util/runtime-profile.h
index 98a02ed..6c90c2d 100644
--- a/be/src/util/runtime-profile.h
+++ b/be/src/util/runtime-profile.h
@@ -320,7 +320,7 @@ class RuntimeProfile {
/// Map from parent counter name to a set of child counter name.
/// All top level counters are the child of "" (root).
- typedef std::map<std::string, std::set<std::string> > ChildCounterMap;
+ typedef std::map<std::string, std::set<std::string>> ChildCounterMap;
ChildCounterMap child_counter_map_;
/// A set of bucket counters registered in this runtime profile.
@@ -336,7 +336,7 @@ class RuntimeProfile {
ChildMap child_map_;
/// Vector of (profile, indentation flag).
- typedef std::vector<std::pair<RuntimeProfile*, bool> > ChildVector;
+ typedef std::vector<std::pair<RuntimeProfile*, bool>> ChildVector;
ChildVector children_;
/// Protects child_map_ and children_.
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/55b2a639/be/src/util/summary-util.cc
----------------------------------------------------------------------
diff --git a/be/src/util/summary-util.cc b/be/src/util/summary-util.cc
index 8ba42fc..30849ca 100644
--- a/be/src/util/summary-util.cc
+++ b/be/src/util/summary-util.cc
@@ -33,7 +33,7 @@ using namespace impala;
// for the current node and on return, will contain the id of the next node.
void PrintExecSummary(const TExecSummary& exec_summary, int indent_level,
int new_indent_level, int* node_idx,
- vector<vector<string> >* result) {
+ vector<vector<string>>* result) {
DCHECK_LT(*node_idx, exec_summary.nodes.size());
const TPlanNodeExecSummary& node = exec_summary.nodes[*node_idx];
const TExecStats& est_stats = node.estimated_stats;
@@ -94,7 +94,7 @@ void PrintExecSummary(const TExecSummary& exec_summary, int indent_level,
if (node.num_children == 0) return;
// Print the non-left children to the stream first.
- vector<vector<string> > child0_result;
+ vector<vector<string>> child0_result;
PrintExecSummary(exec_summary, indent_level, false, node_idx, &child0_result);
for (int i = 1; i < node.num_children; ++i) {
@@ -121,7 +121,7 @@ string impala::PrintExecSummary(const TExecSummary& exec_summary) {
printer.AddColumn("Est. Peak Mem", false);
printer.AddColumn("Detail", true);
- vector<vector<string> > rows;
+ vector<vector<string>> rows;
int node_idx = 0;
::PrintExecSummary(exec_summary, 0, false, &node_idx, &rows);
for (int i = 0; i < rows.size(); ++i) {
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/55b2a639/be/src/util/table-printer.h
----------------------------------------------------------------------
diff --git a/be/src/util/table-printer.h b/be/src/util/table-printer.h
index 69c954c..8e23dab 100644
--- a/be/src/util/table-printer.h
+++ b/be/src/util/table-printer.h
@@ -50,7 +50,7 @@ class TablePrinter {
/// -1 to indicate unlimited.
int max_output_width_;
- std::vector<std::vector<std::string> > rows_;
+ std::vector<std::vector<std::string>> rows_;
std::vector<int> max_col_widths_;
/// Helper function to print one row to ss.
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/55b2a639/be/src/util/thread-pool.h
----------------------------------------------------------------------
diff --git a/be/src/util/thread-pool.h b/be/src/util/thread-pool.h
index 0f1c656..63ad41a 100644
--- a/be/src/util/thread-pool.h
+++ b/be/src/util/thread-pool.h
@@ -161,11 +161,11 @@ class ThreadPool {
};
/// Utility thread-pool that accepts callable work items, and simply invokes them.
-class CallableThreadPool : public ThreadPool<boost::function<void()> > {
+class CallableThreadPool : public ThreadPool<boost::function<void()>> {
public:
CallableThreadPool(const std::string& group, const std::string& thread_prefix,
uint32_t num_threads, uint32_t queue_size) :
- ThreadPool<boost::function<void()> >(
+ ThreadPool<boost::function<void()>>(
group, thread_prefix, num_threads, queue_size, &CallableThreadPool::Worker) {
}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/55b2a639/be/src/util/url-coding.cc
----------------------------------------------------------------------
diff --git a/be/src/util/url-coding.cc b/be/src/util/url-coding.cc
index b64e609..20e5783 100644
--- a/be/src/util/url-coding.cc
+++ b/be/src/util/url-coding.cc
@@ -107,7 +107,7 @@ bool UrlDecode(const string& in, string* out, bool hive_compat) {
}
static inline void Base64Encode(const char* in, int in_len, stringstream* out) {
- typedef base64_from_binary<transform_width<const char*, 6, 8> > base64_encode;
+ typedef base64_from_binary<transform_width<const char*, 6, 8>> base64_encode;
// Base64 encodes 8 byte chars as 6 bit values.
stringstream::pos_type len_before = out->tellp();
copy(base64_encode(in), base64_encode(in + in_len), std::ostream_iterator<char>(*out));
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/55b2a639/fe/src/test/java/com/cloudera/impala/catalog/HdfsPartitionTest.java
----------------------------------------------------------------------
diff --git a/fe/src/test/java/com/cloudera/impala/catalog/HdfsPartitionTest.java b/fe/src/test/java/com/cloudera/impala/catalog/HdfsPartitionTest.java
index 9959c8b..7f5054f 100644
--- a/fe/src/test/java/com/cloudera/impala/catalog/HdfsPartitionTest.java
+++ b/fe/src/test/java/com/cloudera/impala/catalog/HdfsPartitionTest.java
@@ -57,7 +57,7 @@ public class HdfsPartitionTest {
@Test
public void testCompare() {
- List< List<LiteralExpr> > allLists = Lists.newArrayList();
+ List<List<LiteralExpr>> allLists = Lists.newArrayList();
allLists.add(valuesNull_);
allLists.add(valuesDecimal_);
allLists.add(valuesDecimal1_);
[16/18] incubator-impala git commit: IMPALA-3817: Ensure filter hash
function is the same on all hardware.
Posted by ta...@apache.org.
IMPALA-3817: Ensure filter hash function is the same on all hardware.
If some nodes in a cluster support the hardware CRC instruction, then
those nodes would use that hash function when testing filters. This
change uses Murmur2 always. Murmur2 is slower, but a perf run on a
16-node cluster showed negligible performance differences.
This impacts DataStreamMgr's hash function, but that that also doesn't
show a perf difference.
(Also, remove unused hash function from HashUtil that was made dead
code by an earlier commit.)
Change-Id: Ia09b67f9e987af3e2c8ac12c347b95a7e09ce6fa
Reviewed-on: http://gerrit.cloudera.org:8080/3566
Reviewed-by: Jim Apple <jb...@cloudera.com>
Tested-by: Internal Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/26266fda
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/26266fda
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/26266fda
Branch: refs/heads/master
Commit: 26266fda4ca00ab0034ae67dfc0dfde26f62ad70
Parents: 79641a8
Author: Jim Apple <jb...@cloudera.com>
Authored: Sun Jul 3 10:22:44 2016 -0700
Committer: Taras Bobrovytsky <ta...@apache.org>
Committed: Thu Jul 14 19:04:45 2016 +0000
----------------------------------------------------------------------
be/src/runtime/raw-value.inline.h | 69 ++++++++--------------------------
be/src/util/hash-util.h | 8 ----
2 files changed, 16 insertions(+), 61 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/26266fda/be/src/runtime/raw-value.inline.h
----------------------------------------------------------------------
diff --git a/be/src/runtime/raw-value.inline.h b/be/src/runtime/raw-value.inline.h
index 2d08294..446ae71 100644
--- a/be/src/runtime/raw-value.inline.h
+++ b/be/src/runtime/raw-value.inline.h
@@ -108,11 +108,7 @@ inline uint32_t RawValue::GetHashValueNonNull<int8_t>(const int8_t* v,
const ColumnType& type, uint32_t seed) {
DCHECK_EQ(type.type, TYPE_TINYINT);
DCHECK(v != NULL);
- if (LIKELY(CpuInfo::IsSupported(CpuInfo::SSE4_2))) {
- return HashUtil::CrcHash1(v, seed);
- } else {
- return HashUtil::MurmurHash2_64(v, 1, seed);
- }
+ return HashUtil::MurmurHash2_64(v, 1, seed);
}
template<>
@@ -120,11 +116,7 @@ inline uint32_t RawValue::GetHashValueNonNull<int16_t>(const int16_t* v,
const ColumnType& type, uint32_t seed) {
DCHECK_EQ(type.type, TYPE_SMALLINT);
DCHECK(v != NULL);
- if (LIKELY(CpuInfo::IsSupported(CpuInfo::SSE4_2))) {
- return HashUtil::CrcHash2(v, seed);
- } else {
- return HashUtil::MurmurHash2_64(v, 2, seed);
- }
+ return HashUtil::MurmurHash2_64(v, 2, seed);
}
template<>
@@ -132,11 +124,7 @@ inline uint32_t RawValue::GetHashValueNonNull<int32_t>(const int32_t* v,
const ColumnType& type, uint32_t seed) {
DCHECK_EQ(type.type, TYPE_INT);
DCHECK(v != NULL);
- if (LIKELY(CpuInfo::IsSupported(CpuInfo::SSE4_2))) {
- return HashUtil::CrcHash4(v, seed);
- } else {
- return HashUtil::MurmurHash2_64(v, 4, seed);
- }
+ return HashUtil::MurmurHash2_64(v, 4, seed);
}
template<>
@@ -144,11 +132,7 @@ inline uint32_t RawValue::GetHashValueNonNull<int64_t>(const int64_t* v,
const ColumnType& type, uint32_t seed) {
DCHECK_EQ(type.type, TYPE_BIGINT);
DCHECK(v != NULL);
- if (LIKELY(CpuInfo::IsSupported(CpuInfo::SSE4_2))) {
- return HashUtil::CrcHash8(v, seed);
- } else {
- return HashUtil::MurmurHash2_64(v, 8, seed);
- }
+ return HashUtil::MurmurHash2_64(v, 8, seed);
}
template<>
@@ -156,11 +140,7 @@ inline uint32_t RawValue::GetHashValueNonNull<float>(const float* v,
const ColumnType& type, uint32_t seed) {
DCHECK_EQ(type.type, TYPE_FLOAT);
DCHECK(v != NULL);
- if (LIKELY(CpuInfo::IsSupported(CpuInfo::SSE4_2))) {
- return HashUtil::CrcHash4(v, seed);
- } else {
- return HashUtil::MurmurHash2_64(v, 4, seed);
- }
+ return HashUtil::MurmurHash2_64(v, 4, seed);
}
template<>
@@ -168,11 +148,7 @@ inline uint32_t RawValue::GetHashValueNonNull<double>(const double* v,
const ColumnType& type, uint32_t seed) {
DCHECK_EQ(type.type, TYPE_DOUBLE);
DCHECK(v != NULL);
- if (LIKELY(CpuInfo::IsSupported(CpuInfo::SSE4_2))) {
- return HashUtil::CrcHash8(v, seed);
- } else {
- return HashUtil::MurmurHash2_64(v, 8, seed);
- }
+ return HashUtil::MurmurHash2_64(v, 8, seed);
}
template<>
@@ -180,14 +156,15 @@ inline uint32_t RawValue::GetHashValueNonNull<impala::StringValue>(
const impala::StringValue* v,const ColumnType& type, uint32_t seed) {
DCHECK(v != NULL);
if (type.type == TYPE_CHAR) {
- return HashUtil::Hash(StringValue::CharSlotToPtr(
- reinterpret_cast<const void*>(v), type),type.len, seed);
+ return HashUtil::MurmurHash2_64(
+ StringValue::CharSlotToPtr(reinterpret_cast<const void*>(v), type), type.len,
+ seed);
} else {
DCHECK(type.type == TYPE_STRING || type.type == TYPE_VARCHAR);
if (v->len == 0) {
return HashUtil::HashCombine32(HASH_VAL_EMPTY, seed);
}
- return HashUtil::Hash(v->ptr, v->len, seed);
+ return HashUtil::MurmurHash2_64(v->ptr, v->len, seed);
}
}
@@ -196,11 +173,7 @@ inline uint32_t RawValue::GetHashValueNonNull<TimestampValue>(
const TimestampValue* v, const ColumnType& type, uint32_t seed) {
DCHECK_EQ(type.type, TYPE_TIMESTAMP);
DCHECK(v != NULL);
- if (LIKELY(CpuInfo::IsSupported(CpuInfo::SSE4_2))) {
- return HashUtil::CrcHash12(v, seed);
- } else {
- return HashUtil::MurmurHash2_64(v, 12, seed);
- }
+ return HashUtil::MurmurHash2_64(v, 12, seed);
}
template<>
@@ -208,11 +181,7 @@ inline uint32_t RawValue::GetHashValueNonNull<Decimal4Value>(
const Decimal4Value* v, const ColumnType& type, uint32_t seed) {
DCHECK_EQ(type.type, TYPE_DECIMAL);
DCHECK(v != NULL);
- if (LIKELY(CpuInfo::IsSupported(CpuInfo::SSE4_2))) {
- return HashUtil::CrcHash4(v, seed);
- } else {
- return HashUtil::MurmurHash2_64(v, 4, seed);
- }
+ return HashUtil::MurmurHash2_64(v, 4, seed);
}
template<>
@@ -220,11 +189,7 @@ inline uint32_t RawValue::GetHashValueNonNull<Decimal8Value>(
const Decimal8Value* v, const ColumnType& type, uint32_t seed) {
DCHECK_EQ(type.type, TYPE_DECIMAL);
DCHECK(v != NULL);
- if (LIKELY(CpuInfo::IsSupported(CpuInfo::SSE4_2))) {
- return HashUtil::CrcHash8(v, seed);
- } else {
- return HashUtil::MurmurHash2_64(v, 8, seed);
- }
+ return HashUtil::MurmurHash2_64(v, 8, seed);
}
template<>
@@ -232,11 +197,7 @@ inline uint32_t RawValue::GetHashValueNonNull<Decimal16Value>(
const Decimal16Value* v, const ColumnType& type, uint32_t seed) {
DCHECK_EQ(type.type, TYPE_DECIMAL);
DCHECK(v != NULL);
- if (LIKELY(CpuInfo::IsSupported(CpuInfo::SSE4_2))) {
- return HashUtil::CrcHash16(v, seed);
- } else {
- return HashUtil::MurmurHash2_64(v, 16, seed);
- }
+ return HashUtil::MurmurHash2_64(v, 16, seed);
}
template<typename T>
@@ -249,6 +210,8 @@ inline uint32_t RawValue::GetHashValue(const T* v, const ColumnType& type,
inline uint32_t RawValue::GetHashValue(const void* v, const ColumnType& type,
uint32_t seed) {
+ //The choice of hash function needs to be consistent across all hosts of the cluster.
+
// Use HashCombine with arbitrary constant to ensure we don't return seed.
if (v == NULL) return HashUtil::HashCombine32(HASH_VAL_NULL, seed);
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/26266fda/be/src/util/hash-util.h
----------------------------------------------------------------------
diff --git a/be/src/util/hash-util.h b/be/src/util/hash-util.h
index 1b49855..d8d7b0c 100644
--- a/be/src/util/hash-util.h
+++ b/be/src/util/hash-util.h
@@ -228,14 +228,6 @@ class HashUtil {
// Rehash32to32(hash2) is minimal.
return (static_cast<uint64_t>(hash) * m + a) >> 32;
}
-
- static inline uint64_t Rehash32to64(const uint32_t hash) {
- static const uint64_t m1 = 0x47b6137a44974d91ull, m2 = 0x8824ad5ba2b7289cull,
- a1 = 0x705495c62df1424aull, a2 = 0x9efc49475c6bfb31ull;
- const uint64_t hash1 = (static_cast<uint64_t>(hash) * m1 + a1) >> 32;
- const uint64_t hash2 = (static_cast<uint64_t>(hash) * m2 + a2) >> 32;
- return hash1 | (hash2 << 32);
- }
};
}
[15/18] incubator-impala git commit: IMPALA-1671: Print time and link
to coordinator web UI once query is submitted in shell
Posted by ta...@apache.org.
IMPALA-1671: Print time and link to coordinator web UI once query is submitted in shell
To help supportability and debugging, it's helpful to have the impala
shell print out the coordinator time and the link to the coordinator
web UI once the query is submitted.
This is done by calling the PingImpalaService() routine everytime a
query is submitted, which returns the coordinator's hostname,
webserver port and the coordinator epoch time at that moment which the
shell then formats and prints out.
Added tests to verify these new messages.
Change-Id: I704eb64546e27c367830120241311fea6091266b
Reviewed-on: http://gerrit.cloudera.org:8080/3507
Reviewed-by: Sailesh Mukil <sa...@cloudera.com>
Tested-by: Internal Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/900f1480
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/900f1480
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/900f1480
Branch: refs/heads/master
Commit: 900f148078b4fdb8375e3a3107d62fb3ca74e24f
Parents: 144cc68
Author: Sailesh Mukil <sa...@cloudera.com>
Authored: Mon Jun 27 12:59:34 2016 -0700
Committer: Taras Bobrovytsky <ta...@apache.org>
Committed: Thu Jul 14 19:04:45 2016 +0000
----------------------------------------------------------------------
be/src/service/impala-beeswax-server.cc | 2 ++
be/src/util/webserver.cc | 5 +++++
be/src/util/webserver.h | 3 +++
common/thrift/ImpalaService.thrift | 6 ++++++
shell/impala_client.py | 8 +++++---
shell/impala_shell.py | 23 ++++++++++++++++++-----
tests/shell/test_shell_commandline.py | 10 ++++++++++
7 files changed, 49 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/900f1480/be/src/service/impala-beeswax-server.cc
----------------------------------------------------------------------
diff --git a/be/src/service/impala-beeswax-server.cc b/be/src/service/impala-beeswax-server.cc
index 988386d..f7c1dc4 100644
--- a/be/src/service/impala-beeswax-server.cc
+++ b/be/src/service/impala-beeswax-server.cc
@@ -498,6 +498,8 @@ void ImpalaServer::PingImpalaService(TPingImpalaServiceResp& return_val) {
VLOG_RPC << "PingImpalaService()";
return_val.version = GetVersionString(true);
+ return_val.webserver_address = ExecEnv::GetInstance()->webserver()->Url();
+ return_val.epoch_time = reinterpret_cast<int64_t>(std::time(0));
VLOG_RPC << "PingImpalaService(): return_val=" << ThriftDebugString(return_val);
}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/900f1480/be/src/util/webserver.cc
----------------------------------------------------------------------
diff --git a/be/src/util/webserver.cc b/be/src/util/webserver.cc
index 318a189..b514c36 100644
--- a/be/src/util/webserver.cc
+++ b/be/src/util/webserver.cc
@@ -215,6 +215,11 @@ bool Webserver::IsSecure() const {
return !FLAGS_webserver_certificate_file.empty();
}
+string Webserver::Url() {
+ return Substitute("$0://$1:$2", IsSecure() ? "https" : "http",
+ http_address_.hostname, http_address_.port);
+}
+
Status Webserver::Start() {
LOG(INFO) << "Starting webserver on " << http_address_;
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/900f1480/be/src/util/webserver.h
----------------------------------------------------------------------
diff --git a/be/src/util/webserver.h b/be/src/util/webserver.h
index 347fd94..014267f 100644
--- a/be/src/util/webserver.h
+++ b/be/src/util/webserver.h
@@ -87,6 +87,9 @@ class Webserver {
/// True if serving all traffic over SSL, false otherwise
bool IsSecure() const;
+ /// Returns the URL to the webserver as a string.
+ string Url();
+
private:
/// Contains all information relevant to rendering one Url. Each Url has one callback
/// that produces the output to render. The callback either produces a Json document
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/900f1480/common/thrift/ImpalaService.thrift
----------------------------------------------------------------------
diff --git a/common/thrift/ImpalaService.thrift b/common/thrift/ImpalaService.thrift
index 0777c32..218a339 100644
--- a/common/thrift/ImpalaService.thrift
+++ b/common/thrift/ImpalaService.thrift
@@ -240,6 +240,12 @@ struct TInsertResult {
struct TPingImpalaServiceResp {
// The Impala service's version string.
1: string version
+
+ // The Impalad's webserver address.
+ 2: string webserver_address
+
+ // The Impalad's server time.
+ 3: i64 epoch_time
}
// Parameters for a ResetTable request which will invalidate a table's metadata.
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/900f1480/shell/impala_client.py
----------------------------------------------------------------------
diff --git a/shell/impala_client.py b/shell/impala_client.py
index aa5e9fc..63195f0 100755
--- a/shell/impala_client.py
+++ b/shell/impala_client.py
@@ -229,10 +229,12 @@ class ImpalaClient(object):
self.transport.open()
protocol = TBinaryProtocol.TBinaryProtocol(self.transport)
self.imp_service = ImpalaService.Client(protocol)
- result = self.imp_service.PingImpalaService()
- server_version = result.version
+ result = self.ping_impala_service()
self.connected = True
- return server_version
+ return result.version
+
+ def ping_impala_service(self):
+ return self.imp_service.PingImpalaService()
def close_connection(self):
"""Close the transport if it's still open"""
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/900f1480/shell/impala_shell.py
----------------------------------------------------------------------
diff --git a/shell/impala_shell.py b/shell/impala_shell.py
index deb65d8..eda4e05 100755
--- a/shell/impala_shell.py
+++ b/shell/impala_shell.py
@@ -676,9 +676,7 @@ class ImpalaShell(cmd.Cmd):
def _connect(self):
try:
- server_version = self.imp_client.connect()
- if server_version:
- self.server_version = server_version
+ self.server_version = self.imp_client.connect()
except TApplicationException:
# We get a TApplicationException if the transport is valid,
# but the RPC does not exist.
@@ -859,13 +857,28 @@ class ImpalaShell(cmd.Cmd):
The execution time is printed and the query is closed if it hasn't been already
"""
+
+ self._print_if_verbose("Query: %s" % (query.query,))
+ # TODO: Clean up this try block and refactor it (IMPALA-3814)
try:
- self._print_if_verbose("Query: %s" % (query.query,))
- start_time = time.time()
+ # Get the hostname, webserver port and the current coordinator time.
+ coordinator = self.imp_client.ping_impala_service()
+ # If the coordinator is on a different time zone, the epoch time returned will be
+ # different than from this system, so time.localtime(coordinator.epoch_time) will
+ # return the timestamp of the server.
+ self._print_if_verbose("Query submitted at: %s (Coordinator: %s)" % (time.strftime(
+ "%Y-%m-%d %H:%M:%S", time.localtime(coordinator.epoch_time)),
+ coordinator.webserver_address))
+ start_time = time.time()
self.last_query_handle = self.imp_client.execute_query(query)
self.query_handle_closed = False
self.last_summary = time.time()
+ if coordinator.webserver_address:
+ self._print_if_verbose(
+ "Query progress can be monitored at: %s/query_plan?query_id=%s" %
+ (coordinator.webserver_address, self.last_query_handle.id))
+
wait_to_finish = self.imp_client.wait_to_finish(self.last_query_handle,
self._periodic_wait_callback)
# Reset the progress stream.
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/900f1480/tests/shell/test_shell_commandline.py
----------------------------------------------------------------------
diff --git a/tests/shell/test_shell_commandline.py b/tests/shell/test_shell_commandline.py
index f14e3ec..fa4fc44 100644
--- a/tests/shell/test_shell_commandline.py
+++ b/tests/shell/test_shell_commandline.py
@@ -417,3 +417,13 @@ class TestImpalaShell(ImpalaTestSuite):
% (os.path.join(QUERY_FILE_PATH, 'test_var_substitution.sql'))
result = run_impala_shell_cmd(args, expect_success=True)
assert_var_substitution(result)
+
+ def test_query_start_time_message(self):
+ results = run_impala_shell_cmd('--query="select 1"')
+ assert "Query submitted at: " in results.stderr
+
+ def test_query_coordinator_link_message(self):
+ results = run_impala_shell_cmd('--query="select 1"')
+ assert "(Coordinator: " in results.stderr
+ assert "Query progress can be monitored at: " in results.stderr
+
[09/18] incubator-impala git commit: IMPALA-2885:
ScannerContext::Stream objects should be owned by ScannerContext
Posted by ta...@apache.org.
IMPALA-2885: ScannerContext::Stream objects should be owned by ScannerContext
Each scanner context has at least one stream which corresponds to
a scan range. For parquet scanner, there can be multiple streams.
These Stream objects are stored in the RuntimeState's object pool
even though they have the same life span of the scanner threads.
This change makes ScannerContext the owner of the Stream objects
so they will be freed when ScannerContext is destroyed.
Change-Id: Ic5440d414ecc0ca19676c553275aeb85231d6045
Reviewed-on: http://gerrit.cloudera.org:8080/3590
Reviewed-by: Michael Ho <kw...@cloudera.com>
Tested-by: Internal Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/59cdec21
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/59cdec21
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/59cdec21
Branch: refs/heads/master
Commit: 59cdec21bf2a890e6f7fe856a8e1c616e8bebec3
Parents: f4fbd79
Author: Michael Ho <kw...@cloudera.com>
Authored: Wed Jul 6 19:20:34 2016 -0700
Committer: Taras Bobrovytsky <ta...@apache.org>
Committed: Thu Jul 14 19:04:44 2016 +0000
----------------------------------------------------------------------
be/src/exec/hdfs-scanner.cc | 2 ++
be/src/exec/hdfs-text-scanner.cc | 5 +++--
be/src/exec/scanner-context.cc | 6 +++---
be/src/exec/scanner-context.h | 6 +++---
4 files changed, 11 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/59cdec21/be/src/exec/hdfs-scanner.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/hdfs-scanner.cc b/be/src/exec/hdfs-scanner.cc
index 275956c..5abd346 100644
--- a/be/src/exec/hdfs-scanner.cc
+++ b/be/src/exec/hdfs-scanner.cc
@@ -209,6 +209,8 @@ Status HdfsScanner::CommitRows(int num_rows) {
void HdfsScanner::AddFinalRowBatch() {
DCHECK(batch_ != NULL);
+ // Cannot DCHECK(stream_ != NULL) as parquet scanner sets it to NULL in ProcessSplit().
+ stream_ = NULL;
context_->ReleaseCompletedResources(batch_, /* done */ true);
scan_node_->AddMaterializedRowBatch(batch_);
batch_ = NULL;
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/59cdec21/be/src/exec/hdfs-text-scanner.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/hdfs-text-scanner.cc b/be/src/exec/hdfs-text-scanner.cc
index 6cc308d..dcd3081 100644
--- a/be/src/exec/hdfs-text-scanner.cc
+++ b/be/src/exec/hdfs-text-scanner.cc
@@ -180,6 +180,7 @@ void HdfsTextScanner::Close() {
decompressor_->Close();
decompressor_.reset(NULL);
}
+ THdfsCompression::type compression = stream_->file_desc()->file_compression;
if (batch_ != NULL) {
AttachPool(data_buffer_pool_.get(), false);
AttachPool(boundary_pool_.get(), false);
@@ -189,9 +190,9 @@ void HdfsTextScanner::Close() {
DCHECK_EQ(data_buffer_pool_.get()->total_allocated_bytes(), 0);
DCHECK_EQ(boundary_pool_.get()->total_allocated_bytes(), 0);
DCHECK_EQ(context_->num_completed_io_buffers(), 0);
+ // Must happen after AddFinalRowBatch() is called.
if (!only_parsing_header_) {
- scan_node_->RangeComplete(
- THdfsFileFormat::TEXT, stream_->file_desc()->file_compression);
+ scan_node_->RangeComplete(THdfsFileFormat::TEXT, compression);
}
HdfsScanner::Close();
}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/59cdec21/be/src/exec/scanner-context.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/scanner-context.cc b/be/src/exec/scanner-context.cc
index f7f65be..4935f4c 100644
--- a/be/src/exec/scanner-context.cc
+++ b/be/src/exec/scanner-context.cc
@@ -63,7 +63,7 @@ ScannerContext::Stream::Stream(ScannerContext* parent)
}
ScannerContext::Stream* ScannerContext::AddStream(DiskIoMgr::ScanRange* range) {
- Stream* stream = state_->obj_pool()->Add(new Stream(this));
+ std::unique_ptr<Stream> stream(new Stream(this));
stream->scan_range_ = range;
stream->file_desc_ = scan_node_->GetFileDesc(stream->filename());
stream->file_len_ = stream->file_desc_->file_length;
@@ -76,8 +76,8 @@ ScannerContext::Stream* ScannerContext::AddStream(DiskIoMgr::ScanRange* range) {
stream->output_buffer_bytes_left_ =
const_cast<int64_t*>(&OUTPUT_BUFFER_BYTES_LEFT_INIT);
stream->contains_tuple_data_ = scan_node_->tuple_desc()->ContainsStringData();
- streams_.push_back(stream);
- return stream;
+ streams_.push_back(std::move(stream));
+ return streams_.back().get();
}
void ScannerContext::Stream::ReleaseCompletedResources(RowBatch* batch, bool done) {
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/59cdec21/be/src/exec/scanner-context.h
----------------------------------------------------------------------
diff --git a/be/src/exec/scanner-context.h b/be/src/exec/scanner-context.h
index 2ab35e6..b90d512 100644
--- a/be/src/exec/scanner-context.h
+++ b/be/src/exec/scanner-context.h
@@ -264,7 +264,7 @@ class ScannerContext {
Stream* GetStream(int idx = 0) {
DCHECK_GE(idx, 0);
DCHECK_LT(idx, streams_.size());
- return streams_[idx];
+ return streams_[idx].get();
}
/// If a non-NULL 'batch' is passed, attaches completed io buffers and boundary mem pools
@@ -304,8 +304,8 @@ class ScannerContext {
HdfsPartitionDescriptor* partition_desc_;
- /// Vector of streams. Non-columnar formats will always have one stream per context.
- std::vector<Stream*> streams_;
+ /// Vector of streams. Non-columnar formats will always have one stream per context.
+ std::vector<std::unique_ptr<Stream>> streams_;
/// Always equal to the sum of completed_io_buffers_.size() across all streams.
int num_completed_io_buffers_;
[03/18] incubator-impala git commit: IMPALA-3734: C++11 - Replace
boost:shared_ptr with std:: equivalent
Posted by ta...@apache.org.
IMPALA-3734: C++11 - Replace boost:shared_ptr with std:: equivalent
We retain boost::shared_ptr in some files to maintain compatibility with
Thrift's API.
Change-Id: I6b52ec7167999d6ac1d1c56a0ee9cef52dc1615e
Reviewed-on: http://gerrit.cloudera.org:8080/3563
Reviewed-by: Henry Robinson <he...@cloudera.com>
Tested-by: Internal Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/f4fbd791
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/f4fbd791
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/f4fbd791
Branch: refs/heads/master
Commit: f4fbd7914c8756e8b657d630a3e5d7d1f010dfa1
Parents: 4680252
Author: Henry Robinson <he...@cloudera.com>
Authored: Fri Jul 1 16:45:53 2016 -0700
Committer: Taras Bobrovytsky <ta...@apache.org>
Committed: Thu Jul 14 19:04:44 2016 +0000
----------------------------------------------------------------------
be/src/benchmarks/network-perf-benchmark.cc | 7 ++--
be/src/catalog/catalogd-main.cc | 4 +-
be/src/common/names.h | 22 +++++-----
be/src/exec/exchange-node.h | 2 +-
be/src/exec/hash-table.h | 8 ++--
be/src/resourcebroker/resource-broker.cc | 2 +-
be/src/rpc/authentication.cc | 15 +++----
be/src/rpc/thrift-server-test.cc | 6 +--
be/src/rpc/thrift-server.cc | 43 ++++++++++----------
be/src/rpc/thrift-thread.cc | 12 +++---
be/src/rpc/thrift-util.cc | 5 +--
be/src/runtime/buffered-block-mgr.h | 6 +--
be/src/runtime/client-cache.h | 12 +++---
be/src/runtime/coordinator.h | 2 +-
be/src/runtime/data-stream-mgr.h | 11 +++--
be/src/runtime/data-stream-sender.cc | 1 -
be/src/runtime/data-stream-test.cc | 4 +-
be/src/runtime/mem-tracker.cc | 5 ++-
be/src/runtime/mem-tracker.h | 6 +--
be/src/runtime/plan-fragment-executor.h | 1 -
be/src/runtime/runtime-state.h | 7 ++--
be/src/runtime/test-env.cc | 6 ++-
be/src/runtime/test-env.h | 2 +-
be/src/scheduling/query-resource-mgr.h | 9 ++---
be/src/service/fragment-mgr.h | 5 +--
be/src/service/impala-hs2-server.cc | 2 +-
be/src/service/impala-server.cc | 21 +++++-----
be/src/service/impala-server.h | 51 ++++++++++++------------
be/src/service/query-exec-state.h | 4 +-
be/src/statestore/statestore-subscriber.cc | 5 ++-
be/src/statestore/statestore-subscriber.h | 2 +-
be/src/statestore/statestore.h | 2 +-
be/src/statestore/statestored-main.cc | 4 +-
be/src/testutil/impalad-query-executor.h | 1 -
be/src/testutil/in-process-servers.cc | 3 +-
be/src/testutil/in-process-servers.h | 1 -
be/src/transport/TSaslServerTransport.cpp | 16 ++++----
be/src/transport/TSaslServerTransport.h | 7 ++--
be/src/udf/uda-test-harness-impl.h | 6 +--
be/src/udf/uda-test-harness.h | 1 -
be/src/util/parquet-reader.cc | 9 +++--
41 files changed, 167 insertions(+), 171 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f4fbd791/be/src/benchmarks/network-perf-benchmark.cc
----------------------------------------------------------------------
diff --git a/be/src/benchmarks/network-perf-benchmark.cc b/be/src/benchmarks/network-perf-benchmark.cc
index b218c97..188419d 100644
--- a/be/src/benchmarks/network-perf-benchmark.cc
+++ b/be/src/benchmarks/network-perf-benchmark.cc
@@ -214,9 +214,10 @@ int main(int argc, char** argv) {
}
// Start up server and client shell
- shared_ptr<TestServer> handler(new TestServer);
- shared_ptr<ThreadFactory> thread_factory(new ThriftThreadFactory("test", "test"));
- shared_ptr<TProcessor> processor(new NetworkTestServiceProcessor(handler));
+ boost::shared_ptr<TestServer> handler(new TestServer);
+ boost::shared_ptr<ThreadFactory> thread_factory(
+ new ThriftThreadFactory("test", "test"));
+ boost::shared_ptr<TProcessor> processor(new NetworkTestServiceProcessor(handler));
ThriftServer* server = new ThriftServer("Network Test Server", processor,
FLAGS_port, NULL, NULL, 100, ThriftServer::ThreadPool);
thread* server_thread = new thread(&TestServer::Server, handler.get(), server);
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f4fbd791/be/src/catalog/catalogd-main.cc
----------------------------------------------------------------------
diff --git a/be/src/catalog/catalogd-main.cc b/be/src/catalog/catalogd-main.cc
index 5e4b186..2a84390 100644
--- a/be/src/catalog/catalogd-main.cc
+++ b/be/src/catalog/catalogd-main.cc
@@ -75,9 +75,9 @@ int CatalogdMain(int argc, char** argv) {
CatalogServer catalog_server(metrics.get());
ABORT_IF_ERROR(catalog_server.Start());
catalog_server.RegisterWebpages(webserver.get());
- shared_ptr<TProcessor> processor(
+ boost::shared_ptr<TProcessor> processor(
new CatalogServiceProcessor(catalog_server.thrift_iface()));
- shared_ptr<TProcessorEventHandler> event_handler(
+ boost::shared_ptr<TProcessorEventHandler> event_handler(
new RpcEventHandler("catalog-server", metrics.get()));
processor->setEventHandler(event_handler);
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f4fbd791/be/src/common/names.h
----------------------------------------------------------------------
diff --git a/be/src/common/names.h b/be/src/common/names.h
index 7817e84..12240cb 100644
--- a/be/src/common/names.h
+++ b/be/src/common/names.h
@@ -12,14 +12,15 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-/// The motivation for the using declarations below is to allow accessing the most relevant
-/// and most frequently used library classes without having to explicitly pull them into
-/// the global namespace. The goal is that when readers sees a usage of vector (etc.) without
-/// any further specialization they can rely on the fact that it will be a std::vector.
+/// The motivation for the using declarations below is to allow accessing the most
+/// relevant and most frequently used library classes without having to explicitly pull
+/// them into the global namespace. The goal is that when readers sees a usage of vector
+/// (etc.) without any further specialization they can rely on the fact that it will be a
+/// std::vector.
//
/// Instead of actually including the header files for the symbols, this file only checks
-/// if certain include guards are defined before applying the using declaration. This makes sure
-/// that including this file has no impact on the compile time.
+/// if certain include guards are defined before applying the using declaration. This
+/// makes sure that including this file has no impact on the compile time.
//
/// Please make sure that this file is included last in the cc file's include list to make
/// sure that all relevant include guards are defined.
@@ -109,6 +110,11 @@ using std::max;
using std::sort;
#endif
+#ifdef _GLIBCXX_MEMORY
+using std::make_shared;
+using std::shared_ptr;
+#endif
+
#ifdef BOOST_THREAD_THREAD_COMMON_HPP
using boost::thread;
#endif
@@ -147,10 +153,6 @@ using boost::shared_lock;
using boost::upgrade_lock;
#endif
-#ifdef BOOST_SMART_PTR_SHARED_PTR_HPP_INCLUDED
-using boost::shared_ptr;
-#endif
-
#ifdef BOOST_SMART_PTR_SCOPED_PTR_HPP_INCLUDED
using boost::scoped_ptr;
#endif
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f4fbd791/be/src/exec/exchange-node.h
----------------------------------------------------------------------
diff --git a/be/src/exec/exchange-node.h b/be/src/exec/exchange-node.h
index 0e8fc86..79e9484 100644
--- a/be/src/exec/exchange-node.h
+++ b/be/src/exec/exchange-node.h
@@ -69,7 +69,7 @@ class ExchangeNode : public ExecNode {
/// The underlying DataStreamRecvr instance. Ownership is shared between this
/// exchange node instance and the DataStreamMgr used to create the receiver.
/// stream_recvr_->Close() must be called before this instance is destroyed.
- boost::shared_ptr<DataStreamRecvr> stream_recvr_;
+ std::shared_ptr<DataStreamRecvr> stream_recvr_;
/// our input rows are a prefix of the rows we produce
RowDescriptor input_row_desc_;
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f4fbd791/be/src/exec/hash-table.h
----------------------------------------------------------------------
diff --git a/be/src/exec/hash-table.h b/be/src/exec/hash-table.h
index 5b466dc..b6c0b8c 100644
--- a/be/src/exec/hash-table.h
+++ b/be/src/exec/hash-table.h
@@ -51,10 +51,10 @@ class HashTable;
/// joins and aggregation where the input/build tuple row descriptor is different from the
/// find/probe descriptor. The implementation is designed to allow codegen for some paths.
//
-/// In addition to the hash table there is also an accompanying hash table context that
-/// is used for insertions and probes. For example, the hash table context stores
-/// evaluated expr results for the current row being processed when possible into a
-/// contiguous memory buffer. This allows for efficient hash computation.
+/// In addition to the hash table there is also an accompanying hash table context that is
+/// used for insertions and probes. For example, the hash table context stores evaluated
+/// expr results for the current row being processed when possible into a contiguous
+/// memory buffer. This allows for efficient hash computation.
//
/// The hash table does not support removes. The hash table is not thread safe.
/// The table is optimized for the partition hash aggregation and hash joins and is not
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f4fbd791/be/src/resourcebroker/resource-broker.cc
----------------------------------------------------------------------
diff --git a/be/src/resourcebroker/resource-broker.cc b/be/src/resourcebroker/resource-broker.cc
index e4a933a..8a70ab1 100644
--- a/be/src/resourcebroker/resource-broker.cc
+++ b/be/src/resourcebroker/resource-broker.cc
@@ -151,7 +151,7 @@ Status ResourceBroker::Init() {
return Status("No Llama addresses configured (see --llama_addresses)");
}
- shared_ptr<TProcessor> llama_callback_proc(
+ boost::shared_ptr<TProcessor> llama_callback_proc(
new llama::LlamaNotificationServiceProcessor(llama_callback_thrift_iface_));
llama_callback_server_.reset(new ThriftServer("llama-callback", llama_callback_proc,
llama_callback_address_.port, NULL, metrics_, 5));
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f4fbd791/be/src/rpc/authentication.cc
----------------------------------------------------------------------
diff --git a/be/src/rpc/authentication.cc b/be/src/rpc/authentication.cc
index afd4a7c..5c02168 100644
--- a/be/src/rpc/authentication.cc
+++ b/be/src/rpc/authentication.cc
@@ -830,7 +830,7 @@ Status SaslAuthProvider::Start() {
}
Status SaslAuthProvider::GetServerTransportFactory(
- shared_ptr<TTransportFactory>* factory) {
+ boost::shared_ptr<TTransportFactory>* factory) {
DCHECK(!principal_.empty() || has_ldap_);
// This is the heart of the link between this file and thrift. Here we
@@ -868,10 +868,10 @@ Status SaslAuthProvider::GetServerTransportFactory(
}
Status SaslAuthProvider::WrapClientTransport(const string& hostname,
- shared_ptr<TTransport> raw_transport, const string& service_name,
- shared_ptr<TTransport>* wrapped_transport) {
+ boost::shared_ptr<TTransport> raw_transport, const string& service_name,
+ boost::shared_ptr<TTransport>* wrapped_transport) {
- shared_ptr<sasl::TSasl> sasl_client;
+ boost::shared_ptr<sasl::TSasl> sasl_client;
const map<string, string> props; // Empty; unused by thrift
const string auth_id; // Empty; unused by thrift
@@ -898,15 +898,16 @@ Status SaslAuthProvider::WrapClientTransport(const string& hostname,
return Status::OK();
}
-Status NoAuthProvider::GetServerTransportFactory(shared_ptr<TTransportFactory>* factory) {
+Status NoAuthProvider::GetServerTransportFactory(
+ boost::shared_ptr<TTransportFactory>* factory) {
// No Sasl - yawn. Here, have a regular old buffered transport.
factory->reset(new ThriftServer::BufferedTransportFactory());
return Status::OK();
}
Status NoAuthProvider::WrapClientTransport(const string& hostname,
- shared_ptr<TTransport> raw_transport, const string& dummy_service,
- shared_ptr<TTransport>* wrapped_transport) {
+ boost::shared_ptr<TTransport> raw_transport, const string& dummy_service,
+ boost::shared_ptr<TTransport>* wrapped_transport) {
// No Sasl - yawn. Don't do any transport wrapping for clients.
*wrapped_transport = raw_transport;
return Status::OK();
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f4fbd791/be/src/rpc/thrift-server-test.cc
----------------------------------------------------------------------
diff --git a/be/src/rpc/thrift-server-test.cc b/be/src/rpc/thrift-server-test.cc
index 6b010d6..f1e45ad 100644
--- a/be/src/rpc/thrift-server-test.cc
+++ b/be/src/rpc/thrift-server-test.cc
@@ -52,9 +52,9 @@ class DummyStatestoreService : public StatestoreServiceIf {
}
};
-shared_ptr<TProcessor> MakeProcessor() {
- shared_ptr<DummyStatestoreService> service(new DummyStatestoreService());
- return shared_ptr<TProcessor>(new StatestoreServiceProcessor(service));
+boost::shared_ptr<TProcessor> MakeProcessor() {
+ boost::shared_ptr<DummyStatestoreService> service(new DummyStatestoreService());
+ return boost::shared_ptr<TProcessor>(new StatestoreServiceProcessor(service));
}
int GetServerPort() {
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f4fbd791/be/src/rpc/thrift-server.cc
----------------------------------------------------------------------
diff --git a/be/src/rpc/thrift-server.cc b/be/src/rpc/thrift-server.cc
index f3ec770..f1b2ff3 100644
--- a/be/src/rpc/thrift-server.cc
+++ b/be/src/rpc/thrift-server.cc
@@ -91,14 +91,15 @@ class ThriftServer::ThriftServerEventProcessor : public TServerEventHandler {
// Called when a client connects; we create per-client state and call any
// ConnectionHandlerIf handler.
- virtual void* createContext(shared_ptr<TProtocol> input, shared_ptr<TProtocol> output);
+ virtual void* createContext(boost::shared_ptr<TProtocol> input,
+ boost::shared_ptr<TProtocol> output);
// Called when a client starts an RPC; we set the thread-local connection context.
- virtual void processContext(void* context, shared_ptr<TTransport> output);
+ virtual void processContext(void* context, boost::shared_ptr<TTransport> output);
// Called when a client disconnects; we call any ConnectionHandlerIf handler.
- virtual void deleteContext(void* serverContext, shared_ptr<TProtocol> input,
- shared_ptr<TProtocol> output);
+ virtual void deleteContext(void* serverContext, boost::shared_ptr<TProtocol> input,
+ boost::shared_ptr<TProtocol> output);
// Waits for a timeout of TIMEOUT_MS for a server to signal that it has started
// correctly.
@@ -218,12 +219,12 @@ const ThriftServer::ConnectionContext* ThriftServer::GetThreadConnectionContext(
return __connection_context__;
}
-void* ThriftServer::ThriftServerEventProcessor::createContext(shared_ptr<TProtocol> input,
- shared_ptr<TProtocol> output) {
+void* ThriftServer::ThriftServerEventProcessor::createContext(
+ boost::shared_ptr<TProtocol> input, boost::shared_ptr<TProtocol> output) {
TSocket* socket = NULL;
TTransport* transport = input->getTransport().get();
- shared_ptr<ConnectionContext> connection_ptr =
- shared_ptr<ConnectionContext>(new ConnectionContext);
+ boost::shared_ptr<ConnectionContext> connection_ptr =
+ boost::shared_ptr<ConnectionContext>(new ConnectionContext);
TTransport* underlying_transport =
(static_cast<TBufferedTransport*>(transport))->getUnderlyingTransport().get();
if (!thrift_server_->auth_provider_->is_sasl()) {
@@ -267,12 +268,12 @@ void* ThriftServer::ThriftServerEventProcessor::createContext(shared_ptr<TProtoc
}
void ThriftServer::ThriftServerEventProcessor::processContext(void* context,
- shared_ptr<TTransport> transport) {
+ boost::shared_ptr<TTransport> transport) {
__connection_context__ = reinterpret_cast<ConnectionContext*>(context);
}
void ThriftServer::ThriftServerEventProcessor::deleteContext(void* serverContext,
- shared_ptr<TProtocol> input, shared_ptr<TProtocol> output) {
+ boost::shared_ptr<TProtocol> input, boost::shared_ptr<TProtocol> output) {
__connection_context__ = (ConnectionContext*) serverContext;
if (thrift_server_->connection_handler_ != NULL) {
@@ -289,9 +290,9 @@ void ThriftServer::ThriftServerEventProcessor::deleteContext(void* serverContext
}
}
-ThriftServer::ThriftServer(const string& name, const shared_ptr<TProcessor>& processor,
- int port, AuthProvider* auth_provider, MetricGroup* metrics, int num_worker_threads,
- ServerType server_type)
+ThriftServer::ThriftServer(const string& name,
+ const boost::shared_ptr<TProcessor>& processor, int port, AuthProvider* auth_provider,
+ MetricGroup* metrics, int num_worker_threads, ServerType server_type)
: started_(false),
port_(port),
ssl_enabled_(false),
@@ -336,11 +337,11 @@ class ImpalaSslSocketFactory : public TSSLSocketFactory {
const string password_;
};
-Status ThriftServer::CreateSocket(shared_ptr<TServerTransport>* socket) {
+Status ThriftServer::CreateSocket(boost::shared_ptr<TServerTransport>* socket) {
if (ssl_enabled()) {
// This 'factory' is only called once, since CreateSocket() is only called from
// Start()
- shared_ptr<TSSLSocketFactory> socket_factory(
+ boost::shared_ptr<TSSLSocketFactory> socket_factory(
new ImpalaSslSocketFactory(key_password_));
socket_factory->overrideDefaultPasswordCallback();
try {
@@ -390,20 +391,20 @@ Status ThriftServer::EnableSsl(const string& certificate, const string& private_
Status ThriftServer::Start() {
DCHECK(!started_);
- shared_ptr<TProtocolFactory> protocol_factory(new TBinaryProtocolFactory());
- shared_ptr<ThreadFactory> thread_factory(
+ boost::shared_ptr<TProtocolFactory> protocol_factory(new TBinaryProtocolFactory());
+ boost::shared_ptr<ThreadFactory> thread_factory(
new ThriftThreadFactory("thrift-server", name_));
// Note - if you change the transport types here, you must check that the
// logic in createContext is still accurate.
- shared_ptr<TServerTransport> server_socket;
- shared_ptr<TTransportFactory> transport_factory;
+ boost::shared_ptr<TServerTransport> server_socket;
+ boost::shared_ptr<TTransportFactory> transport_factory;
RETURN_IF_ERROR(CreateSocket(&server_socket));
RETURN_IF_ERROR(auth_provider_->GetServerTransportFactory(&transport_factory));
switch (server_type_) {
case ThreadPool:
{
- shared_ptr<ThreadManager> thread_mgr(
+ boost::shared_ptr<ThreadManager> thread_mgr(
ThreadManager::newSimpleThreadManager(num_worker_threads_));
thread_mgr->threadFactory(thread_factory);
thread_mgr->start();
@@ -421,7 +422,7 @@ Status ThriftServer::Start() {
LOG(ERROR) << error_msg.str();
return Status(error_msg.str());
}
- shared_ptr<ThriftServer::ThriftServerEventProcessor> event_processor(
+ boost::shared_ptr<ThriftServer::ThriftServerEventProcessor> event_processor(
new ThriftServer::ThriftServerEventProcessor(this));
server_->setServerEventHandler(event_processor);
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f4fbd791/be/src/rpc/thrift-thread.cc
----------------------------------------------------------------------
diff --git a/be/src/rpc/thrift-thread.cc b/be/src/rpc/thrift-thread.cc
index e76d8dc..e94cbf0 100644
--- a/be/src/rpc/thrift-thread.cc
+++ b/be/src/rpc/thrift-thread.cc
@@ -41,17 +41,17 @@ void ThriftThread::join() {
impala_thread_->Join();
}
-shared_ptr<atc::Thread> ThriftThreadFactory::newThread(
- shared_ptr<atc::Runnable> runnable) const {
+boost::shared_ptr<atc::Thread> ThriftThreadFactory::newThread(
+ boost::shared_ptr<atc::Runnable> runnable) const {
stringstream name;
name << prefix_ << "-" << count_++;
- shared_ptr<ThriftThread> result =
- shared_ptr<ThriftThread>(new ThriftThread(group_, name.str(), runnable));
+ boost::shared_ptr<ThriftThread> result =
+ boost::shared_ptr<ThriftThread>(new ThriftThread(group_, name.str(), runnable));
runnable->thread(result);
return result;
}
-void ThriftThread::RunRunnable(shared_ptr<atc::Runnable> runnable,
+void ThriftThread::RunRunnable(boost::shared_ptr<atc::Runnable> runnable,
Promise<atc::Thread::id_t>* promise) {
promise->Set(get_current());
// Passing runnable in to this method (rather than reading from this->runnable())
@@ -66,7 +66,7 @@ atc::Thread::id_t ThriftThreadFactory::getCurrentThreadId() const {
}
ThriftThread::ThriftThread(const string& group, const string& name,
- shared_ptr<atc::Runnable> runnable)
+ boost::shared_ptr<atc::Runnable> runnable)
: group_(group), name_(name) {
// Sets this::runnable (and no, I don't know why it's not protected in atc::Thread)
this->Thread::runnable(runnable);
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f4fbd791/be/src/rpc/thrift-util.cc
----------------------------------------------------------------------
diff --git a/be/src/rpc/thrift-util.cc b/be/src/rpc/thrift-util.cc
index db965f2..4677237 100644
--- a/be/src/rpc/thrift-util.cc
+++ b/be/src/rpc/thrift-util.cc
@@ -14,7 +14,6 @@
#include "rpc/thrift-util.h"
-#include <boost/shared_ptr.hpp>
#include <boost/thread.hpp>
#include "util/hash-util.h"
@@ -74,8 +73,8 @@ ThriftSerializer::ThriftSerializer(bool compact, int initial_buffer_size) :
}
}
-shared_ptr<TProtocol> CreateDeserializeProtocol(
- shared_ptr<TMemoryBuffer> mem, bool compact) {
+boost::shared_ptr<TProtocol> CreateDeserializeProtocol(
+ boost::shared_ptr<TMemoryBuffer> mem, bool compact) {
if (compact) {
TCompactProtocolFactoryT<TMemoryBuffer> tproto_factory;
return tproto_factory.getProtocol(mem);
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f4fbd791/be/src/runtime/buffered-block-mgr.h
----------------------------------------------------------------------
diff --git a/be/src/runtime/buffered-block-mgr.h b/be/src/runtime/buffered-block-mgr.h
index 416751e..141943f 100644
--- a/be/src/runtime/buffered-block-mgr.h
+++ b/be/src/runtime/buffered-block-mgr.h
@@ -15,8 +15,6 @@
#ifndef IMPALA_RUNTIME_BUFFERED_BLOCK_MGR
#define IMPALA_RUNTIME_BUFFERED_BLOCK_MGR
-#include <boost/shared_ptr.hpp>
-
#include "runtime/disk-io-mgr.h"
#include "runtime/tmp-file-mgr.h"
@@ -305,7 +303,7 @@ class BufferedBlockMgr {
/// - buffer_size: maximum size of each buffer.
static Status Create(RuntimeState* state, MemTracker* parent,
RuntimeProfile* profile, TmpFileMgr* tmp_file_mgr, int64_t mem_limit,
- int64_t buffer_size, boost::shared_ptr<BufferedBlockMgr>* block_mgr);
+ int64_t buffer_size, std::shared_ptr<BufferedBlockMgr>* block_mgr);
~BufferedBlockMgr();
@@ -637,7 +635,7 @@ class BufferedBlockMgr {
/// map contains only weak ptrs. BufferedBlockMgrs that are handed out are shared ptrs.
/// When all the shared ptrs are no longer referenced, the BufferedBlockMgr
/// d'tor will be called at which point the weak ptr will be removed from the map.
- typedef boost::unordered_map<TUniqueId, boost::weak_ptr<BufferedBlockMgr>>
+ typedef boost::unordered_map<TUniqueId, std::weak_ptr<BufferedBlockMgr>>
BlockMgrsMap;
static BlockMgrsMap query_to_block_mgrs_;
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f4fbd791/be/src/runtime/client-cache.h
----------------------------------------------------------------------
diff --git a/be/src/runtime/client-cache.h b/be/src/runtime/client-cache.h
index 350bdd6..0028167 100644
--- a/be/src/runtime/client-cache.h
+++ b/be/src/runtime/client-cache.h
@@ -141,20 +141,20 @@ class ClientCacheHelper {
boost::mutex cache_lock_;
/// Map from an address to a PerHostCache containing a list of keys that have entries in
- /// client_map_ for that host. The value type is wrapped in a shared_ptr so that the copy
- /// c'tor for PerHostCache is not required.
+ /// client_map_ for that host. The value type is wrapped in a shared_ptr so that the
+ /// copy c'tor for PerHostCache is not required.
typedef boost::unordered_map<
- TNetworkAddress, boost::shared_ptr<PerHostCache>> PerHostCacheMap;
+ TNetworkAddress, std::shared_ptr<PerHostCache>> PerHostCacheMap;
PerHostCacheMap per_host_caches_;
/// Protects client_map_.
boost::mutex client_map_lock_;
/// Map from client key back to its associated ThriftClientImpl transport. This is where
- /// all the clients are actually stored, and client instances are owned by this class and
- /// persist for exactly as long as they are present in this map.
+ /// all the clients are actually stored, and client instances are owned by this class
+ /// and persist for exactly as long as they are present in this map.
/// We use a map (vs. unordered_map) so we get iterator consistency across operations.
- typedef std::map<ClientKey, boost::shared_ptr<ThriftClientImpl>> ClientMap;
+ typedef std::map<ClientKey, std::shared_ptr<ThriftClientImpl>> ClientMap;
ClientMap client_map_;
/// Number of attempts to make to open a connection. 0 means retry indefinitely.
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f4fbd791/be/src/runtime/coordinator.h
----------------------------------------------------------------------
diff --git a/be/src/runtime/coordinator.h b/be/src/runtime/coordinator.h
index 9994e1e..015fcaf 100644
--- a/be/src/runtime/coordinator.h
+++ b/be/src/runtime/coordinator.h
@@ -260,7 +260,7 @@ class Coordinator {
/// is no coordinator fragment (i.e. executor_ == NULL). If executor_ is not NULL,
/// this->runtime_state()->query_mem_tracker() returns the query mem tracker.
/// (See this->query_mem_tracker())
- boost::shared_ptr<MemTracker> query_mem_tracker_;
+ std::shared_ptr<MemTracker> query_mem_tracker_;
/// owned by plan root, which resides in runtime_state_'s pool
const RowDescriptor* row_desc_;
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f4fbd791/be/src/runtime/data-stream-mgr.h
----------------------------------------------------------------------
diff --git a/be/src/runtime/data-stream-mgr.h b/be/src/runtime/data-stream-mgr.h
index 2b2b253..b4c422f 100644
--- a/be/src/runtime/data-stream-mgr.h
+++ b/be/src/runtime/data-stream-mgr.h
@@ -19,7 +19,6 @@
#include <list>
#include <set>
#include <boost/thread/mutex.hpp>
-#include <boost/shared_ptr.hpp>
#include <boost/thread/condition_variable.hpp>
#include <boost/unordered_map.hpp>
#include <boost/unordered_set.hpp>
@@ -71,7 +70,7 @@ class DataStreamMgr {
/// single stream.
/// Ownership of the receiver is shared between this DataStream mgr instance and the
/// caller.
- boost::shared_ptr<DataStreamRecvr> CreateRecvr(
+ std::shared_ptr<DataStreamRecvr> CreateRecvr(
RuntimeState* state, const RowDescriptor& row_desc,
const TUniqueId& fragment_instance_id, PlanNodeId dest_node_id,
int num_senders, int buffer_size, RuntimeProfile* profile,
@@ -122,7 +121,7 @@ class DataStreamMgr {
/// we don't want to create a map<pair<TUniqueId, PlanNodeId>, DataStreamRecvr*>,
/// because that requires a bunch of copying of ids for lookup
typedef boost::unordered_multimap<uint32_t,
- boost::shared_ptr<DataStreamRecvr>> RecvrMap;
+ std::shared_ptr<DataStreamRecvr>> RecvrMap;
RecvrMap receiver_map_;
/// (Fragment instance id, Plan node id) pair that uniquely identifies a stream.
@@ -155,7 +154,7 @@ class DataStreamMgr {
/// Return the receiver for given fragment_instance_id/node_id, or NULL if not found. If
/// 'acquire_lock' is false, assumes lock_ is already being held and won't try to
/// acquire it.
- boost::shared_ptr<DataStreamRecvr> FindRecvr(const TUniqueId& fragment_instance_id,
+ std::shared_ptr<DataStreamRecvr> FindRecvr(const TUniqueId& fragment_instance_id,
PlanNodeId node_id, bool acquire_lock = true);
/// Calls FindRecvr(), but if NULL is returned, wait for up to
@@ -172,7 +171,7 @@ class DataStreamMgr {
///
/// 2. *already_unregistered == false: the receiver has yet to arrive when this method
/// returns, and the timeout has expired
- boost::shared_ptr<DataStreamRecvr> FindRecvrOrWait(
+ std::shared_ptr<DataStreamRecvr> FindRecvrOrWait(
const TUniqueId& fragment_instance_id, PlanNodeId node_id,
bool* already_unregistered);
@@ -182,7 +181,7 @@ class DataStreamMgr {
inline uint32_t GetHashValue(const TUniqueId& fragment_instance_id, PlanNodeId node_id);
/// The coordination primitive used to signal the arrival of a waited-for receiver
- typedef Promise<boost::shared_ptr<DataStreamRecvr>> RendezvousPromise;
+ typedef Promise<std::shared_ptr<DataStreamRecvr>> RendezvousPromise;
/// A reference-counted promise-wrapper used to coordinate between senders and
/// receivers. The ref_count field tracks the number of senders waiting for the arrival
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f4fbd791/be/src/runtime/data-stream-sender.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/data-stream-sender.cc b/be/src/runtime/data-stream-sender.cc
index 0a10963..61e8b85 100644
--- a/be/src/runtime/data-stream-sender.cc
+++ b/be/src/runtime/data-stream-sender.cc
@@ -15,7 +15,6 @@
#include "runtime/data-stream-sender.h"
#include <iostream>
-#include <boost/shared_ptr.hpp>
#include <thrift/protocol/TDebugProtocol.h>
#include "common/logging.h"
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f4fbd791/be/src/runtime/data-stream-test.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/data-stream-test.cc b/be/src/runtime/data-stream-test.cc
index 54b38b1..57dfeef 100644
--- a/be/src/runtime/data-stream-test.cc
+++ b/be/src/runtime/data-stream-test.cc
@@ -446,8 +446,8 @@ class DataStreamTest : public testing::Test {
// Start backend in separate thread.
void StartBackend() {
- shared_ptr<ImpalaTestBackend> handler(new ImpalaTestBackend(stream_mgr_));
- shared_ptr<TProcessor> processor(new ImpalaInternalServiceProcessor(handler));
+ boost::shared_ptr<ImpalaTestBackend> handler(new ImpalaTestBackend(stream_mgr_));
+ boost::shared_ptr<TProcessor> processor(new ImpalaInternalServiceProcessor(handler));
server_ = new ThriftServer("DataStreamTest backend", processor, FLAGS_port, NULL);
server_->Start();
}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f4fbd791/be/src/runtime/mem-tracker.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/mem-tracker.cc b/be/src/runtime/mem-tracker.cc
index 66b0a8d..9749599 100644
--- a/be/src/runtime/mem-tracker.cc
+++ b/be/src/runtime/mem-tracker.cc
@@ -206,8 +206,9 @@ shared_ptr<MemTracker> MemTracker::GetQueryMemTracker(
} else {
// First time this id registered, make a new object. Give a shared ptr to
// the caller and put a weak ptr in the map.
- shared_ptr<MemTracker> tracker(new MemTracker(byte_limit, rm_reserved_limit,
- Substitute("Query($0) Limit", lexical_cast<string>(id)), parent));
+ shared_ptr<MemTracker> tracker = make_shared<MemTracker>(byte_limit,
+ rm_reserved_limit, Substitute("Query($0) Limit", lexical_cast<string>(id)),
+ parent);
tracker->auto_unregister_ = true;
tracker->query_id_ = id;
request_to_mem_trackers_[id] = tracker;
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f4fbd791/be/src/runtime/mem-tracker.h
----------------------------------------------------------------------
diff --git a/be/src/runtime/mem-tracker.h b/be/src/runtime/mem-tracker.h
index 7700b06..aae37c5 100644
--- a/be/src/runtime/mem-tracker.h
+++ b/be/src/runtime/mem-tracker.h
@@ -19,8 +19,6 @@
#include <stdint.h>
#include <map>
#include <vector>
-#include <boost/shared_ptr.hpp>
-#include <boost/weak_ptr.hpp>
#include <boost/thread/mutex.hpp>
#include <boost/unordered_map.hpp>
@@ -92,7 +90,7 @@ class MemTracker {
/// 'parent' as the parent tracker.
/// byte_limit and parent must be the same for all GetMemTracker() calls with the
/// same id.
- static boost::shared_ptr<MemTracker> GetQueryMemTracker(const TUniqueId& id,
+ static std::shared_ptr<MemTracker> GetQueryMemTracker(const TUniqueId& id,
int64_t byte_limit, int64_t rm_reserved_limit, MemTracker* parent,
QueryResourceMgr* res_mgr);
@@ -408,7 +406,7 @@ class MemTracker {
/// contains only weak ptrs. MemTrackers that are handed out via GetQueryMemTracker()
/// are shared ptrs. When all the shared ptrs are no longer referenced, the MemTracker
/// d'tor will be called at which point the weak ptr will be removed from the map.
- typedef boost::unordered_map<TUniqueId, boost::weak_ptr<MemTracker>>
+ typedef boost::unordered_map<TUniqueId, std::weak_ptr<MemTracker>>
RequestTrackersMap;
static RequestTrackersMap request_to_mem_trackers_;
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f4fbd791/be/src/runtime/plan-fragment-executor.h
----------------------------------------------------------------------
diff --git a/be/src/runtime/plan-fragment-executor.h b/be/src/runtime/plan-fragment-executor.h
index 9511a3e..7bbe319 100644
--- a/be/src/runtime/plan-fragment-executor.h
+++ b/be/src/runtime/plan-fragment-executor.h
@@ -18,7 +18,6 @@
#include <vector>
#include <boost/scoped_ptr.hpp>
-#include <boost/shared_ptr.hpp>
#include <boost/function.hpp>
#include "common/status.h"
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f4fbd791/be/src/runtime/runtime-state.h
----------------------------------------------------------------------
diff --git a/be/src/runtime/runtime-state.h b/be/src/runtime/runtime-state.h
index f104724..a5e560e 100644
--- a/be/src/runtime/runtime-state.h
+++ b/be/src/runtime/runtime-state.h
@@ -17,7 +17,6 @@
#define IMPALA_RUNTIME_RUNTIME_STATE_H
#include <boost/scoped_ptr.hpp>
-#include <boost/shared_ptr.hpp>
#include <vector>
#include <string>
@@ -266,7 +265,7 @@ class RuntimeState {
Status CreateCodegen();
/// Use a custom block manager for the query for testing purposes.
- void set_block_mgr(const boost::shared_ptr<BufferedBlockMgr>& block_mgr) {
+ void set_block_mgr(const std::shared_ptr<BufferedBlockMgr>& block_mgr) {
block_mgr_ = block_mgr;
}
@@ -326,7 +325,7 @@ class RuntimeState {
/// MemTracker that is shared by all fragment instances running on this host.
/// The query mem tracker must be released after the instance_mem_tracker_.
- boost::shared_ptr<MemTracker> query_mem_tracker_;
+ std::shared_ptr<MemTracker> query_mem_tracker_;
/// Memory usage of this fragment instance
boost::scoped_ptr<MemTracker> instance_mem_tracker_;
@@ -350,7 +349,7 @@ class RuntimeState {
/// BufferedBlockMgr object used to allocate and manage blocks of input data in memory
/// with a fixed memory budget.
/// The block mgr is shared by all fragments for this query.
- boost::shared_ptr<BufferedBlockMgr> block_mgr_;
+ std::shared_ptr<BufferedBlockMgr> block_mgr_;
/// This is the node id of the root node for this plan fragment. This is used as the
/// hash seed and has two useful properties:
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f4fbd791/be/src/runtime/test-env.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/test-env.cc b/be/src/runtime/test-env.cc
index 223ecad..7a3c896 100644
--- a/be/src/runtime/test-env.cc
+++ b/be/src/runtime/test-env.cc
@@ -16,7 +16,11 @@
#include "util/disk-info.h"
#include "util/impalad-metrics.h"
-using namespace boost;
+#include <memory>
+
+#include "common/names.h"
+
+using boost::scoped_ptr;
namespace impala {
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f4fbd791/be/src/runtime/test-env.h
----------------------------------------------------------------------
diff --git a/be/src/runtime/test-env.h b/be/src/runtime/test-env.h
index 900bb3e..69e1eb8 100644
--- a/be/src/runtime/test-env.h
+++ b/be/src/runtime/test-env.h
@@ -74,7 +74,7 @@ class TestEnv {
boost::scoped_ptr<TmpFileMgr> tmp_file_mgr_;
/// Per-query states with associated block managers.
- vector<boost::shared_ptr<RuntimeState>> query_states_;
+ vector<std::shared_ptr<RuntimeState>> query_states_;
};
}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f4fbd791/be/src/scheduling/query-resource-mgr.h
----------------------------------------------------------------------
diff --git a/be/src/scheduling/query-resource-mgr.h b/be/src/scheduling/query-resource-mgr.h
index c6080e8..d29394b 100644
--- a/be/src/scheduling/query-resource-mgr.h
+++ b/be/src/scheduling/query-resource-mgr.h
@@ -28,7 +28,6 @@
#include <boost/thread/mutex.hpp>
#include <boost/unordered_map.hpp>
#include <boost/unordered_set.hpp>
-#include <boost/shared_ptr.hpp>
#include <string>
namespace impala {
@@ -195,12 +194,12 @@ class QueryResourceMgr {
/// parent QueryResourceMgr has been destroyed.
/// TODO: Combine with ShouldExit(), and replace with AtomicBool when we have such a
/// thing.
- boost::shared_ptr<AtomicInt32> early_exit_;
+ std::shared_ptr<AtomicInt32> early_exit_;
/// Signals to the destructor that the vcore acquisition thread is currently in an
/// Expand() RPC. If so, the destructor does not need to wait for the acquisition thread
/// to exit.
- boost::shared_ptr<AtomicInt32> thread_in_expand_;
+ std::shared_ptr<AtomicInt32> thread_in_expand_;
/// Creates the llama resource for the memory and/or cores specified, associated with
/// the reservation context.
@@ -209,8 +208,8 @@ class QueryResourceMgr {
/// Run as a thread owned by acquire_cpu_thread_. Waits for notification from
/// NotifyThreadUsageChange(), then checks the subscription level to decide if more
/// VCores are needed, and starts a new expansion request if so.
- void AcquireVcoreResources(boost::shared_ptr<AtomicInt32 > thread_in_expand,
- boost::shared_ptr<AtomicInt32> early_exit);
+ void AcquireVcoreResources(std::shared_ptr<AtomicInt32 > thread_in_expand,
+ std::shared_ptr<AtomicInt32> early_exit);
/// True if thread:VCore subscription is too high, meaning more VCores are required.
/// Must be called holding threads_running_ lock.
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f4fbd791/be/src/service/fragment-mgr.h
----------------------------------------------------------------------
diff --git a/be/src/service/fragment-mgr.h b/be/src/service/fragment-mgr.h
index 68eac15..4ec2253 100644
--- a/be/src/service/fragment-mgr.h
+++ b/be/src/service/fragment-mgr.h
@@ -15,7 +15,6 @@
#ifndef IMPALA_SERVICE_FRAGMENT_MGR_H
#define IMPALA_SERVICE_FRAGMENT_MGR_H
-#include <boost/shared_ptr.hpp>
#include <boost/thread/mutex.hpp>
#include <boost/unordered_map.hpp>
@@ -58,7 +57,7 @@ class FragmentMgr {
/// Returns a shared pointer to the FragmentExecState if one can be found for the
/// given id. If the id is not found, the shared pointer will contain NULL.
- boost::shared_ptr<FragmentExecState> GetFragmentExecState(
+ std::shared_ptr<FragmentExecState> GetFragmentExecState(
const TUniqueId& fragment_instance_id);
/// Publishes a global runtime filter to a local fragment.
@@ -71,7 +70,7 @@ class FragmentMgr {
/// Map from fragment instance id to exec state; FragmentExecState is owned by us and
/// referenced as a shared_ptr to allow asynchronous calls to CancelPlanFragment()
- typedef boost::unordered_map<TUniqueId, boost::shared_ptr<FragmentExecState>>
+ typedef boost::unordered_map<TUniqueId, std::shared_ptr<FragmentExecState>>
FragmentExecStateMap;
FragmentExecStateMap fragment_exec_state_map_;
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f4fbd791/be/src/service/impala-hs2-server.cc
----------------------------------------------------------------------
diff --git a/be/src/service/impala-hs2-server.cc b/be/src/service/impala-hs2-server.cc
index 803242e..709c733 100644
--- a/be/src/service/impala-hs2-server.cc
+++ b/be/src/service/impala-hs2-server.cc
@@ -590,7 +590,7 @@ void ImpalaServer::OpenSession(TOpenSessionResp& return_val,
// query options.
// TODO: put secret in session state map and check it
// TODO: Fix duplication of code between here and ConnectionStart().
- shared_ptr<SessionState> state(new SessionState());
+ shared_ptr<SessionState> state = make_shared<SessionState>();
state->closed = false;
state->start_time = TimestampValue::LocalTime();
state->session_type = TSessionType::HIVESERVER2;
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f4fbd791/be/src/service/impala-server.cc
----------------------------------------------------------------------
diff --git a/be/src/service/impala-server.cc b/be/src/service/impala-server.cc
index e092f4b..a4e7cda 100644
--- a/be/src/service/impala-server.cc
+++ b/be/src/service/impala-server.cc
@@ -1069,7 +1069,7 @@ Status ImpalaServer::GetSessionState(const TUniqueId& session_id,
lock_guard<mutex> l(session_state_map_lock_);
SessionStateMap::iterator i = session_state_map_.find(session_id);
if (i == session_state_map_.end()) {
- *session_state = boost::shared_ptr<SessionState>();
+ *session_state = std::shared_ptr<SessionState>();
return Status("Invalid session id");
} else {
if (mark_active) {
@@ -1839,13 +1839,13 @@ Status CreateImpalaServer(ExecEnv* exec_env, int beeswax_port, int hs2_port, int
DCHECK((hs2_port == 0) == (hs2_server == NULL));
DCHECK((be_port == 0) == (be_server == NULL));
- shared_ptr<ImpalaServer> handler(new ImpalaServer(exec_env));
+ boost::shared_ptr<ImpalaServer> handler(new ImpalaServer(exec_env));
if (beeswax_port != 0 && beeswax_server != NULL) {
// Beeswax FE must be a TThreadPoolServer because ODBC and Hue only support
// TThreadPoolServer.
- shared_ptr<TProcessor> beeswax_processor(new ImpalaServiceProcessor(handler));
- shared_ptr<TProcessorEventHandler> event_handler(
+ boost::shared_ptr<TProcessor> beeswax_processor(new ImpalaServiceProcessor(handler));
+ boost::shared_ptr<TProcessorEventHandler> event_handler(
new RpcEventHandler("beeswax", exec_env->metrics()));
beeswax_processor->setEventHandler(event_handler);
*beeswax_server = new ThriftServer(BEESWAX_SERVER_NAME, beeswax_processor,
@@ -1864,9 +1864,9 @@ Status CreateImpalaServer(ExecEnv* exec_env, int beeswax_port, int hs2_port, int
if (hs2_port != 0 && hs2_server != NULL) {
// HiveServer2 JDBC driver does not support non-blocking server.
- shared_ptr<TProcessor> hs2_fe_processor(
+ boost::shared_ptr<TProcessor> hs2_fe_processor(
new ImpalaHiveServer2ServiceProcessor(handler));
- shared_ptr<TProcessorEventHandler> event_handler(
+ boost::shared_ptr<TProcessorEventHandler> event_handler(
new RpcEventHandler("hs2", exec_env->metrics()));
hs2_fe_processor->setEventHandler(event_handler);
@@ -1885,11 +1885,12 @@ Status CreateImpalaServer(ExecEnv* exec_env, int beeswax_port, int hs2_port, int
}
if (be_port != 0 && be_server != NULL) {
- shared_ptr<FragmentMgr> fragment_mgr(new FragmentMgr());
- shared_ptr<ImpalaInternalService> thrift_if(
+ boost::shared_ptr<FragmentMgr> fragment_mgr(new FragmentMgr());
+ boost::shared_ptr<ImpalaInternalService> thrift_if(
new ImpalaInternalService(handler, fragment_mgr));
- shared_ptr<TProcessor> be_processor(new ImpalaInternalServiceProcessor(thrift_if));
- shared_ptr<TProcessorEventHandler> event_handler(
+ boost::shared_ptr<TProcessor> be_processor(
+ new ImpalaInternalServiceProcessor(thrift_if));
+ boost::shared_ptr<TProcessorEventHandler> event_handler(
new RpcEventHandler("backend", exec_env->metrics()));
be_processor->setEventHandler(event_handler);
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f4fbd791/be/src/service/impala-server.h
----------------------------------------------------------------------
diff --git a/be/src/service/impala-server.h b/be/src/service/impala-server.h
index 9069d75..35b866d 100644
--- a/be/src/service/impala-server.h
+++ b/be/src/service/impala-server.h
@@ -314,11 +314,11 @@ class ImpalaServer : public ImpalaServiceIf, public ImpalaHiveServer2ServiceIf,
/// Return exec state for given query_id, or NULL if not found.
/// If 'lock' is true, the returned exec state's lock() will be acquired before
/// the query_exec_state_map_lock_ is released.
- boost::shared_ptr<QueryExecState> GetQueryExecState(
+ std::shared_ptr<QueryExecState> GetQueryExecState(
const TUniqueId& query_id, bool lock);
- /// Writes the session id, if found, for the given query to the output parameter. Returns
- /// false if no query with the given ID is found.
+ /// Writes the session id, if found, for the given query to the output
+ /// parameter. Returns false if no query with the given ID is found.
bool GetSessionIdForQuery(const TUniqueId& query_id, TUniqueId* session_id);
/// Updates the number of databases / tables metrics from the FE catalog
@@ -333,20 +333,20 @@ class ImpalaServer : public ImpalaServiceIf, public ImpalaHiveServer2ServiceIf,
/// query_session_state is a snapshot of session state that changes when the
/// query was run. (e.g. default database).
Status Execute(TQueryCtx* query_ctx,
- boost::shared_ptr<SessionState> session_state,
- boost::shared_ptr<QueryExecState>* exec_state);
+ std::shared_ptr<SessionState> session_state,
+ std::shared_ptr<QueryExecState>* exec_state);
/// Implements Execute() logic, but doesn't unregister query on error.
Status ExecuteInternal(const TQueryCtx& query_ctx,
- boost::shared_ptr<SessionState> session_state,
+ std::shared_ptr<SessionState> session_state,
bool* registered_exec_state,
- boost::shared_ptr<QueryExecState>* exec_state);
+ std::shared_ptr<QueryExecState>* exec_state);
/// Registers the query exec state with query_exec_state_map_ using the globally
/// unique query_id and add the query id to session state's open query list.
/// The caller must have checked out the session state.
- Status RegisterQuery(boost::shared_ptr<SessionState> session_state,
- const boost::shared_ptr<QueryExecState>& exec_state);
+ Status RegisterQuery(std::shared_ptr<SessionState> session_state,
+ const std::shared_ptr<QueryExecState>& exec_state);
/// Adds the query to the set of in-flight queries for the session. The query remains
/// in-flight until the query is unregistered. Until a query is in-flight, an attempt
@@ -357,8 +357,8 @@ class ImpalaServer : public ImpalaServiceIf, public ImpalaHiveServer2ServiceIf,
/// (e.g. via an RPC) and the session close path can close (cancel and unregister) it.
/// The query must have already been registered using RegisterQuery(). The caller
/// must have checked out the session state.
- Status SetQueryInflight(boost::shared_ptr<SessionState> session_state,
- const boost::shared_ptr<QueryExecState>& exec_state);
+ Status SetQueryInflight(std::shared_ptr<SessionState> session_state,
+ const std::shared_ptr<QueryExecState>& exec_state);
/// Unregister the query by cancelling it, removing exec_state from
/// query_exec_state_map_, and removing the query id from session state's in-flight
@@ -695,7 +695,7 @@ class ImpalaServer : public ImpalaServiceIf, public ImpalaHiveServer2ServiceIf,
/// map from query id to exec state; QueryExecState is owned by us and referenced
/// as a shared_ptr to allow asynchronous deletion
- typedef boost::unordered_map<TUniqueId, boost::shared_ptr<QueryExecState>>
+ typedef boost::unordered_map<TUniqueId, std::shared_ptr<QueryExecState>>
QueryExecStateMap;
QueryExecStateMap query_exec_state_map_;
boost::mutex query_exec_state_map_lock_; // protects query_exec_state_map_
@@ -792,11 +792,11 @@ class ImpalaServer : public ImpalaServiceIf, public ImpalaHiveServer2ServiceIf,
public:
ScopedSessionState(ImpalaServer* impala) : impala_(impala) { }
- /// Marks a session as in-use, and saves it so that it can be unmarked when this object
- /// goes out of scope. Returns OK unless there is an error in GetSessionState.
+ /// Marks a session as in-use, and saves it so that it can be unmarked when this
+ /// object goes out of scope. Returns OK unless there is an error in GetSessionState.
/// Must only be called once per ScopedSessionState.
Status WithSession(const TUniqueId& session_id,
- boost::shared_ptr<SessionState>* session = NULL) {
+ std::shared_ptr<SessionState>* session = NULL) {
DCHECK(session_.get() == NULL);
RETURN_IF_ERROR(impala_->GetSessionState(session_id, &session_, true));
if (session != NULL) (*session) = session_;
@@ -812,7 +812,7 @@ class ImpalaServer : public ImpalaServiceIf, public ImpalaHiveServer2ServiceIf,
private:
/// Reference-counted pointer to the session state object.
- boost::shared_ptr<SessionState> session_;
+ std::shared_ptr<SessionState> session_;
/// Saved so that we can access ImpalaServer methods to get / return session state.
ImpalaServer* impala_;
@@ -822,22 +822,21 @@ class ImpalaServer : public ImpalaServiceIf, public ImpalaHiveServer2ServiceIf,
friend class ScopedSessionState;
/// Protects session_state_map_. Should be taken before any query exec-state locks,
- /// including query_exec_state_map_lock_. Should be taken before individual session-state
- /// locks.
+ /// including query_exec_state_map_lock_. Should be taken before individual
+ /// session-state locks.
boost::mutex session_state_map_lock_;
/// A map from session identifier to a structure containing per-session information
- typedef boost::unordered_map<TUniqueId, boost::shared_ptr<SessionState>>
- SessionStateMap;
+ typedef boost::unordered_map<TUniqueId, std::shared_ptr<SessionState>> SessionStateMap;
SessionStateMap session_state_map_;
/// Protects connection_to_sessions_map_. May be taken before session_state_map_lock_.
boost::mutex connection_to_sessions_map_lock_;
- /// Map from a connection ID to the associated list of sessions so that all can be closed
- /// when the connection ends. HS2 allows for multiplexing several sessions across a
- /// single connection. If a session has already been closed (only possible via HS2) it is
- /// not removed from this map to avoid the cost of looking it up.
+ /// Map from a connection ID to the associated list of sessions so that all can be
+ /// closed when the connection ends. HS2 allows for multiplexing several sessions across
+ /// a single connection. If a session has already been closed (only possible via HS2) it
+ /// is not removed from this map to avoid the cost of looking it up.
typedef boost::unordered_map<TUniqueId, std::vector<TUniqueId>>
ConnectionToSessionMap;
ConnectionToSessionMap connection_to_sessions_map_;
@@ -847,11 +846,11 @@ class ImpalaServer : public ImpalaServiceIf, public ImpalaHiveServer2ServiceIf,
/// If mark_active is true, also checks if the session is expired or closed and
/// increments the session's reference counter if it is still alive.
Status GetSessionState(const TUniqueId& session_id,
- boost::shared_ptr<SessionState>* session_state, bool mark_active = false);
+ std::shared_ptr<SessionState>* session_state, bool mark_active = false);
/// Decrement the session's reference counter and mark last_accessed_ms so that state
/// expiration can proceed.
- inline void MarkSessionInactive(boost::shared_ptr<SessionState> session) {
+ inline void MarkSessionInactive(std::shared_ptr<SessionState> session) {
boost::lock_guard<boost::mutex> l(session->lock);
DCHECK_GT(session->ref_count, 0);
--session->ref_count;
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f4fbd791/be/src/service/query-exec-state.h
----------------------------------------------------------------------
diff --git a/be/src/service/query-exec-state.h b/be/src/service/query-exec-state.h
index c2459cf..0c56451 100644
--- a/be/src/service/query-exec-state.h
+++ b/be/src/service/query-exec-state.h
@@ -56,7 +56,7 @@ class QueryExecStateCleaner;
class ImpalaServer::QueryExecState {
public:
QueryExecState(const TQueryCtx& query_ctx, ExecEnv* exec_env, Frontend* frontend,
- ImpalaServer* server, boost::shared_ptr<ImpalaServer::SessionState> session);
+ ImpalaServer* server, std::shared_ptr<ImpalaServer::SessionState> session);
~QueryExecState();
@@ -228,7 +228,7 @@ class ImpalaServer::QueryExecState {
bool is_block_on_wait_joining_;
/// Session that this query is from
- boost::shared_ptr<SessionState> session_;
+ std::shared_ptr<SessionState> session_;
/// Resource assignment determined by scheduler. Owned by obj_pool_.
boost::scoped_ptr<QuerySchedule> schedule_;
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f4fbd791/be/src/statestore/statestore-subscriber.cc
----------------------------------------------------------------------
diff --git a/be/src/statestore/statestore-subscriber.cc b/be/src/statestore/statestore-subscriber.cc
index 4df9aa7..aac51a0 100644
--- a/be/src/statestore/statestore-subscriber.cc
+++ b/be/src/statestore/statestore-subscriber.cc
@@ -178,8 +178,9 @@ Status StatestoreSubscriber::Start() {
LOG(INFO) << "Starting statestore subscriber";
// Backend must be started before registration
- shared_ptr<TProcessor> processor(new StatestoreSubscriberProcessor(thrift_iface_));
- shared_ptr<TProcessorEventHandler> event_handler(
+ boost::shared_ptr<TProcessor> processor(
+ new StatestoreSubscriberProcessor(thrift_iface_));
+ boost::shared_ptr<TProcessorEventHandler> event_handler(
new RpcEventHandler("statestore-subscriber", metrics_));
processor->setEventHandler(event_handler);
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f4fbd791/be/src/statestore/statestore-subscriber.h
----------------------------------------------------------------------
diff --git a/be/src/statestore/statestore-subscriber.h b/be/src/statestore/statestore-subscriber.h
index ddf663e..be0c37e 100644
--- a/be/src/statestore/statestore-subscriber.h
+++ b/be/src/statestore/statestore-subscriber.h
@@ -133,7 +133,7 @@ class StatestoreSubscriber {
boost::shared_ptr<StatestoreSubscriberIf> thrift_iface_;
/// Container for the heartbeat server.
- boost::shared_ptr<ThriftServer> heartbeat_server_;
+ std::shared_ptr<ThriftServer> heartbeat_server_;
/// Failure detector that tracks heartbeat messages from the statestore.
boost::scoped_ptr<impala::TimeoutFailureDetector> failure_detector_;
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f4fbd791/be/src/statestore/statestore.h
----------------------------------------------------------------------
diff --git a/be/src/statestore/statestore.h b/be/src/statestore/statestore.h
index 2061e38..edc9d39 100644
--- a/be/src/statestore/statestore.h
+++ b/be/src/statestore/statestore.h
@@ -350,7 +350,7 @@ class Statestore {
/// Subscribers are held in shared_ptrs so that RegisterSubscriber() may overwrite their
/// entry in this map while UpdateSubscriber() tries to update an existing registration
/// without risk of use-after-free.
- typedef boost::unordered_map<SubscriberId, boost::shared_ptr<Subscriber>>
+ typedef boost::unordered_map<SubscriberId, std::shared_ptr<Subscriber>>
SubscriberMap;
SubscriberMap subscribers_;
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f4fbd791/be/src/statestore/statestored-main.cc
----------------------------------------------------------------------
diff --git a/be/src/statestore/statestored-main.cc b/be/src/statestore/statestored-main.cc
index 450459c..c344cea 100644
--- a/be/src/statestore/statestored-main.cc
+++ b/be/src/statestore/statestored-main.cc
@@ -70,9 +70,9 @@ int StatestoredMain(int argc, char** argv) {
Statestore statestore(metrics.get());
statestore.RegisterWebpages(webserver.get());
- shared_ptr<TProcessor> processor(
+ boost::shared_ptr<TProcessor> processor(
new StatestoreServiceProcessor(statestore.thrift_iface()));
- shared_ptr<TProcessorEventHandler> event_handler(
+ boost::shared_ptr<TProcessorEventHandler> event_handler(
new RpcEventHandler("statestore", metrics.get()));
processor->setEventHandler(event_handler);
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f4fbd791/be/src/testutil/impalad-query-executor.h
----------------------------------------------------------------------
diff --git a/be/src/testutil/impalad-query-executor.h b/be/src/testutil/impalad-query-executor.h
index f0448f8..137092f 100644
--- a/be/src/testutil/impalad-query-executor.h
+++ b/be/src/testutil/impalad-query-executor.h
@@ -19,7 +19,6 @@
#include <string>
#include <vector>
#include <boost/scoped_ptr.hpp>
-#include <boost/shared_ptr.hpp>
#include <boost/thread/thread.hpp>
#include "rpc/thrift-client.h"
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f4fbd791/be/src/testutil/in-process-servers.cc
----------------------------------------------------------------------
diff --git a/be/src/testutil/in-process-servers.cc b/be/src/testutil/in-process-servers.cc
index c4bbcbd..7ff1136 100644
--- a/be/src/testutil/in-process-servers.cc
+++ b/be/src/testutil/in-process-servers.cc
@@ -34,7 +34,6 @@ DECLARE_string(ssl_private_key);
using namespace apache::thrift;
using namespace impala;
-using boost::shared_ptr;
InProcessImpalaServer* InProcessImpalaServer::StartWithEphemeralPorts(
const string& statestore_host, int statestore_port) {
@@ -153,7 +152,7 @@ InProcessStatestore::InProcessStatestore(int statestore_port, int webserver_port
Status InProcessStatestore::Start() {
webserver_->Start();
- shared_ptr<TProcessor> processor(
+ boost::shared_ptr<TProcessor> processor(
new StatestoreServiceProcessor(statestore_->thrift_iface()));
statestore_server_.reset(new ThriftServer("StatestoreService", processor,
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f4fbd791/be/src/testutil/in-process-servers.h
----------------------------------------------------------------------
diff --git a/be/src/testutil/in-process-servers.h b/be/src/testutil/in-process-servers.h
index f670115..49acc8a 100644
--- a/be/src/testutil/in-process-servers.h
+++ b/be/src/testutil/in-process-servers.h
@@ -16,7 +16,6 @@
#define IMPALA_TESTUTIL_IN_PROCESS_SERVERS_H
#include <boost/scoped_ptr.hpp>
-#include <boost/shared_ptr.hpp>
#include "common/status.h"
#include "util/thread.h"
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f4fbd791/be/src/transport/TSaslServerTransport.cpp
----------------------------------------------------------------------
diff --git a/be/src/transport/TSaslServerTransport.cpp b/be/src/transport/TSaslServerTransport.cpp
index 1dfa99e..b867c31 100644
--- a/be/src/transport/TSaslServerTransport.cpp
+++ b/be/src/transport/TSaslServerTransport.cpp
@@ -39,7 +39,7 @@
using namespace sasl;
namespace apache { namespace thrift { namespace transport {
-TSaslServerTransport::TSaslServerTransport(shared_ptr<TTransport> transport)
+TSaslServerTransport::TSaslServerTransport(boost::shared_ptr<TTransport> transport)
: TSaslTransport(transport) {
}
@@ -50,7 +50,7 @@ TSaslServerTransport::TSaslServerTransport(const string& mechanism,
unsigned flags,
const map<string, string>& props,
const vector<struct sasl_callback>& callbacks,
- shared_ptr<TTransport> transport)
+ boost::shared_ptr<TTransport> transport)
: TSaslTransport(transport) {
addServerDefinition(mechanism, protocol, serverName, realm, flags,
props, callbacks);
@@ -112,8 +112,8 @@ void TSaslServerTransport::handleSaslStartMessage() {
}
-shared_ptr<TTransport> TSaslServerTransport::Factory::getTransport(
- shared_ptr<TTransport> trans) {
+boost::shared_ptr<TTransport> TSaslServerTransport::Factory::getTransport(
+ boost::shared_ptr<TTransport> trans) {
lock_guard<mutex> l(transportMap_mutex_);
// Thrift servers use both an input and an output transport to communicate with
// clients. In principal, these can be different, but for SASL clients we require them
@@ -135,13 +135,13 @@ shared_ptr<TTransport> TSaslServerTransport::Factory::getTransport(
// ensure that when ret_transport is eventually deleted, its corresponding map entry is
// removed. That is likely to be error prone given the locking involved; for now we go
// with the simple solution.
- map<shared_ptr<TTransport>, shared_ptr<TBufferedTransport>>::iterator trans_map =
- transportMap_.find(trans);
+ TransportMap::iterator trans_map = transportMap_.find(trans);
VLOG_EVERY_N(2, 100) << "getTransport(): transportMap_ size is: "
<< transportMap_.size();
- shared_ptr<TBufferedTransport> ret_transport;
+ boost::shared_ptr<TBufferedTransport> ret_transport;
if (trans_map == transportMap_.end()) {
- shared_ptr<TTransport> wrapped(new TSaslServerTransport(serverDefinitionMap_, trans));
+ boost::shared_ptr<TTransport> wrapped(
+ new TSaslServerTransport(serverDefinitionMap_, trans));
ret_transport.reset(new TBufferedTransport(wrapped,
impala::ThriftServer::BufferedTransportFactory::DEFAULT_BUFFER_SIZE_BYTES));
ret_transport.get()->open();
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f4fbd791/be/src/transport/TSaslServerTransport.h
----------------------------------------------------------------------
diff --git a/be/src/transport/TSaslServerTransport.h b/be/src/transport/TSaslServerTransport.h
index 31041b3..28d17e6 100644
--- a/be/src/transport/TSaslServerTransport.h
+++ b/be/src/transport/TSaslServerTransport.h
@@ -33,7 +33,7 @@
namespace apache { namespace thrift { namespace transport {
/**
- *
+ *
* This transport implements the Simple Authentication and Security Layer (SASL).
* see: http://www.ietf.org/rfc/rfc2222.txt. It is based on and depends
* on the presence of the cyrus-sasl library.
@@ -173,8 +173,9 @@ class TSaslServerTransport : public TSaslTransport {
std::map<std::string, TSaslServerDefinition*> serverDefinitionMap_;
/* Map from a transport to its Sasl Transport (wrapped by a TBufferedTransport). */
- std::map<boost::shared_ptr<TTransport>,
- boost::shared_ptr<TBufferedTransport>> transportMap_;
+ typedef std::map<boost::shared_ptr<TTransport>,
+ boost::shared_ptr<TBufferedTransport>> TransportMap;
+ TransportMap transportMap_;
/* Lock to synchronize the transport map. */
boost::mutex transportMap_mutex_;
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f4fbd791/be/src/udf/uda-test-harness-impl.h
----------------------------------------------------------------------
diff --git a/be/src/udf/uda-test-harness-impl.h b/be/src/udf/uda-test-harness-impl.h
index 8be05ca..ecd4db8 100644
--- a/be/src/udf/uda-test-harness-impl.h
+++ b/be/src/udf/uda-test-harness-impl.h
@@ -20,8 +20,6 @@
#include <sstream>
#include <vector>
-#include <boost/shared_ptr.hpp>
-
namespace impala_udf {
/// Utility class to help test UDAs. This can be used to test the correctness of the
@@ -181,7 +179,7 @@ RESULT UdaTestHarnessBase<RESULT, INTERMEDIATE>::ExecuteSingleNode(
template<typename RESULT, typename INTERMEDIATE>
RESULT UdaTestHarnessBase<RESULT, INTERMEDIATE>::ExecuteOneLevel(int num_nodes,
ScopedFunctionContext* result_context) {
- std::vector<boost::shared_ptr<ScopedFunctionContext>> contexts;
+ std::vector<std::shared_ptr<ScopedFunctionContext>> contexts;
std::vector<INTERMEDIATE> intermediates;
contexts.resize(num_nodes);
@@ -237,7 +235,7 @@ RESULT UdaTestHarnessBase<RESULT, INTERMEDIATE>::ExecuteOneLevel(int num_nodes,
template<typename RESULT, typename INTERMEDIATE>
RESULT UdaTestHarnessBase<RESULT, INTERMEDIATE>::ExecuteTwoLevel(
int num1, int num2, ScopedFunctionContext* result_context) {
- std::vector<boost::shared_ptr<ScopedFunctionContext>> level1_contexts, level2_contexts;
+ std::vector<std::shared_ptr<ScopedFunctionContext>> level1_contexts, level2_contexts;
std::vector<INTERMEDIATE> level1_intermediates, level2_intermediates;
level1_contexts.resize(num1);
level2_contexts.resize(num2);
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f4fbd791/be/src/udf/uda-test-harness.h
----------------------------------------------------------------------
diff --git a/be/src/udf/uda-test-harness.h b/be/src/udf/uda-test-harness.h
index 5391e4d..772acb4 100644
--- a/be/src/udf/uda-test-harness.h
+++ b/be/src/udf/uda-test-harness.h
@@ -21,7 +21,6 @@
#include <vector>
#include <boost/scoped_ptr.hpp>
-#include <boost/shared_ptr.hpp>
#include "udf/udf.h"
#include "udf/udf-debug.h"
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f4fbd791/be/src/util/parquet-reader.cc
----------------------------------------------------------------------
diff --git a/be/src/util/parquet-reader.cc b/be/src/util/parquet-reader.cc
index 47f69a5..e4bec3e 100644
--- a/be/src/util/parquet-reader.cc
+++ b/be/src/util/parquet-reader.cc
@@ -48,8 +48,8 @@ using std::min;
// Some code is replicated to make this more stand-alone.
const uint8_t PARQUET_VERSION_NUMBER[] = {'P', 'A', 'R', '1'};
-shared_ptr<TProtocol> CreateDeserializeProtocol(
- shared_ptr<TMemoryBuffer> mem, bool compact) {
+boost::shared_ptr<TProtocol> CreateDeserializeProtocol(
+ boost::shared_ptr<TMemoryBuffer> mem, bool compact) {
if (compact) {
TCompactProtocolFactoryT<TMemoryBuffer> tproto_factory;
return tproto_factory.getProtocol(mem);
@@ -66,8 +66,9 @@ template <class T>
bool DeserializeThriftMsg(uint8_t* buf, uint32_t* len, bool compact,
T* deserialized_msg) {
// Deserialize msg bytes into c++ thrift msg using memory transport.
- shared_ptr<TMemoryBuffer> tmem_transport(new TMemoryBuffer(buf, *len));
- shared_ptr<TProtocol> tproto = CreateDeserializeProtocol(tmem_transport, compact);
+ boost::shared_ptr<TMemoryBuffer> tmem_transport(new TMemoryBuffer(buf, *len));
+ boost::shared_ptr<TProtocol> tproto =
+ CreateDeserializeProtocol(tmem_transport, compact);
try {
deserialized_msg->read(tproto.get());
} catch (apache::thrift::protocol::TProtocolException& e) {
[06/18] incubator-impala git commit: IMPALA-3677: Write minidump on
SIGUSR1
Posted by ta...@apache.org.
IMPALA-3677: Write minidump on SIGUSR1
Sending SIGUSR1 to any of the impala daemons (catalogd, impalad,
statestored) will trigger a minidump write.
The hotspot JVM also uses SIGUSR1 internally. However the documentation
explains, that existing signal handlers will be transparently wrapped by
the JVM and no spurious signals should be received by the daemon signal
handler:
http://www.oracle.com/technetwork/java/javase/signals-139944.html
Example: killall -SIGUSR1 catalogd
Change-Id: I40149e48e391451de21a5c8bda18e2307fc89513
Reviewed-on: http://gerrit.cloudera.org:8080/3312
Reviewed-by: Lars Volker <lv...@cloudera.com>
Tested-by: Internal Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/948a6c34
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/948a6c34
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/948a6c34
Branch: refs/heads/master
Commit: 948a6c34fcaa7ba481254e0a7b0d2cf987a4b690
Parents: db4294c
Author: Lars Volker <lv...@cloudera.com>
Authored: Sat Jun 4 22:18:04 2016 +0200
Committer: Taras Bobrovytsky <ta...@apache.org>
Committed: Thu Jul 14 19:04:44 2016 +0000
----------------------------------------------------------------------
be/src/util/minidump.cc | 29 ++++++++++++++--
tests/custom_cluster/test_breakpad.py | 55 ++++++++++++++++++++++++++----
2 files changed, 74 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/948a6c34/be/src/util/minidump.cc
----------------------------------------------------------------------
diff --git a/be/src/util/minidump.cc b/be/src/util/minidump.cc
index 3a7577f..f516a56 100644
--- a/be/src/util/minidump.cc
+++ b/be/src/util/minidump.cc
@@ -21,10 +21,11 @@
#include <google_breakpad/common/minidump_format.h>
#include <third_party/lss/linux_syscall_support.h>
#include <ctime>
+#include <fstream>
#include <glob.h>
#include <iomanip>
-#include <fstream>
#include <map>
+#include <signal.h>
#include "common/logging.h"
#include "common/version.h"
@@ -48,6 +49,10 @@ DECLARE_int32(minidump_size_limit_hint_kb);
namespace impala {
+/// Breakpad ExceptionHandler. It registers its own signal handlers to write minidump
+/// files during process crashes, but also can be used to write minidumps directly.
+static google_breakpad::ExceptionHandler* minidump_exception_handler = NULL;
+
/// Callback for breakpad. It is called by breakpad whenever a minidump file has been
/// written and should not be called directly. It logs the event before breakpad crashes
/// the process. Due to the process being in a failed state we write to stdout/stderr and
@@ -80,6 +85,21 @@ static bool DumpCallback(const google_breakpad::MinidumpDescriptor& descriptor,
return succeeded;
}
+/// Signal handler to write a minidump file outside of crashes.
+static void HandleSignal(int signal) {
+ minidump_exception_handler->WriteMinidump(FLAGS_minidump_path, DumpCallback, NULL);
+}
+
+/// Register our signal handler to write minidumps on SIGUSR1.
+static void SetupSignalHandler() {
+ DCHECK(minidump_exception_handler != NULL);
+ struct sigaction sig_action;
+ memset(&sig_action, 0, sizeof(sig_action));
+ sigemptyset(&sig_action.sa_mask);
+ sig_action.sa_handler = &HandleSignal;
+ sigaction(SIGUSR1, &sig_action, NULL);
+}
+
/// Check the number of minidump files and removes the oldest ones to maintain an upper
/// bound on the number of files.
static void CheckAndRemoveMinidumps(int max_minidumps) {
@@ -201,9 +221,12 @@ Status RegisterMinidump(const char* cmd_line_path) {
}
// Intentionally leaked. We want this to have the lifetime of the process.
- google_breakpad::ExceptionHandler* eh =
+ DCHECK(minidump_exception_handler == NULL);
+ minidump_exception_handler =
new google_breakpad::ExceptionHandler(desc, NULL, DumpCallback, NULL, true, -1);
- (void)eh;
+
+ // Setup signal handler for SIGUSR1.
+ SetupSignalHandler();
return Status::OK();
}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/948a6c34/tests/custom_cluster/test_breakpad.py
----------------------------------------------------------------------
diff --git a/tests/custom_cluster/test_breakpad.py b/tests/custom_cluster/test_breakpad.py
index 84cbbe1..91a5f9f 100644
--- a/tests/custom_cluster/test_breakpad.py
+++ b/tests/custom_cluster/test_breakpad.py
@@ -21,7 +21,7 @@ import tempfile
import time
from resource import setrlimit, RLIMIT_CORE, RLIM_INFINITY
-from signal import SIGSEGV, SIGKILL
+from signal import SIGKILL, SIGSEGV, SIGUSR1
from tests.common.skip import SkipIfBuildType
from subprocess import CalledProcessError
@@ -85,12 +85,12 @@ class TestBreakpad(CustomClusterTestSuite):
processes = self.cluster.impalads + [self.cluster.catalogd, self.cluster.statestored]
processes = filter(None, processes)
self.kill_processes(processes, signal)
- self.assert_all_processes_killed()
+ signal is SIGUSR1 or self.assert_all_processes_killed()
def kill_processes(self, processes, signal):
for process in processes:
process.kill(signal)
- self.wait_for_all_processes_dead(processes)
+ signal is SIGUSR1 or self.wait_for_all_processes_dead(processes)
def wait_for_all_processes_dead(self, processes, timeout=300):
for process in processes:
@@ -104,6 +104,25 @@ class TestBreakpad(CustomClusterTestSuite):
raise RuntimeError("Unable to kill %s (pid %d) after %d seconds." %
(psutil_process.name, psutil_process.pid, timeout))
+ def get_num_processes(self, daemon):
+ self.cluster.refresh()
+ if daemon == 'impalad':
+ return len(self.cluster.impalads)
+ elif daemon == 'catalogd':
+ return self.cluster.catalogd and 1 or 0
+ elif daemon == 'statestored':
+ return self.cluster.statestored and 1 or 0
+ raise RuntimeError("Unknown daemon name: %s" % daemon)
+
+ def wait_for_num_processes(self, daemon, num_expected, timeout=60):
+ end = time.time() + timeout
+ self.cluster.refresh()
+ num_processes = self.get_num_processes(daemon)
+ while num_processes != num_expected and time.time() <= end:
+ time.sleep(1)
+ num_processes = self.get_num_processes(daemon)
+ return num_processes
+
def assert_all_processes_killed(self):
self.cluster.refresh()
assert not self.cluster.impalads
@@ -130,7 +149,7 @@ class TestBreakpad(CustomClusterTestSuite):
assert self.count_all_minidumps() == 0
self.start_cluster()
assert self.count_all_minidumps() == 0
- cluster_size = len(self.cluster.impalads)
+ cluster_size = self.get_num_processes('impalad')
self.kill_cluster(SIGSEGV)
self.assert_num_logfile_entries(1)
assert self.count_minidumps('impalad') == cluster_size
@@ -138,6 +157,28 @@ class TestBreakpad(CustomClusterTestSuite):
assert self.count_minidumps('catalogd') == 1
@pytest.mark.execute_serially
+ def test_sigusr1_writes_minidump(self):
+ """Check that when a daemon receives SIGUSR1 it writes a minidump file."""
+ assert self.count_all_minidumps() == 0
+ self.start_cluster()
+ assert self.count_all_minidumps() == 0
+ cluster_size = self.get_num_processes('impalad')
+ self.kill_cluster(SIGUSR1)
+ # Breakpad forks to write its minidump files, wait for all the clones to terminate.
+ assert self.wait_for_num_processes('impalad', cluster_size, 5) == cluster_size
+ assert self.wait_for_num_processes('catalogd', 1, 5) == 1
+ assert self.wait_for_num_processes('statestored', 1, 5) == 1
+ # Make sure impalad still answers queries.
+ client = self.create_impala_client()
+ self.execute_query_expect_success(client, "SELECT COUNT(*) FROM functional.alltypes")
+ # Kill the cluster. Sending SIGKILL will not trigger minidumps to be written.
+ self.kill_cluster(SIGKILL)
+ self.assert_num_logfile_entries(1)
+ assert self.count_minidumps('impalad') == cluster_size
+ assert self.count_minidumps('statestored') == 1
+ assert self.count_minidumps('catalogd') == 1
+
+ @pytest.mark.execute_serially
def test_minidump_relative_path(self):
"""Check that setting 'minidump_path' to a relative value results in minidump files
written to 'log_dir'.
@@ -148,7 +189,7 @@ class TestBreakpad(CustomClusterTestSuite):
# configuration, which is a FLAGS_log_dir/minidumps.
self.start_cluster_with_args()
assert self.count_all_minidumps(minidump_base_dir) == 0
- cluster_size = len(self.cluster.impalads)
+ cluster_size = self.get_num_processes('impalad')
self.kill_cluster(SIGSEGV)
self.assert_num_logfile_entries(1)
assert self.count_minidumps('impalad', minidump_base_dir) == cluster_size
@@ -164,7 +205,7 @@ class TestBreakpad(CustomClusterTestSuite):
self.kill_cluster(SIGSEGV)
self.assert_num_logfile_entries(1)
self.start_cluster()
- expected_impalads = min(len(self.cluster.impalads), 2)
+ expected_impalads = min(self.get_num_processes('impalad'), 2)
assert self.count_minidumps('impalad') == expected_impalads
assert self.count_minidumps('statestored') == 1
assert self.count_minidumps('catalogd') == 1
@@ -183,7 +224,7 @@ class TestBreakpad(CustomClusterTestSuite):
the cluster. Clean up the single minidump file and return its size.
"""
self.cluster.refresh()
- assert len(self.cluster.impalads) > 0
+ assert self.get_num_processes('impalad') > 0
# Make one impalad write a minidump.
self.kill_processes(self.cluster.impalads[:1], SIGSEGV)
# Kill the rest of the cluster.
[11/18] incubator-impala git commit: IMPALA-3715: Include more info
by default in Impala debug memz webpage
Posted by ta...@apache.org.
IMPALA-3715: Include more info by default in Impala debug memz webpage
Testing: ran locally and looked at memz page
Change-Id: I0d3f7de64ab321f3b9c41a119fb3d744b60c78d1
Reviewed-on: http://gerrit.cloudera.org:8080/3540
Tested-by: Internal Jenkins
Reviewed-by: Kathy Sun <ka...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/148c23a3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/148c23a3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/148c23a3
Branch: refs/heads/master
Commit: 148c23a3cc0c588fe85f3d933e92cba0c0a72323
Parents: 761c39c
Author: kathy.sun <ka...@cloudera.com>
Authored: Wed Jun 29 16:53:22 2016 -0700
Committer: Taras Bobrovytsky <ta...@apache.org>
Committed: Thu Jul 14 19:04:44 2016 +0000
----------------------------------------------------------------------
be/src/util/default-path-handlers.cc | 9 ++++-----
www/memz.tmpl | 2 --
2 files changed, 4 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/148c23a3/be/src/util/default-path-handlers.cc
----------------------------------------------------------------------
diff --git a/be/src/util/default-path-handlers.cc b/be/src/util/default-path-handlers.cc
index 706f31a..8954fcc 100644
--- a/be/src/util/default-path-handlers.cc
+++ b/be/src/util/default-path-handlers.cc
@@ -101,11 +101,10 @@ void MemUsageHandler(MemTracker* mem_tracker, const Webserver::ArgumentMap& args
Value overview(ss.str().c_str(), document->GetAllocator());
document->AddMember("overview", overview, document->GetAllocator());
- if (args.find("detailed") != args.end()) {
- // Dump all mem trackers.
- Value detailed(mem_tracker->LogUsage().c_str(), document->GetAllocator());
- document->AddMember("detailed", detailed, document->GetAllocator());
- }
+ // Dump all mem trackers.
+ Value detailed(mem_tracker->LogUsage().c_str(), document->GetAllocator());
+ document->AddMember("detailed", detailed, document->GetAllocator());
+
}
void impala::AddDefaultUrlCallbacks(
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/148c23a3/www/memz.tmpl
----------------------------------------------------------------------
diff --git a/www/memz.tmpl b/www/memz.tmpl
index 629fe94..5b4636f 100644
--- a/www/memz.tmpl
+++ b/www/memz.tmpl
@@ -22,9 +22,7 @@ Memory consumption / limit: <strong>{{consumption}}</strong> / <strong>{{mem_lim
<h3>tcmalloc</h2>
<pre>{{overview}}</pre>
-{{?detailed}}
<h3>Breakdown</h3>
<pre>{{detailed}}</pre>
-{{/detailed}}
{{> www/common-footer.tmpl }}
[17/18] incubator-impala git commit: IMPALA-3715: Include total usage
of JVM memory
Posted by ta...@apache.org.
IMPALA-3715: Include total usage of JVM memory
Get jvm metrics by global handler ExecEnv
and filter out items with "total"
Testing: Ran locally and looked at memz page
Change-Id: Ib44c25eb5a5d70f10a6a120501eec2d50fad5ce9
Reviewed-on: http://gerrit.cloudera.org:8080/3625
Reviewed-by: Henry Robinson <he...@cloudera.com>
Tested-by: Internal Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/74410326
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/74410326
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/74410326
Branch: refs/heads/master
Commit: 744103264546e5fd695c0d4661a804da247e86c4
Parents: f129dfd
Author: kathy.sun <ka...@cloudera.com>
Authored: Tue Jul 12 14:53:36 2016 -0700
Committer: Taras Bobrovytsky <ta...@apache.org>
Committed: Thu Jul 14 19:04:45 2016 +0000
----------------------------------------------------------------------
be/src/util/default-path-handlers.cc | 11 +++++++++++
www/memz.tmpl | 31 ++++++++++++++++++++++++++++++-
2 files changed, 41 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/74410326/be/src/util/default-path-handlers.cc
----------------------------------------------------------------------
diff --git a/be/src/util/default-path-handlers.cc b/be/src/util/default-path-handlers.cc
index 8954fcc..740077b 100644
--- a/be/src/util/default-path-handlers.cc
+++ b/be/src/util/default-path-handlers.cc
@@ -23,6 +23,7 @@
#include <gutil/strings/substitute.h>
#include "common/logging.h"
+#include "runtime/exec-env.h"
#include "runtime/mem-tracker.h"
#include "util/debug-util.h"
#include "util/pprof-path-handlers.h"
@@ -105,6 +106,16 @@ void MemUsageHandler(MemTracker* mem_tracker, const Webserver::ArgumentMap& args
Value detailed(mem_tracker->LogUsage().c_str(), document->GetAllocator());
document->AddMember("detailed", detailed, document->GetAllocator());
+ Value jvm(kObjectType);
+ ExecEnv::GetInstance()->metrics()->GetChildGroup("jvm")->ToJson(false, document, &jvm);
+
+ Value total(kArrayType);
+ for(SizeType i = 0; i < jvm["metrics"].Size(); ++i){
+ if (strstr(jvm["metrics"][i]["name"].GetString(), "total") != nullptr){
+ total.PushBack(jvm["metrics"][i], document->GetAllocator());
+ }
+ }
+ document->AddMember("jvm", total, document->GetAllocator());
}
void impala::AddDefaultUrlCallbacks(
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/74410326/www/memz.tmpl
----------------------------------------------------------------------
diff --git a/www/memz.tmpl b/www/memz.tmpl
index 5b4636f..5709a89 100644
--- a/www/memz.tmpl
+++ b/www/memz.tmpl
@@ -19,10 +19,39 @@ limitations under the License.
Memory consumption / limit: <strong>{{consumption}}</strong> / <strong>{{mem_limit}}</strong>
-<h3>tcmalloc</h2>
+<h3>tcmalloc</h3>
<pre>{{overview}}</pre>
<h3>Breakdown</h3>
<pre>{{detailed}}</pre>
+<h3>JVM memory total usage</h3>
+
+<table class='table table-bordered table-hover'>
+ <tr>
+ <th>Name</th>
+ <th>Value</th>
+ <th>Description</th>
+ </tr>
+ {{#jvm}}
+ <tr>
+ <td><tt>{{name}}</tt></td>
+ {{! Is this a stats metric? }}
+ {{?mean}}
+ <td>
+ Last (of {{count}}): <strong>{{last}}</strong>.
+ Min: {{min}}, max: {{max}}, avg: {{mean}}</td>
+ {{/mean}}
+ {{^mean}}
+ <td>
+ {{human_readable}}
+ </td>
+ {{/mean}}
+ <td>
+ {{description}}
+ </td>
+ </tr>
+ {{/jvm}}
+</table>
+
{{> www/common-footer.tmpl }}
[10/18] incubator-impala git commit: IMPALA-3852: Remove Derby and
Shiro FE dependencies
Posted by ta...@apache.org.
IMPALA-3852: Remove Derby and Shiro FE dependencies
Neither of these dependencies are apparently needed by the FE, and so
should be removed to make Impala's packages smaller.
Testing: Passed a clean data-load build and an exhaustive test run.
Change-Id: Ib337bf51150089d6441d1e63119e8598fd85efec
Reviewed-on: http://gerrit.cloudera.org:8080/3624
Reviewed-by: Alex Behm <al...@cloudera.com>
Tested-by: Internal Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/79641a87
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/79641a87
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/79641a87
Branch: refs/heads/master
Commit: 79641a873d2d12c2d8646349e58eff2ac0b1952d
Parents: 148c23a
Author: Henry Robinson <he...@cloudera.com>
Authored: Mon Jul 11 14:47:35 2016 -0700
Committer: Taras Bobrovytsky <ta...@apache.org>
Committed: Thu Jul 14 19:04:44 2016 +0000
----------------------------------------------------------------------
fe/pom.xml | 12 ------------
1 file changed, 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/79641a87/fe/pom.xml
----------------------------------------------------------------------
diff --git a/fe/pom.xml b/fe/pom.xml
index a3c1385..e2be27c 100644
--- a/fe/pom.xml
+++ b/fe/pom.xml
@@ -184,12 +184,6 @@
</dependency>
<dependency>
- <groupId>org.apache.shiro</groupId>
- <artifactId>shiro-core</artifactId>
- <version>1.2.1</version>
- </dependency>
-
- <dependency>
<groupId>commons-lang</groupId>
<artifactId>commons-lang</artifactId>
<version>2.6</version>
@@ -232,12 +226,6 @@
</dependency>
<dependency>
- <groupId>org.apache.derby</groupId>
- <artifactId>derby</artifactId>
- <version>10.4.2.0</version>
- </dependency>
-
- <dependency>
<groupId>org.apache.hive</groupId>
<artifactId>hive-hbase-handler</artifactId>
<version>${hive.version}</version>
[07/18] incubator-impala git commit: IMPALA-3772: Fix admission
control flaky stress test
Posted by ta...@apache.org.
IMPALA-3772: Fix admission control flaky stress test
The admission control stress test could occasionally fail
with some queued queries timing out. At some point the
default statestore topic update rpc frequency was increased
to 2sec, and because the updates trigger the admission
control queue check (to determine if queued queries can be
dequeued), it is possible that the topic updates weren't
frequent enough for a large admission control queue to fully
drain within the 60sec queue timeout.
In the test that failed, queries were submitted round-robin
so the admission control behavior is non-deterministic.
There isn't enough information to prove that this is
definitely the issue, but it seems likely and improving the
test seems valuable regardless.
This change reduces the statestore topic update frequency to
500ms so that state can be shared faster and the queued
requests have more chances to drain within the 60sec queue
timeout. The code was also a bit confusing before because it
was waiting for statestore topic updates but the test had
only changed the heartbeat frequency (which is a different
RPC). This now sets the lower frequency for both and makes
the code a bit more clear.
Change-Id: I235a14c4674240dc0a01dabb664da87c752153cf
Reviewed-on: http://gerrit.cloudera.org:8080/3450
Reviewed-by: Matthew Jacobs <mj...@cloudera.com>
Tested-by: Matthew Jacobs <mj...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/4680252a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/4680252a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/4680252a
Branch: refs/heads/master
Commit: 4680252a9e4fc9ad63060010050d11e2c8355705
Parents: 55b2a63
Author: Matthew Jacobs <mj...@cloudera.com>
Authored: Wed Jun 22 13:09:59 2016 -0700
Committer: Taras Bobrovytsky <ta...@apache.org>
Committed: Thu Jul 14 19:04:44 2016 +0000
----------------------------------------------------------------------
tests/custom_cluster/test_admission_controller.py | 18 ++++++++++--------
1 file changed, 10 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/4680252a/tests/custom_cluster/test_admission_controller.py
----------------------------------------------------------------------
diff --git a/tests/custom_cluster/test_admission_controller.py b/tests/custom_cluster/test_admission_controller.py
index 5baf35e..dfcdcc7 100644
--- a/tests/custom_cluster/test_admission_controller.py
+++ b/tests/custom_cluster/test_admission_controller.py
@@ -46,8 +46,8 @@ ROUND_ROBIN_SUBMISSION = [True, False]
# pool with the parameters below.
POOL_NAME = "default-pool"
-# The statestore heartbeat of the impala cluster the test is executing against
-STATESTORE_HEARTBEAT_MS = 500
+# The statestore heartbeat and topic update frequency (ms). Set low for testing.
+STATESTORE_RPC_FREQUENCY_MS = 500
# The number of queries that can execute concurrently in the pool POOL_NAME.
MAX_NUM_CONCURRENT_QUERIES = 5
@@ -58,7 +58,9 @@ MAX_NUM_QUEUED_QUERIES = 10
# Mem limit (bytes) used in the mem limit test
MEM_TEST_LIMIT = 12 * 1024 * 1024 * 1024
-_STATESTORED_ARGS = "-statestore_heartbeat_frequency_ms=%s" % (STATESTORE_HEARTBEAT_MS)
+_STATESTORED_ARGS = "-statestore_heartbeat_frequency_ms=%s "\
+ "-statestore_update_frequency_ms=%s" %\
+ (STATESTORE_RPC_FREQUENCY_MS, STATESTORE_RPC_FREQUENCY_MS)
# Key in the query profile for the query options.
PROFILE_QUERY_OPTIONS_KEY = "Query Options (non default): "
@@ -410,7 +412,7 @@ class TestAdmissionControllerStress(TestAdmissionControllerBase):
"Timed out waiting %s seconds for metrics" % (timeout,)
sleep(1)
- def wait_for_heartbeats(self, heartbeats, timeout=30):
+ def wait_for_statestore_updates(self, heartbeats, timeout=30):
"""Waits for a number of statestore heartbeats from all impalads."""
start_time = time()
num_impalads = len(self.impalads)
@@ -422,7 +424,7 @@ class TestAdmissionControllerStress(TestAdmissionControllerBase):
curr[impalad] = init[impalad]
while True:
- LOG.debug("wait_for_heartbeats: curr=%s, init=%s, d=%s", curr.values(),
+ LOG.debug("wait_for_statestore_updates: curr=%s, init=%s, d=%s", curr.values(),
init.values(), [curr[i] - init[i] for i in self.impalads])
if all([curr[i] - init[i] >= heartbeats for i in self.impalads]): break
for impalad in self.impalads:
@@ -430,7 +432,7 @@ class TestAdmissionControllerStress(TestAdmissionControllerBase):
'statestore-subscriber.topic-update-interval-time')['count']
assert (time() - start_time < timeout),\
"Timed out waiting %s seconds for heartbeats" % (timeout,)
- sleep(0.5)
+ sleep(STATESTORE_RPC_FREQUENCY_MS / float(1000))
LOG.debug("Waited %s for %s heartbeats", round(time() - start_time, 1), heartbeats)
def wait_for_admitted_threads(self, num_threads, timeout=30):
@@ -628,9 +630,9 @@ class TestAdmissionControllerStress(TestAdmissionControllerBase):
(metric_deltas, _) = self.wait_for_metric_changes(['admitted'], curr_metrics,
expected_admitted)
self.wait_for_admitted_threads(metric_deltas['admitted'])
- # Wait a few heartbeats to ensure the admission controllers have reached a steady
+ # Wait a few topic updates to ensure the admission controllers have reached a steady
# state or we may find an impalad dequeue more requests after we capture metrics.
- self.wait_for_heartbeats(4)
+ self.wait_for_statestore_updates(10)
final_metrics = self.get_admission_metrics();
log_metrics("Final metrics: ", final_metrics, logging.INFO);