You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ta...@apache.org on 2016/05/14 08:30:07 UTC
[1/5] incubator-impala git commit: Renamed conjunct_ordering.test to
primitive_conjunct_ordering.test in targeted-perf
Repository: incubator-impala
Updated Branches:
refs/heads/master ff0cd823c -> 4c9c74dd3
Renamed conjunct_ordering.test to primitive_conjunct_ordering.test in targeted-perf
This is needed because the workload runner required a prefix of query names to run.
Change-Id: Ica8db68141ef653b0b01a7cfa7773302717a35a2
Reviewed-on: http://gerrit.cloudera.org:8080/3021
Tested-by: Internal Jenkins
Reviewed-by: Thomas Tauber-Marshall <tm...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/11ea79c5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/11ea79c5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/11ea79c5
Branch: refs/heads/master
Commit: 11ea79c5251a15d7a6902084f8a537fdab649cfc
Parents: ff0cd82
Author: Thomas Tauber-Marshall <tm...@cloudera.com>
Authored: Tue May 10 16:25:00 2016 -0700
Committer: Tim Armstrong <ta...@cloudera.com>
Committed: Sat May 14 01:30:00 2016 -0700
----------------------------------------------------------------------
.../queries/conjunct_ordering.test | 73 --------------------
.../queries/primitive_conjunct_ordering.test | 73 ++++++++++++++++++++
2 files changed, 73 insertions(+), 73 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/11ea79c5/testdata/workloads/targeted-perf/queries/conjunct_ordering.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/targeted-perf/queries/conjunct_ordering.test b/testdata/workloads/targeted-perf/queries/conjunct_ordering.test
deleted file mode 100644
index 155aebe..0000000
--- a/testdata/workloads/targeted-perf/queries/conjunct_ordering.test
+++ /dev/null
@@ -1,73 +0,0 @@
-====
----- QUERY: conjunct_ordering_1
--- Description : A simple select with one expensive conjunct and one cheap conjunct.
-SELECT *
-FROM lineitem
-WHERE (l_shipdate < current_timestamp() - interval 1 day) AND l_orderkey = -1;
-====
----- QUERY: conjunct_ordering_2
--- Description: Based on TPCH-Q19
-SELECT sum(l_extendedprice * (1 - l_discount)) AS revenue
-FROM lineitem, part
-WHERE p_partkey = l_partkey
- AND ((p_brand = 'Brand#12'
- AND p_container IN ('SM CASE', 'SM BOX', 'SM PACK', 'SM PKG')
- AND l_quantity >= 1 AND l_quantity <= 11
- AND p_size BETWEEN 1 AND 5
- AND l_shipmode IN ('AIR', 'AIR REG')
- AND l_shipinstruct = 'DELIVER IN PERSON')
- OR (p_brand = 'Brand#23'
- AND p_container IN ('MED BAG', 'MED BOX', 'MED PKG', 'MED PACK')
- AND l_quantity >= 10 AND l_quantity <= 20
- AND p_size BETWEEN 1 AND 10
- AND l_shipmode IN ('AIR', 'AIR REG')
- AND l_shipinstruct = 'DELIVER IN PERSON')
- OR (p_brand = 'Brand#34'
- AND p_container IN ('LG CASE', 'LG BOX', 'LG PACK', 'LG PKG')
- AND l_quantity >= 20 AND l_quantity <= 30
- AND p_size BETWEEN 1 AND 15
- AND l_shipmode IN ('AIR', 'AIR REG')
- AND l_shipinstruct = 'DELIVER IN PERSON'))
- AND (p_partkey = 0 OR l_partkey = 0)
-====
----- QUERY: conjunct_ordering_3
--- Description: Based on TPCDS-Q28
-SELECT avg(l_extendedprice), count(l_extendedprice), count(distinct l_extendedprice)
-FROM lineitem
-WHERE
- (l_discount BETWEEN 0.2 AND 0.3
- OR l_discount BETWEEN 0.4 AND 0.5
- OR l_tax BETWEEN 0.1 AND 0.2
- OR l_tax BETWEEN 0 AND 0.09)
- AND l_extendedprice BETWEEN 100 AND 500
-====
----- QUERY: conjunct_ordering_4
--- Description: Based on TPCDS-Q41
-SELECT p_mfgr, count(*) as item_cnt
-FROM part
-WHERE ((p_type LIKE '%STEEL%'
- AND p_size BETWEEN 10 AND 40
- AND p_container IN ('LG BOX', 'LG BAG', 'LG CASE'))
- OR (p_type LIKE '%COPPER%'
- AND p_size BETWEEN 20 AND 50
- AND p_container IN ('JUMBO BOX', 'JUMBO BAG', 'JUMBO CASE'))
- OR (p_type LIKE '%TIN%'
- AND p_size BETWEEN 0 AND 30
- AND p_container IN ('MED BOX', 'MED BAG', 'MED CASE'))
- OR (p_type LIKE '%BRASS%'
- AND p_size BETWEEN 0 AND 20
- AND p_container IN ('SMALL BOX', 'SMALL BAG', 'SMALL CASE')))
- AND p_partkey = -1
-GROUP BY p_mfgr
-====
----- QUERY: conjunct_ordering_5
--- Description: Based on TPCDS-Q48
-SELECT sum(l_quantity)
-FROM orders, lineitem,
- (SELECT * FROM customer WHERE c_comment LIKE c_comment AND c_nationkey IN (5, 20)) as v1
-WHERE l_orderkey = o_orderkey
- AND (v1.c_custkey = o_custkey
- AND (o_totalprice BETWEEN 1000 AND 2000
- OR o_totalprice BETWEEN 3000 AND 4000
- OR o_totalprice BETWEEN 5000 AND 6000))
-====
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/11ea79c5/testdata/workloads/targeted-perf/queries/primitive_conjunct_ordering.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/targeted-perf/queries/primitive_conjunct_ordering.test b/testdata/workloads/targeted-perf/queries/primitive_conjunct_ordering.test
new file mode 100644
index 0000000..71be122
--- /dev/null
+++ b/testdata/workloads/targeted-perf/queries/primitive_conjunct_ordering.test
@@ -0,0 +1,73 @@
+====
+---- QUERY: primitive_conjunct_ordering_1
+-- Description : A simple select with one expensive conjunct and one cheap conjunct.
+SELECT *
+FROM lineitem
+WHERE (l_shipdate < current_timestamp() - interval 1 day) AND l_orderkey = -1;
+====
+---- QUERY: primitive_conjunct_ordering_2
+-- Description: Based on TPCH-Q19
+SELECT sum(l_extendedprice * (1 - l_discount)) AS revenue
+FROM lineitem, part
+WHERE p_partkey = l_partkey
+ AND ((p_brand = 'Brand#12'
+ AND p_container IN ('SM CASE', 'SM BOX', 'SM PACK', 'SM PKG')
+ AND l_quantity >= 1 AND l_quantity <= 11
+ AND p_size BETWEEN 1 AND 5
+ AND l_shipmode IN ('AIR', 'AIR REG')
+ AND l_shipinstruct = 'DELIVER IN PERSON')
+ OR (p_brand = 'Brand#23'
+ AND p_container IN ('MED BAG', 'MED BOX', 'MED PKG', 'MED PACK')
+ AND l_quantity >= 10 AND l_quantity <= 20
+ AND p_size BETWEEN 1 AND 10
+ AND l_shipmode IN ('AIR', 'AIR REG')
+ AND l_shipinstruct = 'DELIVER IN PERSON')
+ OR (p_brand = 'Brand#34'
+ AND p_container IN ('LG CASE', 'LG BOX', 'LG PACK', 'LG PKG')
+ AND l_quantity >= 20 AND l_quantity <= 30
+ AND p_size BETWEEN 1 AND 15
+ AND l_shipmode IN ('AIR', 'AIR REG')
+ AND l_shipinstruct = 'DELIVER IN PERSON'))
+ AND (p_partkey = 0 OR l_partkey = 0)
+====
+---- QUERY: primitive_conjunct_ordering_3
+-- Description: Based on TPCDS-Q28
+SELECT avg(l_extendedprice), count(l_extendedprice), count(distinct l_extendedprice)
+FROM lineitem
+WHERE
+ (l_discount BETWEEN 0.2 AND 0.3
+ OR l_discount BETWEEN 0.4 AND 0.5
+ OR l_tax BETWEEN 0.1 AND 0.2
+ OR l_tax BETWEEN 0 AND 0.09)
+ AND l_extendedprice BETWEEN 100 AND 500
+====
+---- QUERY: primitive_conjunct_ordering_4
+-- Description: Based on TPCDS-Q41
+SELECT p_mfgr, count(*) as item_cnt
+FROM part
+WHERE ((p_type LIKE '%STEEL%'
+ AND p_size BETWEEN 10 AND 40
+ AND p_container IN ('LG BOX', 'LG BAG', 'LG CASE'))
+ OR (p_type LIKE '%COPPER%'
+ AND p_size BETWEEN 20 AND 50
+ AND p_container IN ('JUMBO BOX', 'JUMBO BAG', 'JUMBO CASE'))
+ OR (p_type LIKE '%TIN%'
+ AND p_size BETWEEN 0 AND 30
+ AND p_container IN ('MED BOX', 'MED BAG', 'MED CASE'))
+ OR (p_type LIKE '%BRASS%'
+ AND p_size BETWEEN 0 AND 20
+ AND p_container IN ('SMALL BOX', 'SMALL BAG', 'SMALL CASE')))
+ AND p_partkey = -1
+GROUP BY p_mfgr
+====
+---- QUERY: primitive_conjunct_ordering_5
+-- Description: Based on TPCDS-Q48
+SELECT sum(l_quantity)
+FROM orders, lineitem,
+ (SELECT * FROM customer WHERE c_comment LIKE c_comment AND c_nationkey IN (5, 20)) as v1
+WHERE l_orderkey = o_orderkey
+ AND (v1.c_custkey = o_custkey
+ AND (o_totalprice BETWEEN 1000 AND 2000
+ OR o_totalprice BETWEEN 3000 AND 4000
+ OR o_totalprice BETWEEN 5000 AND 6000))
+====
[5/5] incubator-impala git commit: Kudu: Exclude non-Kudu symbols
during stub client generation
Posted by ta...@apache.org.
Kudu: Exclude non-Kudu symbols during stub client generation
Previously boost related symbols (and others) would get defined in the
Kudu client stub with a non-functional implementation. If these
implementations were used at runtime they would crash Impala.
Change-Id: I54292095692ce38c255a8df48cf8f3f655d797b0
Reviewed-on: http://gerrit.cloudera.org:8080/2864
Reviewed-by: Tim Armstrong <ta...@cloudera.com>
Tested-by: Internal Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/cd193f06
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/cd193f06
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/cd193f06
Branch: refs/heads/master
Commit: cd193f063f02bf7921c2d0f5b1231f91deed6694
Parents: 11ea79c
Author: Casey Ching <ca...@cloudera.com>
Authored: Mon Apr 25 13:54:57 2016 -0700
Committer: Tim Armstrong <ta...@cloudera.com>
Committed: Sat May 14 01:30:01 2016 -0700
----------------------------------------------------------------------
bin/bootstrap_toolchain.py | 174 +++++++++++++++++++++-------------------
1 file changed, 91 insertions(+), 83 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/cd193f06/bin/bootstrap_toolchain.py
----------------------------------------------------------------------
diff --git a/bin/bootstrap_toolchain.py b/bin/bootstrap_toolchain.py
index 3009332..2589d1f 100755
--- a/bin/bootstrap_toolchain.py
+++ b/bin/bootstrap_toolchain.py
@@ -124,6 +124,80 @@ def bootstrap(packages):
write_version_file(toolchain_root, pkg_name, pkg_version, compiler,
get_platform_release_label())
+def check_output(cmd_args):
+ """Run the command and return the output. Raise an exception if the command returns
+ a non-zero return code. Similar to subprocess.check_output() which is only provided
+ in python 2.7.
+ """
+ process = subprocess.Popen(cmd_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+ stdout, _ = process.communicate()
+ if process.wait() != 0:
+ raise Exception("Command with args '%s' failed with exit code %s:\n%s"
+ % (cmd_args, process.returncode, stdout))
+ return stdout
+
+def package_directory(toolchain_root, pkg_name, pkg_version):
+ dir_name = "{0}-{1}".format(pkg_name, pkg_version)
+ return os.path.join(toolchain_root, dir_name)
+
+def version_file_path(toolchain_root, pkg_name, pkg_version):
+ return os.path.join(package_directory(toolchain_root, pkg_name, pkg_version),
+ "toolchain_package_version.txt")
+
+def check_custom_toolchain(toolchain_root, packages):
+ missing = []
+ for p in packages:
+ pkg_name, pkg_version = unpack_name_and_version(p)
+ pkg_dir = package_directory(toolchain_root, pkg_name, pkg_version)
+ if not os.path.isdir(pkg_dir):
+ missing.append((p, pkg_dir))
+
+ if missing:
+ print("The following packages are not in their expected locations.")
+ for p, pkg_dir in missing:
+ print(" %s (expected directory %s to exist)" % (p, pkg_dir))
+ print("Pre-built toolchain archives not available for your platform.")
+ print("Clone and build native toolchain from source using this repository:")
+ print(" https://github.com/cloudera/native-toolchain")
+ raise Exception("Toolchain bootstrap failed: required packages were missing")
+
+def check_for_existing_package(toolchain_root, pkg_name, pkg_version, compiler):
+ """Return true if toolchain_root already contains the package with the correct
+ version and compiler.
+ """
+ version_file = version_file_path(toolchain_root, pkg_name, pkg_version)
+ if not os.path.exists(version_file):
+ return False
+
+ label = get_platform_release_label()
+ pkg_version_string = "{0}-{1}-{2}-{3}".format(pkg_name, pkg_version, compiler, label)
+ with open(version_file) as f:
+ return f.read().strip() == pkg_version_string
+
+def write_version_file(toolchain_root, pkg_name, pkg_version, compiler, label):
+ with open(version_file_path(toolchain_root, pkg_name, pkg_version), 'w') as f:
+ f.write("{0}-{1}-{2}-{3}".format(pkg_name, pkg_version, compiler, label))
+
+def remove_existing_package(toolchain_root, pkg_name, pkg_version):
+ dir_path = package_directory(toolchain_root, pkg_name, pkg_version)
+ if os.path.exists(dir_path):
+ print "Removing existing package directory {0}".format(dir_path)
+ shutil.rmtree(dir_path)
+
+def unpack_name_and_version(package):
+ """A package definition is either a string where the version is fetched from the
+ environment or a tuple where the package name and the package version are fully
+ specified.
+ """
+ if isinstance(package, basestring):
+ env_var = "IMPALA_{0}_VERSION".format(package).replace("-", "_").upper()
+ try:
+ return package, os.environ[env_var]
+ except KeyError:
+ raise Exception("Could not find version for {0} in environment var {1}".format(
+ package, env_var))
+ return package[0], package[1]
+
def build_kudu_stub(toolchain_root, kudu_version, compiler):
# When Kudu isn't supported, the CentOS 7 package will be downloaded and the client
# lib will be replaced with a stubbed client.
@@ -159,7 +233,7 @@ def build_kudu_stub(toolchain_root, kudu_version, compiler):
# Extract the symbols and write the stubbed client source. There is a special method
# kudu::client::GetShortVersionString() that is overridden so that the stub can be
# identified by the caller.
- get_short_version_symbol = "_ZN4kudu6client21GetShortVersionStringEv"
+ get_short_version_sig = "kudu::client::GetShortVersionString()"
nm_out = check_output([nm_path, "--defined-only", "-D", client_lib_path])
stub_build_dir = tempfile.mkdtemp()
stub_client_src_file = open(os.path.join(stub_build_dir, "kudu_client.cc"), "w")
@@ -178,22 +252,30 @@ std::string GetShortVersionString() { return kFakeKuduVersion; }
}}
""")
found_start_version_symbol = False
+ cpp_filt_path = os.path.join(binutils_dir, "bin", "c++filt")
for line in nm_out.splitlines():
- addr, sym_type, name = line.split(" ")
- if name in ["_init", "_fini"]:
+ addr, sym_type, mangled_name = line.split(" ")
+ # Skip special functions an anything that isn't a strong symbol. Any symbols that
+ # get passed this check must be related to Kudu. If a symbol unrelated to Kudu
+ # (ex: a boost symbol) gets defined in the stub, there's a chance the symbol could
+ # get used and crash Impala.
+ if mangled_name in ["_init", "_fini"] or sym_type not in "Tt":
continue
- if name == get_short_version_symbol:
+ demangled_name = check_output([cpp_filt_path, mangled_name]).strip()
+ assert "kudu" in demangled_name, \
+ "Symbol doesn't appear to be related to Kudu: " + demangled_name
+ if demangled_name == get_short_version_sig:
found_start_version_symbol = True
continue
- if sym_type.upper() in "TW":
- stub_client_src_file.write("""
+ stub_client_src_file.write("""
extern "C" void %s() {
KuduNotSupported();
}
-""" % name)
+""" % mangled_name)
+
if not found_start_version_symbol:
- raise Exception("Expected to find symbol " + get_short_version_symbol +
- " corresponding to kudu::client::GetShortVersionString() but it was not found.")
+ raise Exception("Expected to find symbol a corresponding to"
+ " %s but it was not found." % get_short_version_sig)
stub_client_src_file.flush()
# The soname is needed to avoid problem in packaging builds. Without the soname,
@@ -221,80 +303,6 @@ extern "C" void %s() {
finally:
shutil.rmtree(stub_build_dir)
-def check_output(cmd_args):
- """Run the command and return the output. Raise an exception if the command returns
- a non-zero return code. Similar to subprocess.check_output() which is only provided
- in python 2.7.
- """
- process = subprocess.Popen(cmd_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
- stdout, _ = process.communicate()
- if process.wait() != 0:
- raise Exception("Command with args '%s' failed with exit code %s:\n%s"
- % (cmd_args, process.returncode, stdout))
- return stdout
-
-def package_directory(toolchain_root, pkg_name, pkg_version):
- dir_name = "{0}-{1}".format(pkg_name, pkg_version)
- return os.path.join(toolchain_root, dir_name)
-
-def version_file_path(toolchain_root, pkg_name, pkg_version):
- return os.path.join(package_directory(toolchain_root, pkg_name, pkg_version),
- "toolchain_package_version.txt")
-
-def check_custom_toolchain(toolchain_root, packages):
- missing = []
- for p in packages:
- pkg_name, pkg_version = unpack_name_and_version(p)
- pkg_dir = package_directory(toolchain_root, pkg_name, pkg_version)
- if not os.path.isdir(pkg_dir):
- missing.append((p, pkg_dir))
-
- if missing:
- print("The following packages are not in their expected locations.")
- for p, pkg_dir in missing:
- print(" %s (expected directory %s to exist)" % (p, pkg_dir))
- print("Pre-built toolchain archives not available for your platform.")
- print("Clone and build native toolchain from source using this repository:")
- print(" https://github.com/cloudera/native-toolchain")
- raise Exception("Toolchain bootstrap failed: required packages were missing")
-
-def check_for_existing_package(toolchain_root, pkg_name, pkg_version, compiler):
- """Return true if toolchain_root already contains the package with the correct
- version and compiler.
- """
- version_file = version_file_path(toolchain_root, pkg_name, pkg_version)
- if not os.path.exists(version_file):
- return False
-
- label = get_platform_release_label()
- pkg_version_string = "{0}-{1}-{2}-{3}".format(pkg_name, pkg_version, compiler, label)
- with open(version_file) as f:
- return f.read().strip() == pkg_version_string
-
-def write_version_file(toolchain_root, pkg_name, pkg_version, compiler, label):
- with open(version_file_path(toolchain_root, pkg_name, pkg_version), 'w') as f:
- f.write("{0}-{1}-{2}-{3}".format(pkg_name, pkg_version, compiler, label))
-
-def remove_existing_package(toolchain_root, pkg_name, pkg_version):
- dir_path = package_directory(toolchain_root, pkg_name, pkg_version)
- if os.path.exists(dir_path):
- print "Removing existing package directory {0}".format(dir_path)
- shutil.rmtree(dir_path)
-
-def unpack_name_and_version(package):
- """A package definition is either a string where the version is fetched from the
- environment or a tuple where the package name and the package version are fully
- specified.
- """
- if isinstance(package, basestring):
- env_var = "IMPALA_{0}_VERSION".format(package).replace("-", "_").upper()
- try:
- return package, os.environ[env_var]
- except KeyError:
- raise Exception("Could not find version for {0} in environment var {1}".format(
- package, env_var))
- return package[0], package[1]
-
if __name__ == "__main__":
packages = ["avro", "binutils", "boost", "breakpad", "bzip2", "gcc", "gflags", "glog",
"gperftools", "gtest", "kudu", "llvm", ("llvm", "3.8.0-asserts-p1"), "lz4",
[2/5] incubator-impala git commit: IMPALA-3511: Fix race setting up
TestKuduOperations
Posted by ta...@apache.org.
IMPALA-3511: Fix race setting up TestKuduOperations
A couple of tests could both attempt to create/destroy the same
database if they were running in parallel. Several other related
tests were marked as requiring serial execution, these needed to be
marked for serial execution as well.
Change-Id: If0573a755cd371363c2e43c001d5c1ba499793c6
Reviewed-on: http://gerrit.cloudera.org:8080/3063
Reviewed-by: Casey Ching <ca...@cloudera.com>
Tested-by: Internal Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/e61b5bc1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/e61b5bc1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/e61b5bc1
Branch: refs/heads/master
Commit: e61b5bc119f712fdcb9b0326302c466add65a217
Parents: cd193f0
Author: Casey Ching <ca...@cloudera.com>
Authored: Fri May 13 12:49:38 2016 -0700
Committer: Tim Armstrong <ta...@cloudera.com>
Committed: Sat May 14 01:30:01 2016 -0700
----------------------------------------------------------------------
tests/query_test/test_kudu.py | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/e61b5bc1/tests/query_test/test_kudu.py
----------------------------------------------------------------------
diff --git a/tests/query_test/test_kudu.py b/tests/query_test/test_kudu.py
index a8d7545..cc478fe 100644
--- a/tests/query_test/test_kudu.py
+++ b/tests/query_test/test_kudu.py
@@ -70,6 +70,7 @@ class TestKuduOperations(ImpalaTestSuite):
def teardown_method(self, method):
self.cleanup_db("kududb_test")
+ @pytest.mark.execute_serially
def test_kudu_scan_node(self, vector):
self.run_test_case('QueryTest/kudu-scan-node', vector, use_db="functional_kudu",
wait_secs_between_stmts=1)
@@ -87,6 +88,7 @@ class TestKuduOperations(ImpalaTestSuite):
def test_kudu_alter_table(self, vector):
self.run_test_case('QueryTest/kudu_alter', vector, use_db="kududb_test")
+ @pytest.mark.execute_serially
def test_kudu_stats(self, vector):
self.run_test_case('QueryTest/kudu_stats', vector, use_db="kududb_test")
[3/5] incubator-impala git commit: IMPALA-3539: Return error status
if def/rep level caches failed to allocate.
Posted by ta...@apache.org.
IMPALA-3539: Return error status if def/rep level caches failed to allocate.
The information in the JIRA is consistent with a failure to allocate
memory for the def level cache. There was a bug where this failure status
was not properly propagated, so eventually a DCHECK was hit that expected
the cache memory to be allocated.
Change-Id: I38856e6e1f5fbdbf5327cf31a2a109e6c930901d
Reviewed-on: http://gerrit.cloudera.org:8080/3065
Reviewed-by: Alex Behm <al...@cloudera.com>
Tested-by: Internal Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/b4558d38
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/b4558d38
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/b4558d38
Branch: refs/heads/master
Commit: b4558d384ee103e8cc1b7456e504ac1ef07180be
Parents: e61b5bc
Author: Alex Behm <al...@cloudera.com>
Authored: Fri May 13 14:03:58 2016 -0700
Committer: Tim Armstrong <ta...@cloudera.com>
Committed: Sat May 14 01:30:01 2016 -0700
----------------------------------------------------------------------
be/src/exec/hdfs-parquet-scanner.cc | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b4558d38/be/src/exec/hdfs-parquet-scanner.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/hdfs-parquet-scanner.cc b/be/src/exec/hdfs-parquet-scanner.cc
index 3fcb693..a70fc8a 100644
--- a/be/src/exec/hdfs-parquet-scanner.cc
+++ b/be/src/exec/hdfs-parquet-scanner.cc
@@ -1490,17 +1490,17 @@ Status HdfsParquetScanner::BaseScalarColumnReader::ReadDataPage() {
}
// Initialize the repetition level data
- rep_levels_.Init(filename(),
+ RETURN_IF_ERROR(rep_levels_.Init(filename(),
current_page_header_.data_page_header.repetition_level_encoding,
parent_->level_cache_pool_.get(), parent_->state_->batch_size(),
max_rep_level(), num_buffered_values_,
- &data_, &data_size);
+ &data_, &data_size));
// Initialize the definition level data
- def_levels_.Init(filename(),
+ RETURN_IF_ERROR(def_levels_.Init(filename(),
current_page_header_.data_page_header.definition_level_encoding,
parent_->level_cache_pool_.get(), parent_->state_->batch_size(),
- max_def_level(), num_buffered_values_, &data_, &data_size);
+ max_def_level(), num_buffered_values_, &data_, &data_size));
// Data can be empty if the column contains all NULLs
if (data_size != 0) RETURN_IF_ERROR(InitDataPage(data_, data_size));
[4/5] incubator-impala git commit: IMPALA-3532: S3:
test_truncate_cleans_hdfs_files fails because we skip INSERT staging
Posted by ta...@apache.org.
IMPALA-3532: S3: test_truncate_cleans_hdfs_files fails because we skip INSERT staging
On the introduction of IMPALA-3452, we defaulted to skipping the
INSERT staging for S3. test_truncate_cleans_hdfs_files assumes that
it will always see the _impala_insert_staging folder but we will not
see that on S3 runs.
This patch deletes the staging folder if it exists and continues the
test without taking into account the staging folder.
Change-Id: I3580f03690e29fe99f441b26bc9baa4c0964d79c
Reviewed-on: http://gerrit.cloudera.org:8080/3049
Reviewed-by: Sailesh Mukil <sa...@cloudera.com>
Tested-by: Internal Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/4c9c74dd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/4c9c74dd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/4c9c74dd
Branch: refs/heads/master
Commit: 4c9c74dd33e0b84c3a3328df0a1721d627267192
Parents: b4558d3
Author: Sailesh Mukil <sa...@cloudera.com>
Authored: Thu May 12 13:07:58 2016 -0700
Committer: Tim Armstrong <ta...@cloudera.com>
Committed: Sat May 14 01:30:01 2016 -0700
----------------------------------------------------------------------
tests/metadata/test_ddl.py | 54 +++++++++++++++++++++++------------------
1 file changed, 30 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/4c9c74dd/tests/metadata/test_ddl.py
----------------------------------------------------------------------
diff --git a/tests/metadata/test_ddl.py b/tests/metadata/test_ddl.py
index 0a1900c..6179d1a 100644
--- a/tests/metadata/test_ddl.py
+++ b/tests/metadata/test_ddl.py
@@ -183,30 +183,36 @@ class TestDdlStatements(ImpalaTestSuite):
# Verify the table directory exists
assert self.filesystem_client.exists("test-warehouse/truncate_table_test_db.db/t1/")
- # Should have created one file in the table's dir
- self.client.execute("insert into %s.t1 values (1)" % TRUNCATE_DB)
- assert len(self.filesystem_client.ls("test-warehouse/%s.db/t1/" % TRUNCATE_DB)) == 2
-
- # Truncating the table removes the data files and preserves the table's directory
- self.client.execute("truncate table %s.t1" % TRUNCATE_DB)
- assert len(self.filesystem_client.ls("test-warehouse/%s.db/t1/" % TRUNCATE_DB)) == 1
-
- self.client.execute(
- "create table %s.t2(i int) partitioned by (p int)" % TRUNCATE_DB)
- # Verify the table directory exists
- assert self.filesystem_client.exists("test-warehouse/%s.db/t2/" % TRUNCATE_DB)
-
- # Should have created the partition dir, which should contain exactly one file
- self.client.execute(
- "insert into %s.t2 partition(p=1) values (1)" % TRUNCATE_DB)
- assert len(self.filesystem_client.ls(
- "test-warehouse/%s.db/t2/p=1" % TRUNCATE_DB)) == 1
-
- # Truncating the table removes the data files and preserves the partition's directory
- self.client.execute("truncate table %s.t2" % TRUNCATE_DB)
- assert self.filesystem_client.exists("test-warehouse/%s.db/t2/p=1" % TRUNCATE_DB)
- assert len(self.filesystem_client.ls(
- "test-warehouse/%s.db/t2/p=1" % TRUNCATE_DB)) == 0
+ try:
+ # If we're testing S3, we want the staging directory to be created.
+ self.client.execute("set s3_skip_insert_staging=false")
+ # Should have created one file in the table's dir
+ self.client.execute("insert into %s.t1 values (1)" % TRUNCATE_DB)
+ assert len(self.filesystem_client.ls("test-warehouse/%s.db/t1/" % TRUNCATE_DB)) == 2
+
+ # Truncating the table removes the data files and preserves the table's directory
+ self.client.execute("truncate table %s.t1" % TRUNCATE_DB)
+ assert len(self.filesystem_client.ls("test-warehouse/%s.db/t1/" % TRUNCATE_DB)) == 1
+
+ self.client.execute(
+ "create table %s.t2(i int) partitioned by (p int)" % TRUNCATE_DB)
+ # Verify the table directory exists
+ assert self.filesystem_client.exists("test-warehouse/%s.db/t2/" % TRUNCATE_DB)
+
+ # Should have created the partition dir, which should contain exactly one file
+ self.client.execute(
+ "insert into %s.t2 partition(p=1) values (1)" % TRUNCATE_DB)
+ assert len(self.filesystem_client.ls(
+ "test-warehouse/%s.db/t2/p=1" % TRUNCATE_DB)) == 1
+
+ # Truncating the table removes the data files and preserves the partition's directory
+ self.client.execute("truncate table %s.t2" % TRUNCATE_DB)
+ assert self.filesystem_client.exists("test-warehouse/%s.db/t2/p=1" % TRUNCATE_DB)
+ assert len(self.filesystem_client.ls(
+ "test-warehouse/%s.db/t2/p=1" % TRUNCATE_DB)) == 0
+ finally:
+ # Reset to its default value.
+ self.client.execute("set s3_skip_insert_staging=true")
@pytest.mark.execute_serially
def test_truncate_table(self, vector):