You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by jo...@apache.org on 2020/04/01 14:49:06 UTC
[impala] branch branch-3.4.0 updated (45f89da -> 8842ccf)
This is an automated email from the ASF dual-hosted git repository.
joemcdonnell pushed a change to branch branch-3.4.0
in repository https://gitbox.apache.org/repos/asf/impala.git.
from 45f89da IMPALA-9560: Fix TestStatsExtrapolation for release versions
new 946fa63 IMPALA-9451: Fix test_hive_text_codec_interop.py failure in CDP build
new 8842ccf IMPALA-9582: Upgrade thrift_sasl to 0.4.2 for impala-shell
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
LICENSE.txt | 2 +-
bin/rat_exclude_files.txt | 2 +-
infra/python/deps/compiled-requirements.txt | 2 +-
shell/.gitignore | 6 +++---
.../CHANGELOG.md | 5 +++++
.../{thrift_sasl-0.4.1 => thrift_sasl-0.4.2}/LICENSE | 0
.../README.md | 0
.../{thrift_sasl-0.4.1 => thrift_sasl-0.4.2}/setup.py | 2 +-
.../thrift_sasl/__init__.py | 17 ++++++++++++-----
shell/packaging/requirements.txt | 2 +-
tests/custom_cluster/test_hive_text_codec_interop.py | 19 +++++++++++++++++--
11 files changed, 42 insertions(+), 15 deletions(-)
rename shell/ext-py/{thrift_sasl-0.4.1 => thrift_sasl-0.4.2}/CHANGELOG.md (64%)
rename shell/ext-py/{thrift_sasl-0.4.1 => thrift_sasl-0.4.2}/LICENSE (100%)
rename shell/ext-py/{thrift_sasl-0.4.1 => thrift_sasl-0.4.2}/README.md (100%)
rename shell/ext-py/{thrift_sasl-0.4.1 => thrift_sasl-0.4.2}/setup.py (98%)
rename shell/ext-py/{thrift_sasl-0.4.1 => thrift_sasl-0.4.2}/thrift_sasl/__init__.py (94%)
[impala] 01/02: IMPALA-9451: Fix test_hive_text_codec_interop.py
failure in CDP build
Posted by jo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
joemcdonnell pushed a commit to branch branch-3.4.0
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 946fa636fc05f3f6b32f52291edc1ea4105a2b0d
Author: xiaomeng <xi...@cloudera.com>
AuthorDate: Fri Mar 20 17:30:54 2020 -0700
IMPALA-9451: Fix test_hive_text_codec_interop.py failure in CDP build
In CDP build we use Hive3 which has a bug HIVE-22371 (CTAS puts
files in the wrong place). It causes failure of newly added test as
CTAS creates empty table.
Workaround by explicitly creating an external table when hive
version >= 3.
Tested:
Run this test in newest CDP build using job
impala-private-basic-parameterized.
Change-Id: Ief8e583aae82f548754f41e07efac5d7bca4b930
Reviewed-on: http://gerrit.cloudera.org:8080/15520
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
(cherry picked from commit 66de7808c7e1b017e442939d6dd6fd58017e4a1b)
---
tests/custom_cluster/test_hive_text_codec_interop.py | 19 +++++++++++++++++--
1 file changed, 17 insertions(+), 2 deletions(-)
diff --git a/tests/custom_cluster/test_hive_text_codec_interop.py b/tests/custom_cluster/test_hive_text_codec_interop.py
index daaad8f..556a0e0 100644
--- a/tests/custom_cluster/test_hive_text_codec_interop.py
+++ b/tests/custom_cluster/test_hive_text_codec_interop.py
@@ -20,6 +20,7 @@
import pytest
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
+from tests.common.environ import HIVE_MAJOR_VERSION
from tests.common.skip import SkipIfS3
from tests.common.test_dimensions import create_exec_option_dimension
from tests.common.test_result_verifier import verify_query_result_is_equal
@@ -66,6 +67,15 @@ class TestTextInterop(CustomClusterTestSuite):
"insert into {0}(id) values (7777), (8888), (9999), (11111), (22222), (33333)"
.format(source_table))
+ # For Hive 3+, workaround for HIVE-22371 (CTAS puts files in the wrong place) by
+ # explicitly creating an external table so that files are in the external warehouse
+ # directory. Use external.table.purge=true so that it is equivalent to a Hive 2
+ # managed table. Hive 2 stays the same.
+ external = ""
+ tblproperties = ""
+ if HIVE_MAJOR_VERSION >= 3:
+ external = "external"
+ tblproperties = "TBLPROPERTIES('external.table.purge'='TRUE')"
# Loop through the compression codecs and run interop tests.
for codec in TEXT_CODECS:
# Write data in Hive and read from Impala
@@ -83,8 +93,13 @@ class TestTextInterop(CustomClusterTestSuite):
self.run_stmt_in_hive("drop table if exists {0}".format(hive_table))
self.run_stmt_in_hive("set hive.exec.compress.output=true;\
set mapreduce.output.fileoutputformat.compress.codec={0};\
- create table {1} stored as textfile as select * from {2}"
- .format(switcher.get(codec, 'Invalid codec'), hive_table, source_table))
+ create {1} table {2} stored as textfile {3} as select * from {4}"
+ .format(switcher.get(codec, 'Invalid codec'), external, hive_table,
+ tblproperties, source_table))
+
+ # Make sure hive CTAS table is not empty
+ assert self.run_stmt_in_hive("select count(*) from {0}".format(
+ hive_table)).split("\n")[1] != "0", "CTAS created Hive table is empty."
# Make sure Impala's metadata is in sync.
if cluster_properties.is_catalog_v2_cluster():
[impala] 02/02: IMPALA-9582: Upgrade thrift_sasl to 0.4.2 for
impala-shell
Posted by jo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
joemcdonnell pushed a commit to branch branch-3.4.0
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 8842ccf351c4f7f643e25aa00a9001ee0822e9af
Author: David Knupp <dk...@cloudera.com>
AuthorDate: Mon Mar 30 18:29:44 2020 -0700
IMPALA-9582: Upgrade thrift_sasl to 0.4.2 for impala-shell
Change-Id: Iff739ebeaf5b022a7418883b638b5c5d17885f3b
Reviewed-on: http://gerrit.cloudera.org:8080/15610
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
(cherry picked from commit 5c541512f080327686fe0ad6fcf4ef90ca048007)
---
LICENSE.txt | 2 +-
bin/rat_exclude_files.txt | 2 +-
infra/python/deps/compiled-requirements.txt | 2 +-
shell/.gitignore | 6 +++---
.../CHANGELOG.md | 5 +++++
.../{thrift_sasl-0.4.1 => thrift_sasl-0.4.2}/LICENSE | 0
.../{thrift_sasl-0.4.1 => thrift_sasl-0.4.2}/README.md | 0
.../{thrift_sasl-0.4.1 => thrift_sasl-0.4.2}/setup.py | 2 +-
.../thrift_sasl/__init__.py | 17 ++++++++++++-----
shell/packaging/requirements.txt | 2 +-
10 files changed, 25 insertions(+), 13 deletions(-)
diff --git a/LICENSE.txt b/LICENSE.txt
index 6786ce0..c12850c 100644
--- a/LICENSE.txt
+++ b/LICENSE.txt
@@ -697,7 +697,7 @@ be/src/thirdparty/mustache: Apache 2.0 license
be/src/thirdparty/pcg-cpp-0.98: Apache 2.0 license
be/src/expr/hll-bias.h: Apache 2.0 license
shell/ext-py/sasl-0.1.1: Apache 2.0 license
-shell/ext-py/thrift_sasl-0.4.1: Apache 2.0 license
+shell/ext-py/thrift_sasl-0.4.2: Apache 2.0 license
--------------------------------------------------------------------------------
diff --git a/bin/rat_exclude_files.txt b/bin/rat_exclude_files.txt
index 0a4f6c5..29851b6 100644
--- a/bin/rat_exclude_files.txt
+++ b/bin/rat_exclude_files.txt
@@ -46,7 +46,7 @@ shell/ext-py/prettytable-0.7.1/*
shell/ext-py/sasl-0.1.1/*
shell/ext-py/six-1.14.0/*
shell/ext-py/sqlparse-0.1.19/*
-shell/ext-py/thrift_sasl-0.4.1/*
+shell/ext-py/thrift_sasl-0.4.2/*
www/d3.v3.min.js
www/jquery/jquery-3.4.1.min.js
tests/comparison/leopard/static/css/hljs.css
diff --git a/infra/python/deps/compiled-requirements.txt b/infra/python/deps/compiled-requirements.txt
index 0ff2586..061e286 100644
--- a/infra/python/deps/compiled-requirements.txt
+++ b/infra/python/deps/compiled-requirements.txt
@@ -23,7 +23,7 @@ impyla == 0.16.2
bitarray == 1.2.1
sasl == 0.2.1
six == 1.14.0
- thrift_sasl == 0.4.1
+ thrift_sasl == 0.4.2
psutil == 5.6.3
# Required for Kudu:
Cython == 0.23.4
diff --git a/shell/.gitignore b/shell/.gitignore
index 60a2275..380b3fd 100644
--- a/shell/.gitignore
+++ b/shell/.gitignore
@@ -13,9 +13,9 @@ ext-py/sqlparse-0.1.19/build/
ext-py/sqlparse-0.1.19/sqlparse.egg-info/
ext-py/bitarray-0.9.0/bitarray.egg-info/
ext-py/bitarray-0.9.0/dist/
-ext-py/thrift_sasl-0.4.1/dist/
-ext-py/thrift_sasl-0.4.1/build/
-ext-py/thrift_sasl-0.4.1/six.egg-info/
+ext-py/thrift_sasl-0.4.2/dist/
+ext-py/thrift_sasl-0.4.2/build/
+ext-py/thrift_sasl-0.4.2/six.egg-info/
# This file is used by buildall.sh to find files that need to be removed during the
# clean phase. Previous version of deps should be kept here for cleaning otherwise they
diff --git a/shell/ext-py/thrift_sasl-0.4.1/CHANGELOG.md b/shell/ext-py/thrift_sasl-0.4.2/CHANGELOG.md
similarity index 64%
rename from shell/ext-py/thrift_sasl-0.4.1/CHANGELOG.md
rename to shell/ext-py/thrift_sasl-0.4.2/CHANGELOG.md
index 08ac33a..28b90cf 100644
--- a/shell/ext-py/thrift_sasl-0.4.1/CHANGELOG.md
+++ b/shell/ext-py/thrift_sasl-0.4.2/CHANGELOG.md
@@ -1,6 +1,11 @@
Changelog
=========
+0.4.2
+------
+* **Bug Fixes**
+ - Fixes a bug where Thrift transport was not reading all data (#22)
+
0.4.1
------
* **Bug Fixes**
diff --git a/shell/ext-py/thrift_sasl-0.4.1/LICENSE b/shell/ext-py/thrift_sasl-0.4.2/LICENSE
similarity index 100%
rename from shell/ext-py/thrift_sasl-0.4.1/LICENSE
rename to shell/ext-py/thrift_sasl-0.4.2/LICENSE
diff --git a/shell/ext-py/thrift_sasl-0.4.1/README.md b/shell/ext-py/thrift_sasl-0.4.2/README.md
similarity index 100%
rename from shell/ext-py/thrift_sasl-0.4.1/README.md
rename to shell/ext-py/thrift_sasl-0.4.2/README.md
diff --git a/shell/ext-py/thrift_sasl-0.4.1/setup.py b/shell/ext-py/thrift_sasl-0.4.2/setup.py
similarity index 98%
rename from shell/ext-py/thrift_sasl-0.4.1/setup.py
rename to shell/ext-py/thrift_sasl-0.4.2/setup.py
index 26c51db..81d8d80 100644
--- a/shell/ext-py/thrift_sasl-0.4.1/setup.py
+++ b/shell/ext-py/thrift_sasl-0.4.2/setup.py
@@ -26,7 +26,7 @@ description = ("Thrift SASL Python module that implements SASL transports for "
setup(
name='thrift_sasl',
- version='0.4.1',
+ version='0.4.2',
description=description,
long_description=description,
url='https://github.com/cloudera/thrift_sasl',
diff --git a/shell/ext-py/thrift_sasl-0.4.1/thrift_sasl/__init__.py b/shell/ext-py/thrift_sasl-0.4.2/thrift_sasl/__init__.py
similarity index 94%
rename from shell/ext-py/thrift_sasl-0.4.1/thrift_sasl/__init__.py
rename to shell/ext-py/thrift_sasl-0.4.2/thrift_sasl/__init__.py
index 534b7b2..17fde01 100644
--- a/shell/ext-py/thrift_sasl-0.4.1/thrift_sasl/__init__.py
+++ b/shell/ext-py/thrift_sasl-0.4.2/thrift_sasl/__init__.py
@@ -109,10 +109,10 @@ class TSaslClientTransport(TTransportBase, CReadableTransport):
self._trans.flush()
def _recv_sasl_message(self):
- header = self._trans.read(5)
+ header = self._trans_read_all(5)
status, length = struct.unpack(">BI", header)
if length > 0:
- payload = self._trans.read(length)
+ payload = self._trans_read_all(length)
else:
payload = ""
return status, payload
@@ -174,22 +174,29 @@ class TSaslClientTransport(TTransportBase, CReadableTransport):
return ret + self.__rbuf.read(sz - len(ret))
def _read_frame(self):
- header = self._trans.read(4)
+ header = self._trans_read_all(4)
(length,) = struct.unpack(">I", header)
if self.encode:
# If the frames are encoded (i.e. you're using a QOP of auth-int or
# auth-conf), then make sure to include the header in the bytes you send to
# sasl.decode()
- encoded = header + self._trans.read(length)
+ encoded = header + self._trans_read_all(length)
success, decoded = self.sasl.decode(encoded)
if not success:
raise TTransportException(type=TTransportException.UNKNOWN,
message=self.sasl.getError())
else:
# If the frames are not encoded, just pass it through
- decoded = self._trans.read(length)
+ decoded = self._trans_read_all(length)
self.__rbuf = BufferIO(decoded)
+ def _trans_read_all(self, sz):
+ try:
+ read_all = self._trans.readAll # Thrift
+ except AttributeError:
+ read_all = self._trans.read # thriftpy
+ return read_all(sz)
+
def close(self):
self._trans.close()
self.sasl = None
diff --git a/shell/packaging/requirements.txt b/shell/packaging/requirements.txt
index cc84d3f..878b2d5 100644
--- a/shell/packaging/requirements.txt
+++ b/shell/packaging/requirements.txt
@@ -5,4 +5,4 @@ setuptools>=36.8.0
six==1.14.0
sqlparse==0.1.19
thrift==0.9.3
-thrift_sasl==0.4.1
+thrift_sasl==0.4.2