You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by jo...@apache.org on 2023/02/28 17:12:53 UTC
[impala] 06/06: IMPALA-11952 (part 4): Fix odds and ends: Octals, long, lambda, etc.
This is an automated email from the ASF dual-hosted git repository.
joemcdonnell pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit ba3518366a503efceb859102c8e6d6450083ae38
Author: Joe McDonnell <jo...@cloudera.com>
AuthorDate: Sun Feb 26 14:40:24 2023 -0800
IMPALA-11952 (part 4): Fix odds and ends: Octals, long, lambda, etc.
There are a variety of small python 3 syntax differences:
- Octal constants need to start with 0o rather than just 0
- Long constants are not supported (i.e. numbers ending with L)
- Lambda syntax is slightly different
- The 'ur' string mode is no longer supported
Testing:
- check-python-syntax.sh now passes
Change-Id: Ie027a50ddf6a2a0db4b34ec9b49484ce86947f20
Reviewed-on: http://gerrit.cloudera.org:8080/19554
Reviewed-by: Joe McDonnell <jo...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Reviewed-by: Michael Smith <mi...@cloudera.com>
---
bin/check-rat-report.py | 2 +-
docker/test-with-docker.py | 2 +-
tests/comparison/data_generator_mapred_common.py | 2 +-
tests/comparison/leopard/front_end.py | 2 +-
tests/comparison/leopard/impala_docker_env.py | 3 +-
tests/comparison/leopard/report.py | 40 ++++++++++++------------
tests/custom_cluster/test_breakpad.py | 2 +-
tests/custom_cluster/test_client_ssl.py | 2 +-
tests/custom_cluster/test_thrift_socket.py | 2 +-
tests/util/get_parquet_metadata.py | 2 +-
10 files changed, 29 insertions(+), 30 deletions(-)
diff --git a/bin/check-rat-report.py b/bin/check-rat-report.py
index 3c6870bee..0346a16ee 100755
--- a/bin/check-rat-report.py
+++ b/bin/check-rat-report.py
@@ -66,7 +66,7 @@ for r in resources:
if not excluded:
typename = r.findall('type')[0].attrib['name']
if not (clean_name[0:9] == 'testdata/' and typename in ['archive', 'binary']
- and clean_name[-4:] <> '.jar'):
+ and clean_name[-4:] != '.jar'):
sys.stderr.write(
"%s: %s\n" %
('UNAPPROVED' if approvals else "NO APPROVALS; " + typename, clean_name))
diff --git a/docker/test-with-docker.py b/docker/test-with-docker.py
index 47e505a7e..55ae96372 100755
--- a/docker/test-with-docker.py
+++ b/docker/test-with-docker.py
@@ -790,7 +790,7 @@ class TestSuiteRunner(object):
# io-file-mgr-test expects a real-ish file system at /tmp;
# we mount a temporary directory into the container to appease it.
tmpdir = tempfile.mkdtemp(prefix=test_with_docker.name + "-" + self.name)
- os.chmod(tmpdir, 01777)
+ os.chmod(tmpdir, 0o1777)
# Container names are sometimes used as hostnames, and DNS names shouldn't
# have underscores.
container_name = test_with_docker.name + "-" + self.name.replace("_", "-")
diff --git a/tests/comparison/data_generator_mapred_common.py b/tests/comparison/data_generator_mapred_common.py
index bcf162089..df6c087b5 100644
--- a/tests/comparison/data_generator_mapred_common.py
+++ b/tests/comparison/data_generator_mapred_common.py
@@ -62,7 +62,7 @@ class TextTableDataGenerator(object):
val_generator = col_val_generators[col_idx]
for idx in xrange(val_buffer_size):
val = next(val_generator)
- val_buffer[idx] = "\N" if val is None else val
+ val_buffer[idx] = r"\N" if val is None else val
for col_idx, col in enumerate(cols):
if col_idx > 0:
# Postgres doesn't seem to have an option to specify that the last column value
diff --git a/tests/comparison/leopard/front_end.py b/tests/comparison/leopard/front_end.py
index 9752f7b98..ada5bd130 100755
--- a/tests/comparison/leopard/front_end.py
+++ b/tests/comparison/leopard/front_end.py
@@ -309,7 +309,7 @@ def front_page():
'index.template',
assets=ASSETS,
reports=sorted(
- app.reports.items(), key=lambda (k, report): report.run_date, reverse=True),
+ app.reports.items(), key=lambda k, report: report.run_date, reverse=True),
schedule_items=schedule_items)
if __name__ == '__main__':
diff --git a/tests/comparison/leopard/impala_docker_env.py b/tests/comparison/leopard/impala_docker_env.py
index 8cb60101b..63fc33199 100755
--- a/tests/comparison/leopard/impala_docker_env.py
+++ b/tests/comparison/leopard/impala_docker_env.py
@@ -302,6 +302,7 @@ class ImpalaDockerEnv(object):
# data as a volume to bypass AUFS. See also the README for Leopard.
LOG.info('Warming testdata cluster external volume')
self.start_new_container()
+ volume_map = None
with settings(
warn_only=True,
host_string=self.host,
@@ -325,8 +326,6 @@ class ImpalaDockerEnv(object):
volume_map = {
HOST_TESTDATA_EXTERNAL_VOLUME_PATH: DOCKER_TESTDATA_VOLUME_PATH,
}
- else:
- volume_map = None
self.start_new_container(volume_map=volume_map)
LOG.info('Container Started')
diff --git a/tests/comparison/leopard/report.py b/tests/comparison/leopard/report.py
index d1bbf5cca..db463c993 100644
--- a/tests/comparison/leopard/report.py
+++ b/tests/comparison/leopard/report.py
@@ -49,23 +49,23 @@ class Report(object):
def classify_error(self, error):
d = {
- ur'LINE \d+:': 'Postgres_error',
- ur'Permission denied': 'permission_denied',
- ur'^AnalysisException': 'AnalysisException',
- ur'^Column \d+ in row \d+ does not match': 'mismatch',
- ur'^Could not connect': 'could_not_connect',
- ur'^IllegalStateException': 'IllegalStateException',
- ur'^Invalid query handle: ': 'invalid_query_handle',
- ur'^Invalid or unknown query handle: ': 'invalid_query_handle',
- ur'^Known issue:': 'known_issue',
- ur'^Operation is in ERROR_STATE': 'error_state',
- ur'^Query timed out after \d+ seconds': 'timeout',
- ur'^Row counts do not match': 'row_counts',
- ur'^Too much data': 'too_much_data',
- ur'^Unknown expr node type: \d+': 'unkown_node',
- ur'^Year is out of valid range': 'year_range',
- ur'^[A-Za-z]+ out of range': 'out_of_range',
- ur'^division by zero': 'division_by_zero'}
+ r'LINE \d+:': 'Postgres_error',
+ r'Permission denied': 'permission_denied',
+ r'^AnalysisException': 'AnalysisException',
+ r'^Column \d+ in row \d+ does not match': 'mismatch',
+ r'^Could not connect': 'could_not_connect',
+ r'^IllegalStateException': 'IllegalStateException',
+ r'^Invalid query handle: ': 'invalid_query_handle',
+ r'^Invalid or unknown query handle: ': 'invalid_query_handle',
+ r'^Known issue:': 'known_issue',
+ r'^Operation is in ERROR_STATE': 'error_state',
+ r'^Query timed out after \d+ seconds': 'timeout',
+ r'^Row counts do not match': 'row_counts',
+ r'^Too much data': 'too_much_data',
+ r'^Unknown expr node type: \d+': 'unkown_node',
+ r'^Year is out of valid range': 'year_range',
+ r'^[A-Za-z]+ out of range': 'out_of_range',
+ r'^division by zero': 'division_by_zero'}
for r in d:
if re.search(r, error):
@@ -94,7 +94,7 @@ class Report(object):
stack = query_result['formatted_stack']
if stack:
for line in stack.split('\n'):
- match = re.search(ur'(impala::.*) \(', line)
+ match = re.search(r'(impala::.*) \(', line)
if match:
return match.group(1)
else:
@@ -106,10 +106,10 @@ class Report(object):
def clean_frame(frame):
#remove memory address from each frame
- reg = re.match(ur'#\d+ *0x[0123456789abcdef]* in (.*)', frame)
+ reg = re.match(r'#\d+ *0x[0123456789abcdef]* in (.*)', frame)
if reg: return reg.group(1)
# this is for matching lines like "#7 SLL_Next (this=0x9046780, src=0x90467c8...
- reg = re.match(ur'#\d+ *(\S.*)', frame)
+ reg = re.match(r'#\d+ *(\S.*)', frame)
if reg: return reg.group(1)
return frame
diff --git a/tests/custom_cluster/test_breakpad.py b/tests/custom_cluster/test_breakpad.py
index aee47772c..1d17f8570 100644
--- a/tests/custom_cluster/test_breakpad.py
+++ b/tests/custom_cluster/test_breakpad.py
@@ -463,7 +463,7 @@ class TestLogging(TestBreakpadBase):
test_error_msg = ('123456789abcde_' * 64) # 1 KB error message
test_debug_actions = 'LOG_MAINTENANCE_STDERR:FAIL@1.0@' + test_error_msg
daemon = 'impalad'
- os.chmod(self.tmp_dir, 0744)
+ os.chmod(self.tmp_dir, 0o744)
expected_log_max_bytes = int(1.2 * 1024**2) # 1.2 MB
self.assert_logs(daemon, 0, expected_log_max_bytes)
diff --git a/tests/custom_cluster/test_client_ssl.py b/tests/custom_cluster/test_client_ssl.py
index b2a5267d3..b02a688aa 100644
--- a/tests/custom_cluster/test_client_ssl.py
+++ b/tests/custom_cluster/test_client_ssl.py
@@ -35,7 +35,7 @@ from tests.common.test_dimensions import create_client_protocol_dimension
from tests.shell.util import run_impala_shell_cmd, run_impala_shell_cmd_no_expect, \
ImpalaShell, create_impala_shell_executable_dimension
-REQUIRED_MIN_OPENSSL_VERSION = 0x10001000L
+REQUIRED_MIN_OPENSSL_VERSION = 0x10001000
# Python supports TLSv1.2 from 2.7.9 officially but on Red Hat/CentOS Python2.7.5
# with newer python-libs (eg python-libs-2.7.5-77) supports TLSv1.2 already
if IS_REDHAT_DERIVATIVE:
diff --git a/tests/custom_cluster/test_thrift_socket.py b/tests/custom_cluster/test_thrift_socket.py
index 42d5fa142..c276384fd 100644
--- a/tests/custom_cluster/test_thrift_socket.py
+++ b/tests/custom_cluster/test_thrift_socket.py
@@ -27,7 +27,7 @@ from tests.common.test_vector import ImpalaTestVector
from tests.common.test_dimensions import create_client_protocol_dimension
from tests.shell.util import ImpalaShell
-REQUIRED_MIN_OPENSSL_VERSION = 0x10001000L
+REQUIRED_MIN_OPENSSL_VERSION = 0x10001000
# Python supports TLSv1.2 from 2.7.9 officially but on Red Hat/CentOS Python2.7.5
# with newer python-libs (eg python-libs-2.7.5-77) supports TLSv1.2 already
if IS_REDHAT_DERIVATIVE:
diff --git a/tests/util/get_parquet_metadata.py b/tests/util/get_parquet_metadata.py
index 107b33934..db4ef3f41 100644
--- a/tests/util/get_parquet_metadata.py
+++ b/tests/util/get_parquet_metadata.py
@@ -41,7 +41,7 @@ def julian_day_to_date(julian_day):
arbitrarily and can be validated with an online converter like
http://aa.usno.navy.mil/jdconverter?ID=AA&jd=2457755
"""
- return date(2017, 01, 01) + timedelta(julian_day - 2457755)
+ return date(2017, 1, 1) + timedelta(julian_day - 2457755)
def nanos_to_time(nanos):