You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by jo...@apache.org on 2023/03/09 17:22:49 UTC
[impala] branch master updated (148888e3e -> 0c7c6a335)
This is an automated email from the ASF dual-hosted git repository.
joemcdonnell pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
from 148888e3e IMPALA-11822: Optimize the Refresh/Invalidate event processing by skipping unnecessary events
new 566df8089 IMPALA-11959: Add Python 3 virtualenv
new 82bd087fb IMPALA-11973: Add absolute_import, division to all eligible Python files
new eb66d00f9 IMPALA-11974: Fix lazy list operators for Python 3 compatibility
new c233634d7 IMPALA-11975: Fix Dictionary methods to work with Python 3
new aa4050b4d IMPALA-11976: Fix use of deprecated functions/fields removed in Python 3
new 0c7c6a335 IMPALA-11977: Fix Python 3 broken imports and object model differences
The 6 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
CMakeLists.txt | 6 +-
bin/banned_py3k_warnings.txt | 26 ++++
bin/bootstrap_toolchain.py | 3 +
bin/check-pylint-py3k.sh | 140 +++++++++++++++++++
bin/dump_breakpad_symbols.py | 1 +
bin/generate_minidump_collection_testdata.py | 6 +-
bin/get_code_size.py | 3 +-
bin/impala-config.sh | 3 +-
testdata/bin/kill-mini-dfs.sh => bin/impala-pip3 | 4 +-
testdata/bin/kill-mini-dfs.sh => bin/impala-pylint | 4 +-
bin/{impala-py.test => impala-python3} | 6 +-
...a-python-common.sh => impala-python3-common.sh} | 6 +-
bin/init-impala-python.sh | 26 +++-
bin/inline_pom.py | 2 +-
bin/load-data.py | 6 +-
bin/parse-thrift-profile.py | 4 +-
bin/rat_exclude_files.txt | 1 +
bin/run-workload.py | 8 +-
bin/single_node_perf_run.py | 3 +-
bin/start-impala-cluster.py | 12 +-
docker/setup_build_context.py | 5 +-
infra/python/bootstrap_virtualenv.py | 152 ++++++++++++++-------
infra/python/deps/pip_download.py | 3 +-
.../python/deps/py2-requirements.txt | 26 ++--
.../python/deps/py3-requirements.txt | 23 ++--
infra/python/deps/requirements.txt | 20 +--
infra/python/deps/setuptools-requirements.txt | 2 +-
lib/python/impala_py_lib/gdb/impala-gdb.py | 6 +-
lib/python/impala_py_lib/helpers.py | 1 +
lib/python/impala_py_lib/profiles.py | 2 +-
testdata/bin/check-hbase-nodes.py | 3 +-
testdata/bin/generate-schema-statements.py | 9 +-
testdata/bin/generate-test-vectors.py | 6 +-
testdata/bin/load-tpc-kudu.py | 2 +-
testdata/bin/load_nested.py | 6 +-
testdata/bin/random_avro_schema.py | 2 +
testdata/bin/rewrite-iceberg-metadata.py | 14 +-
testdata/bin/wait-for-hiveserver2.py | 6 +-
testdata/bin/wait-for-metastore.py | 6 +-
testdata/common/cgroups.py | 3 +-
testdata/common/text_delims_table.py | 5 +-
testdata/common/widetable.py | 13 +-
tests/authorization/test_authorization.py | 1 +
tests/authorization/test_authorized_proxy.py | 1 +
tests/authorization/test_provider.py | 1 +
tests/authorization/test_ranger.py | 4 +-
tests/beeswax/impala_beeswax.py | 14 +-
tests/benchmark/plugins/clear_buffer_cache.py | 1 +
tests/benchmark/plugins/vtune_plugin.py | 4 +-
tests/benchmark/report_benchmark_results.py | 17 ++-
.../catalog_service/test_catalog_service_client.py | 1 +
tests/catalog_service/test_large_num_partitions.py | 1 +
tests/common/base_test_suite.py | 1 +
tests/common/custom_cluster_test_suite.py | 81 +++++------
tests/common/environ.py | 3 +-
tests/common/file_utils.py | 1 +
tests/common/iceberg_test_suite.py | 1 +
tests/common/impala_cluster.py | 7 +-
tests/common/impala_connection.py | 12 +-
tests/common/impala_service.py | 1 +
tests/common/impala_test_suite.py | 13 +-
tests/common/kudu_test_suite.py | 6 +-
tests/common/network.py | 1 +
tests/common/parametrize.py | 1 +
tests/common/patterns.py | 1 +
tests/common/resource_pool_config.py | 2 +-
tests/common/skip.py | 1 +
tests/common/test_dimensions.py | 4 +-
tests/common/test_result_verifier.py | 29 +++-
tests/common/test_vector.py | 5 +-
tests/comparison/cli_options.py | 1 +
tests/comparison/cluster.py | 40 +++---
tests/comparison/common.py | 13 +-
tests/comparison/compat.py | 1 +
tests/comparison/data_generator.py | 13 +-
tests/comparison/data_generator_mapred_common.py | 14 +-
tests/comparison/db_connection.py | 16 +--
tests/comparison/db_types.py | 8 +-
tests/comparison/discrepancy_searcher.py | 24 ++--
tests/comparison/funcs.py | 5 +-
tests/comparison/leopard/controller.py | 3 +-
tests/comparison/leopard/front_end.py | 5 +-
tests/comparison/leopard/impala_docker_env.py | 4 +-
tests/comparison/leopard/job.py | 4 +-
tests/comparison/leopard/report.py | 1 +
tests/comparison/leopard/schedule_item.py | 1 +
tests/comparison/model_translator.py | 1 +
tests/comparison/query.py | 17 +--
tests/comparison/query_flattener.py | 1 +
tests/comparison/query_generator.py | 30 ++--
tests/comparison/query_profile.py | 27 ++--
tests/comparison/random_val_generator.py | 1 +
tests/comparison/statement_generator.py | 4 +-
tests/comparison/tests/conftest.py | 1 +
tests/comparison/tests/fake_query.py | 1 +
.../hive/test_hive_create_agg_or_analytic_tree.py | 1 +
.../test_hive_create_relational_join_condition.py | 1 +
tests/comparison/tests/query_object_testdata.py | 1 +
tests/comparison/tests/test_cluster.py | 1 +
tests/comparison/tests/test_cursor.py | 1 +
tests/comparison/tests/test_query_generator.py | 3 +-
tests/comparison/tests/test_query_objects.py | 1 +
tests/comparison/tests/test_use_nested_with.py | 1 +
tests/comparison/util/verify-oracle-connection.py | 2 +-
tests/conftest.py | 11 +-
tests/custom_cluster/test_admission_controller.py | 53 +++----
tests/custom_cluster/test_alloc_fail.py | 1 +
tests/custom_cluster/test_always_false_filter.py | 1 +
tests/custom_cluster/test_auto_scaling.py | 2 +
.../custom_cluster/test_automatic_invalidation.py | 1 +
tests/custom_cluster/test_blacklist.py | 5 +-
.../test_blacklisted_dbs_and_tables.py | 1 +
tests/custom_cluster/test_breakpad.py | 11 +-
tests/custom_cluster/test_catalog_hms_failures.py | 2 +-
tests/custom_cluster/test_catalog_wait.py | 1 +
tests/custom_cluster/test_client_ssl.py | 2 +-
tests/custom_cluster/test_codegen_cache.py | 2 +
.../custom_cluster/test_compact_catalog_updates.py | 1 +
tests/custom_cluster/test_concurrent_ddls.py | 8 +-
.../custom_cluster/test_concurrent_kudu_create.py | 4 +-
tests/custom_cluster/test_coordinators.py | 1 +
tests/custom_cluster/test_custom_hive_configs.py | 1 +
tests/custom_cluster/test_custom_statestore.py | 4 +-
tests/custom_cluster/test_data_cache.py | 1 +
tests/custom_cluster/test_delegation.py | 1 +
.../test_disable_catalog_data_ops.py | 1 +
tests/custom_cluster/test_disable_features.py | 1 +
.../test_disk_spill_configurations.py | 1 +
tests/custom_cluster/test_events_custom_configs.py | 5 +-
.../test_exchange_deferred_batches.py | 2 +
tests/custom_cluster/test_exchange_delays.py | 1 +
tests/custom_cluster/test_exchange_eos.py | 1 +
tests/custom_cluster/test_executor_groups.py | 12 +-
.../test_frontend_connection_limit.py | 3 +-
tests/custom_cluster/test_geospatial_library.py | 2 +
.../custom_cluster/test_hbase_hms_column_order.py | 1 +
tests/custom_cluster/test_hdfs_fd_caching.py | 2 +
tests/custom_cluster/test_hdfs_timeout.py | 1 +
tests/custom_cluster/test_hedged_reads.py | 1 +
.../test_hive_parquet_codec_interop.py | 1 +
.../test_hive_parquet_timestamp_conversion.py | 1 +
.../custom_cluster/test_hive_text_codec_interop.py | 1 +
tests/custom_cluster/test_hs2.py | 1 +
tests/custom_cluster/test_hs2_fault_injection.py | 4 +-
.../test_incremental_metadata_updates.py | 2 +
tests/custom_cluster/test_insert_behaviour.py | 1 +
tests/custom_cluster/test_jvm_mem_tracking.py | 1 +
tests/custom_cluster/test_krpc_mem_usage.py | 1 +
tests/custom_cluster/test_krpc_metrics.py | 1 +
tests/custom_cluster/test_krpc_options.py | 1 +
tests/custom_cluster/test_krpc_socket.py | 1 +
tests/custom_cluster/test_kudu.py | 1 +
tests/custom_cluster/test_kudu_not_available.py | 1 +
.../test_kudu_table_create_without_hms.py | 1 +
tests/custom_cluster/test_lineage.py | 1 +
tests/custom_cluster/test_local_catalog.py | 15 +-
tests/custom_cluster/test_local_tz_conversion.py | 1 +
tests/custom_cluster/test_logging.py | 1 +
tests/custom_cluster/test_mem_reservations.py | 6 +-
.../test_metadata_no_events_processing.py | 1 +
tests/custom_cluster/test_metadata_replicas.py | 6 +-
.../test_metastore_events_cleanup.py | 1 +
tests/custom_cluster/test_metastore_service.py | 2 +
tests/custom_cluster/test_mt_dop.py | 1 +
tests/custom_cluster/test_observability.py | 1 +
.../custom_cluster/test_parquet_max_page_header.py | 10 +-
tests/custom_cluster/test_partition.py | 1 +
tests/custom_cluster/test_pause_monitor.py | 1 +
tests/custom_cluster/test_permanent_udfs.py | 1 +
tests/custom_cluster/test_preload_table_types.py | 2 +
tests/custom_cluster/test_process_failures.py | 6 +-
tests/custom_cluster/test_query_concurrency.py | 1 +
tests/custom_cluster/test_query_event_hooks.py | 1 +
tests/custom_cluster/test_query_expiration.py | 10 +-
tests/custom_cluster/test_query_retries.py | 9 +-
tests/custom_cluster/test_re2_max_mem.py | 1 +
tests/custom_cluster/test_redaction.py | 1 +
.../custom_cluster/test_reserved_words_version.py | 1 +
tests/custom_cluster/test_restart_services.py | 15 +-
tests/custom_cluster/test_result_spooling.py | 1 +
tests/custom_cluster/test_rpc_exception.py | 1 +
tests/custom_cluster/test_rpc_timeout.py | 4 +-
tests/custom_cluster/test_runtime_profile.py | 1 +
tests/custom_cluster/test_s3a_access.py | 1 +
tests/custom_cluster/test_saml2_sso.py | 33 +++--
tests/custom_cluster/test_scheduler_locality.py | 1 +
tests/custom_cluster/test_scratch_disk.py | 5 +-
tests/custom_cluster/test_seq_file_filtering.py | 1 +
tests/custom_cluster/test_services_rpc_errors.py | 1 +
tests/custom_cluster/test_session_expiration.py | 1 +
tests/custom_cluster/test_set_and_unset.py | 5 +-
tests/custom_cluster/test_shared_tzdb.py | 1 +
tests/custom_cluster/test_shell_commandline.py | 1 +
tests/custom_cluster/test_shell_interactive.py | 1 +
.../test_shell_interactive_reconnect.py | 1 +
.../test_startup_filesystem_checks.py | 1 +
tests/custom_cluster/test_stats_extrapolation.py | 1 +
.../test_thrift_debug_string_exception.py | 1 +
tests/custom_cluster/test_thrift_socket.py | 1 +
.../custom_cluster/test_topic_update_frequency.py | 3 +-
tests/custom_cluster/test_udf_concurrency.py | 11 +-
tests/custom_cluster/test_web_pages.py | 2 +-
tests/custom_cluster/test_wide_table_operations.py | 2 +
tests/data_errors/test_data_errors.py | 1 +
tests/experiments/test_targeted_perf.py | 1 +
tests/failure/test_failpoints.py | 2 +
tests/hs2/hs2_test_suite.py | 6 +-
tests/hs2/test_fetch.py | 1 +
tests/hs2/test_fetch_first.py | 14 +-
tests/hs2/test_fetch_timeout.py | 1 +
tests/hs2/test_hs2.py | 11 +-
tests/hs2/test_json_endpoints.py | 6 +-
tests/infra/test_perf_infra.py | 1 +
tests/infra/test_stress_infra.py | 1 +
tests/infra/test_utils.py | 1 +
tests/metadata/test_catalogd_debug_actions.py | 1 +
tests/metadata/test_compute_stats.py | 4 +-
tests/metadata/test_ddl.py | 26 ++--
tests/metadata/test_ddl_base.py | 1 +
tests/metadata/test_event_processing.py | 1 +
tests/metadata/test_explain.py | 1 +
tests/metadata/test_hdfs_encryption.py | 1 +
tests/metadata/test_hdfs_permissions.py | 1 +
tests/metadata/test_hidden_files.py | 1 +
tests/metadata/test_hms_integration.py | 7 +-
tests/metadata/test_last_ddl_time_update.py | 10 +-
tests/metadata/test_load.py | 8 +-
tests/metadata/test_metadata_query_statements.py | 1 +
tests/metadata/test_partition_metadata.py | 1 +
tests/metadata/test_recover_partitions.py | 16 ++-
tests/metadata/test_recursive_listing.py | 2 +
tests/metadata/test_refresh_partition.py | 1 +
tests/metadata/test_reset_metadata.py | 1 +
tests/metadata/test_reuse_partitions.py | 1 +
tests/metadata/test_set.py | 1 +
tests/metadata/test_show_create_table.py | 1 +
tests/metadata/test_stale_metadata.py | 1 +
tests/metadata/test_stats_extrapolation.py | 6 +-
tests/metadata/test_testcase_builder.py | 1 +
tests/metadata/test_views_compatibility.py | 1 +
tests/observability/test_jvm_metrics.py | 1 +
tests/observability/test_log_fragments.py | 1 +
tests/observability/test_profile_tool.py | 1 +
tests/performance/query.py | 4 +
tests/performance/query_exec_functions.py | 5 +-
tests/performance/query_executor.py | 1 +
tests/performance/scheduler.py | 8 +-
tests/performance/workload.py | 3 +-
tests/performance/workload_runner.py | 1 +
tests/query_test/test_acid.py | 1 +
tests/query_test/test_acid_row_validation.py | 1 +
tests/query_test/test_aggregation.py | 11 +-
tests/query_test/test_analytic_tpcds.py | 1 +
tests/query_test/test_async_codegen.py | 1 +
tests/query_test/test_avro_schema_resolution.py | 2 +
tests/query_test/test_beeswax.py | 1 +
tests/query_test/test_cancellation.py | 10 +-
tests/query_test/test_cast_with_format.py | 2 +
tests/query_test/test_chars.py | 1 +
tests/query_test/test_codegen.py | 1 +
tests/query_test/test_compressed_formats.py | 5 +-
tests/query_test/test_datasketches.py | 1 +
tests/query_test/test_datastream_sender.py | 1 +
tests/query_test/test_date_queries.py | 1 +
tests/query_test/test_decimal_casting.py | 16 ++-
tests/query_test/test_decimal_fuzz.py | 14 +-
tests/query_test/test_decimal_queries.py | 1 +
tests/query_test/test_delimited_text.py | 1 +
tests/query_test/test_errorlog.py | 1 +
tests/query_test/test_exprs.py | 14 +-
tests/query_test/test_fetch.py | 1 +
tests/query_test/test_geospatial_functions.py | 1 +
tests/query_test/test_hash_join_timer.py | 5 +-
tests/query_test/test_hbase_queries.py | 1 +
tests/query_test/test_hdfs_caching.py | 5 +-
tests/query_test/test_hdfs_file_mods.py | 1 +
tests/query_test/test_iceberg.py | 2 +
tests/query_test/test_insert.py | 1 +
tests/query_test/test_insert_behaviour.py | 2 +
tests/query_test/test_insert_parquet.py | 18 ++-
tests/query_test/test_insert_permutation.py | 6 +-
tests/query_test/test_invalid_test_header.py | 1 +
tests/query_test/test_io_metrics.py | 1 +
tests/query_test/test_join_queries.py | 1 +
tests/query_test/test_kudu.py | 7 +-
tests/query_test/test_lifecycle.py | 1 +
tests/query_test/test_limit.py | 2 +-
tests/query_test/test_limit_pushdown_analytic.py | 1 +
tests/query_test/test_local_fs.py | 1 +
tests/query_test/test_mem_usage_scaling.py | 3 +
tests/query_test/test_mt_dop.py | 1 +
tests/query_test/test_multiple_filesystems.py | 1 +
tests/query_test/test_nested_types.py | 1 +
tests/query_test/test_observability.py | 1 +
tests/query_test/test_orc_stats.py | 1 +
tests/query_test/test_parquet_bloom_filter.py | 2 +
.../test_parquet_late_materialization.py | 1 +
tests/query_test/test_parquet_page_index.py | 1 +
tests/query_test/test_parquet_stats.py | 1 +
tests/query_test/test_partitioning.py | 1 +
tests/query_test/test_queries.py | 1 +
tests/query_test/test_query_compilation.py | 1 +
tests/query_test/test_query_mem_limit.py | 7 +-
tests/query_test/test_query_opts.py | 1 +
tests/query_test/test_resource_limits.py | 1 +
tests/query_test/test_result_spooling.py | 1 +
tests/query_test/test_rows_availability.py | 1 +
tests/query_test/test_runtime_filters.py | 1 +
tests/query_test/test_scanners.py | 3 +-
tests/query_test/test_scanners_fuzz.py | 6 +-
tests/query_test/test_scratch_limit.py | 1 +
tests/query_test/test_sfs.py | 1 +
tests/query_test/test_sort.py | 19 ++-
tests/query_test/test_spilling.py | 1 +
tests/query_test/test_tablesample.py | 1 +
tests/query_test/test_tpcds_queries.py | 1 +
tests/query_test/test_tpch_nested_queries.py | 1 +
tests/query_test/test_tpch_queries.py | 4 +-
tests/query_test/test_udfs.py | 1 +
tests/query_test/test_utf8_strings.py | 1 +
tests/run-tests.py | 4 +-
tests/shell/test_cookie_util.py | 1 +
tests/shell/test_shell_client.py | 7 +-
tests/shell/test_shell_commandline.py | 12 +-
tests/shell/test_shell_interactive.py | 42 +++---
tests/shell/util.py | 2 +-
tests/statestore/test_statestore.py | 17 ++-
tests/stress/concurrent_select.py | 45 +++---
tests/stress/extract_min_mem.py | 4 +-
tests/stress/mem_broker.py | 1 +
tests/stress/queries.py | 6 +-
tests/stress/query_retries_stress_runner.py | 12 +-
tests/stress/query_runner.py | 14 +-
tests/stress/runtime_info.py | 12 +-
tests/stress/stress_util.py | 1 +
tests/stress/test_acid_stress.py | 28 ++--
tests/stress/test_ddl_stress.py | 6 +-
tests/stress/test_insert_stress.py | 15 +-
tests/stress/util.py | 2 +-
tests/unittests/test_command.py | 1 +
tests/unittests/test_file_parser.py | 4 +-
tests/unittests/test_result_verifier.py | 5 +-
tests/util/acid_txn.py | 2 +-
tests/util/adls_util.py | 1 +
tests/util/auto_scaler.py | 3 +-
tests/util/calculation_util.py | 6 +-
tests/util/cancel_util.py | 1 +
tests/util/cluster_controller.py | 1 +
tests/util/compute_table_stats.py | 1 +
tests/util/concurrent_workload.py | 14 +-
tests/util/event_processor_utils.py | 1 +
tests/util/failpoints_util.py | 1 +
tests/util/filesystem_base.py | 6 +-
tests/util/filesystem_utils.py | 1 +
tests/util/get_parquet_metadata.py | 5 +-
tests/util/hdfs_util.py | 11 +-
tests/util/iceberg_util.py | 1 +
tests/util/parse_util.py | 1 +
tests/util/plugin_runner.py | 1 +
tests/util/run_impyla_http_query.py | 2 +-
tests/util/shell_util.py | 1 +
tests/util/ssh_util.py | 8 +-
tests/util/test_file_parser.py | 12 +-
tests/util/thrift_util.py | 1 +
tests/util/web_pages_util.py | 1 +
tests/verifiers/mem_usage_verifier.py | 1 +
tests/verifiers/metric_verifier.py | 1 +
tests/verifiers/test_verify_metrics.py | 1 +
tests/webserver/test_web_pages.py | 1 +
369 files changed, 1409 insertions(+), 690 deletions(-)
create mode 100644 bin/banned_py3k_warnings.txt
create mode 100755 bin/check-pylint-py3k.sh
copy testdata/bin/kill-mini-dfs.sh => bin/impala-pip3 (88%)
copy testdata/bin/kill-mini-dfs.sh => bin/impala-pylint (90%)
copy bin/{impala-py.test => impala-python3} (91%)
copy bin/{impala-python-common.sh => impala-python3-common.sh} (87%)
copy be/src/exec/sequence/CMakeLists.txt => infra/python/deps/py2-requirements.txt (65%)
copy be/src/exec/sequence/CMakeLists.txt => infra/python/deps/py3-requirements.txt (70%)
[impala] 03/06: IMPALA-11974: Fix lazy list operators for Python 3 compatibility
Posted by jo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
joemcdonnell pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit eb66d00f9f43ddaa6a9547574a150e0b1436f4d4
Author: Joe McDonnell <jo...@cloudera.com>
AuthorDate: Sat Mar 4 09:02:36 2023 -0800
IMPALA-11974: Fix lazy list operators for Python 3 compatibility
Python 3 changes list operators such as range, map, and filter
to be lazy. Some code that expects the list operators to happen
immediately will fail. e.g.
Python 2:
range(0,5) == [0,1,2,3,4]
True
Python 3:
range(0,5) == [0,1,2,3,4]
False
The fix is to wrap locations with list(). i.e.
Python 3:
list(range(0,5)) == [0,1,2,3,4]
True
Since the base operators are now lazy, Python 3 also removes the
old lazy versions (e.g. xrange, ifilter, izip, etc). This uses
future's builtins package to convert the code to the Python 3
behavior (i.e. xrange -> future's builtins.range).
Most of the changes were done via these futurize fixes:
- libfuturize.fixes.fix_xrange_with_import
- lib2to3.fixes.fix_map
- lib2to3.fixes.fix_filter
This eliminates the pylint warnings:
- xrange-builtin
- range-builtin-not-iterating
- map-builtin-not-iterating
- zip-builtin-not-iterating
- filter-builtin-not-iterating
- reduce-builtin
- deprecated-itertools-function
Testing:
- Ran core job
Change-Id: Ic7c082711f8eff451a1b5c085e97461c327edb5f
Reviewed-on: http://gerrit.cloudera.org:8080/19589
Reviewed-by: Joe McDonnell <jo...@cloudera.com>
Tested-by: Joe McDonnell <jo...@cloudera.com>
---
bin/banned_py3k_warnings.txt | 7 ++++++
bin/generate_minidump_collection_testdata.py | 5 ++--
bin/get_code_size.py | 1 +
bin/load-data.py | 2 +-
bin/run-workload.py | 6 ++---
bin/single_node_perf_run.py | 1 +
bin/start-impala-cluster.py | 5 ++--
testdata/bin/generate-schema-statements.py | 2 +-
testdata/bin/generate-test-vectors.py | 1 +
testdata/bin/load_nested.py | 5 ++--
testdata/bin/random_avro_schema.py | 1 +
testdata/bin/rewrite-iceberg-metadata.py | 12 +++++-----
testdata/common/cgroups.py | 1 +
testdata/common/text_delims_table.py | 3 ++-
testdata/common/widetable.py | 7 +++---
tests/authorization/test_ranger.py | 3 ++-
tests/beeswax/impala_beeswax.py | 5 ++--
tests/benchmark/plugins/vtune_plugin.py | 3 ++-
tests/benchmark/report_benchmark_results.py | 3 ++-
tests/common/environ.py | 2 +-
tests/common/impala_cluster.py | 1 +
tests/common/impala_test_suite.py | 3 ++-
tests/common/kudu_test_suite.py | 5 ++--
tests/common/test_dimensions.py | 3 ++-
tests/common/test_result_verifier.py | 13 ++++++-----
tests/comparison/cluster.py | 8 +++----
tests/comparison/data_generator.py | 8 ++++---
tests/comparison/data_generator_mapred_common.py | 5 ++--
tests/comparison/db_connection.py | 9 ++++----
tests/comparison/discrepancy_searcher.py | 16 ++++++-------
tests/comparison/funcs.py | 4 ++--
tests/comparison/query.py | 3 ++-
tests/comparison/query_generator.py | 18 +++++++--------
tests/comparison/query_profile.py | 12 ++++++----
tests/comparison/statement_generator.py | 3 ++-
tests/conftest.py | 5 ++--
tests/custom_cluster/test_admission_controller.py | 19 +++++++--------
tests/custom_cluster/test_auto_scaling.py | 1 +
tests/custom_cluster/test_blacklist.py | 3 ++-
tests/custom_cluster/test_breakpad.py | 5 ++--
tests/custom_cluster/test_codegen_cache.py | 1 +
tests/custom_cluster/test_concurrent_ddls.py | 7 +++---
.../custom_cluster/test_concurrent_kudu_create.py | 3 ++-
tests/custom_cluster/test_custom_statestore.py | 3 ++-
tests/custom_cluster/test_events_custom_configs.py | 3 ++-
.../test_exchange_deferred_batches.py | 1 +
tests/custom_cluster/test_executor_groups.py | 3 ++-
tests/custom_cluster/test_hdfs_fd_caching.py | 1 +
.../test_incremental_metadata_updates.py | 1 +
tests/custom_cluster/test_local_catalog.py | 7 +++---
tests/custom_cluster/test_mem_reservations.py | 5 ++--
tests/custom_cluster/test_metadata_replicas.py | 5 ++--
tests/custom_cluster/test_metastore_service.py | 1 +
.../custom_cluster/test_parquet_max_page_header.py | 5 ++--
tests/custom_cluster/test_preload_table_types.py | 1 +
tests/custom_cluster/test_process_failures.py | 5 ++--
tests/custom_cluster/test_query_expiration.py | 9 ++++----
tests/custom_cluster/test_query_retries.py | 7 +++---
tests/custom_cluster/test_restart_services.py | 5 ++--
tests/custom_cluster/test_rpc_timeout.py | 3 ++-
tests/custom_cluster/test_scratch_disk.py | 3 ++-
tests/custom_cluster/test_set_and_unset.py | 4 ++--
.../custom_cluster/test_topic_update_frequency.py | 1 +
tests/custom_cluster/test_udf_concurrency.py | 5 ++--
tests/custom_cluster/test_wide_table_operations.py | 1 +
tests/failure/test_failpoints.py | 1 +
tests/hs2/hs2_test_suite.py | 3 ++-
tests/hs2/test_fetch_first.py | 13 ++++++-----
tests/hs2/test_hs2.py | 5 ++--
tests/metadata/test_compute_stats.py | 3 ++-
tests/metadata/test_ddl.py | 25 ++++++++++----------
tests/metadata/test_hms_integration.py | 5 ++--
tests/metadata/test_load.py | 7 +++---
tests/metadata/test_recover_partitions.py | 15 ++++++------
tests/metadata/test_recursive_listing.py | 1 +
tests/metadata/test_stats_extrapolation.py | 5 ++--
tests/performance/scheduler.py | 7 +++---
tests/query_test/test_aggregation.py | 9 ++++----
tests/query_test/test_avro_schema_resolution.py | 1 +
tests/query_test/test_cancellation.py | 7 +++---
tests/query_test/test_cast_with_format.py | 1 +
tests/query_test/test_compressed_formats.py | 1 +
tests/query_test/test_decimal_casting.py | 13 ++++++-----
tests/query_test/test_decimal_fuzz.py | 7 +++---
tests/query_test/test_exprs.py | 13 ++++++-----
tests/query_test/test_hdfs_caching.py | 3 ++-
tests/query_test/test_iceberg.py | 1 +
tests/query_test/test_insert_behaviour.py | 1 +
tests/query_test/test_insert_parquet.py | 6 +++--
tests/query_test/test_insert_permutation.py | 5 ++--
tests/query_test/test_kudu.py | 1 +
tests/query_test/test_mem_usage_scaling.py | 1 +
tests/query_test/test_parquet_bloom_filter.py | 1 +
tests/query_test/test_scanners.py | 1 +
tests/query_test/test_scanners_fuzz.py | 3 ++-
tests/query_test/test_sort.py | 18 ++++++++++++---
tests/query_test/test_tpch_queries.py | 3 ++-
tests/shell/test_shell_commandline.py | 3 ++-
tests/statestore/test_statestore.py | 7 +++---
tests/stress/concurrent_select.py | 9 ++++----
tests/stress/queries.py | 3 ++-
tests/stress/query_retries_stress_runner.py | 7 +++---
tests/stress/test_acid_stress.py | 27 ++++++++++++----------
tests/stress/test_ddl_stress.py | 5 ++--
tests/stress/test_insert_stress.py | 14 ++++++-----
tests/util/calculation_util.py | 1 +
tests/util/concurrent_workload.py | 3 ++-
tests/util/get_parquet_metadata.py | 4 +++-
tests/util/ssh_util.py | 1 +
tests/util/test_file_parser.py | 10 +++++---
110 files changed, 358 insertions(+), 231 deletions(-)
diff --git a/bin/banned_py3k_warnings.txt b/bin/banned_py3k_warnings.txt
index c5e7bdc12..01d54fe73 100644
--- a/bin/banned_py3k_warnings.txt
+++ b/bin/banned_py3k_warnings.txt
@@ -1,2 +1,9 @@
no-absolute-import
old-division
+xrange-builtin
+range-builtin-not-iterating
+map-builtin-not-iterating
+zip-builtin-not-iterating
+filter-builtin-not-iterating
+reduce-builtin
+deprecated-itertools-function
diff --git a/bin/generate_minidump_collection_testdata.py b/bin/generate_minidump_collection_testdata.py
index 09341f539..9149e814d 100755
--- a/bin/generate_minidump_collection_testdata.py
+++ b/bin/generate_minidump_collection_testdata.py
@@ -28,6 +28,7 @@
# making the files easily compressible by having some repeated data.
from __future__ import absolute_import, division, print_function
+from builtins import range
import errno
import os
import random
@@ -99,7 +100,7 @@ def random_bytes(num):
def write_minidump(common_data, timestamp, target_dir):
'''Generate and write the minidump into the target_dir. atime and mtime of the minidump
will be set to timestamp.'''
- file_name = ''.join(random.choice('abcdefghijklmnopqrstuvwxyz') for _ in xrange(10))
+ file_name = ''.join(random.choice('abcdefghijklmnopqrstuvwxyz') for _ in range(10))
with open(os.path.join(target_dir, file_name), 'wb') as f:
# We want the minidump to be pretty similar to each other. The number 8192 was chosen
# arbitratily and seemed like a reasonable guess.
@@ -130,7 +131,7 @@ def generate_minidumps():
interval = 0
else:
interval = (end_timestamp - start_timestamp) // (options.num_minidumps - 1)
- for i in xrange(options.num_minidumps):
+ for i in range(options.num_minidumps):
write_minidump(common_data,
start_timestamp + interval * i,
os.path.join(minidump_dir, role_name))
diff --git a/bin/get_code_size.py b/bin/get_code_size.py
index 44afba04f..c69007057 100755
--- a/bin/get_code_size.py
+++ b/bin/get_code_size.py
@@ -20,6 +20,7 @@
# This tool walks the build directory (release by default) and will print the text, data,
# and bss section sizes of the archives.
from __future__ import absolute_import, division, print_function
+from builtins import range
import fnmatch
import os
import re
diff --git a/bin/load-data.py b/bin/load-data.py
index 3c35cfc59..535a4f8bf 100755
--- a/bin/load-data.py
+++ b/bin/load-data.py
@@ -427,7 +427,7 @@ def main():
def log_file_list(header, file_list):
if (len(file_list) == 0): return
LOG.debug(header)
- map(LOG.debug, map(os.path.basename, file_list))
+ list(map(LOG.debug, list(map(os.path.basename, file_list))))
LOG.debug("\n")
log_file_list("Impala Create Files:", impala_create_files)
diff --git a/bin/run-workload.py b/bin/run-workload.py
index 1da5dd0f5..612c83436 100755
--- a/bin/run-workload.py
+++ b/bin/run-workload.py
@@ -172,8 +172,8 @@ def prettytable_print(results, failed=False):
def print_result_summary(results):
"""Print failed and successfull queries for a given result list"""
- failed_results = filter(lambda x: x.success == False, results)
- successful_results = filter(lambda x: x.success == True, results)
+ failed_results = [x for x in results if not x.success]
+ successful_results = [x for x in results if x.success]
prettytable_print(successful_results)
if failed_results: prettytable_print(failed_results, failed=True)
@@ -195,7 +195,7 @@ def get_workload_scale_factor():
def split_and_strip(input_string, delim=","):
"""Convert a string into a list using the given delimiter"""
if not input_string: return list()
- return map(str.strip, input_string.split(delim))
+ return list(map(str.strip, input_string.split(delim)))
def create_workload_config():
"""Parse command line inputs.
diff --git a/bin/single_node_perf_run.py b/bin/single_node_perf_run.py
index f64d94e6c..1452c9fee 100755
--- a/bin/single_node_perf_run.py
+++ b/bin/single_node_perf_run.py
@@ -70,6 +70,7 @@
# --ninja use ninja, rather than Make, as the build tool
from __future__ import absolute_import, division, print_function
+from builtins import range
from optparse import OptionParser
from tempfile import mkdtemp
diff --git a/bin/start-impala-cluster.py b/bin/start-impala-cluster.py
index 6c5fa07d4..5fac2c2d6 100755
--- a/bin/start-impala-cluster.py
+++ b/bin/start-impala-cluster.py
@@ -21,6 +21,7 @@
# ImpalaD instances. Each ImpalaD runs on a different port allowing this to be run
# on a single machine.
from __future__ import absolute_import, division, print_function
+from builtins import range
import getpass
import itertools
import json
@@ -548,7 +549,7 @@ class MiniClusterOperations(object):
cluster_size, num_coordinators, use_exclusive_coordinators, remap_ports=True,
start_idx=start_idx)
assert cluster_size == len(impalad_arg_lists)
- for i in xrange(start_idx, start_idx + cluster_size):
+ for i in range(start_idx, start_idx + cluster_size):
service_name = impalad_service_name(i)
LOG.info("Starting Impala Daemon logging to {log_dir}/{service_name}.INFO".format(
log_dir=options.log_dir, service_name=service_name))
@@ -624,7 +625,7 @@ class DockerMiniClusterOperations(object):
use_exclusive_coordinators, remap_ports=False, admissiond_host="admissiond")
assert cluster_size == len(impalad_arg_lists)
mem_limit = compute_impalad_mem_limit(cluster_size)
- for i in xrange(cluster_size):
+ for i in range(cluster_size):
chosen_ports = choose_impalad_ports(i)
port_map = {DEFAULT_BEESWAX_PORT: chosen_ports['beeswax_port'],
DEFAULT_HS2_PORT: chosen_ports['hs2_port'],
diff --git a/testdata/bin/generate-schema-statements.py b/testdata/bin/generate-schema-statements.py
index 23cb88556..69683a2f4 100755
--- a/testdata/bin/generate-schema-statements.py
+++ b/testdata/bin/generate-schema-statements.py
@@ -276,7 +276,7 @@ def parse_table_properties(file_format, table_properties):
r'(?:(\w+):)?' +
# Required key=value, capturing the key and value
r'(.+?)=(.*)')
- for table_property in filter(None, table_properties.split("\n")):
+ for table_property in [_f for _f in table_properties.split("\n") if _f]:
m = TABLE_PROPERTY_RE.match(table_property)
if not m:
raise Exception("Invalid table property line: {0}", format(table_property))
diff --git a/testdata/bin/generate-test-vectors.py b/testdata/bin/generate-test-vectors.py
index 00e7228e8..9b5272c37 100755
--- a/testdata/bin/generate-test-vectors.py
+++ b/testdata/bin/generate-test-vectors.py
@@ -41,6 +41,7 @@
# downloaded from http://pypi.python.org/pypi/AllPairs/2.0.1
#
from __future__ import absolute_import, division, print_function
+from builtins import range
import collections
import csv
import math
diff --git a/testdata/bin/load_nested.py b/testdata/bin/load_nested.py
index 7a4faceed..929a421b7 100755
--- a/testdata/bin/load_nested.py
+++ b/testdata/bin/load_nested.py
@@ -21,6 +21,7 @@
loaded.
'''
from __future__ import absolute_import, division, print_function
+from builtins import range
import logging
import os
@@ -105,7 +106,7 @@ def load():
"external": external}
# Split table creation into multiple queries or "chunks" so less memory is needed.
- for chunk_idx in xrange(chunks):
+ for chunk_idx in range(chunks):
sql_params["chunk_idx"] = chunk_idx
# Create the nested data in text format. The \00#'s are nested field terminators,
@@ -144,7 +145,7 @@ def load():
else:
impala.execute("INSERT INTO TABLE tmp_orders_string " + tmp_orders_sql)
- for chunk_idx in xrange(chunks):
+ for chunk_idx in range(chunks):
sql_params["chunk_idx"] = chunk_idx
tmp_customer_sql = r"""
SELECT STRAIGHT_JOIN
diff --git a/testdata/bin/random_avro_schema.py b/testdata/bin/random_avro_schema.py
index e065b1dcc..ef084edbc 100755
--- a/testdata/bin/random_avro_schema.py
+++ b/testdata/bin/random_avro_schema.py
@@ -18,6 +18,7 @@
# under the License.
from __future__ import absolute_import, division, print_function
+from builtins import range
from random import choice, randint, random, shuffle
from os.path import join as join_path
from optparse import OptionParser
diff --git a/testdata/bin/rewrite-iceberg-metadata.py b/testdata/bin/rewrite-iceberg-metadata.py
index 2f8e22e32..d0c4d40dc 100755
--- a/testdata/bin/rewrite-iceberg-metadata.py
+++ b/testdata/bin/rewrite-iceberg-metadata.py
@@ -18,6 +18,7 @@
# under the License.
from __future__ import absolute_import, division, print_function
+from builtins import map
import glob
import json
import os
@@ -52,8 +53,7 @@ def add_prefix_to_snapshot(snapshot):
if 'manifest-list' in snapshot:
snapshot['manifest-list'] = generate_new_path(prefix, snapshot['manifest-list'])
if 'manifests' in snapshot:
- snapshot['manifests'] = map(lambda m: generate_new_path(prefix, m),
- snapshot['manifests'])
+ snapshot['manifests'] = [generate_new_path(prefix, m) for m in snapshot['manifests']]
return snapshot
@@ -99,11 +99,11 @@ for arg in args[1:]:
# snapshots: optional
if 'snapshots' in metadata:
- metadata['snapshots'] = map(add_prefix_to_snapshot, metadata['snapshots'])
+ metadata['snapshots'] = list(map(add_prefix_to_snapshot, metadata['snapshots']))
# metadata-log: optional
if 'metadata-log' in metadata:
- metadata['metadata-log'] = map(add_prefix_to_mlog, metadata['metadata-log'])
+ metadata['metadata-log'] = list(map(add_prefix_to_mlog, metadata['metadata-log']))
with open(mfile + '.tmp', 'w') as f:
json.dump(metadata, f, indent=2)
@@ -113,7 +113,7 @@ for arg in args[1:]:
with open(afile, 'rb') as f:
with DataFileReader(f, DatumReader()) as reader:
schema = reader.datum_reader.writers_schema
- lines = map(add_prefix_to_snapshot_entry, reader)
+ lines = list(map(add_prefix_to_snapshot_entry, reader))
with open(afile + '.tmp', 'wb') as f:
with DataFileWriter(f, DatumWriter(), schema) as writer:
@@ -127,7 +127,7 @@ for arg in args[1:]:
with open(snapfile, 'rb') as f:
with DataFileReader(f, DatumReader()) as reader:
schema = reader.datum_reader.writers_schema
- lines = map(fix_manifest_length, reader)
+ lines = list(map(fix_manifest_length, reader))
with open(snapfile + '.tmp', 'wb') as f:
with DataFileWriter(f, DatumWriter(), schema) as writer:
diff --git a/testdata/common/cgroups.py b/testdata/common/cgroups.py
index 36fe5b75f..5e5d5c048 100755
--- a/testdata/common/cgroups.py
+++ b/testdata/common/cgroups.py
@@ -20,6 +20,7 @@
# Utility code for creating cgroups for the Impala development environment.
# May be used as a library or as a command-line utility for manual testing.
from __future__ import absolute_import, division, print_function
+from builtins import range
import os
import sys
import errno
diff --git a/testdata/common/text_delims_table.py b/testdata/common/text_delims_table.py
index a5b06acf0..5a605e548 100755
--- a/testdata/common/text_delims_table.py
+++ b/testdata/common/text_delims_table.py
@@ -23,6 +23,7 @@
# print a SQL load statement to incorporate into dataload SQL script generation.
from __future__ import absolute_import, division, print_function
+from builtins import range
from shutil import rmtree
from optparse import OptionParser
from contextlib import contextmanager
@@ -35,7 +36,7 @@ parser.add_option("--only_newline", dest="only_newline", default=False, action="
parser.add_option("--file_len", dest="file_len", type="int")
def generate_testescape_files(table_location, only_newline, file_len):
- data = ''.join(["1234567890" for _ in xrange(1 + file_len // 10)])
+ data = ''.join(["1234567890" for _ in range(1 + file_len // 10)])
suffix_list = ["\\", ",", "a"]
if only_newline:
diff --git a/testdata/common/widetable.py b/testdata/common/widetable.py
index f04b5cc69..405bdedf3 100755
--- a/testdata/common/widetable.py
+++ b/testdata/common/widetable.py
@@ -23,6 +23,7 @@
# into dataload SQL script generation.
from __future__ import absolute_import, division, print_function
+from builtins import range
from datetime import datetime, timedelta
import itertools
import optparse
@@ -51,7 +52,7 @@ def get_columns(num_cols):
iter = itertools.cycle(templates)
# Produces [bool_col1, tinyint_col1, ..., bool_col2, tinyint_col2, ...]
# The final list has 'num_cols' elements.
- return [iter.next() % (i // len(templates) + 1) for i in xrange(num_cols)]
+ return [iter.next() % (i // len(templates) + 1) for i in range(num_cols)]
# Data generators for different types. Each generator yields an infinite number of
# value strings suitable for writing to a CSV file.
@@ -100,11 +101,11 @@ def get_data(num_cols, num_rows, delimiter=',', quote_strings=False):
]
# Create a generator instance for each column, cycling through the different types
iter = itertools.cycle(generators)
- column_generators = [iter.next()() for i in xrange(num_cols)]
+ column_generators = [iter.next()() for i in range(num_cols)]
# Populate each row using column_generators
rows = []
- for i in xrange(num_rows):
+ for i in range(num_rows):
vals = [gen.next() for gen in column_generators]
rows.append(delimiter.join(vals))
return rows
diff --git a/tests/authorization/test_ranger.py b/tests/authorization/test_ranger.py
index 03669f081..9f94cc1a9 100644
--- a/tests/authorization/test_ranger.py
+++ b/tests/authorization/test_ranger.py
@@ -18,6 +18,7 @@
# Client tests for SQL statement authorization
from __future__ import absolute_import, division, print_function
+from builtins import map, range
import os
import grp
import json
@@ -1017,7 +1018,7 @@ class TestRanger(CustomClusterTestSuite):
cols = row.split("\t")
return cols[0:len(cols) - 1]
- assert map(columns, result.data) == expected
+ assert list(map(columns, result.data)) == expected
def _refresh_authorization(self, client, statement):
if statement is not None:
diff --git a/tests/beeswax/impala_beeswax.py b/tests/beeswax/impala_beeswax.py
index f7c2eb3f3..c58bb7109 100644
--- a/tests/beeswax/impala_beeswax.py
+++ b/tests/beeswax/impala_beeswax.py
@@ -26,6 +26,7 @@
# result = client.execute(query_string)
# where result is an object of the class ImpalaBeeswaxResult.
from __future__ import absolute_import, division, print_function
+from builtins import filter, map, range
import logging
import time
import shlex
@@ -336,7 +337,7 @@ class ImpalaBeeswaxClient(object):
idx = \
self.__build_summary_table(
summary, idx, False, indent_level, False, first_child_output)
- for child_idx in xrange(1, node.num_children):
+ for child_idx in range(1, node.num_children):
# All other children are indented (we only have 0, 1 or 2 children for every exec
# node at the moment)
idx = self.__build_summary_table(
@@ -498,7 +499,7 @@ class ImpalaBeeswaxClient(object):
return tokens[0].lower()
# Because the WITH clause may precede INSERT or SELECT queries,
# just checking the first token is insufficient.
- if filter(self.INSERT_REGEX.match, tokens):
+ if list(filter(self.INSERT_REGEX.match, tokens)):
return "insert"
return tokens[0].lower()
diff --git a/tests/benchmark/plugins/vtune_plugin.py b/tests/benchmark/plugins/vtune_plugin.py
index ada5e1069..aad27298c 100644
--- a/tests/benchmark/plugins/vtune_plugin.py
+++ b/tests/benchmark/plugins/vtune_plugin.py
@@ -16,6 +16,7 @@
# under the License.
from __future__ import absolute_import, division, print_function
+from builtin import filter, map
from os import environ
from tests.util.cluster_controller import ClusterController
from tests.benchmark.plugins import Plugin
@@ -117,7 +118,7 @@ class VTunePlugin(Plugin):
def _kill_vtune(self, host_dict):
# This method kills threads that are still hanging around after timeout
- kill_list = filter(self.__is_not_none_or_empty_str, host_dict.keys())
+ kill_list = list(filter(self.__is_not_none_or_empty_str, host_dict.keys()))
if kill_list:
self.cluster_controller.deprecated_run_cmd(self.KILL_CMD, hosts=kill_list)
diff --git a/tests/benchmark/report_benchmark_results.py b/tests/benchmark/report_benchmark_results.py
index 680a871f9..b7b1c50a5 100755
--- a/tests/benchmark/report_benchmark_results.py
+++ b/tests/benchmark/report_benchmark_results.py
@@ -29,6 +29,7 @@
# if necessary (2.5).
from __future__ import absolute_import, division, print_function
+from builtins import range
import difflib
import json
import logging
@@ -914,7 +915,7 @@ class ExecSummaryComparison(object):
table.align = 'l'
table.float_format = '.2'
table_contains_at_least_one_row = False
- for row in filter(lambda row: is_significant(row), self.rows):
+ for row in [row for row in self.rows if is_significant(row)]:
table_row = [row[OPERATOR],
'{0:.2%}'.format(row[PERCENT_OF_QUERY]),
'{0:.2%}'.format(row[RSTD]),
diff --git a/tests/common/environ.py b/tests/common/environ.py
index 80c2750ac..d5c7f8f9f 100644
--- a/tests/common/environ.py
+++ b/tests/common/environ.py
@@ -96,7 +96,7 @@ kernel_release = os.uname()[2]
kernel_version_regex = re.compile(r'(\d+)\.(\d+)\.(\d+)\-(\d+).*')
kernel_version_match = kernel_version_regex.match(kernel_release)
if kernel_version_match is not None and len(kernel_version_match.groups()) == 4:
- kernel_version = map(lambda x: int(x), list(kernel_version_match.groups()))
+ kernel_version = [int(x) for x in list(kernel_version_match.groups())]
IS_BUGGY_EL6_KERNEL = 'el6' in kernel_release and kernel_version < [2, 6, 32, 674]
class ImpalaBuildFlavors:
diff --git a/tests/common/impala_cluster.py b/tests/common/impala_cluster.py
index 0b43189eb..8a02e952f 100644
--- a/tests/common/impala_cluster.py
+++ b/tests/common/impala_cluster.py
@@ -18,6 +18,7 @@
# Basic object model of an Impala cluster (set of Impala processes).
from __future__ import absolute_import, division, print_function
+from builtins import map, range
import json
import logging
import os
diff --git a/tests/common/impala_test_suite.py b/tests/common/impala_test_suite.py
index 605d9894a..92099170e 100644
--- a/tests/common/impala_test_suite.py
+++ b/tests/common/impala_test_suite.py
@@ -18,6 +18,7 @@
# The base class that should be used for almost all Impala tests
from __future__ import absolute_import, division, print_function
+from builtins import range
import glob
import grp
import json
@@ -467,7 +468,7 @@ class ImpalaTestSuite(BaseTestSuite):
# Parse the /varz endpoint to get the flag information.
varz = self.get_debug_page(VARZ_URL)
assert 'flags' in varz.keys()
- filtered_varz = filter(lambda flag: flag['name'] == var, varz['flags'])
+ filtered_varz = [flag for flag in varz['flags'] if flag['name'] == var]
assert len(filtered_varz) == 1
assert 'current' in filtered_varz[0].keys()
return filtered_varz[0]['current'].strip()
diff --git a/tests/common/kudu_test_suite.py b/tests/common/kudu_test_suite.py
index 5292b31eb..0d8eca371 100644
--- a/tests/common/kudu_test_suite.py
+++ b/tests/common/kudu_test_suite.py
@@ -16,6 +16,7 @@
# under the License.
from __future__ import absolute_import, division, print_function
+from builtins import range
import os
import pytest
import requests
@@ -110,7 +111,7 @@ class KuduTestSuite(ImpalaTestSuite):
@classmethod
def random_table_name(cls):
- return "".join(choice(string.lowercase) for _ in xrange(10))
+ return "".join(choice(string.lowercase) for _ in range(10))
@classmethod
def to_kudu_table_name(cls, db_name, tbl_name):
@@ -144,7 +145,7 @@ class KuduTestSuite(ImpalaTestSuite):
if not col_names:
if len(col_types) > 26:
raise Exception("Too many columns for default naming")
- col_names = [chr(97 + i) for i in xrange(len(col_types))]
+ col_names = [chr(97 + i) for i in range(len(col_types))]
schema_builder = SchemaBuilder()
for i, t in enumerate(col_types):
column_spec = schema_builder.add_column(col_names[i], type_=t)
diff --git a/tests/common/test_dimensions.py b/tests/common/test_dimensions.py
index 48d147541..3f9bf23fa 100644
--- a/tests/common/test_dimensions.py
+++ b/tests/common/test_dimensions.py
@@ -18,6 +18,7 @@
# Common test dimensions and associated utility functions.
from __future__ import absolute_import, division, print_function
+from builtins import range
import copy
import os
from itertools import product
@@ -83,7 +84,7 @@ class TableFormatInfo(object):
raise ValueError('Table format string cannot be None')
format_parts = table_format_string.strip().split('/')
- if len(format_parts) not in range(2, 4):
+ if len(format_parts) not in list(range(2, 4)):
raise ValueError('Invalid table format %s' % table_format_string)
file_format, compression_codec = format_parts[:2]
diff --git a/tests/common/test_result_verifier.py b/tests/common/test_result_verifier.py
index 59f637d9a..e3b90aad9 100644
--- a/tests/common/test_result_verifier.py
+++ b/tests/common/test_result_verifier.py
@@ -18,6 +18,7 @@
# This modules contians utility functions used to help verify query test results.
from __future__ import absolute_import, division, print_function
+from builtins import map, range
import logging
import math
import re
@@ -457,7 +458,7 @@ def verify_raw_results(test_section, exec_result, file_format, result_section,
expected_results_list = re.findall(r'\[(.*?)\]', expected_results, flags=re.DOTALL)
if not is_raw_string:
# Needs escaping
- expected_results_list = map(lambda s: s.replace('\n', '\\n'), expected_results_list)
+ expected_results_list = [s.replace('\n', '\\n') for s in expected_results_list]
else:
expected_results_list = split_section_lines(expected_results)
expected = QueryTestResult(expected_results_list, expected_types,
@@ -503,7 +504,7 @@ def parse_result_rows(exec_result, escape_strings=True):
cols = row.split('\t')
assert len(cols) == len(col_types)
new_cols = list()
- for i in xrange(len(cols)):
+ for i in range(len(cols)):
if col_types[i] in ['STRING', 'CHAR', 'VARCHAR', 'BINARY']:
col = cols[i]
if isinstance(col, str):
@@ -588,7 +589,7 @@ def compute_aggregation(function, field, runtime_profile):
if (field_regex_re.search(line)):
match_list.extend(re.findall(field_regex, line))
- int_match_list = map(int, match_list)
+ int_match_list = list(map(int, match_list))
result = None
if function == 'SUM':
result = sum(int_match_list)
@@ -620,7 +621,7 @@ def verify_runtime_profile(expected, actual, update_section=False):
# Check the expected and actual rows pairwise.
for line in actual.splitlines():
- for i in xrange(len(expected_lines)):
+ for i in range(len(expected_lines)):
if matched[i]: continue
if expected_regexes[i] is not None:
match = expected_regexes[i].match(line)
@@ -638,7 +639,7 @@ def verify_runtime_profile(expected, actual, update_section=False):
break
unmatched_lines = []
- for i in xrange(len(expected_lines)):
+ for i in range(len(expected_lines)):
if not matched[i] and unexpected_regexes[i] is None:
unmatched_lines.append(expected_lines[i])
assert len(unmatched_lines) == 0, ("Did not find matches for lines in runtime profile:"
@@ -650,7 +651,7 @@ def verify_runtime_profile(expected, actual, update_section=False):
updated_aggregations = []
# Compute the aggregations and check against values
- for i in xrange(len(expected_aggregations)):
+ for i in range(len(expected_aggregations)):
if (expected_aggregations[i] is None): continue
function, field, op, expected_value = expected_aggregations[i]
actual_value = compute_aggregation(function, field, actual)
diff --git a/tests/comparison/cluster.py b/tests/comparison/cluster.py
index c8bb416d8..f95de563d 100644
--- a/tests/comparison/cluster.py
+++ b/tests/comparison/cluster.py
@@ -21,6 +21,7 @@
# module depends on db_connection which use some query generator classes.
from __future__ import absolute_import, division, print_function
+from builtins import range, zip
import hdfs
import logging
import os
@@ -33,7 +34,6 @@ from collections import defaultdict
from collections import OrderedDict
from contextlib import contextmanager
from getpass import getuser
-from itertools import izip
from multiprocessing.pool import ThreadPool
from random import choice
from StringIO import StringIO
@@ -227,7 +227,7 @@ class MiniCluster(Cluster):
hs2_base_port = 21050
web_ui_base_port = 25000
impalads = [MiniClusterImpalad(hs2_base_port + p, web_ui_base_port + p)
- for p in xrange(self.num_impalads)]
+ for p in range(self.num_impalads)]
self._impala = Impala(self, impalads)
class MiniHiveCluster(MiniCluster):
@@ -615,7 +615,7 @@ class Impala(Service):
return dict.fromkeys(stopped_impalads)
messages = OrderedDict()
impalads_with_message = dict()
- for i, message in izip(stopped_impalads, self.for_each_impalad(
+ for i, message in zip(stopped_impalads, self.for_each_impalad(
lambda i: i.find_last_crash_message(start_time), impalads=stopped_impalads)):
if message:
impalads_with_message[i] = "%s crashed:\n%s" % (i.host_name, message)
@@ -631,7 +631,7 @@ class Impala(Service):
# Python doesn't handle ctrl-c well unless a timeout is provided.
results = promise.get(maxint)
if as_dict:
- results = dict(izip(impalads, results))
+ results = dict(zip(impalads, results))
return results
def restart(self):
diff --git a/tests/comparison/data_generator.py b/tests/comparison/data_generator.py
index 45dd611b7..9e744f96b 100755
--- a/tests/comparison/data_generator.py
+++ b/tests/comparison/data_generator.py
@@ -27,6 +27,7 @@
'''
from __future__ import absolute_import, division, print_function
+from builtins import filter, range
import os
from copy import deepcopy
from logging import getLogger
@@ -110,7 +111,7 @@ class DbPopulator(object):
hdfs = self.cluster.hdfs.create_client()
table_and_generators = list()
- for table_idx in xrange(table_count):
+ for table_idx in range(table_count):
table = self._create_random_table(
'table_%s' % (table_idx + 1),
self.min_col_count,
@@ -183,9 +184,10 @@ class DbPopulator(object):
# doesn't actually modify the table's columns. 'table.cols' should be changed
# to allow access to the real columns.
cols = table.cols
- for col_idx in xrange(col_count):
+ for col_idx in range(col_count):
col_type = choice(allowed_types)
- col_type = choice(filter(lambda type_: issubclass(type_, col_type), EXACT_TYPES))
+ col_type = \
+ choice(list(filter(lambda type_: issubclass(type_, col_type), EXACT_TYPES)))
if issubclass(col_type, VarChar) and not issubclass(col_type, String):
col_type = get_varchar_class(randint(1, VarChar.MAX))
elif issubclass(col_type, Char) and not issubclass(col_type, String):
diff --git a/tests/comparison/data_generator_mapred_common.py b/tests/comparison/data_generator_mapred_common.py
index dfc811147..78d7d8d38 100644
--- a/tests/comparison/data_generator_mapred_common.py
+++ b/tests/comparison/data_generator_mapred_common.py
@@ -24,6 +24,7 @@
'''
from __future__ import absolute_import, division, print_function
+from base import range
import base64
import pickle
import StringIO
@@ -55,13 +56,13 @@ class TextTableDataGenerator(object):
col_val_generators = [self._create_val_generator(c.exact_type) for c in cols]
val_buffer_size = 1024
col_val_buffers = [[None] * val_buffer_size for c in cols]
- for row_idx in xrange(self.row_count):
+ for row_idx in range(self.row_count):
val_buffer_idx = row_idx % val_buffer_size
if val_buffer_idx == 0:
for col_idx, col in enumerate(cols):
val_buffer = col_val_buffers[col_idx]
val_generator = col_val_generators[col_idx]
- for idx in xrange(val_buffer_size):
+ for idx in range(val_buffer_size):
val = next(val_generator)
val_buffer[idx] = r"\N" if val is None else val
for col_idx, col in enumerate(cols):
diff --git a/tests/comparison/db_connection.py b/tests/comparison/db_connection.py
index e641b15c9..0177deb6a 100644
--- a/tests/comparison/db_connection.py
+++ b/tests/comparison/db_connection.py
@@ -22,6 +22,7 @@
'''
from __future__ import absolute_import, division, print_function
+from builtins import filter, map, range, zip
import hashlib
import impala.dbapi
import re
@@ -29,7 +30,7 @@ import shelve
from abc import ABCMeta, abstractmethod
from contextlib import closing
from decimal import Decimal as PyDecimal
-from itertools import combinations, ifilter, izip
+from itertools import combinations
from logging import getLogger
from os import symlink, unlink
from pyparsing import (
@@ -123,7 +124,7 @@ class DbCursor(object):
table_keys=table.primary_key_names))
mismatch = True
break
- for left, right in izip(common_table.cols, table.cols):
+ for left, right in zip(common_table.cols, table.cols):
if not (left.name == right.name and left.type == right.type):
LOG.debug('Ignoring table %s. It has different columns %s vs %s.' %
(table_name, left, right))
@@ -513,10 +514,10 @@ class DbCursor(object):
table = self.describe_table(table_name)
sql_templ = 'SELECT COUNT(*) FROM %s GROUP BY %%s HAVING COUNT(*) > 1' % table.name
unique_cols = list()
- for current_depth in xrange(1, depth + 1):
+ for current_depth in range(1, depth + 1):
for cols in combinations(table.cols, current_depth): # redundant combos excluded
cols = set(cols)
- if any(ifilter(lambda unique_subset: unique_subset < cols, unique_cols)):
+ if any(filter(lambda unique_subset: unique_subset < cols, unique_cols)):
# cols contains a combo known to be unique
continue
col_names = ', '.join(col.name for col in cols)
diff --git a/tests/comparison/discrepancy_searcher.py b/tests/comparison/discrepancy_searcher.py
index ce56d7961..2e1fdb143 100755
--- a/tests/comparison/discrepancy_searcher.py
+++ b/tests/comparison/discrepancy_searcher.py
@@ -25,9 +25,9 @@
# TODO: IMPALA-4600: refactor this module
from __future__ import absolute_import, division, print_function
+from builtins import range, zip
from copy import deepcopy
from decimal import Decimal
-from itertools import izip
from logging import getLogger
from math import isinf, isnan
from os import getenv, symlink, unlink
@@ -182,8 +182,8 @@ class QueryResultComparator(object):
data_set.sort(cmp=self.row_sort_cmp)
found_data = False # Will be set to True if the result contains non-zero/NULL data
- for ref_row, test_row in izip(ref_data_set, test_data_set):
- for col_idx, (ref_val, test_val) in enumerate(izip(ref_row, test_row)):
+ for ref_row, test_row in zip(ref_data_set, test_data_set):
+ for col_idx, (ref_val, test_val) in enumerate(zip(ref_row, test_row)):
if ref_val or test_val: # Ignores zeros, ex "SELECT COUNT(*) ... WHERE FALSE"
found_data = True
if self.vals_are_equal(ref_val, test_val):
@@ -222,7 +222,7 @@ class QueryResultComparator(object):
def row_sort_cmp(self, ref_row, test_row):
'''Comparison used for sorting. '''
- for ref_val, test_val in izip(ref_row, test_row):
+ for ref_val, test_val in zip(ref_row, test_row):
if ref_val is None and test_val is not None:
return -1
if ref_val is not None and test_val is None:
@@ -368,7 +368,7 @@ class QueryExecutor(object):
self._table_or_view_name = query.dml_table.name
query_threads = list()
- for sql_writer, cursor, log_file in izip(
+ for sql_writer, cursor, log_file in zip(
self.sql_writers, self.cursors, self.query_logs
):
if self.ENABLE_RANDOM_QUERY_OPTIONS and cursor.db_type == IMPALA:
@@ -387,7 +387,7 @@ class QueryExecutor(object):
query_threads.append(query_thread)
end_time = time() + self.query_timeout_seconds
- for query_thread, cursor in izip(query_threads, self.cursors):
+ for query_thread, cursor in zip(query_threads, self.cursors):
join_time = end_time - time()
if join_time > 0:
query_thread.join(join_time)
@@ -481,7 +481,7 @@ class QueryExecutor(object):
def _create_random_table_name(self):
char_choices = ascii_lowercase
chars = list()
- for idx in xrange(4): # will result in ~1M combinations
+ for idx in range(4): # will result in ~1M combinations
if idx == 1:
char_choices += '_' + digits
chars.append(choice(char_choices))
@@ -585,7 +585,7 @@ class FrontendExceptionSearcher(object):
LOG.error("Error generating explain plan for test db:\n%s" % sql)
raise e
- for idx in xrange(number_of_test_queries):
+ for idx in range(number_of_test_queries):
LOG.info("Explaining query #%s" % (idx + 1))
statement_type = self.query_profile.choose_statement()
statement_generator = get_generator(statement_type)(self.query_profile)
diff --git a/tests/comparison/funcs.py b/tests/comparison/funcs.py
index 7a0f37272..ad523fe6d 100644
--- a/tests/comparison/funcs.py
+++ b/tests/comparison/funcs.py
@@ -16,8 +16,8 @@
# under the License.
from __future__ import absolute_import, division, print_function
+from builtins import filter
from copy import deepcopy
-from itertools import ifilter
from tests.comparison.common import ValExpr
from tests.comparison.db_types import (
@@ -419,7 +419,7 @@ def create_func(name, returns=None, accepts=[], signatures=[], base_type=Func):
for arg_idx, arg in enumerate(signature.args):
replacement_signature = None
if arg.is_subquery:
- if any(ifilter(lambda type_: type_ == Number, arg.type)):
+ if any(filter(lambda type_: type_ == Number, arg.type)):
raise Exception('Number not accepted in subquery signatures')
elif arg.type == Number:
for replacement_type in [Decimal, Int, Float]:
diff --git a/tests/comparison/query.py b/tests/comparison/query.py
index c3e80352f..dd73f895e 100644
--- a/tests/comparison/query.py
+++ b/tests/comparison/query.py
@@ -16,6 +16,7 @@
# under the License.
from __future__ import absolute_import, division, print_function
+from builtins import range
from abc import ABCMeta, abstractproperty
from copy import deepcopy
from logging import getLogger
@@ -43,7 +44,7 @@ class StatementExecutionMode(object):
DML_SETUP,
# a DML statement that's actually a test
DML_TEST,
- ) = xrange(5)
+ ) = range(5)
class AbstractStatement(object):
diff --git a/tests/comparison/query_generator.py b/tests/comparison/query_generator.py
index f4150fc86..3da76a1ab 100644
--- a/tests/comparison/query_generator.py
+++ b/tests/comparison/query_generator.py
@@ -16,9 +16,9 @@
# under the License.
from __future__ import absolute_import, division, print_function
+from builtins import filter, range
from collections import defaultdict
from copy import deepcopy
-from itertools import ifilter
from logging import getLogger
from random import shuffle, choice, randint, randrange
@@ -254,7 +254,7 @@ class QueryGenerator(object):
table_exprs = TableExprList(table_exprs)
with_clause_inline_views = TableExprList()
for with_clause_inline_view_idx \
- in xrange(self.profile.get_with_clause_table_ref_count()):
+ in range(self.profile.get_with_clause_table_ref_count()):
query = self.generate_statement(table_exprs,
allow_with_clause=self.profile.use_nested_with())
with_clause_alias_count = getattr(self.root_query, 'with_clause_alias_count', 0) + 1
@@ -933,7 +933,7 @@ class QueryGenerator(object):
else:
excluded_designs.append('DETERMINISTIC_ORDER')
- allow_agg = any(ifilter(lambda expr: expr.contains_agg, select_item_exprs))
+ allow_agg = any(filter(lambda expr: expr.contains_agg, select_item_exprs))
value = self._create_analytic_func_tree(return_type, excluded_designs, allow_agg)
value = self.populate_func_with_vals(
value,
@@ -1125,7 +1125,7 @@ class QueryGenerator(object):
table_expr.alias = self.get_next_id()
from_clause = FromClause(table_expr)
- for idx in xrange(1, table_count):
+ for idx in range(1, table_count):
join_clause = self._create_join_clause(from_clause, table_exprs)
join_clause.table_expr.alias = self.get_next_id()
from_clause.join_clauses.append(join_clause)
@@ -1294,7 +1294,7 @@ class QueryGenerator(object):
% (arg_stop_idx, arg_start_idx))
if null_args is None:
null_args = list()
- for idx in xrange(arg_start_idx, arg_stop_idx):
+ for idx in range(arg_start_idx, arg_stop_idx):
arg = func.args[idx]
if arg.is_constant and issubclass(arg.type, allowed_types):
assert arg.val is None
@@ -1422,7 +1422,7 @@ class QueryGenerator(object):
if not relational_col_types:
relational_col_types = tuple()
- for _ in xrange(func_count):
+ for _ in range(func_count):
is_relational = False
if and_or_count > 0:
@@ -1495,12 +1495,12 @@ def generate_queries_for_manual_inspection():
tables = list()
data_types = list(TYPES)
data_types.remove(Float)
- for table_idx in xrange(NUM_TABLES):
+ for table_idx in range(NUM_TABLES):
table = Table('table_%s' % table_idx)
tables.append(table)
cols = table.cols
col_idx = 0
- for _ in xrange(NUM_COLS_EACH_TYPE):
+ for _ in range(NUM_COLS_EACH_TYPE):
for col_type in data_types:
col = Column(table, '%s_col_%s' % (col_type.__name__.lower(), col_idx), col_type)
cols.append(col)
@@ -1512,7 +1512,7 @@ def generate_queries_for_manual_inspection():
sql_writer = SqlWriter.create(dialect='IMPALA')
ref_writer = SqlWriter.create(dialect='POSTGRESQL',
nulls_order_asc=query_profile.nulls_order_asc())
- for _ in xrange(NUM_QUERIES):
+ for _ in range(NUM_QUERIES):
query = query_generator.generate_statement(tables)
print("Test db")
print(sql_writer.write_query(query) + '\n')
diff --git a/tests/comparison/query_profile.py b/tests/comparison/query_profile.py
index f5003732a..e47f7a6c6 100644
--- a/tests/comparison/query_profile.py
+++ b/tests/comparison/query_profile.py
@@ -16,6 +16,8 @@
# under the License.
from __future__ import absolute_import, division, print_function
+from builtins import filter
+from functools import reduce
from logging import getLogger
from random import choice, randint, random, shuffle
@@ -303,14 +305,14 @@ class DefaultProfile(object):
return choice_
numeric_choice -= weight
- def _choose_from_filtered_weights(self, filter, *weights):
+ def _choose_from_filtered_weights(self, filter_fn, *weights):
'''Convenience method, apply the given filter before choosing a value.'''
if isinstance(weights[0], str):
weights = self.weights(*weights)
else:
weights = weights[0]
return self._choose_from_weights(dict((choice_, weight) for choice_, weight
- in weights.iteritems() if filter(choice_)))
+ in weights.iteritems() if filter_fn(choice_)))
def _decide_from_probability(self, *keys):
return random() < self.probability(*keys)
@@ -490,11 +492,11 @@ class DefaultProfile(object):
'''
if not signatures:
raise Exception('At least one signature is required')
- filtered_signatures = filter(
+ filtered_signatures = list(filter(
lambda s: s.return_type == Boolean \
and len(s.args) > 1 \
and not any(a.is_subquery for a in s.args),
- signatures)
+ signatures))
if not filtered_signatures:
raise Exception(
'None of the provided signatures corresponded to a relational function')
@@ -717,7 +719,7 @@ class TestFunctionProfile(DefaultProfile):
def choose_func_signature(self, signatures):
if not signatures:
raise Exception('At least one signature is required')
- preferred_signatures = filter(lambda s: "DistinctFrom" in s.func._NAME, signatures)
+ preferred_signatures = [s for s in signatures if "DistinctFrom" in s.func._NAME]
if preferred_signatures:
signatures = preferred_signatures
return super(TestFunctionProfile, self).choose_func_signature(signatures)
diff --git a/tests/comparison/statement_generator.py b/tests/comparison/statement_generator.py
index 3cfa713c4..81e956710 100644
--- a/tests/comparison/statement_generator.py
+++ b/tests/comparison/statement_generator.py
@@ -16,6 +16,7 @@
# under the License.
from __future__ import absolute_import, division, print_function
+from builtins import range
from copy import deepcopy
from tests.comparison.common import Table
@@ -114,7 +115,7 @@ class InsertStatementGenerator(object):
null.
"""
values_rows = []
- for _ in xrange(self.profile.choose_insert_values_row_count()):
+ for _ in range(self.profile.choose_insert_values_row_count()):
values_row = []
for col in columns:
if col.is_primary_key:
diff --git a/tests/conftest.py b/tests/conftest.py
index ae8fc0a3c..d7bfd4372 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -18,6 +18,7 @@
# py.test configuration module
#
from __future__ import absolute_import, division, print_function
+from builtins import map, range
from impala.dbapi import connect as impala_connect
from kudu import connect as kudu_connect
from random import choice, sample
@@ -231,7 +232,7 @@ def pytest_generate_tests(metafunc):
LOG.warning("No test vectors generated for test '%s'. Check constraints and "
"input vectors" % metafunc.function.func_name)
- vector_names = map(str, vectors)
+ vector_names = list(map(str, vectors))
# In the case this is a test result update or sanity run, select a single test vector
# to run. This is okay for update_results because results are expected to be the same
# for all test vectors.
@@ -665,7 +666,7 @@ def pytest_collection_modifyitems(items, config, session):
return
num_items = len(items)
- this_shard, num_shards = map(int, config.option.shard_tests.split("/"))
+ this_shard, num_shards = list(map(int, config.option.shard_tests.split("/")))
assert 0 <= this_shard <= num_shards
if this_shard == num_shards:
this_shard = 0
diff --git a/tests/custom_cluster/test_admission_controller.py b/tests/custom_cluster/test_admission_controller.py
index b10f8e4ef..79e4cc3a7 100644
--- a/tests/custom_cluster/test_admission_controller.py
+++ b/tests/custom_cluster/test_admission_controller.py
@@ -18,6 +18,7 @@
# Tests admission control
from __future__ import absolute_import, division, print_function
+from builtins import range
import itertools
import logging
import os
@@ -854,7 +855,7 @@ class TestAdmissionController(TestAdmissionControllerBase, HS2TestSuite):
EXPECTED_REASON = \
"Latest admission queue reason: number of running queries 1 is at or over limit 1"
NUM_QUERIES = 5
- profiles = self._execute_and_collect_profiles([STMT for i in xrange(NUM_QUERIES)],
+ profiles = self._execute_and_collect_profiles([STMT for i in range(NUM_QUERIES)],
TIMEOUT_S)
num_reasons = len([profile for profile in profiles if EXPECTED_REASON in profile])
@@ -891,7 +892,7 @@ class TestAdmissionController(TestAdmissionControllerBase, HS2TestSuite):
NUM_QUERIES = 5
# IMPALA-9856: Disable query result spooling so that we can run queries with low
# mem_limit.
- profiles = self._execute_and_collect_profiles([STMT for i in xrange(NUM_QUERIES)],
+ profiles = self._execute_and_collect_profiles([STMT for i in range(NUM_QUERIES)],
TIMEOUT_S, {'mem_limit': '9mb', 'spool_query_results': '0'})
num_reasons = len([profile for profile in profiles if EXPECTED_REASON in profile])
@@ -933,7 +934,7 @@ class TestAdmissionController(TestAdmissionControllerBase, HS2TestSuite):
NUM_QUERIES = 5
# IMPALA-9856: Disable query result spooling so that we can run queries with low
# mem_limit.
- profiles = self._execute_and_collect_profiles([STMT for i in xrange(NUM_QUERIES)],
+ profiles = self._execute_and_collect_profiles([STMT for i in range(NUM_QUERIES)],
TIMEOUT_S, {'mem_limit': '2mb', 'spool_query_results': '0'}, True)
EXPECTED_REASON = """.*Admission for query exceeded timeout 1000ms in pool """\
@@ -967,7 +968,7 @@ class TestAdmissionController(TestAdmissionControllerBase, HS2TestSuite):
NUM_QUERIES = 5
# IMPALA-9856: Disable query result spooling so that we can run queries with low
# mem_limit.
- profiles = self._execute_and_collect_profiles([STMT for i in xrange(NUM_QUERIES)],
+ profiles = self._execute_and_collect_profiles([STMT for i in range(NUM_QUERIES)],
TIMEOUT_S, {'mem_limit': '2mb', 'spool_query_results': '0'}, True)
EXPECTED_REASON = """.*Admission for query exceeded timeout 1000ms in pool """\
@@ -1001,7 +1002,7 @@ class TestAdmissionController(TestAdmissionControllerBase, HS2TestSuite):
"admission-controller.total-dequeue-failed-coordinator-limited"
original_metric_value = self.get_ac_process().service.get_metric_value(
coordinator_limited_metric)
- profiles = self._execute_and_collect_profiles([STMT for i in xrange(NUM_QUERIES)],
+ profiles = self._execute_and_collect_profiles([STMT for i in range(NUM_QUERIES)],
TIMEOUT_S, config_options={"mt_dop": 4})
num_reasons = len([profile for profile in profiles if EXPECTED_REASON in profile])
@@ -1403,7 +1404,7 @@ class TestAdmissionController(TestAdmissionControllerBase, HS2TestSuite):
STMT = "select sleep(100)"
TIMEOUT_S = 60
NUM_QUERIES = 5
- profiles = self._execute_and_collect_profiles([STMT for i in xrange(NUM_QUERIES)],
+ profiles = self._execute_and_collect_profiles([STMT for i in range(NUM_QUERIES)],
TIMEOUT_S, allow_query_failure=True)
ADMITTED_STALENESS_WARNING = \
"Warning: admission control information from statestore is stale"
@@ -1795,7 +1796,7 @@ class TestAdmissionControllerStress(TestAdmissionControllerBase):
num_submitted queries. See IMPALA-6227 for an example of problems with inconsistent
metrics where a dequeued query is reflected in dequeued but not admitted."""
ATTEMPTS = 5
- for i in xrange(ATTEMPTS):
+ for i in range(ATTEMPTS):
metrics = self.get_admission_metrics()
admitted_immediately = num_submitted - metrics['queued'] - metrics['rejected']
if admitted_immediately + metrics['dequeued'] == metrics['admitted']:
@@ -1891,7 +1892,7 @@ class TestAdmissionControllerStress(TestAdmissionControllerBase):
# Request admitted clients to end their queries
current_executing_queries = []
- for i in xrange(num_queries):
+ for i in range(num_queries):
# pop() is thread-safe, it's OK if another thread is appending concurrently.
thread = self.executing_threads.pop(0)
LOG.info("Cancelling query %s", thread.query_num)
@@ -2100,7 +2101,7 @@ class TestAdmissionControllerStress(TestAdmissionControllerBase):
initial_metrics = self.get_admission_metrics()
log_metrics("Initial metrics: ", initial_metrics)
- for query_num in xrange(num_queries):
+ for query_num in range(num_queries):
impalad = self.impalads[query_num % len(self.impalads)]
query_end_behavior = QUERY_END_BEHAVIORS[query_num % len(QUERY_END_BEHAVIORS)]
thread = self.SubmitQueryThread(impalad, additional_query_options, vector,
diff --git a/tests/custom_cluster/test_auto_scaling.py b/tests/custom_cluster/test_auto_scaling.py
index 86dda7687..5a9f2cbbf 100644
--- a/tests/custom_cluster/test_auto_scaling.py
+++ b/tests/custom_cluster/test_auto_scaling.py
@@ -18,6 +18,7 @@
# under the License.
from __future__ import absolute_import, division, print_function
+from builtins import range
import logging
import pytest
from time import sleep, time
diff --git a/tests/custom_cluster/test_blacklist.py b/tests/custom_cluster/test_blacklist.py
index d98cd8221..be8d6999f 100644
--- a/tests/custom_cluster/test_blacklist.py
+++ b/tests/custom_cluster/test_blacklist.py
@@ -16,6 +16,7 @@
# under the License.
from __future__ import absolute_import, division, print_function
+from builtins import range
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
import pytest
@@ -223,7 +224,7 @@ class TestBlacklistFaultyDisk(CustomClusterTestSuite):
def __generate_scratch_dir(self, num):
result = []
- for i in xrange(num):
+ for i in range(num):
dir_path = tempfile.mkdtemp()
self.created_dirs.append(dir_path)
result.append(dir_path)
diff --git a/tests/custom_cluster/test_breakpad.py b/tests/custom_cluster/test_breakpad.py
index c44d3259b..6e43c42b5 100644
--- a/tests/custom_cluster/test_breakpad.py
+++ b/tests/custom_cluster/test_breakpad.py
@@ -16,6 +16,7 @@
# under the License.
from __future__ import absolute_import, division, print_function
+from builtins import range
import glob
import os
import psutil
@@ -75,7 +76,7 @@ class TestBreakpadBase(CustomClusterTestSuite):
def kill_cluster(self, signal):
self.cluster.refresh()
processes = self.cluster.impalads + [self.cluster.catalogd, self.cluster.statestored]
- processes = filter(None, processes)
+ processes = [_f for _f in processes if _f]
self.kill_processes(processes, signal)
signal is SIGUSR1 or self.assert_all_processes_killed()
@@ -317,7 +318,7 @@ class TestBreakpadExhaustive(TestBreakpadBase):
cluster_size = self.get_num_processes('impalad')
# We trigger several rounds of minidump creation to make sure that all daemons wrote
# enough files to trigger rotation.
- for i in xrange(max_minidumps + 1):
+ for i in range(max_minidumps + 1):
self.kill_cluster(SIGUSR1)
# Breakpad forks to write its minidump files, sleep briefly to allow the forked
# processes to start.
diff --git a/tests/custom_cluster/test_codegen_cache.py b/tests/custom_cluster/test_codegen_cache.py
index 72b8e28d6..611ba5a03 100644
--- a/tests/custom_cluster/test_codegen_cache.py
+++ b/tests/custom_cluster/test_codegen_cache.py
@@ -16,6 +16,7 @@
# under the License.
from __future__ import absolute_import, division, print_function
+from builtins import range
import pytest
from copy import copy
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
diff --git a/tests/custom_cluster/test_concurrent_ddls.py b/tests/custom_cluster/test_concurrent_ddls.py
index 8d61e2735..4052de682 100644
--- a/tests/custom_cluster/test_concurrent_ddls.py
+++ b/tests/custom_cluster/test_concurrent_ddls.py
@@ -16,6 +16,7 @@
# under the License.
from __future__ import absolute_import, division, print_function
+from builtins import range
import pytest
import threading
@@ -140,9 +141,9 @@ class TestConcurrentDdls(CustomClusterTestSuite):
# Run DDLs with invalidate metadata in parallel
NUM_ITERS = 16
worker = [None] * (NUM_ITERS + 1)
- for i in xrange(1, NUM_ITERS + 1):
+ for i in range(1, NUM_ITERS + 1):
worker[i] = pool.apply_async(run_ddls, (i,))
- for i in xrange(1, NUM_ITERS + 1):
+ for i in range(1, NUM_ITERS + 1):
try:
worker[i].get(timeout=100)
except TimeoutError:
@@ -185,7 +186,7 @@ class TestConcurrentDdls(CustomClusterTestSuite):
NUM_ITERS = 20
pool = ThreadPool(processes=2)
- for i in xrange(NUM_ITERS):
+ for i in range(NUM_ITERS):
# Run two INVALIDATE METADATA commands in parallel
r1 = pool.apply_async(run_invalidate_metadata)
r2 = pool.apply_async(run_invalidate_metadata)
diff --git a/tests/custom_cluster/test_concurrent_kudu_create.py b/tests/custom_cluster/test_concurrent_kudu_create.py
index ad86ee925..d27db8eba 100644
--- a/tests/custom_cluster/test_concurrent_kudu_create.py
+++ b/tests/custom_cluster/test_concurrent_kudu_create.py
@@ -16,6 +16,7 @@
# under the License.
from __future__ import absolute_import, division, print_function
+from builtins import range
import pytest
import threading
import time
@@ -50,7 +51,7 @@ class TestConcurrentKuduCreate(CustomClusterTestSuite):
self.execute_query("drop table if exists %s" % table_name)
NUM_ITERS = 20
pool = ThreadPool(processes=3)
- for i in xrange(NUM_ITERS):
+ for i in range(NUM_ITERS):
# Run several commands by specific time interval to reproduce this bug
r1 = pool.apply_async(run_create_table_if_not_exists)
r2 = pool.apply_async(run_create_table_if_not_exists)
diff --git a/tests/custom_cluster/test_custom_statestore.py b/tests/custom_cluster/test_custom_statestore.py
index 185e3f049..4a5db8149 100644
--- a/tests/custom_cluster/test_custom_statestore.py
+++ b/tests/custom_cluster/test_custom_statestore.py
@@ -19,6 +19,7 @@
# Tests statestore with non-default startup options
from __future__ import absolute_import, division, print_function
+from builtins import range
import logging
import os
import pytest
@@ -79,7 +80,7 @@ class TestCustomStatestore(CustomClusterTestSuite):
exceeded."""
# With a statestore_max_subscribers of 3, we should hit the registration error
# pretty quick.
- for x in xrange(20):
+ for x in range(20):
response = self.__register_subscriber()
if response.status.status_code == TErrorCode.OK:
self.registration_id = response.registration_id
diff --git a/tests/custom_cluster/test_events_custom_configs.py b/tests/custom_cluster/test_events_custom_configs.py
index 703f11b61..dc7ac51c9 100644
--- a/tests/custom_cluster/test_events_custom_configs.py
+++ b/tests/custom_cluster/test_events_custom_configs.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
from __future__ import absolute_import, division, print_function
+from builtins import range
import logging
import pytest
@@ -199,7 +200,7 @@ class TestEventProcessingCustomConfigs(CustomClusterTestSuite):
removed_metric_val_before = EventProcessorUtils.get_int_metric(removed_metric_name, 0)
events_skipped_before = EventProcessorUtils.get_int_metric('events-skipped', 0)
num_iters = 100
- for iter in xrange(num_iters):
+ for iter in range(num_iters):
for q in queries:
try:
self.execute_query_expect_success(self.create_impala_client(), q)
diff --git a/tests/custom_cluster/test_exchange_deferred_batches.py b/tests/custom_cluster/test_exchange_deferred_batches.py
index 52a70a074..05c97ed47 100644
--- a/tests/custom_cluster/test_exchange_deferred_batches.py
+++ b/tests/custom_cluster/test_exchange_deferred_batches.py
@@ -16,6 +16,7 @@
# under the License.
from __future__ import absolute_import, division, print_function
+from builtins import range
import pytest
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
from tests.common.skip import SkipIfBuildType
diff --git a/tests/custom_cluster/test_executor_groups.py b/tests/custom_cluster/test_executor_groups.py
index f0d3c3293..5d624d969 100644
--- a/tests/custom_cluster/test_executor_groups.py
+++ b/tests/custom_cluster/test_executor_groups.py
@@ -18,6 +18,7 @@
# under the License.
from __future__ import absolute_import, division, print_function
+from builtins import range
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
from tests.util.concurrent_workload import ConcurrentWorkload
@@ -375,7 +376,7 @@ class TestExecutorGroups(CustomClusterTestSuite):
# a new query to fit (see IMPALA-9073).
NUM_SAMPLES = 30
executor_slots_in_use = []
- for _ in xrange(NUM_SAMPLES):
+ for _ in range(NUM_SAMPLES):
backends_json = json.loads(
self.impalad_test_service.read_debug_webpage('backends?json'))
for backend in backends_json['backends']:
diff --git a/tests/custom_cluster/test_hdfs_fd_caching.py b/tests/custom_cluster/test_hdfs_fd_caching.py
index c2c66be49..b5e5db5e8 100644
--- a/tests/custom_cluster/test_hdfs_fd_caching.py
+++ b/tests/custom_cluster/test_hdfs_fd_caching.py
@@ -16,6 +16,7 @@
# under the License.
from __future__ import absolute_import, division, print_function
+from builtins import range
import pytest
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
diff --git a/tests/custom_cluster/test_incremental_metadata_updates.py b/tests/custom_cluster/test_incremental_metadata_updates.py
index 82334f080..2f330f3d6 100755
--- a/tests/custom_cluster/test_incremental_metadata_updates.py
+++ b/tests/custom_cluster/test_incremental_metadata_updates.py
@@ -16,6 +16,7 @@
# under the License.
from __future__ import absolute_import, division, print_function
+from builtins import range
import pytest
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
diff --git a/tests/custom_cluster/test_local_catalog.py b/tests/custom_cluster/test_local_catalog.py
index baeb453bc..25e0c9f62 100644
--- a/tests/custom_cluster/test_local_catalog.py
+++ b/tests/custom_cluster/test_local_catalog.py
@@ -18,6 +18,7 @@
# Test behaviors specific to --use_local_catalog being enabled.
from __future__ import absolute_import, division, print_function
+from builtins import range
import pytest
import Queue
import random
@@ -144,7 +145,7 @@ class TestCompactCatalogUpdates(CustomClusterTestSuite):
# catalog pushes a new topic update.
self.cluster.catalogd.start()
NUM_ATTEMPTS = 30
- for attempt in xrange(NUM_ATTEMPTS):
+ for attempt in range(NUM_ATTEMPTS):
try:
self.assert_impalad_log_contains('WARNING', 'Detected catalog service restart')
err = self.execute_query_expect_failure(client, "select * from %s" % view)
@@ -445,7 +446,7 @@ class TestLocalCatalogRetries(CustomClusterTestSuite):
# Prior to fixing IMPALA-7534, this test would fail within 20-30 iterations,
# so 100 should be quite reliable as a regression test.
NUM_ITERS = 100
- for i in t.imap_unordered(do_table, xrange(NUM_ITERS)):
+ for i in t.imap_unordered(do_table, range(NUM_ITERS)):
pass
class TestObservability(CustomClusterTestSuite):
@@ -493,7 +494,7 @@ class TestObservability(CustomClusterTestSuite):
"explain select count(*) from functional.alltypes",
"create table %s (a int)" % test_table_name,
"drop table %s" % test_table_name]
- for _ in xrange(0, 10):
+ for _ in range(0, 10):
for query in queries_to_test:
ret = self.execute_query_expect_success(client, query)
assert ret.runtime_profile.count("Frontend:") == 1
diff --git a/tests/custom_cluster/test_mem_reservations.py b/tests/custom_cluster/test_mem_reservations.py
index f6e66ae81..36996f910 100644
--- a/tests/custom_cluster/test_mem_reservations.py
+++ b/tests/custom_cluster/test_mem_reservations.py
@@ -16,6 +16,7 @@
# under the License.
from __future__ import absolute_import, division, print_function
+from builtins import range
import pytest
import threading
@@ -84,7 +85,7 @@ class TestMemReservations(CustomClusterTestSuite):
client = self.coordinator.service.create_beeswax_client()
try:
client.set_configuration(CONFIG_MAP)
- for i in xrange(20):
+ for i in range(20):
result = client.execute(self.query)
assert result.success
assert len(result.data) == 1
@@ -96,7 +97,7 @@ class TestMemReservations(CustomClusterTestSuite):
# Create two threads to submit COORDINATOR_QUERY to two different coordinators concurrently.
# They should both succeed.
threads = [QuerySubmitThread(COORDINATOR_QUERY, self.cluster.impalads[i])
- for i in xrange(2)]
+ for i in range(2)]
for t in threads: t.start()
for t in threads:
t.join()
diff --git a/tests/custom_cluster/test_metadata_replicas.py b/tests/custom_cluster/test_metadata_replicas.py
index 4afa4249f..47e628eea 100644
--- a/tests/custom_cluster/test_metadata_replicas.py
+++ b/tests/custom_cluster/test_metadata_replicas.py
@@ -16,6 +16,7 @@
# under the License.
from __future__ import absolute_import, division, print_function
+from builtins import range
import pytest
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
from tests.common.skip import SkipIfFS
@@ -53,7 +54,7 @@ class TestMetadataReplicas(CustomClusterTestSuite):
# Issue several invalidates to boost the version for the current incarnation of the
# catalog. As a result, the table we'll add to Hive will get a version that's easier
# to see is higher than the highest version of the restarted catalogd incarnation.
- for i in xrange(0, 50):
+ for i in range(0, 50):
self.client.execute("invalidate metadata functional.alltypes")
assert self.cluster.catalogd.service.get_catalog_version() >= 50
# Creates a database and table with Hive and makes it visible to Impala.
@@ -86,7 +87,7 @@ class TestMetadataReplicas(CustomClusterTestSuite):
c_objects = self.cluster.catalogd.service.get_catalog_objects()
i_objects = [proc.service.get_catalog_objects() for proc in self.cluster.impalads]
- for idx in xrange(0, len(i_objects)):
+ for idx in range(0, len(i_objects)):
i_obj = i_objects[idx]
diff = self.__diff_catalog_objects(c_objects, i_obj)
assert diff[0] == {},\
diff --git a/tests/custom_cluster/test_metastore_service.py b/tests/custom_cluster/test_metastore_service.py
index c74ab63fd..2ce83907c 100644
--- a/tests/custom_cluster/test_metastore_service.py
+++ b/tests/custom_cluster/test_metastore_service.py
@@ -16,6 +16,7 @@
# under the License.
from __future__ import absolute_import, division, print_function
+from builtins import range
import pytest
from hive_metastore.ttypes import Database
from hive_metastore.ttypes import FieldSchema
diff --git a/tests/custom_cluster/test_parquet_max_page_header.py b/tests/custom_cluster/test_parquet_max_page_header.py
index 73496487b..52e13c690 100644
--- a/tests/custom_cluster/test_parquet_max_page_header.py
+++ b/tests/custom_cluster/test_parquet_max_page_header.py
@@ -18,6 +18,7 @@
# Tests for IMPALA-2273
from __future__ import absolute_import, division, print_function
+from builtins import range
import os
import pytest
import random
@@ -91,9 +92,9 @@ class TestParquetMaxPageHeader(CustomClusterTestSuite):
file_name = os.path.join(dir, file)
# Create two 10MB long strings.
random_text1 = "".join([random.choice(string.letters)
- for i in xrange(self.MAX_STRING_LENGTH)])
+ for i in range(self.MAX_STRING_LENGTH)])
random_text2 = "".join([random.choice(string.letters)
- for i in xrange(self.MAX_STRING_LENGTH)])
+ for i in range(self.MAX_STRING_LENGTH)])
put = subprocess.Popen(["hdfs", "dfs", "-put", "-d", "-f", "-", file_name],
stdin=subprocess.PIPE, bufsize=-1)
put.stdin.write(random_text1 + "\n")
diff --git a/tests/custom_cluster/test_preload_table_types.py b/tests/custom_cluster/test_preload_table_types.py
index f86427eac..bf57200de 100644
--- a/tests/custom_cluster/test_preload_table_types.py
+++ b/tests/custom_cluster/test_preload_table_types.py
@@ -16,6 +16,7 @@
# under the License.
from __future__ import absolute_import, division, print_function
+from builtins import range
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
diff --git a/tests/custom_cluster/test_process_failures.py b/tests/custom_cluster/test_process_failures.py
index 21464ed3f..c9e9cd3ea 100644
--- a/tests/custom_cluster/test_process_failures.py
+++ b/tests/custom_cluster/test_process_failures.py
@@ -16,6 +16,7 @@
# under the License.
from __future__ import absolute_import, division, print_function
+from builtins import range
import pytest
from beeswaxd.BeeswaxService import QueryState
@@ -76,7 +77,7 @@ class TestProcessFailures(CustomClusterTestSuite):
handles = []
# Run num_concurrent_queries asynchronously
- for _ in xrange(num_concurrent_queries):
+ for _ in range(num_concurrent_queries):
handles.append(client.execute_async(query))
# Wait for the queries to start running
@@ -87,7 +88,7 @@ class TestProcessFailures(CustomClusterTestSuite):
impalad.kill()
# Assert that all executors have 0 in-flight fragments
- for i in xrange(1, len(self.cluster.impalads)):
+ for i in range(1, len(self.cluster.impalads)):
self.cluster.impalads[i].service.wait_for_metric_value(
"impala-server.num-fragments-in-flight", 0, timeout=30)
diff --git a/tests/custom_cluster/test_query_expiration.py b/tests/custom_cluster/test_query_expiration.py
index d7dd5a0c3..7e08dc81c 100644
--- a/tests/custom_cluster/test_query_expiration.py
+++ b/tests/custom_cluster/test_query_expiration.py
@@ -18,6 +18,7 @@
# Tests for query expiration.
from __future__ import absolute_import, division, print_function
+from builtins import range
import pytest
import re
import threading
@@ -246,14 +247,14 @@ class TestQueryExpiration(CustomClusterTestSuite):
num_expired = impalad.service.get_metric_value('impala-server.num-queries-expired')
non_expiring_threads = \
[NonExpiringQueryThread(impalad.service.create_beeswax_client())
- for _ in xrange(5)]
+ for _ in range(5)]
expiring_threads = [ExpiringQueryThread(impalad.service.create_beeswax_client())
- for _ in xrange(5)]
+ for _ in range(5)]
time_limit_threads = [TimeLimitThread(impalad.service.create_beeswax_client())
- for _ in xrange(5)]
+ for _ in range(5)]
non_expiring_time_limit_threads = [
NonExpiringTimeLimitThread(impalad.service.create_beeswax_client())
- for _ in xrange(5)]
+ for _ in range(5)]
all_threads = non_expiring_threads + expiring_threads + time_limit_threads +\
non_expiring_time_limit_threads
for t in all_threads:
diff --git a/tests/custom_cluster/test_query_retries.py b/tests/custom_cluster/test_query_retries.py
index 5d321ceed..56e852970 100644
--- a/tests/custom_cluster/test_query_retries.py
+++ b/tests/custom_cluster/test_query_retries.py
@@ -22,6 +22,7 @@
# TODO: Add a test that cancels queries while a retry is running
from __future__ import absolute_import, division, print_function
+from builtins import map, range
import pytest
import re
import shutil
@@ -205,14 +206,14 @@ class TestQueryRetries(CustomClusterTestSuite):
# Launch a set of concurrent queries.
num_concurrent_queries = 3
handles = []
- for _ in xrange(num_concurrent_queries):
+ for _ in range(num_concurrent_queries):
handle = self.execute_query_async(self._shuffle_heavy_query,
query_options={'retry_failed_queries': 'true'})
handles.append(handle)
# Wait for each query to start running.
running_state = self.client.QUERY_STATES['RUNNING']
- map(lambda handle: self.wait_for_state(handle, running_state, 60), handles)
+ list(map(lambda handle: self.wait_for_state(handle, running_state, 60), handles))
# Kill a random impalad.
killed_impalad = self.__kill_random_impalad()
@@ -1180,7 +1181,7 @@ class TestQueryRetriesFaultyDisk(CustomClusterTestSuite):
def __generate_scratch_dir(self, num):
result = []
- for i in xrange(num):
+ for i in range(num):
dir_path = tempfile.mkdtemp()
self.created_dirs.append(dir_path)
result.append(dir_path)
diff --git a/tests/custom_cluster/test_restart_services.py b/tests/custom_cluster/test_restart_services.py
index 7d9c003d1..1492089f5 100644
--- a/tests/custom_cluster/test_restart_services.py
+++ b/tests/custom_cluster/test_restart_services.py
@@ -16,6 +16,7 @@
# under the License.
from __future__ import absolute_import, division, print_function
+from builtins import range
import logging
import os
import pytest
@@ -59,7 +60,7 @@ class TestRestart(CustomClusterTestSuite):
# existing metrics yet so we wait for some time here.
wait_time_s = build_flavor_timeout(60, slow_build_timeout=100)
sleep(wait_time_s)
- for retry in xrange(wait_time_s):
+ for retry in range(wait_time_s):
try:
cursor.execute("describe database functional")
return
@@ -82,7 +83,7 @@ class TestRestart(CustomClusterTestSuite):
client = self.cluster.impalads[0].service.create_beeswax_client()
assert client is not None
- for i in xrange(5):
+ for i in range(5):
self.execute_query_expect_success(client, "select * from functional.alltypes")
node_to_restart = 1 + (i % 2)
self.cluster.impalads[node_to_restart].restart()
diff --git a/tests/custom_cluster/test_rpc_timeout.py b/tests/custom_cluster/test_rpc_timeout.py
index 410e22e9e..375a4b9c4 100644
--- a/tests/custom_cluster/test_rpc_timeout.py
+++ b/tests/custom_cluster/test_rpc_timeout.py
@@ -16,6 +16,7 @@
# under the License.
from __future__ import absolute_import, division, print_function
+from builtins import range
import pytest
from tests.beeswax.impala_beeswax import ImpalaBeeswaxException
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
@@ -141,7 +142,7 @@ class TestRPCTimeout(CustomClusterTestSuite):
@pytest.mark.execute_serially
@CustomClusterTestSuite.with_args("--backend_client_rpc_timeout_ms=1000"
" --datastream_sender_timeout_ms=30000 --debug_actions=%s" %
- "|".join(map(lambda rpc: "%s_DELAY:JITTER@3000@0.1" % rpc, all_rpcs)))
+ "|".join(["%s_DELAY:JITTER@3000@0.1" % rpc for rpc in all_rpcs]))
def test_random_rpc_timeout(self, vector):
self.execute_query_verify_metrics(self.TEST_QUERY, None, 10)
diff --git a/tests/custom_cluster/test_scratch_disk.py b/tests/custom_cluster/test_scratch_disk.py
index dc3797b8e..db1c1c08c 100644
--- a/tests/custom_cluster/test_scratch_disk.py
+++ b/tests/custom_cluster/test_scratch_disk.py
@@ -18,6 +18,7 @@
# Tests for query expiration.
from __future__ import absolute_import, division, print_function
+from builtins import range
import os
import pytest
import re
@@ -74,7 +75,7 @@ class TestScratchDir(CustomClusterTestSuite):
def generate_dirs(self, num, writable=True, non_existing=False):
result = []
- for i in xrange(num):
+ for i in range(num):
dir_path = tempfile.mkdtemp()
if non_existing:
shutil.rmtree(dir_path)
diff --git a/tests/custom_cluster/test_set_and_unset.py b/tests/custom_cluster/test_set_and_unset.py
index 3425a5895..2abe2586b 100644
--- a/tests/custom_cluster/test_set_and_unset.py
+++ b/tests/custom_cluster/test_set_and_unset.py
@@ -171,5 +171,5 @@ class TestSetAndUnset(CustomClusterTestSuite, HS2TestSuite):
fetch_results_req.operationHandle = execute_statement_resp.operationHandle
fetch_results_req.maxRows = 100
fetch_results_resp = self.hs2_client.FetchResults(fetch_results_req)
- return zip(fetch_results_resp.results.columns[0].stringVal.values,
- fetch_results_resp.results.columns[1].stringVal.values)
+ return list(zip(fetch_results_resp.results.columns[0].stringVal.values,
+ fetch_results_resp.results.columns[1].stringVal.values))
diff --git a/tests/custom_cluster/test_topic_update_frequency.py b/tests/custom_cluster/test_topic_update_frequency.py
index f5e6c92f5..f7e2b1581 100644
--- a/tests/custom_cluster/test_topic_update_frequency.py
+++ b/tests/custom_cluster/test_topic_update_frequency.py
@@ -10,6 +10,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import absolute_import, division, print_function
+from builtins import range
from multiprocessing.pool import ThreadPool
import pytest
diff --git a/tests/custom_cluster/test_udf_concurrency.py b/tests/custom_cluster/test_udf_concurrency.py
index e9f9b8575..aa35eac39 100644
--- a/tests/custom_cluster/test_udf_concurrency.py
+++ b/tests/custom_cluster/test_udf_concurrency.py
@@ -16,6 +16,7 @@
# under the License.
from __future__ import absolute_import, division, print_function
+from builtins import range
import os
import pytest
import random
@@ -114,11 +115,11 @@ class TestUdfConcurrency(CustomClusterTestSuite):
# create threads to use native function.
runner_threads = []
- for i in xrange(num_uses):
+ for i in range(num_uses):
runner_threads.append(threading.Thread(target=use_fn_method))
# create threads to drop/create native functions.
- for i in xrange(num_loads):
+ for i in range(num_loads):
runner_threads.append(threading.Thread(target=load_fn_method))
# launch all runner threads.
diff --git a/tests/custom_cluster/test_wide_table_operations.py b/tests/custom_cluster/test_wide_table_operations.py
index 26daf46aa..85657dcea 100644
--- a/tests/custom_cluster/test_wide_table_operations.py
+++ b/tests/custom_cluster/test_wide_table_operations.py
@@ -16,6 +16,7 @@
# under the License.
from __future__ import absolute_import, division, print_function
+from builtins import range
import os
import pytest
from subprocess import call
diff --git a/tests/failure/test_failpoints.py b/tests/failure/test_failpoints.py
index e95aa1e86..38f0919e5 100644
--- a/tests/failure/test_failpoints.py
+++ b/tests/failure/test_failpoints.py
@@ -19,6 +19,7 @@
# two types of failures - cancellation of the query and a failure test hook.
#
from __future__ import absolute_import, division, print_function
+from builtins import range
import pytest
import re
from time import sleep
diff --git a/tests/hs2/hs2_test_suite.py b/tests/hs2/hs2_test_suite.py
index b8acbc502..3c6720529 100644
--- a/tests/hs2/hs2_test_suite.py
+++ b/tests/hs2/hs2_test_suite.py
@@ -18,6 +18,7 @@
# Superclass of all HS2 tests containing commonly used functions.
from __future__ import absolute_import, division, print_function
+from builtins import range
from getpass import getuser
from TCLIService import TCLIService
from ImpalaService import ImpalaHiveServer2Service
@@ -281,7 +282,7 @@ class HS2TestSuite(ImpalaTestSuite):
num_rows = len(typed_col.values)
break
- for i in xrange(num_rows):
+ for i in range(num_rows):
row = []
for c in columns:
for col_type in HS2TestSuite.HS2_V6_COLUMN_TYPES:
diff --git a/tests/hs2/test_fetch_first.py b/tests/hs2/test_fetch_first.py
index d40b1d0bb..6fccc80f0 100644
--- a/tests/hs2/test_fetch_first.py
+++ b/tests/hs2/test_fetch_first.py
@@ -21,6 +21,7 @@
# succeed as long all previously fetched rows fit into the bounded result cache.
from __future__ import absolute_import, division, print_function
+from builtins import range
import pytest
from ImpalaService import ImpalaHiveServer2Service
@@ -199,7 +200,7 @@ class TestFetchFirst(HS2TestSuite):
"SELECT * FROM functional.alltypessmall ORDER BY id LIMIT 30"
execute_statement_resp = self.hs2_client.ExecuteStatement(execute_statement_req)
HS2TestSuite.check_response(execute_statement_resp)
- for i in xrange(1, 5):
+ for i in range(1, 5):
# Fetch 10 rows with the FETCH_NEXT orientation.
expected_num_rows = 10
if i == 4:
@@ -220,7 +221,7 @@ class TestFetchFirst(HS2TestSuite):
execute_statement_req.statement =\
"SELECT * FROM functional.alltypessmall ORDER BY id LIMIT 30"
execute_statement_resp = self.hs2_client.ExecuteStatement(execute_statement_req)
- for _ in xrange(1, 5):
+ for _ in range(1, 5):
self.fetch_until(execute_statement_resp.operationHandle,
TCLIService.TFetchOrientation.FETCH_FIRST, 30)
self.__verify_num_cached_rows(30)
@@ -339,7 +340,7 @@ class TestFetchFirst(HS2TestSuite):
execute_statement_req.statement =\
"SELECT * FROM functional.alltypessmall ORDER BY id LIMIT 0"
execute_statement_resp = self.hs2_client.ExecuteStatement(execute_statement_req)
- for i in xrange(0, 3):
+ for i in range(0, 3):
# Fetch some rows. Expect to get 0 rows.
self.fetch_at_most(execute_statement_resp.operationHandle,
TCLIService.TFetchOrientation.FETCH_NEXT, i * 10, 0)
@@ -358,7 +359,7 @@ class TestFetchFirst(HS2TestSuite):
self.fetch_at_most(execute_statement_resp.operationHandle,
TCLIService.TFetchOrientation.FETCH_FIRST, 100, 1)
self.__verify_num_cached_rows(1)
- for i in xrange(0, 3):
+ for i in range(0, 3):
# Fetch some rows with FETCH_FIRST. Expect to get 1 row.
self.fetch_at_most(execute_statement_resp.operationHandle,
TCLIService.TFetchOrientation.FETCH_FIRST, i * 10, 1)
@@ -391,7 +392,7 @@ class TestFetchFirst(HS2TestSuite):
execute_statement_req.statement = "show table stats functional.alltypes"
execute_statement_resp = self.hs2_client.ExecuteStatement(execute_statement_req)
HS2TestSuite.check_response(execute_statement_resp)
- for i in xrange(1, 5):
+ for i in range(1, 5):
# Fetch 10 rows with the FETCH_NEXT orientation.
expected_num_rows = 10
if i == 3:
@@ -414,7 +415,7 @@ class TestFetchFirst(HS2TestSuite):
execute_statement_req.statement = "show table stats functional.alltypes"
execute_statement_resp = self.hs2_client.ExecuteStatement(execute_statement_req)
HS2TestSuite.check_response(execute_statement_resp)
- for _ in xrange(1, 5):
+ for _ in range(1, 5):
self.fetch_until(execute_statement_resp.operationHandle,
TCLIService.TFetchOrientation.FETCH_FIRST, 30, 25)
# The results of non-query stmts are not counted as 'cached'.
diff --git a/tests/hs2/test_hs2.py b/tests/hs2/test_hs2.py
index e87877b54..6353fbf95 100644
--- a/tests/hs2/test_hs2.py
+++ b/tests/hs2/test_hs2.py
@@ -18,6 +18,7 @@
# Client tests for Impala's HiveServer2 interface
from __future__ import absolute_import, division, print_function
+from builtins import range
from getpass import getuser
from contextlib import contextmanager
import json
@@ -408,7 +409,7 @@ class TestHS2(HS2TestSuite):
num_sessions = self.impalad_test_service.get_metric_value(
"impala-server.num-open-hiveserver2-sessions")
session_ids = []
- for _ in xrange(5):
+ for _ in range(5):
open_session_req = TCLIService.TOpenSessionReq()
resp = self.hs2_client.OpenSession(open_session_req)
TestHS2.check_response(resp)
@@ -883,7 +884,7 @@ class TestHS2(HS2TestSuite):
args=(profile_fetch_exception, query_uuid, op_handle)))
# Start threads that will race to unregister the query.
- for i in xrange(NUM_UNREGISTER_THREADS):
+ for i in range(NUM_UNREGISTER_THREADS):
socket, client = self._open_hs2_connection()
sockets.append(socket)
threads.append(threading.Thread(target=self._unregister_query,
diff --git a/tests/metadata/test_compute_stats.py b/tests/metadata/test_compute_stats.py
index 7f4bdff9a..7d4007a4c 100644
--- a/tests/metadata/test_compute_stats.py
+++ b/tests/metadata/test_compute_stats.py
@@ -16,6 +16,7 @@
# under the License.
from __future__ import absolute_import, division, print_function
+from builtins import range
import pytest
from subprocess import check_call
@@ -183,7 +184,7 @@ class TestComputeStats(ImpalaTestSuite):
# Check that the template formulated above exists and row count of the table is
# not zero, for all scans.
- for i in xrange(len(explain_result.data)):
+ for i in range(len(explain_result.data)):
if ("SCAN HDFS" in explain_result.data[i]):
assert(hdfs_physical_properties_template in explain_result.data[i + 1])
assert("cardinality=0" not in explain_result.data[i + 2])
diff --git a/tests/metadata/test_ddl.py b/tests/metadata/test_ddl.py
index ff9357315..95dc10176 100644
--- a/tests/metadata/test_ddl.py
+++ b/tests/metadata/test_ddl.py
@@ -16,6 +16,7 @@
# under the License.
from __future__ import absolute_import, division, print_function
+from builtins import map, range
import getpass
import itertools
import pytest
@@ -303,10 +304,10 @@ class TestDdlStatements(TestDdlBase):
def test_create_table_like_file_orc(self, vector, unique_database):
COMPLEXTYPETBL_PATH = 'test-warehouse/managed/functional_orc_def.db/' \
'complextypestbl_orc_def/'
- base_dir = filter(lambda s: s.startswith('base'),
- self.filesystem_client.ls(COMPLEXTYPETBL_PATH))[0]
- bucket_file = filter(lambda s: s.startswith('bucket'),
- self.filesystem_client.ls(COMPLEXTYPETBL_PATH + base_dir))[0]
+ base_dir = list(filter(lambda s: s.startswith('base'),
+ self.filesystem_client.ls(COMPLEXTYPETBL_PATH)))[0]
+ bucket_file = list(filter(lambda s: s.startswith('bucket'),
+ self.filesystem_client.ls(COMPLEXTYPETBL_PATH + base_dir)))[0]
vector.get_value('exec_option')['abort_on_error'] = False
create_table_from_orc(self.client, unique_database,
'timestamp_with_local_timezone')
@@ -623,7 +624,7 @@ class TestDdlStatements(TestDdlBase):
"location '{1}/{0}'".format(fq_tbl_name, WAREHOUSE))
# Add some partitions (first batch of two)
- for i in xrange(num_parts // 5):
+ for i in range(num_parts // 5):
start = time.time()
self.client.execute(
"alter table {0} add partition(j={1}, s='{1}')".format(fq_tbl_name, i))
@@ -645,7 +646,7 @@ class TestDdlStatements(TestDdlBase):
.format(fq_tbl_name, WAREHOUSE))
# Add some more partitions
- for i in xrange(num_parts // 5, num_parts):
+ for i in range(num_parts // 5, num_parts):
start = time.time()
self.client.execute(
"alter table {0} add partition(j={1},s='{1}')".format(fq_tbl_name, i))
@@ -676,8 +677,8 @@ class TestDdlStatements(TestDdlBase):
result = self.execute_query_expect_success(self.client,
"SHOW PARTITIONS %s" % fq_tbl_name)
- assert 1 == len(filter(lambda line: line.find("PARQUET") != -1, result.data))
- assert 2 == len(filter(lambda line: line.find("ORC") != -1, result.data))
+ assert 1 == len([line for line in result.data if line.find("PARQUET") != -1])
+ assert 2 == len([line for line in result.data if line.find("ORC") != -1])
def test_alter_table_create_many_partitions(self, vector, unique_database):
"""
@@ -688,7 +689,7 @@ class TestDdlStatements(TestDdlBase):
"create table {0}.t(i int) partitioned by (p int)".format(unique_database))
MAX_PARTITION_UPDATES_PER_RPC = 500
alter_stmt = "alter table {0}.t add ".format(unique_database) + " ".join(
- "partition(p=%d)" % (i,) for i in xrange(MAX_PARTITION_UPDATES_PER_RPC + 2))
+ "partition(p=%d)" % (i,) for i in range(MAX_PARTITION_UPDATES_PER_RPC + 2))
self.client.execute(alter_stmt)
partitions = self.client.execute("show partitions {0}.t".format(unique_database))
# Show partitions will contain partition HDFS paths, which we expect to contain
@@ -696,8 +697,8 @@ class TestDdlStatements(TestDdlBase):
# paths, converts them to integers, and checks that wehave all the ones we
# expect.
PARTITION_RE = re.compile("p=([0-9]+)")
- assert map(int, PARTITION_RE.findall(str(partitions))) == \
- range(MAX_PARTITION_UPDATES_PER_RPC + 2)
+ assert list(map(int, PARTITION_RE.findall(str(partitions)))) == \
+ list(range(MAX_PARTITION_UPDATES_PER_RPC + 2))
def test_create_alter_tbl_properties(self, vector, unique_database):
fq_tbl_name = unique_database + ".test_alter_tbl"
@@ -1231,7 +1232,7 @@ class TestLibCache(TestDdlBase):
"""
self.client.set_configuration(vector.get_value("exec_option"))
for drop_stmt in drop_stmts: self.client.execute(drop_stmt % ("if exists"))
- for i in xrange(0, num_iterations):
+ for i in range(0, num_iterations):
for create_stmt in create_stmts: self.client.execute(create_stmt)
self.client.execute(select_stmt)
for drop_stmt in drop_stmts: self.client.execute(drop_stmt % (""))
diff --git a/tests/metadata/test_hms_integration.py b/tests/metadata/test_hms_integration.py
index 343f67877..9caa5dda0 100644
--- a/tests/metadata/test_hms_integration.py
+++ b/tests/metadata/test_hms_integration.py
@@ -24,6 +24,7 @@
# Impala, in all the possible ways of validating that metadata.
from __future__ import absolute_import, division, print_function
+from builtins import range
import pytest
import random
import string
@@ -235,7 +236,7 @@ class TestHmsIntegration(ImpalaTestSuite):
dictionary that holds the parsed attributes."""
result = {}
output_lines = output.split('\n')
- stat_names = map(string.strip, output_lines[0].split(','))
+ stat_names = list(map(string.strip, output_lines[0].split(',')))
stat_values = output_lines[3].split(',')
assert len(stat_names) == len(stat_values)
for i in range(0, len(stat_names)):
@@ -247,7 +248,7 @@ class TestHmsIntegration(ImpalaTestSuite):
dictionary that holds the parsed attributes."""
result = {}
for line in output.split('\n'):
- line_elements = map(string.strip, line.split(','))
+ line_elements = list(map(string.strip, line.split(',')))
if len(line_elements) >= 2:
result[line_elements[0]] = line_elements[1]
return result
diff --git a/tests/metadata/test_load.py b/tests/metadata/test_load.py
index 9dc3500e8..272f8bd0a 100644
--- a/tests/metadata/test_load.py
+++ b/tests/metadata/test_load.py
@@ -18,6 +18,7 @@
# Functional tests for LOAD DATA statements.
from __future__ import absolute_import, division, print_function
+from builtins import range
import time
from beeswaxd.BeeswaxService import QueryState
from copy import deepcopy
@@ -77,18 +78,18 @@ class TestLoadData(ImpalaTestSuite):
# - Sub Directories 5-6 have multiple files (4) copied from alltypesaggmultifiles
# - Sub Directory 3 also has hidden files, in both supported formats.
# - All sub-dirs contain a hidden directory
- for i in xrange(1, 6):
+ for i in range(1, 6):
stagingDir = '{0}/{1}'.format(STAGING_PATH, i)
self.filesystem_client.make_dir(stagingDir, permission=777)
self.filesystem_client.make_dir('{0}/_hidden_dir'.format(stagingDir),
permission=777)
# Copy single file partitions from alltypes.
- for i in xrange(1, 4):
+ for i in range(1, 4):
self.filesystem_client.copy(ALLTYPES_PATH,
"{0}/{1}/100101.txt".format(STAGING_PATH, i))
# Copy multi file partitions from alltypesaggmultifiles.
file_names = self.filesystem_client.ls(MULTIAGG_PATH)
- for i in xrange(4, 6):
+ for i in range(4, 6):
for file_ in file_names:
self.filesystem_client.copy(
"{0}/{1}".format(MULTIAGG_PATH, file_),
diff --git a/tests/metadata/test_recover_partitions.py b/tests/metadata/test_recover_partitions.py
index 131c31155..b6700a67b 100644
--- a/tests/metadata/test_recover_partitions.py
+++ b/tests/metadata/test_recover_partitions.py
@@ -18,6 +18,7 @@
# Impala tests for ALTER TABLE RECOVER PARTITIONS statement
from __future__ import absolute_import, division, print_function
+from builtins import range
import os
from six.moves import urllib
from tests.common.impala_test_suite import ImpalaTestSuite
@@ -177,7 +178,7 @@ class TestRecoverPartitions(ImpalaTestSuite):
"CREATE TABLE %s (c int) PARTITIONED BY (s string)" % (FQ_TBL_NAME))
# Create 700 partitions externally
- for i in xrange(1, 700):
+ for i in range(1, 700):
PART_DIR = "s=part%d/" % i
FILE_PATH = "test"
INSERTED_VALUE = "666"
@@ -185,14 +186,14 @@ class TestRecoverPartitions(ImpalaTestSuite):
result = self.execute_query_expect_success(self.client,
"SHOW PARTITIONS %s" % FQ_TBL_NAME)
- for i in xrange(1, 700):
+ for i in range(1, 700):
PART_DIR = "part%d\t" % i
assert not self.has_value(PART_DIR, result.data)
self.execute_query_expect_success(self.client,
"ALTER TABLE %s RECOVER PARTITIONS" % FQ_TBL_NAME)
result = self.execute_query_expect_success(self.client,
"SHOW PARTITIONS %s" % FQ_TBL_NAME)
- for i in xrange(1, 700):
+ for i in range(1, 700):
PART_DIR = "part%d\t" % i
assert self.has_value(PART_DIR, result.data)
@@ -344,7 +345,7 @@ class TestRecoverPartitions(ImpalaTestSuite):
# Running ALTER TABLE RECOVER PARTITIONS multiple times should only produce
# a single partition when adding a single partition.
- for i in xrange(3):
+ for i in range(3):
self.execute_query_expect_success(
self.client, "ALTER TABLE %s RECOVER PARTITIONS" % FQ_TBL_NAME)
result = self.execute_query_expect_success(
@@ -392,7 +393,7 @@ class TestRecoverPartitions(ImpalaTestSuite):
# Adds partition directories.
num_partitions = 10
- for i in xrange(1, num_partitions):
+ for i in range(1, num_partitions):
PART_DIR = "i=%d/s=part%d" % (i,i)
self.filesystem_client.make_dir(TBL_LOCATION + PART_DIR)
@@ -410,7 +411,7 @@ class TestRecoverPartitions(ImpalaTestSuite):
result = self.execute_query_expect_success(self.client,
"SHOW PARTITIONS %s" % FQ_TBL_NAME)
assert num_partitions - 1 == self.count_partition(result.data)
- for i in xrange(1, num_partitions):
+ for i in range(1, num_partitions):
PART_DIR = "part%d\t" % i
assert self.has_value(PART_DIR, result.data)
@@ -457,7 +458,7 @@ class TestRecoverPartitions(ImpalaTestSuite):
def count_value(self, value, lines):
"""Count the number of lines that contain value."""
- return len(filter(lambda line: line.find(value) != -1, lines))
+ return len([line for line in lines if line.find(value) != -1])
def verify_partitions(self, expected_parts, lines):
"""Check if all partition values are expected"""
diff --git a/tests/metadata/test_recursive_listing.py b/tests/metadata/test_recursive_listing.py
index 9273387e4..de6cfc91d 100644
--- a/tests/metadata/test_recursive_listing.py
+++ b/tests/metadata/test_recursive_listing.py
@@ -11,6 +11,7 @@
# limitations under the License.
from __future__ import absolute_import, division, print_function
+from builtins import range
import pytest
import requests
import time
diff --git a/tests/metadata/test_stats_extrapolation.py b/tests/metadata/test_stats_extrapolation.py
index 00aa721ad..88352d4dc 100644
--- a/tests/metadata/test_stats_extrapolation.py
+++ b/tests/metadata/test_stats_extrapolation.py
@@ -16,6 +16,7 @@
# under the License.
from __future__ import absolute_import, division, print_function
+from builtins import range
from os import path
from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.skip import SkipIfEC
@@ -146,7 +147,7 @@ class TestStatsExtrapolation(ImpalaTestSuite):
col_names = [fs.name.upper() for fs in actual.schema.fieldSchemas]
rows_col_idx = col_names.index("#ROWS")
extrap_rows_col_idx = col_names.index("EXTRAP #ROWS")
- for i in xrange(0, len(actual.data)):
+ for i in range(0, len(actual.data)):
act_cols = actual.data[i].split("\t")
exp_cols = expected.data[i].split("\t")
assert int(exp_cols[rows_col_idx]) >= 0
@@ -173,7 +174,7 @@ class TestStatsExtrapolation(ImpalaTestSuite):
assert len(actual.schema.fieldSchemas) == len(expected.schema.fieldSchemas)
col_names = [fs.name.upper() for fs in actual.schema.fieldSchemas]
ndv_col_idx = col_names.index("#DISTINCT VALUES")
- for i in xrange(0, len(actual.data)):
+ for i in range(0, len(actual.data)):
act_cols = actual.data[i].split("\t")
exp_cols = expected.data[i].split("\t")
assert int(exp_cols[ndv_col_idx]) >= 0
diff --git a/tests/performance/scheduler.py b/tests/performance/scheduler.py
index 95d2a7a13..f0850fdaf 100644
--- a/tests/performance/scheduler.py
+++ b/tests/performance/scheduler.py
@@ -20,6 +20,7 @@
# vector. It treats a workload an the unit of parallelism.
from __future__ import absolute_import, division, print_function
+from builtins import range
import logging
from collections import defaultdict
@@ -78,7 +79,7 @@ class Scheduler(object):
Each workload thread is analogus to a client name, and is identified by a unique ID,
the workload that's being run and the table formats it's being run on."""
- for thread_num in xrange(self.num_clients):
+ for thread_num in range(self.num_clients):
thread = Thread(target=self._run_queries, args=[thread_num],
name=self._thread_name % thread_num)
thread.daemon = True
@@ -98,7 +99,7 @@ class Scheduler(object):
# each thread gets its own copy of query_executors
query_executors = deepcopy(sorted(self.query_executors, key=lambda x: x.query.name))
- for j in xrange(self.iterations):
+ for j in range(self.iterations):
# Randomize the order of execution for each iteration if specified.
if self.shuffle: shuffle(query_executors)
results = defaultdict(list)
@@ -106,7 +107,7 @@ class Scheduler(object):
for query_executor in query_executors:
query_name = query_executor.query.name
LOG.info("Running Query: %s" % query_name)
- for i in xrange(self.query_iterations):
+ for i in range(self.query_iterations):
if self._exit.isSet():
LOG.error("Another thread failed, exiting.")
exit(1)
diff --git a/tests/query_test/test_aggregation.py b/tests/query_test/test_aggregation.py
index cf399e20d..1cf2942ff 100644
--- a/tests/query_test/test_aggregation.py
+++ b/tests/query_test/test_aggregation.py
@@ -18,6 +18,7 @@
# Validates all aggregate functions across all datatypes
#
from __future__ import absolute_import, division, print_function
+from builtins import range
import pytest
from testdata.common import widetable
@@ -279,7 +280,7 @@ class TestAggregationQueries(ImpalaTestSuite):
]
# For each possible integer value, genereate one query and test it out.
- for i in xrange(1, 11):
+ for i in range(1, 11):
ndv_stmt = """
select ndv(bool_col, {0}), ndv(tinyint_col, {0}),
ndv(smallint_col, {0}), ndv(int_col, {0}),
@@ -299,7 +300,7 @@ class TestAggregationQueries(ImpalaTestSuite):
# Verify that each ndv() value (one per column for a total of 11) is identical
# to the corresponding known value. Since NDV() invokes Hash64() hash function
# with a fixed seed value, ndv() result is deterministic.
- for j in xrange(0, 11):
+ for j in range(0, 11):
assert(ndv_results[i - 1][j] == int(ndv_vals[j]))
def test_grouping_sets(self, vector):
@@ -393,7 +394,7 @@ class TestAggregationQueriesRunOnce(ImpalaTestSuite):
assert len(sampled_ndv_vals) == len(ndv_vals)
# Low NDV columns. We expect a reasonaby accurate estimate regardless of the
# sampling percent.
- for i in xrange(0, 14):
+ for i in range(0, 14):
self.appx_equals(int(sampled_ndv_vals[i]), int(ndv_vals[i]), 0.1)
# High NDV columns. We expect the estimate to have high variance and error.
# Since we give NDV() and SAMPLED_NDV() the same input data, i.e., we are not
@@ -401,7 +402,7 @@ class TestAggregationQueriesRunOnce(ImpalaTestSuite):
# be bigger than NDV() proportional to the sampling percent.
# For example, the column 'id' is a PK so we expect the result of SAMPLED_NDV()
# with a sampling percent of 0.1 to be approximately 10x of the NDV().
- for i in xrange(14, 16):
+ for i in range(14, 16):
self.appx_equals(int(sampled_ndv_vals[i]) * sample_perc, int(ndv_vals[i]), 2.0)
diff --git a/tests/query_test/test_avro_schema_resolution.py b/tests/query_test/test_avro_schema_resolution.py
index b6ed92bff..e9a6a66f5 100644
--- a/tests/query_test/test_avro_schema_resolution.py
+++ b/tests/query_test/test_avro_schema_resolution.py
@@ -16,6 +16,7 @@
# under the License.
from __future__ import absolute_import, division, print_function
+from builtins import range
from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.skip import SkipIfCatalogV2
diff --git a/tests/query_test/test_cancellation.py b/tests/query_test/test_cancellation.py
index 784c8ef24..c6cfb50ad 100644
--- a/tests/query_test/test_cancellation.py
+++ b/tests/query_test/test_cancellation.py
@@ -19,6 +19,7 @@
#
from __future__ import absolute_import, division, print_function
+from builtins import range
import pytest
import threading
from time import sleep
@@ -74,7 +75,7 @@ JOIN_BEFORE_CLOSE = [False, True]
# Extra dimensions to test order by without limit
SORT_QUERY = 'select * from lineitem order by l_orderkey'
-SORT_CANCEL_DELAY = range(6, 10)
+SORT_CANCEL_DELAY = list(range(6, 10))
SORT_BUFFER_POOL_LIMIT = ['0', '300m'] # Test spilling and non-spilling sorts.
# Test with and without multithreading
@@ -153,7 +154,7 @@ class TestCancellation(ImpalaTestSuite):
wait_action = vector.get_value('wait_action')
fail_rpc_action = vector.get_value('fail_rpc_action')
- debug_action = "|".join(filter(None, [wait_action, fail_rpc_action]))
+ debug_action = "|".join([_f for _f in [wait_action, fail_rpc_action] if _f])
vector.get_value('exec_option')['debug_action'] = debug_action
vector.get_value('exec_option')['buffer_pool_limit'] =\
@@ -162,7 +163,7 @@ class TestCancellation(ImpalaTestSuite):
vector.get_value('exec_option')['mt_dop'] = vector.get_value('mt_dop')
# Execute the query multiple times, cancelling it each time.
- for i in xrange(NUM_CANCELATION_ITERATIONS):
+ for i in range(NUM_CANCELATION_ITERATIONS):
cancel_query_and_validate_state(self.client, query,
vector.get_value('exec_option'), vector.get_value('table_format'),
vector.get_value('cancel_delay'), vector.get_value('join_before_close'))
diff --git a/tests/query_test/test_cast_with_format.py b/tests/query_test/test_cast_with_format.py
index e73a6f9cf..8d49c7d32 100644
--- a/tests/query_test/test_cast_with_format.py
+++ b/tests/query_test/test_cast_with_format.py
@@ -16,6 +16,7 @@
# under the License.
from __future__ import absolute_import, division, print_function
+from builtins import range
from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.test_dimensions import create_client_protocol_dimension
diff --git a/tests/query_test/test_compressed_formats.py b/tests/query_test/test_compressed_formats.py
index 894f237f0..333fc349c 100644
--- a/tests/query_test/test_compressed_formats.py
+++ b/tests/query_test/test_compressed_formats.py
@@ -16,6 +16,7 @@
# under the License.
from __future__ import absolute_import, division, print_function
+from builtins import range
import math
import os
import pytest
diff --git a/tests/query_test/test_decimal_casting.py b/tests/query_test/test_decimal_casting.py
index 8c4ea5912..e0435a2c1 100644
--- a/tests/query_test/test_decimal_casting.py
+++ b/tests/query_test/test_decimal_casting.py
@@ -18,6 +18,7 @@
# Validates that casting to Decimal works.
#
from __future__ import absolute_import, division, print_function
+from builtins import range
import pytest
from decimal import Decimal, getcontext, ROUND_DOWN, ROUND_HALF_UP
from allpairspy import AllPairs as all_pairs
@@ -39,11 +40,11 @@ class TestDecimalCasting(ImpalaTestSuite):
DECIMAL_TYPES_MAP = {
# All possible decimal types.
# (0 < precision <= 38 && 0 <= scale <= 38 && scale <= precision)
- 'exhaustive' : [(p, s) for p in xrange(1, 39) for s in xrange(0, p + 1)],
+ 'exhaustive': [(p, s) for p in range(1, 39) for s in range(0, p + 1)],
# Core only deals with precision 6,16,26 (different integer types)
- 'core' : [(p, s) for p in [6,16,26] for s in xrange(0, p + 1)],
+ 'core': [(p, s) for p in [6, 16, 26] for s in range(0, p + 1)],
# mimics test_vectors.py and takes a subset of all decimal types
- 'pairwise' : all_pairs([(p, s) for p in xrange(1, 39) for s in xrange(0, p + 1)])
+ 'pairwise': all_pairs([(p, s) for p in range(1, 39) for s in range(0, p + 1)])
}
# We can cast for numerics or string types.
CAST_FROM = ['string', 'number']
@@ -121,7 +122,7 @@ class TestDecimalCasting(ImpalaTestSuite):
precision, scale = vector.get_value('decimal_type')
if vector.get_value('cast_from') == 'decimal':
pytest.skip("Casting between the same decimal type isn't interesting")
- for i in xrange(self.iterations):
+ for i in range(self.iterations):
val = self._gen_decimal_val(precision, scale)
cast = self._normalize_cast_expr(val, precision, vector.get_value('cast_from'))\
.format(val, precision, scale)
@@ -132,7 +133,7 @@ class TestDecimalCasting(ImpalaTestSuite):
"""Test to verify that we always return NULL when trying to cast a number with greater
precision that its intended decimal type"""
precision, scale = vector.get_value('decimal_type')
- for i in xrange(self.iterations):
+ for i in range(self.iterations):
# Generate a decimal with a larger precision than the one we're casting to.
from_precision = randint(precision + 1, 39)
val = self._gen_decimal_val(from_precision, scale)
@@ -150,7 +151,7 @@ class TestDecimalCasting(ImpalaTestSuite):
if precision == scale:
pytest.skip("Cannot underflow scale when precision and scale are equal")
- for i in xrange(self.iterations):
+ for i in range(self.iterations):
from_scale = randint(scale + 1, precision)
val = self._gen_decimal_val(precision, from_scale)
cast = self._normalize_cast_expr(val, precision, cast_from)\
diff --git a/tests/query_test/test_decimal_fuzz.py b/tests/query_test/test_decimal_fuzz.py
index 1f468960c..d55251f00 100644
--- a/tests/query_test/test_decimal_fuzz.py
+++ b/tests/query_test/test_decimal_fuzz.py
@@ -19,6 +19,7 @@
# operations return correct results under decimal_v2.
from __future__ import absolute_import, division, print_function
+from builtins import range
import decimal
import math
import pytest
@@ -190,7 +191,7 @@ class TestDecimalFuzz(ImpalaTestSuite):
return True
return False
- for num_digits_after_dot in xrange(39):
+ for num_digits_after_dot in range(39):
# Reduce the number of digits after the dot in the expected_result to different
# amounts. If it matches the actual result in at least one of the cases, we
# consider the actual result to be acceptable.
@@ -245,7 +246,7 @@ class TestDecimalFuzz(ImpalaTestSuite):
assert self.result_equals(expected_result, result)
def test_decimal_ops(self, vector):
- for _ in xrange(self.iterations):
+ for _ in range(self.iterations):
self.execute_one_decimal_op()
def width_bucket(self, val, min_range, max_range, num_buckets):
@@ -299,5 +300,5 @@ class TestDecimalFuzz(ImpalaTestSuite):
raise e
def test_width_bucket(self, vector):
- for _ in xrange(self.iterations):
+ for _ in range(self.iterations):
self.execute_one_width_bucket()
diff --git a/tests/query_test/test_exprs.py b/tests/query_test/test_exprs.py
index f396841d0..569e584e8 100644
--- a/tests/query_test/test_exprs.py
+++ b/tests/query_test/test_exprs.py
@@ -16,6 +16,7 @@
# under the License.
from __future__ import absolute_import, division, print_function
+from builtins import range
import pytest
import re
from random import randint
@@ -102,7 +103,7 @@ class TestExprLimits(ImpalaTestSuite):
def test_expr_child_limit(self, vector):
# IN predicate
in_query = "select 1 IN("
- for i in xrange(0, self.EXPR_CHILDREN_LIMIT - 1):
+ for i in range(0, self.EXPR_CHILDREN_LIMIT - 1):
in_query += str(i)
if (i + 1 != self.EXPR_CHILDREN_LIMIT - 1):
in_query += ","
@@ -111,7 +112,7 @@ class TestExprLimits(ImpalaTestSuite):
# CASE expr
case_query = "select case "
- for i in xrange(0, self.EXPR_CHILDREN_LIMIT // 2):
+ for i in range(0, self.EXPR_CHILDREN_LIMIT // 2):
case_query += " when true then 1"
case_query += " end"
self.__exec_query(case_query)
@@ -181,7 +182,7 @@ class TestExprLimits(ImpalaTestSuite):
def __gen_huge_case(self, col_name, fanout, depth, indent):
toks = ["case\n"]
- for i in xrange(fanout):
+ for i in range(fanout):
add = randint(1, 1000000)
divisor = randint(1, 10000000)
mod = randint(0, divisor)
@@ -200,16 +201,16 @@ class TestExprLimits(ImpalaTestSuite):
def __gen_deep_infix_expr(self, prefix, repeat_suffix):
expr = prefix
- for i in xrange(self.EXPR_DEPTH_LIMIT - 1):
+ for i in range(self.EXPR_DEPTH_LIMIT - 1):
expr += repeat_suffix
return expr
def __gen_deep_func_expr(self, open_func, base_arg, close_func):
expr = ""
- for i in xrange(self.EXPR_DEPTH_LIMIT - 1):
+ for i in range(self.EXPR_DEPTH_LIMIT - 1):
expr += open_func
expr += base_arg
- for i in xrange(self.EXPR_DEPTH_LIMIT - 1):
+ for i in range(self.EXPR_DEPTH_LIMIT - 1):
expr += close_func
return expr
diff --git a/tests/query_test/test_hdfs_caching.py b/tests/query_test/test_hdfs_caching.py
index 105895af2..f90e83be2 100644
--- a/tests/query_test/test_hdfs_caching.py
+++ b/tests/query_test/test_hdfs_caching.py
@@ -18,6 +18,7 @@
# Validates limit on scan nodes
from __future__ import absolute_import, division, print_function
+from builtins import range
import pytest
import re
import time
@@ -103,7 +104,7 @@ class TestHdfsCaching(ImpalaTestSuite):
select * from t1, t2, t3 where t1.x = t2.x and t2.x = t3.x """
# Run this query for some iterations since it is timing dependent.
- for x in xrange(1, num_iters):
+ for x in range(1, num_iters):
result = self.execute_query(query_string)
assert(len(result.data) == 2)
diff --git a/tests/query_test/test_iceberg.py b/tests/query_test/test_iceberg.py
index 76491ce65..6f64ced0e 100644
--- a/tests/query_test/test_iceberg.py
+++ b/tests/query_test/test_iceberg.py
@@ -16,6 +16,7 @@
# under the License.
from __future__ import absolute_import, division, print_function
+from builtins import range
import datetime
import logging
import os
diff --git a/tests/query_test/test_insert_behaviour.py b/tests/query_test/test_insert_behaviour.py
index f4b269c32..9f55a9377 100644
--- a/tests/query_test/test_insert_behaviour.py
+++ b/tests/query_test/test_insert_behaviour.py
@@ -16,6 +16,7 @@
# under the License.
from __future__ import absolute_import, division, print_function
+from builtins import range
import getpass
import grp
import os
diff --git a/tests/query_test/test_insert_parquet.py b/tests/query_test/test_insert_parquet.py
index 33fc83af5..dc72c63b6 100644
--- a/tests/query_test/test_insert_parquet.py
+++ b/tests/query_test/test_insert_parquet.py
@@ -18,6 +18,7 @@
# Targeted Impala insert tests
from __future__ import absolute_import, division, print_function
+from builtins import map, range
import os
from collections import namedtuple
@@ -633,7 +634,8 @@ class TestHdfsParquetTableStatsWriter(ImpalaTestSuite):
num_columns = len(table_stats)
assert num_columns == len(expected_values)
- for col_idx, stats, expected in zip(range(num_columns), table_stats, expected_values):
+ for col_idx, stats, expected in zip(list(range(num_columns)),
+ table_stats, expected_values):
if col_idx in skip_col_idxs:
continue
if not expected:
@@ -931,7 +933,7 @@ class TestHdfsParquetTableStatsWriter(ImpalaTestSuite):
"""Test that writing a Parquet table with too many columns results in an error."""
num_cols = 12000
query = "create table %s.wide stored as parquet as select \n" % unique_database
- query += ", ".join(map(str, xrange(num_cols)))
+ query += ", ".join(map(str, range(num_cols)))
query += ";\n"
result = self.execute_query_expect_failure(self.client, query)
assert "Minimum required block size must be less than 2GB" in str(result)
diff --git a/tests/query_test/test_insert_permutation.py b/tests/query_test/test_insert_permutation.py
index a3973a40e..14cf638a0 100644
--- a/tests/query_test/test_insert_permutation.py
+++ b/tests/query_test/test_insert_permutation.py
@@ -18,6 +18,7 @@
# Targeted Impala insert tests
from __future__ import absolute_import, division, print_function
+from builtins import map
from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.test_dimensions import (
create_exec_option_dimension,
@@ -47,8 +48,8 @@ class TestInsertQueriesWithPermutation(ImpalaTestSuite):
create_uncompressed_text_dimension(cls.get_workload()))
def test_insert_permutation(self, vector):
- map(self.cleanup_db, ["insert_permutation_test"])
+ list(map(self.cleanup_db, ["insert_permutation_test"]))
self.run_test_case('QueryTest/insert_permutation', vector)
def teardown_method(self, method):
- map(self.cleanup_db, ["insert_permutation_test"])
+ list(map(self.cleanup_db, ["insert_permutation_test"]))
diff --git a/tests/query_test/test_kudu.py b/tests/query_test/test_kudu.py
index d637b3c35..f985599e2 100644
--- a/tests/query_test/test_kudu.py
+++ b/tests/query_test/test_kudu.py
@@ -16,6 +16,7 @@
# under the License.
from __future__ import absolute_import, division, print_function
+from builtins import range
from kudu.schema import (
BOOL,
DOUBLE,
diff --git a/tests/query_test/test_mem_usage_scaling.py b/tests/query_test/test_mem_usage_scaling.py
index 8660f9433..2b5982fa1 100644
--- a/tests/query_test/test_mem_usage_scaling.py
+++ b/tests/query_test/test_mem_usage_scaling.py
@@ -17,6 +17,7 @@
#
from __future__ import absolute_import, division, print_function
+from builtins import range
import pytest
from copy import copy
diff --git a/tests/query_test/test_parquet_bloom_filter.py b/tests/query_test/test_parquet_bloom_filter.py
index 902d7e769..601533373 100644
--- a/tests/query_test/test_parquet_bloom_filter.py
+++ b/tests/query_test/test_parquet_bloom_filter.py
@@ -16,6 +16,7 @@
# under the License.
from __future__ import absolute_import, division, print_function
+from builtins import range
import math
import os
diff --git a/tests/query_test/test_scanners.py b/tests/query_test/test_scanners.py
index be372018b..5f3ae18f9 100644
--- a/tests/query_test/test_scanners.py
+++ b/tests/query_test/test_scanners.py
@@ -22,6 +22,7 @@
# explode.
from __future__ import absolute_import, division, print_function
+from builtins import range
import os
import pytest
import random
diff --git a/tests/query_test/test_scanners_fuzz.py b/tests/query_test/test_scanners_fuzz.py
index ec411abc1..ed7ce144c 100644
--- a/tests/query_test/test_scanners_fuzz.py
+++ b/tests/query_test/test_scanners_fuzz.py
@@ -16,6 +16,7 @@
# under the License.
from __future__ import absolute_import, division, print_function
+from builtins import range
from copy import copy
import itertools
import logging
@@ -322,7 +323,7 @@ class TestScannersFuzzing(ImpalaTestSuite):
data = bytearray(f.read())
num_corruptions = rng.randint(0, int(math.log(len(data))))
- for _ in xrange(num_corruptions):
+ for _ in range(num_corruptions):
flip_offset = rng.randint(0, len(data) - 1)
flip_val = rng.randint(0, 255)
LOG.info("corrupt file: Flip byte in {0} at {1} from {2} to {3}".format(
diff --git a/tests/query_test/test_sort.py b/tests/query_test/test_sort.py
index bcb76a766..dca3ec35a 100644
--- a/tests/query_test/test_sort.py
+++ b/tests/query_test/test_sort.py
@@ -22,12 +22,24 @@ from copy import copy, deepcopy
from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.skip import SkipIfNotHdfsMinicluster
+
def transpose_results(result, map_fn=lambda x: x):
"""Given a query result (list of strings, each string represents a row), return a list
- of columns, where each column is a list of strings. Optionally, map_fn can be provided
- to be applied to every value, eg. to convert the strings to their underlying types."""
+ of columns, where each column is a list of strings. Optionally, map_fn can be
+ provided to be applied to every value, eg. to convert the strings to their
+ underlying types."""
+
+ # Split result rows by tab to produce a list of lists. i.e.
+ # [[a1,a2], [b1, b2], [c1, c2]]
split_result = [row.split('\t') for row in result]
- return [map(map_fn, list(l)) for l in zip(*split_result)]
+ column_result = []
+ for col in zip(*split_result):
+ # col is the transposed result, i.e. a1, b1, c1
+ # Apply map_fn to all elements
+ column_result.append([map_fn(x) for x in col])
+
+ return column_result
+
class TestQueryFullSort(ImpalaTestSuite):
"""Test class to do functional validation of sorting when data is spilled to disk."""
diff --git a/tests/query_test/test_tpch_queries.py b/tests/query_test/test_tpch_queries.py
index 593037d5b..201500a2d 100644
--- a/tests/query_test/test_tpch_queries.py
+++ b/tests/query_test/test_tpch_queries.py
@@ -17,6 +17,7 @@
# Functional tests running the TPCH workload.
from __future__ import absolute_import, division, print_function
+from builtins import range
import pytest
from tests.common.impala_test_suite import ImpalaTestSuite
@@ -42,7 +43,7 @@ class TestTpchQuery(ImpalaTestSuite):
def idfn(val):
return "TPC-H: Q{0}".format(val)
- @pytest.mark.parametrize("query", xrange(1, 23), ids=idfn)
+ @pytest.mark.parametrize("query", range(1, 23), ids=idfn)
def test_tpch(self, vector, query):
self.run_test_case('tpch-q{0}'.format(query), vector)
diff --git a/tests/shell/test_shell_commandline.py b/tests/shell/test_shell_commandline.py
index f78fa332c..f02adba18 100644
--- a/tests/shell/test_shell_commandline.py
+++ b/tests/shell/test_shell_commandline.py
@@ -19,6 +19,7 @@
# under the License.
from __future__ import absolute_import, division, print_function
+from builtins import range
import errno
import getpass
import os
@@ -1006,7 +1007,7 @@ class TestImpalaShell(ImpalaTestSuite):
# This generates a sql file size of ~50K.
num_cols = 1000
os.write(sql_file, "select \n")
- for i in xrange(num_cols):
+ for i in range(num_cols):
if i < num_cols:
os.write(sql_file, "col_{0} as a{1},\n".format(i, i))
os.write(sql_file, "col_{0} as b{1},\n".format(i, i))
diff --git a/tests/statestore/test_statestore.py b/tests/statestore/test_statestore.py
index 55cbf9b93..7ca3d713c 100644
--- a/tests/statestore/test_statestore.py
+++ b/tests/statestore/test_statestore.py
@@ -16,6 +16,7 @@
# under the License.
from __future__ import absolute_import, division, print_function
+from builtins import range
from collections import defaultdict
import json
import logging
@@ -115,7 +116,7 @@ class KillableThreadedServer(TServer):
self.processor = None
def wait_until_up(self, num_tries=10):
- for i in xrange(num_tries):
+ for i in range(num_tries):
cnxn = TSocket.TSocket('localhost', self.port)
try:
cnxn.open()
@@ -125,7 +126,7 @@ class KillableThreadedServer(TServer):
time.sleep(0.1)
def wait_until_down(self, num_tries=10):
- for i in xrange(num_tries):
+ for i in range(num_tries):
cnxn = TSocket.TSocket('localhost', self.port)
try:
cnxn.open()
@@ -349,7 +350,7 @@ class TestStatestore():
num_updates=1, clear_topic_entries=False):
topic_entries = [
Subscriber.TTopicItem(key=key_template + str(x), value=value_template + str(x))
- for x in xrange(num_updates)]
+ for x in range(num_updates)]
return Subscriber.TTopicDelta(topic_name=topic_name,
topic_entries=topic_entries,
is_delta=False,
diff --git a/tests/stress/concurrent_select.py b/tests/stress/concurrent_select.py
index 7fa6f6d28..8c15e49ee 100755
--- a/tests/stress/concurrent_select.py
+++ b/tests/stress/concurrent_select.py
@@ -57,6 +57,7 @@
from __future__ import absolute_import, division, print_function
+from builtins import range
import logging
import os
import re
@@ -187,7 +188,7 @@ def print_crash_info_if_exists(impala, start_time):
that evaluates to True if any impalads are stopped.
"""
max_attempts = 5
- for remaining_attempts in xrange(max_attempts - 1, -1, -1):
+ for remaining_attempts in range(max_attempts - 1, -1, -1):
try:
crashed_impalads = impala.find_crashed_impalads(start_time)
break
@@ -388,7 +389,7 @@ class StressRunner(object):
queries_by_type[query.query_type] = []
queries_by_type[query.query_type].append(query)
try:
- for _ in xrange(self._num_queries_to_run):
+ for _ in range(self._num_queries_to_run):
# First randomly determine a query type, then choose a random query of that
# type.
if (
@@ -918,7 +919,7 @@ def populate_runtime_info(query, impala, converted_args, timeout_secs=maxint):
def get_report(desired_outcome=None):
reports_by_outcome = defaultdict(list)
leading_outcome = None
- for remaining_samples in xrange(samples - 1, -1, -1):
+ for remaining_samples in range(samples - 1, -1, -1):
report = runner.run_query(query, mem_limit, run_set_up=True,
timeout_secs=timeout_secs, retain_profile=True)
if report.timed_out:
@@ -1438,7 +1439,7 @@ def main():
impala, file_queries, converted_args))
# Apply tweaks to the query's runtime info as requested by CLI options.
- for idx in xrange(len(queries) - 1, -1, -1):
+ for idx in range(len(queries) - 1, -1, -1):
query = queries[idx]
if query.required_mem_mb_with_spilling:
query.required_mem_mb_with_spilling += int(
diff --git a/tests/stress/queries.py b/tests/stress/queries.py
index c4c01a783..36b3a287c 100644
--- a/tests/stress/queries.py
+++ b/tests/stress/queries.py
@@ -21,6 +21,7 @@
# stress test, loading them and generating them.
from __future__ import absolute_import, division, print_function
+from builtins import range
import logging
import os
from textwrap import dedent
@@ -36,7 +37,7 @@ LOG = logging.getLogger(os.path.splitext(os.path.basename(__file__))[0])
class QueryType(object):
- COMPUTE_STATS, DELETE, INSERT, SELECT, UPDATE, UPSERT = range(6)
+ COMPUTE_STATS, DELETE, INSERT, SELECT, UPDATE, UPSERT = list(range(6))
class Query(object):
diff --git a/tests/stress/query_retries_stress_runner.py b/tests/stress/query_retries_stress_runner.py
index d21496a83..87560d3e2 100755
--- a/tests/stress/query_retries_stress_runner.py
+++ b/tests/stress/query_retries_stress_runner.py
@@ -25,6 +25,7 @@
# the script, it has to be killed manually (e.g. kill [pid]).
from __future__ import absolute_import, division, print_function
+from builtins import map, range
import logging
import pipes
import os
@@ -176,12 +177,12 @@ def run_concurrent_workloads(concurrency, coordinator, database, queries):
# complete.
workload_threads = []
LOG.info("Starting {0} concurrent workloads".format(concurrency))
- for i in xrange(concurrency):
+ for i in range(concurrency):
workload_thread = threading.Thread(target=__run_workload, args=[i],
name="workload_thread_{0}".format(i))
workload_thread.start()
workload_threads.append(workload_thread)
- map(lambda thread: thread.join(), workload_threads)
+ list(map(lambda thread: thread.join(), workload_threads))
# Check if any of the workload runner threads hit an exception, if one did then print
# the error and exit.
@@ -254,7 +255,7 @@ def run_stress_workload(queries, database, workload, start_delay,
start_random_impalad_killer(kill_frequency, start_delay, cluster)
# Run the stress test 'iterations' times.
- for i in xrange(iterations):
+ for i in range(iterations):
LOG.info("Starting iteration {0} of workload {1}".format(i, workload))
run_concurrent_workloads(concurrency, impala_coordinator, database,
queries)
diff --git a/tests/stress/test_acid_stress.py b/tests/stress/test_acid_stress.py
index 49b94b2db..a284717b5 100644
--- a/tests/stress/test_acid_stress.py
+++ b/tests/stress/test_acid_stress.py
@@ -16,6 +16,7 @@
# under the License.
from __future__ import absolute_import, division, print_function
+from builtins import map, range
import pytest
import random
import time
@@ -62,7 +63,7 @@ class TestAcidInsertsBasic(TestAcidStress):
run_max = -1
i_list = []
for line in result.data:
- [run, i] = map(int, (line.split('\t')))
+ [run, i] = list(map(int, (line.split('\t'))))
run_max = max(run_max, run)
i_list.append(i)
assert expected_result["run"] <= run_max # shouldn't see data overwritten in the past
@@ -72,17 +73,18 @@ class TestAcidInsertsBasic(TestAcidStress):
expected_result["i"] = 0
return
assert i_list[-1] >= expected_result["i"]
- assert i_list == range(i_list[-1] + 1) # 'i' should have all values from 0 to max_i
+ # 'i' should have all values from 0 to max_i
+ assert i_list == list(range(i_list[-1] + 1))
expected_result["i"] = i_list[-1]
def _hive_role_write_inserts(self, tbl_name, partitioned):
"""INSERT INTO/OVERWRITE a table several times from Hive."""
part_expr = "partition (p=1)" if partitioned else ""
- for run in xrange(0, NUM_OVERWRITES):
+ for run in range(0, NUM_OVERWRITES):
OVERWRITE_SQL = """insert overwrite table %s %s values (%i, %i)
""" % (tbl_name, part_expr, run, 0)
self.run_stmt_in_hive(OVERWRITE_SQL)
- for i in xrange(1, NUM_INSERTS_PER_OVERWRITE + 1):
+ for i in range(1, NUM_INSERTS_PER_OVERWRITE + 1):
INSERT_SQL = """insert into table %s %s values (%i, %i)
""" % (tbl_name, part_expr, run, i)
self.run_stmt_in_hive(INSERT_SQL)
@@ -92,11 +94,11 @@ class TestAcidInsertsBasic(TestAcidStress):
try:
impalad_client = ImpalaTestSuite.create_impala_client()
part_expr = "partition (p=1)" if partitioned else ""
- for run in xrange(0, NUM_OVERWRITES + 1):
+ for run in range(0, NUM_OVERWRITES + 1):
OVERWRITE_SQL = """insert overwrite table %s %s values (%i, %i)
""" % (tbl_name, part_expr, run, 0)
impalad_client.execute(OVERWRITE_SQL)
- for i in xrange(1, NUM_INSERTS_PER_OVERWRITE + 1):
+ for i in range(1, NUM_INSERTS_PER_OVERWRITE + 1):
INSERT_SQL = """insert into table %s %s values (%i, %i)
""" % (tbl_name, part_expr, run, i)
impalad_client.execute(INSERT_SQL)
@@ -261,11 +263,12 @@ class TestConcurrentAcidInserts(TestAcidStress):
def verify_result_set(result):
wid_to_run = dict()
for line in result.data:
- [wid, i] = map(int, (line.split('\t')))
+ [wid, i] = list(map(int, (line.split('\t'))))
wid_to_run.setdefault(wid, []).append(i)
for wid, run in wid_to_run.items():
sorted_run = sorted(run)
- assert sorted_run == range(sorted_run[0], sorted_run[-1] + 1), "wid: %d" % wid
+ assert sorted_run == list(range(sorted_run[0], sorted_run[-1] + 1)), \
+ "wid: %d" % wid
target_impalad = cid % ImpalaTestSuite.get_impalad_cluster_size()
impalad_client = ImpalaTestSuite.create_client_for_nth_impalad(target_impalad)
@@ -299,10 +302,10 @@ class TestConcurrentAcidInserts(TestAcidStress):
num_checkers = 3
writers = [Task(self._impala_role_concurrent_writer, tbl_name, i, counter)
- for i in xrange(0, num_writers)]
+ for i in range(0, num_writers)]
checkers = [Task(self._impala_role_concurrent_checker, tbl_name, i, counter,
num_writers)
- for i in xrange(0, num_checkers)]
+ for i in range(0, num_checkers)]
run_tasks(writers + checkers)
@@ -369,9 +372,9 @@ class TestFailingAcidInserts(TestAcidStress):
num_checkers = 3
writers = [Task(self._impala_role_insert, tbl_name, partitioned, i, counter)
- for i in xrange(0, num_writers)]
+ for i in range(0, num_writers)]
checkers = [Task(self._impala_role_checker, tbl_name, i, counter, num_writers)
- for i in xrange(0, num_checkers)]
+ for i in range(0, num_checkers)]
run_tasks(writers + checkers)
@SkipIfFS.stress_insert_timeouts
diff --git a/tests/stress/test_ddl_stress.py b/tests/stress/test_ddl_stress.py
index 09480b315..9eb9e0d42 100644
--- a/tests/stress/test_ddl_stress.py
+++ b/tests/stress/test_ddl_stress.py
@@ -16,6 +16,7 @@
# under the License.
from __future__ import absolute_import, division, print_function
+from builtins import range
import pytest
from tests.common.impala_test_suite import ImpalaTestSuite
@@ -25,7 +26,7 @@ from tests.common.skip import SkipIfFS
NUM_TBLS_PER_THREAD = 10
# Each client will get a different test id.
-TEST_INDICES = xrange(10)
+TEST_INDICES = range(10)
# Simple stress test for DDL operations. Attempts to create, cache,
@@ -57,7 +58,7 @@ class TestDdlStress(ImpalaTestSuite):
# rather simultaneously on the same object.
self.client.execute("create database if not exists {0}".format(self.SHARED_DATABASE))
- for i in xrange(NUM_TBLS_PER_THREAD):
+ for i in range(NUM_TBLS_PER_THREAD):
tbl_name = "{db}.test_{checksum}_{i}".format(
db=self.SHARED_DATABASE,
checksum=testid_checksum,
diff --git a/tests/stress/test_insert_stress.py b/tests/stress/test_insert_stress.py
index 93eaf91d6..81611e5a5 100644
--- a/tests/stress/test_insert_stress.py
+++ b/tests/stress/test_insert_stress.py
@@ -16,6 +16,7 @@
# under the License.
from __future__ import absolute_import, division, print_function
+from builtins import map, range
import pytest
import random
import time
@@ -64,11 +65,12 @@ class TestInsertStress(ImpalaTestSuite):
def verify_result_set(result):
wid_to_run = dict()
for line in result.data:
- [wid, i] = map(int, (line.split('\t')))
+ [wid, i] = list(map(int, (line.split('\t'))))
wid_to_run.setdefault(wid, []).append(i)
for wid, run in wid_to_run.items():
sorted_run = sorted(run)
- assert sorted_run == range(sorted_run[0], sorted_run[-1] + 1), "wid: %d" % wid
+ assert sorted_run == list(range(sorted_run[0], sorted_run[-1] + 1)), \
+ "wid: %d" % wid
target_impalad = cid % ImpalaTestSuite.get_impalad_cluster_size()
impalad_client = ImpalaTestSuite.create_client_for_nth_impalad(target_impalad)
@@ -97,10 +99,10 @@ class TestInsertStress(ImpalaTestSuite):
inserts = 50
writers = [Task(self._impala_role_concurrent_writer, tbl_name, i, inserts, counter)
- for i in xrange(0, num_writers)]
+ for i in range(0, num_writers)]
checkers = [Task(self._impala_role_concurrent_checker, tbl_name, i, counter,
num_writers)
- for i in xrange(0, num_checkers)]
+ for i in range(0, num_checkers)]
run_tasks(writers + checkers)
@pytest.mark.execute_serially
@@ -124,8 +126,8 @@ class TestInsertStress(ImpalaTestSuite):
inserts = 30
writers = [Task(self._impala_role_concurrent_writer, tbl_name, i, inserts, counter)
- for i in xrange(0, num_writers)]
+ for i in range(0, num_writers)]
checkers = [Task(self._impala_role_concurrent_checker, tbl_name, i, counter,
num_writers)
- for i in xrange(0, num_checkers)]
+ for i in range(0, num_checkers)]
run_tasks(writers + checkers)
diff --git a/tests/util/calculation_util.py b/tests/util/calculation_util.py
index 657c40a92..2899eecee 100644
--- a/tests/util/calculation_util.py
+++ b/tests/util/calculation_util.py
@@ -21,6 +21,7 @@
# dependencies.
from __future__ import absolute_import, division, print_function
+from builtins import range
import math
import random
import string
diff --git a/tests/util/concurrent_workload.py b/tests/util/concurrent_workload.py
index 025f67da7..2ceb81574 100755
--- a/tests/util/concurrent_workload.py
+++ b/tests/util/concurrent_workload.py
@@ -20,6 +20,7 @@
# This class can be used to drive a concurrent workload against a local minicluster
from __future__ import absolute_import, division, print_function
+from builtins import range
import argparse
import logging
# Needed to work around datetime threading bug:
@@ -103,7 +104,7 @@ class ConcurrentWorkload(object):
def start(self):
"""Starts worker threads to execute queries."""
# Start workers
- for i in xrange(self.num_streams):
+ for i in range(self.num_streams):
t = Thread(target=self.loop_query, args=(self.query, self.output_q, self.stop_ev))
self.threads.append(t)
t.start()
diff --git a/tests/util/get_parquet_metadata.py b/tests/util/get_parquet_metadata.py
index 21099acb9..126f5434e 100644
--- a/tests/util/get_parquet_metadata.py
+++ b/tests/util/get_parquet_metadata.py
@@ -16,11 +16,13 @@
# under the License.
from __future__ import absolute_import, division, print_function
+from builtins import map
import os
import struct
from datetime import date, datetime, time, timedelta
from decimal import Decimal
+from functools import reduce
from parquet.ttypes import ColumnIndex, FileMetaData, OffsetIndex, PageHeader, Type
from subprocess import check_call
from thrift.protocol import TCompactProtocol
@@ -97,7 +99,7 @@ def decode_decimal(schema, value):
assert schema.type_length == len(value)
assert schema.type == Type.FIXED_LEN_BYTE_ARRAY
- numeric = Decimal(reduce(lambda x, y: x * 256 + y, map(ord, value)))
+ numeric = Decimal(reduce(lambda x, y: x * 256 + y, list(map(ord, value))))
# Compute two's complement for negative values.
if (ord(value[0]) > 127):
diff --git a/tests/util/ssh_util.py b/tests/util/ssh_util.py
index 67b33889e..61b249bf8 100644
--- a/tests/util/ssh_util.py
+++ b/tests/util/ssh_util.py
@@ -16,6 +16,7 @@
# under the License.
from __future__ import absolute_import, division, print_function
+from builtins import range
import atexit
import logging
import os
diff --git a/tests/util/test_file_parser.py b/tests/util/test_file_parser.py
index 91973c457..f1746e45c 100644
--- a/tests/util/test_file_parser.py
+++ b/tests/util/test_file_parser.py
@@ -18,6 +18,7 @@
# This module is used for common utilities related to parsing test files
from __future__ import absolute_import, division, print_function
+from builtins import map
import codecs
import collections
import logging
@@ -125,10 +126,10 @@ def parse_table_constraints(constraints_file):
schema_only[f].append(table_name.lower())
elif constraint_type == 'restrict_to':
schema_include[table_name.lower()] +=\
- map(parse_table_format_constraint, table_formats.split(','))
+ list(map(parse_table_format_constraint, table_formats.split(',')))
elif constraint_type == 'exclude':
schema_exclude[table_name.lower()] +=\
- map(parse_table_format_constraint, table_formats.split(','))
+ list(map(parse_table_format_constraint, table_formats.split(',')))
else:
raise ValueError('Unknown constraint type: %s' % constraint_type)
return schema_include, schema_exclude, schema_only
@@ -377,7 +378,10 @@ def load_tpc_queries(workload, include_stress_queries=False, query_name_filters=
file_name_pattern = re.compile(r"(.*)")
query_name_pattern = re.compile(r"(.*)")
- query_name_filters = map(str.strip, query_name_filters) if query_name_filters else []
+ if query_name_filters:
+ query_name_filters = list(map(str.strip, query_name_filters))
+ else:
+ query_name_filters = []
filter_regex = re.compile(r'|'.join(['^%s$' % n for n in query_name_filters]), re.I)
for query_file in os.listdir(query_dir):
[impala] 01/06: IMPALA-11959: Add Python 3 virtualenv
Posted by jo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
joemcdonnell pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 566df808913aef6ff5eecc3849e14df8370bd651
Author: Joe McDonnell <jo...@cloudera.com>
AuthorDate: Tue Apr 6 21:36:07 2021 -0700
IMPALA-11959: Add Python 3 virtualenv
This adds a Python 3 equivalent to the impala-python
virtualenv base on the toolchain Python 3.7.16.
This modifies bootstrap_virtualenv.py to support
the two different modes. This adds py2-requirements.txt
and py3-requirements.txt to allow some differences
between the Python 2 and Python 3 virtualenvs.
Here are some specific package changes:
- allpairs is replaced with allpairspy, as allpairs did
not support Python 3.
- requests is upgraded slightly, because otherwise is has issues
with idna==2.8.
- pylint is limited to Python 3, because we are adding it
and don't need it on both
- flake8 is limited to Python 2, because it will take
some work to switch to a version that works on Python 3
- cm_api is limited to Python 2, because it doesn't support
Python 3
- pytest-random does not support Python 3 and it is unused,
so it is removed
- Bump the version of setuptool-scm to support Python 3
This adds impala-pylint, which can be used to do further
Python 3 checks via --py3k. This also adds a bin/check-pylint-py3k.sh
script to enforce specific py3k checks. The banned py3k warnings
are specified in the bin/banned_py3k_warnings.txt. This is currently
empty, but this can ratchet up the py3k strictness over time
to avoid regressions.
This pulls in a new toolchain with the fix for IMPALA-11956
to get Python 3.7.16.
Testing:
- Hand tested that the allpairs libraries produce the
same results
- The python3 virtualenv has no influence on regular
tests yet
Change-Id: Ica4853f440c9a46a79bd5fb8e0a66730b0b4efc0
Reviewed-on: http://gerrit.cloudera.org:8080/19567
Reviewed-by: Joe McDonnell <jo...@cloudera.com>
Tested-by: Joe McDonnell <jo...@cloudera.com>
---
CMakeLists.txt | 6 +-
bin/banned_py3k_warnings.txt | 0
bin/bootstrap_toolchain.py | 3 +
bin/check-pylint-py3k.sh | 140 +++++++++++++++++++
bin/impala-config.sh | 3 +-
.../setuptools-requirements.txt => bin/impala-pip3 | 8 +-
.../impala-pylint | 8 +-
.../impala-python3 | 10 +-
...t-impala-python.sh => impala-python3-common.sh} | 27 ++--
bin/init-impala-python.sh | 26 +++-
bin/rat_exclude_files.txt | 1 +
infra/python/bootstrap_virtualenv.py | 150 ++++++++++++++-------
infra/python/deps/pip_download.py | 3 +-
...tools-requirements.txt => py2-requirements.txt} | 21 ++-
...tools-requirements.txt => py3-requirements.txt} | 18 ++-
infra/python/deps/requirements.txt | 20 +--
infra/python/deps/setuptools-requirements.txt | 2 +-
testdata/bin/generate-test-vectors.py | 3 +-
tests/common/test_vector.py | 4 +-
tests/custom_cluster/test_hs2_fault_injection.py | 2 -
tests/query_test/test_decimal_casting.py | 2 +-
21 files changed, 349 insertions(+), 108 deletions(-)
diff --git a/CMakeLists.txt b/CMakeLists.txt
index cb35b393a..2bfc50f5d 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -514,6 +514,10 @@ add_custom_target(impala_python ALL
COMMAND "${CMAKE_SOURCE_DIR}/bin/init-impala-python.sh"
)
+add_custom_target(impala_python3 ALL
+ COMMAND "${CMAKE_SOURCE_DIR}/bin/init-impala-python.sh" "-python3"
+)
+
set(IMPALA_PYTHON_INSTALLS "")
if (NOT $ENV{IMPALA_SYSTEM_PYTHON2} EQUAL "")
list(APPEND IMPALA_PYTHON_INSTALLS shell_python2_install)
@@ -524,7 +528,7 @@ endif()
add_custom_target(impala_shell_pypi ALL DEPENDS ${IMPALA_PYTHON_INSTALLS})
add_custom_target(notests_independent_targets DEPENDS
- java cscope tarballs impala_python impala_shell_pypi
+ java cscope tarballs impala_python impala_python3 impala_shell_pypi
)
add_custom_target(notests_regular_targets DEPENDS
impalad statestored catalogd admissiond fesupport loggingsupport ImpalaUdf udasample udfsample impala-profile-tool
diff --git a/bin/banned_py3k_warnings.txt b/bin/banned_py3k_warnings.txt
new file mode 100644
index 000000000..e69de29bb
diff --git a/bin/bootstrap_toolchain.py b/bin/bootstrap_toolchain.py
index 98ec9d7d6..097484822 100755
--- a/bin/bootstrap_toolchain.py
+++ b/bin/bootstrap_toolchain.py
@@ -494,6 +494,9 @@ def get_toolchain_downloads():
"crcutil", "curl", "flatbuffers", "gdb", "gflags", "glog", "gperftools", "gtest",
"jwt-cpp", "libev", "libunwind", "lz4", "openldap", "openssl", "orc", "protobuf",
"python", "rapidjson", "re2", "snappy", "tpc-h", "tpc-ds", "zlib", "zstd"])
+ python3_package = ToolchainPackage(
+ "python", explicit_version=os.environ.get("IMPALA_PYTHON3_VERSION"))
+ toolchain_packages += [python3_package]
toolchain_packages += get_unique_toolchain_downloads(
["thrift:cpp", "thrift:java", "thrift:py"])
protobuf_package_clang = ToolchainPackage(
diff --git a/bin/check-pylint-py3k.sh b/bin/check-pylint-py3k.sh
new file mode 100755
index 000000000..9dca19c87
--- /dev/null
+++ b/bin/check-pylint-py3k.sh
@@ -0,0 +1,140 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -euo pipefail
+
+BINDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+# To allow incrementally banning individual pylint checks, this uses grep
+# expressions to match banned pylint warnings. The grep expressions are stored
+# in the bin/banned_py3k_warnings.txt file.
+BANNED_PY3K_WARNINGS="${BINDIR}/banned_py3k_warnings.txt"
+
+function print_usage {
+ echo "check-pylink-py3k.sh : Checks eligible python files for pylint py3k compliance."
+ echo "Fails if the python files have py3k warnings that match the patterns in "
+ echo "bin/banned_py3k_warnings.txt."
+ echo "[--error_output_file] : (optional) Also output the errors to a file"
+ echo "[--warning_output_file] : (optional) Also output the warnings to a file"
+}
+
+ERROR_OUTPUT_FILE=""
+WARNING_OUTPUT_FILE=""
+while [ -n "$*" ]
+do
+ case "$1" in
+ --error_output_file)
+ ERROR_OUTPUT_FILE="${2-}"
+ shift;
+ ;;
+ --warning_output_file)
+ WARNING_OUTPUT_FILE="${2-}"
+ shift;
+ ;;
+ --help|*)
+ print_usage
+ exit 1
+ ;;
+ esac
+ shift
+done
+
+pushd ${IMPALA_HOME} > /dev/null 2>&1
+
+OUTPUT_TMP_DIR=$(mktemp -d)
+PYLINT_OUTPUT_FILE="${OUTPUT_TMP_DIR}/pylint_output.txt"
+ERROR_OUTPUT_TMP_FILE="${OUTPUT_TMP_DIR}/error_output_tmp.txt"
+WARNING_OUTPUT_TMP_FILE="${OUTPUT_TMP_DIR}/warning_output_tmp.txt"
+
+RETCODE=0
+for file in $(git ls-files '**/*.py'); do
+ # Skip the shell entirely (but cover tests/shell)
+ if [[ "${file}" =~ "shell/" && ! "${file}" =~ "tests/shell" ]]; then
+ continue
+ fi
+ # For the moment, the focus is on enforcing py3k checks on files that use the
+ # impala-python virtualenv. Ignore executable python files that do not
+ # use impala-python. In practice, this tends to be scripts used during the
+ # build or various scripts for developers in bin.
+ FIRST_LINE=$(head -n1 ${file})
+ if [[ "${file}: ${FIRST_LINE}" =~ "#!" ]]; then
+ if [[ "${FIRST_LINE}" =~ "python3" ]]; then
+ >&2 echo "SKIPPING: ${file} is already using python3: ${FIRST_LINE}"
+ continue
+ fi
+ if [[ ! "${FIRST_LINE}" =~ "impala-python" ]]; then
+ >&2 echo "SKIPPING: ${file} is not using impala-python: ${FIRST_LINE}"
+ continue
+ fi
+ fi
+
+ >&2 echo "PROCESSING: ${file}"
+
+ # -s n (skip score for each file)
+ # --exit-zero: don't fail
+ impala-pylint -s n --exit-zero --py3k ${file} >> ${PYLINT_OUTPUT_FILE}
+done
+
+touch "${ERROR_OUTPUT_TMP_FILE}"
+touch "${WARNING_OUTPUT_TMP_FILE}"
+
+# Hitting a banned py3k warning will cause this to return an error
+echo ""
+echo ""
+if grep -f "${BANNED_PY3K_WARNINGS}" "${PYLINT_OUTPUT_FILE}" > /dev/null 2>&1 ; then
+ echo "ERROR: Some python files contain these banned pylint warnings:" | \
+ tee "${ERROR_OUTPUT_TMP_FILE}"
+ grep -f "${BANNED_PY3K_WARNINGS}" "${PYLINT_OUTPUT_FILE}" | \
+ tee -a "${ERROR_OUTPUT_TMP_FILE}"
+ RETCODE=1
+else
+ echo "No errors found" | tee "${ERROR_OUTPUT_TMP_FILE}"
+fi
+
+if [[ -n "${ERROR_OUTPUT_FILE}" ]]; then
+ cp "${ERROR_OUTPUT_TMP_FILE}" "${ERROR_OUTPUT_FILE}"
+fi
+
+# The remaining py3k warnings are interesting, but they are not yet enforced.
+# Pylint produces annoying lines like "************* Module X", so try to filter those out
+echo ""
+echo ""
+if grep -v -e '\*\*\*\*' -f "${BANNED_PY3K_WARNINGS}" \
+ "${PYLINT_OUTPUT_FILE}" > /dev/null 2>&1 ; then
+ echo "WARNING: Some python files contain these unenforced pylint warnings:" | \
+ tee "${WARNING_OUTPUT_TMP_FILE}"
+ grep -v -e '\*\*\*\*' -f "${BANNED_PY3K_WARNINGS}" "${PYLINT_OUTPUT_FILE}" | \
+ tee -a "${WARNING_OUTPUT_TMP_FILE}"
+
+ echo "WARNING SUMMARY table:"
+ cat "${WARNING_OUTPUT_TMP_FILE}" | grep -v "WARNING" | cut -d: -f4- | \
+ sed 's#^ ##' | sort | uniq -c
+else
+ echo "No warnings found" | tee "${WARNING_OUTPUT_TMP_FILE}"
+fi
+
+if [[ -n "${WARNING_OUTPUT_FILE}" ]]; then
+ cp "${WARNING_OUTPUT_TMP_FILE}" "${WARNING_OUTPUT_FILE}"
+fi
+
+rm -rf "${OUTPUT_TMP_DIR}"
+
+popd > /dev/null 2>&1
+
+exit ${RETCODE}
diff --git a/bin/impala-config.sh b/bin/impala-config.sh
index ae9a00a42..1e9f9ff17 100755
--- a/bin/impala-config.sh
+++ b/bin/impala-config.sh
@@ -81,7 +81,7 @@ export USE_APACHE_HIVE=${USE_APACHE_HIVE-false}
# moving to a different build of the toolchain, e.g. when a version is bumped or a
# compile option is changed. The build id can be found in the output of the toolchain
# build jobs, it is constructed from the build number and toolchain git hash prefix.
-export IMPALA_TOOLCHAIN_BUILD_ID=252-b144ba77b5
+export IMPALA_TOOLCHAIN_BUILD_ID=258-821f1d91bd
# Versions of toolchain dependencies.
# -----------------------------------
export IMPALA_AVRO_VERSION=1.7.4-p5
@@ -159,6 +159,7 @@ export IMPALA_POSTGRES_JDBC_DRIVER_VERSION=42.5.1
unset IMPALA_POSTGRES_JDBC_DRIVER_URL
export IMPALA_PYTHON_VERSION=2.7.16
unset IMPALA_PYTHON_URL
+export IMPALA_PYTHON3_VERSION=3.7.16
export IMPALA_RAPIDJSON_VERSION=1.1.0
unset IMPALA_RAPIDJSON_URL
export IMPALA_RE2_VERSION=20190301
diff --git a/infra/python/deps/setuptools-requirements.txt b/bin/impala-pip3
old mode 100644
new mode 100755
similarity index 86%
copy from infra/python/deps/setuptools-requirements.txt
copy to bin/impala-pip3
index 071f9fc54..273555feb
--- a/infra/python/deps/setuptools-requirements.txt
+++ b/bin/impala-pip3
@@ -1,3 +1,5 @@
+#!/bin/bash
+#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -15,7 +17,5 @@
# specific language governing permissions and limitations
# under the License.
-# Newer versions of setuptools don't support Python 2.7
-setuptools == 44.1.1
- wheel == 0.35.1
-setuptools-scm == 4.1.2
+source "$(dirname "$0")/impala-python3-common.sh"
+exec "$PY_ENV_DIR/bin/python3" "$PY_ENV_DIR/bin/pip3" "$@"
diff --git a/infra/python/deps/setuptools-requirements.txt b/bin/impala-pylint
old mode 100644
new mode 100755
similarity index 86%
copy from infra/python/deps/setuptools-requirements.txt
copy to bin/impala-pylint
index 071f9fc54..012f08bc9
--- a/infra/python/deps/setuptools-requirements.txt
+++ b/bin/impala-pylint
@@ -1,3 +1,5 @@
+#!/bin/bash
+#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -15,7 +17,5 @@
# specific language governing permissions and limitations
# under the License.
-# Newer versions of setuptools don't support Python 2.7
-setuptools == 44.1.1
- wheel == 0.35.1
-setuptools-scm == 4.1.2
+source "$(dirname "$0")/impala-python3-common.sh"
+exec "$PY_ENV_DIR/bin/pylint" "$@"
diff --git a/infra/python/deps/setuptools-requirements.txt b/bin/impala-python3
old mode 100644
new mode 100755
similarity index 75%
copy from infra/python/deps/setuptools-requirements.txt
copy to bin/impala-python3
index 071f9fc54..aec831d12
--- a/infra/python/deps/setuptools-requirements.txt
+++ b/bin/impala-python3
@@ -1,3 +1,6 @@
+#!/bin/bash
+#
+##############################################################################
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -14,8 +17,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+##############################################################################
-# Newer versions of setuptools don't support Python 2.7
-setuptools == 44.1.1
- wheel == 0.35.1
-setuptools-scm == 4.1.2
+source "$(dirname "$0")/impala-python3-common.sh"
+exec "$PY_ENV_DIR/bin/python3" "$@"
diff --git a/bin/init-impala-python.sh b/bin/impala-python3-common.sh
old mode 100755
new mode 100644
similarity index 60%
copy from bin/init-impala-python.sh
copy to bin/impala-python3-common.sh
index e1e20f4a4..06bf3a87a
--- a/bin/init-impala-python.sh
+++ b/bin/impala-python3-common.sh
@@ -1,5 +1,3 @@
-#!/usr/bin/env bash
-#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -16,16 +14,19 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-#
-# This is called during the build to initialize the impala-python
-# virtualenv (which involves installing various packages and
-# compiling things). This is not directly in CMake, because
-# this depends on knowing IMPALA_HOME and other environment
-# variables.
-bin=`dirname "$0"`
-bin=`cd "$bin"; pwd`
-. "$bin"/impala-config.sh
+# This file is intended to be sourced to perform common setup for
+# the Python 3 $IMPALA_HOME/bin/impala-py* executables.
+
+set -euo pipefail
+. $IMPALA_HOME/bin/report_build_error.sh
+setup_report_build_error
+
+. $IMPALA_HOME/bin/set-pythonpath.sh
+
+export LD_LIBRARY_PATH="$(python "$IMPALA_HOME/infra/python/bootstrap_virtualenv.py" \
+ --print-ld-library-path)"
-cd $IMPALA_HOME
-bin/impala-python -c 'print("Initialized impala-python")'
+PY_DIR="$(dirname "$0")/../infra/python"
+PY_ENV_DIR="${PY_DIR}/env-gcc${IMPALA_GCC_VERSION}-py3"
+python "$PY_DIR/bootstrap_virtualenv.py" --python3
diff --git a/bin/init-impala-python.sh b/bin/init-impala-python.sh
index e1e20f4a4..360d2df89 100755
--- a/bin/init-impala-python.sh
+++ b/bin/init-impala-python.sh
@@ -27,5 +27,29 @@ bin=`dirname "$0"`
bin=`cd "$bin"; pwd`
. "$bin"/impala-config.sh
+function print_usage {
+ echo "init-impala-python.sh - Script called from CMake to init python venvs"
+ echo "[-python3] : Init the python3 virtualenv (default is python2)"
+}
+
+IS_PYTHON3=false
+while [ -n "$*" ]
+do
+ case "$1" in
+ -python3)
+ IS_PYTHON3=true
+ ;;
+ -help|*)
+ print_usage
+ exit 1
+ ;;
+ esac
+ shift
+done
+
cd $IMPALA_HOME
-bin/impala-python -c 'print("Initialized impala-python")'
+if $IS_PYTHON3 ; then
+ bin/impala-python3 -c 'print("Initialized impala-python3")'
+else
+ bin/impala-python -c 'print("Initialized impala-python")'
+fi
diff --git a/bin/rat_exclude_files.txt b/bin/rat_exclude_files.txt
index 19d4ecbf2..825eef9b5 100644
--- a/bin/rat_exclude_files.txt
+++ b/bin/rat_exclude_files.txt
@@ -27,6 +27,7 @@ shell/packaging/MANIFEST.in
shell/packaging/requirements.txt
testdata/cluster/node_templates/cdh7/etc/init.d/kms
testdata/authentication/*
+bin/banned_py3k_warnings.txt
# See $IMPALA_HOME/LICENSE.txt
be/src/gutil/*
diff --git a/infra/python/bootstrap_virtualenv.py b/infra/python/bootstrap_virtualenv.py
index c3bc59932..bd9c08144 100644
--- a/infra/python/bootstrap_virtualenv.py
+++ b/infra/python/bootstrap_virtualenv.py
@@ -55,7 +55,10 @@ SKIP_TOOLCHAIN_BOOTSTRAP = "SKIP_TOOLCHAIN_BOOTSTRAP"
GCC_VERSION = os.environ["IMPALA_GCC_VERSION"]
DEPS_DIR = os.path.join(os.path.dirname(__file__), "deps")
-ENV_DIR = os.path.join(os.path.dirname(__file__), "env-gcc{0}".format(GCC_VERSION))
+ENV_DIR_PY2 = os.path.join(os.path.dirname(__file__),
+ "env-gcc{0}".format(GCC_VERSION))
+ENV_DIR_PY3 = os.path.join(os.path.dirname(__file__),
+ "env-gcc{0}-py3".format(GCC_VERSION))
# Setuptools requirements file. Setuptools is required during pip install for
# some packages. Newer setuptools dropped python 2 support, and some python
@@ -77,10 +80,16 @@ KUDU_REQS_PATH = os.path.join(DEPS_DIR, "kudu-requirements.txt")
# Interface) being installed by the requirements step.
ADLS_REQS_PATH = os.path.join(DEPS_DIR, "adls-requirements.txt")
+# Extra packages specific to python 3
+PY3_REQS_PATH = os.path.join(DEPS_DIR, "py3-requirements.txt")
-def delete_virtualenv_if_exist():
- if os.path.exists(ENV_DIR):
- shutil.rmtree(ENV_DIR)
+# Extra packages specific to python 2
+PY2_REQS_PATH = os.path.join(DEPS_DIR, "py2-requirements.txt")
+
+
+def delete_virtualenv_if_exist(venv_dir):
+ if os.path.exists(venv_dir):
+ shutil.rmtree(venv_dir)
def detect_virtualenv_version():
@@ -99,8 +108,16 @@ def detect_virtualenv_version():
return None
-def create_virtualenv():
- LOG.info("Creating python virtualenv")
+def create_virtualenv(venv_dir, is_py3):
+ if is_py3:
+ # Python 3 is much simpler, because there is a builtin venv command
+ LOG.info("Creating python3 virtualenv")
+ python_cmd = download_toolchain_python(is_py3)
+ exec_cmd([python_cmd, "-m" "venv", venv_dir])
+ return
+
+ # Python 2
+ LOG.info("Creating python2 virtualenv")
build_dir = tempfile.mkdtemp()
# Try to find the virtualenv version by parsing the requirements file
# Default to "*" if we can't figure it out.
@@ -114,9 +131,9 @@ def create_virtualenv():
for member in file.getmembers():
file.extract(member, build_dir)
file.close()
- python_cmd = download_toolchain_python()
+ python_cmd = download_toolchain_python(is_py3)
exec_cmd([python_cmd, find_file(build_dir, "virtualenv*", "virtualenv.py"), "--quiet",
- "--python", python_cmd, ENV_DIR])
+ "--python", python_cmd, venv_dir])
shutil.rmtree(build_dir)
@@ -147,7 +164,7 @@ def select_cc():
return cc
-def exec_pip_install(args, cc="no-cc-available", env=None):
+def exec_pip_install(venv_dir, is_py3, args, cc="no-cc-available", env=None):
'''Executes "pip install" with the provided command line arguments. If 'cc' is set,
it is used as the C compiler. Otherwise compilation of C/C++ code is disabled by
setting the CC environment variable to a bogus value.
@@ -169,8 +186,12 @@ def exec_pip_install(args, cc="no-cc-available", env=None):
# Don't call the virtualenv pip directly, it uses a hashbang to to call the python
# virtualenv using an absolute path. If the path to the virtualenv is very long, the
# hashbang won't work.
- impala_pip_base_cmd = [os.path.join(ENV_DIR, "bin", "python"),
- os.path.join(ENV_DIR, "bin", "pip"), "install", "-v"]
+ if is_py3:
+ impala_pip_base_cmd = [os.path.join(venv_dir, "bin", "python3"),
+ os.path.join(venv_dir, "bin", "pip3"), "install", "-v"]
+ else:
+ impala_pip_base_cmd = [os.path.join(venv_dir, "bin", "python"),
+ os.path.join(venv_dir, "bin", "pip"), "install", "-v"]
# Passes --no-binary for IMPALA-3767: without this, Cython (and
# several other packages) fail download.
@@ -181,7 +202,9 @@ def exec_pip_install(args, cc="no-cc-available", env=None):
impala_pip_base_cmd[:] + ["--no-binary", ":all:", "--no-cache-dir"]
# When using a custom mirror, we also must use the index of that mirror.
- if "PYPI_MIRROR" in os.environ:
+ # The python 3 virtualenv has trouble with using --index-url with PYPI_MIRROR,
+ # so it falls back to --no-index, which works fine.
+ if "PYPI_MIRROR" in os.environ and not is_py3:
third_party_pkg_install_cmd.extend(["--index-url",
"%s/simple" % os.environ["PYPI_MIRROR"]])
else:
@@ -217,7 +240,7 @@ def find_file(*paths):
return files[0]
-def download_toolchain_python():
+def download_toolchain_python(is_py3):
'''Grabs the Python implementation from the Impala toolchain, using the machinery from
bin/bootstrap_toolchain.py.
Skip the download if SKIP_TOOLCHAIN_BOOTSTRAP=true in the environment. In that case
@@ -229,27 +252,35 @@ def download_toolchain_python():
raise Exception("Impala environment not set up correctly, make sure "
"$IMPALA_TOOLCHAIN_PACKAGES_HOME is set.")
- package = ToolchainPackage("python")
+ if is_py3:
+ package = ToolchainPackage("python",
+ explicit_version=os.environ["IMPALA_PYTHON3_VERSION"])
+ else:
+ package = ToolchainPackage("python")
if package.needs_download() and \
not (os.environ.get(SKIP_TOOLCHAIN_BOOTSTRAP) == 'true'):
package.download()
- python_cmd = os.path.join(package.pkg_directory(), "bin/python")
+ if is_py3:
+ python_cmd = os.path.join(package.pkg_directory(), "bin/python3")
+ else:
+ python_cmd = os.path.join(package.pkg_directory(), "bin/python")
if not os.path.exists(python_cmd):
raise Exception("Unexpected error bootstrapping python from toolchain: {0} does not "
"exist".format(python_cmd))
return python_cmd
-def install_deps():
- LOG.info("Installing setuptools into the virtualenv")
- exec_pip_install(["-r", SETUPTOOLS_REQS_PATH])
+def install_deps(venv_dir, is_py3):
+ py_str = "3" if is_py3 else "2"
+ LOG.info("Installing setuptools into the python{0} virtualenv".format(py_str))
+ exec_pip_install(venv_dir, is_py3, ["-r", SETUPTOOLS_REQS_PATH])
cc = select_cc()
if cc is None:
raise Exception("CC not available")
env = dict(os.environ)
- LOG.info("Installing packages into the virtualenv")
- exec_pip_install(["-r", REQS_PATH], cc=cc, env=env)
- mark_reqs_installed(REQS_PATH)
+ LOG.info("Installing packages into the python{0} virtualenv".format(py_str))
+ exec_pip_install(venv_dir, is_py3, ["-r", REQS_PATH], cc=cc, env=env)
+ mark_reqs_installed(venv_dir, REQS_PATH)
def have_toolchain():
@@ -264,26 +295,44 @@ def toolchain_pkg_dir(pkg_name):
pkg_name + "-" + pkg_version)
-def install_adls_deps():
+def install_adls_deps(venv_dir, is_py3):
# The ADLS dependencies require that the OS is at least CentOS 6.7 or above,
# which is why we break this into a seperate step. If the target filesystem is
# ADLS, the expectation is that the dev environment is running at least CentOS 6.7.
if os.environ.get('TARGET_FILESYSTEM') == "adls":
- if reqs_are_installed(ADLS_REQS_PATH):
+ if reqs_are_installed(venv_dir, ADLS_REQS_PATH):
LOG.debug("Skipping ADLS deps: matching adls-installed-requirements.txt found")
return True
cc = select_cc()
assert cc is not None
- LOG.info("Installing ADLS packages into the virtualenv")
- exec_pip_install(["-r", ADLS_REQS_PATH], cc=cc)
- mark_reqs_installed(ADLS_REQS_PATH)
+ py_str = "3" if is_py3 else "2"
+ LOG.info("Installing ADLS packages into the python{0} virtualenv".format(py_str))
+ exec_pip_install(venv_dir, is_py3, ["-r", ADLS_REQS_PATH], cc=cc)
+ mark_reqs_installed(venv_dir, ADLS_REQS_PATH)
-def install_kudu_client_if_possible():
+def install_py_version_deps(venv_dir, is_py3):
+ cc = select_cc()
+ assert cc is not None
+ if not is_py3:
+ if not reqs_are_installed(venv_dir, PY2_REQS_PATH):
+ # These are extra python2-only packages
+ LOG.info("Installing python2 packages into the virtualenv")
+ exec_pip_install(venv_dir, is_py3, ["-r", PY2_REQS_PATH], cc=cc)
+ mark_reqs_installed(venv_dir, PY2_REQS_PATH)
+ else:
+ if not reqs_are_installed(venv_dir, PY3_REQS_PATH):
+ # These are extra python3-only packages
+ LOG.info("Installing python3 packages into the virtualenv")
+ exec_pip_install(venv_dir, is_py3, ["-r", PY3_REQS_PATH], cc=cc)
+ mark_reqs_installed(venv_dir, PY3_REQS_PATH)
+
+
+def install_kudu_client_if_possible(venv_dir, is_py3):
'''Installs the Kudu python module if possible, which depends on the toolchain and
the compiled requirements in requirements.txt. If the toolchain isn't
available, nothing will be done.'''
- if reqs_are_installed(KUDU_REQS_PATH):
+ if reqs_are_installed(venv_dir, KUDU_REQS_PATH):
LOG.debug("Skipping Kudu: matching kudu-installed-requirements.txt found")
return
kudu_base_dir = os.environ["IMPALA_KUDU_HOME"]
@@ -291,11 +340,13 @@ def install_kudu_client_if_possible():
LOG.debug("Skipping Kudu: %s doesn't exist" % kudu_base_dir)
return
- LOG.info("Installing Kudu into the virtualenv")
+ py_str = "3" if is_py3 else "2"
+ LOG.info("Installing Kudu into the python{0} virtualenv".format(py_str))
# The installation requires that KUDU_HOME/build/latest exists. An empty directory
# structure will be made to satisfy that. The Kudu client headers and lib will be made
# available through GCC environment variables.
- fake_kudu_build_dir = os.path.join(tempfile.gettempdir(), "virtualenv-kudu")
+ fake_kudu_build_dir = os.path.join(tempfile.gettempdir(),
+ "virtualenv-kudu{0}".format(py_str))
try:
artifact_dir = os.path.join(fake_kudu_build_dir, "build", "latest")
if not os.path.exists(artifact_dir):
@@ -312,8 +363,8 @@ def install_kudu_client_if_possible():
env["CPLUS_INCLUDE_PATH"] = os.path.join(kudu_client_dir, "include")
env["LIBRARY_PATH"] = os.path.pathsep.join([os.path.join(kudu_client_dir, 'lib'),
os.path.join(kudu_client_dir, 'lib64')])
- exec_pip_install(["-r", KUDU_REQS_PATH], cc=cc, env=env)
- mark_reqs_installed(KUDU_REQS_PATH)
+ exec_pip_install(venv_dir, is_py3, ["-r", KUDU_REQS_PATH], cc=cc, env=env)
+ mark_reqs_installed(venv_dir, KUDU_REQS_PATH)
finally:
try:
shutil.rmtree(fake_kudu_build_dir)
@@ -353,17 +404,17 @@ def error_if_kudu_client_not_found(install_dir):
raise Exception("%s not found at %s" % (kudu_client_lib, lib_dir))
-def mark_reqs_installed(reqs_path):
+def mark_reqs_installed(venv_dir, reqs_path):
'''Mark that the requirements from the given file are installed by copying it into
the root directory of the virtualenv.'''
- installed_reqs_path = os.path.join(ENV_DIR, os.path.basename(reqs_path))
+ installed_reqs_path = os.path.join(venv_dir, os.path.basename(reqs_path))
shutil.copyfile(reqs_path, installed_reqs_path)
-def reqs_are_installed(reqs_path):
+def reqs_are_installed(venv_dir, reqs_path):
'''Check if the requirements from the given file are installed in the virtualenv by
looking for a matching requirements file in the root directory of the virtualenv.'''
- installed_reqs_path = os.path.join(ENV_DIR, os.path.basename(reqs_path))
+ installed_reqs_path = os.path.join(venv_dir, os.path.basename(reqs_path))
if not os.path.exists(installed_reqs_path):
return False
installed_reqs_file = open(installed_reqs_path)
@@ -381,11 +432,11 @@ def reqs_are_installed(reqs_path):
installed_reqs_file.close()
-def setup_virtualenv_if_not_exists():
- if not (reqs_are_installed(REQS_PATH)):
- delete_virtualenv_if_exist()
- create_virtualenv()
- install_deps()
+def setup_virtualenv_if_not_exists(venv_dir, is_py3):
+ if not (reqs_are_installed(venv_dir, REQS_PATH)):
+ delete_virtualenv_if_exist(venv_dir)
+ create_virtualenv(venv_dir, is_py3)
+ install_deps(venv_dir, is_py3)
LOG.debug("Virtualenv setup complete")
@@ -397,6 +448,8 @@ if __name__ == "__main__":
" the virtualenv even if it exists and appears to be completely up-to-date.")
parser.add_option("--print-ld-library-path", action="store_true", help="Print the"
" LD_LIBRARY_PATH that should be used when running python from the virtualenv.")
+ parser.add_option("--python3", action="store_true", help="Generate the python3"
+ " virtualenv")
options, args = parser.parse_args()
if options.print_ld_library_path:
@@ -411,10 +464,17 @@ if __name__ == "__main__":
sys.exit()
logging.basicConfig(level=getattr(logging, options.log_level))
+
+ if options.python3:
+ venv_dir = ENV_DIR_PY3
+ else:
+ venv_dir = ENV_DIR_PY2
+
if options.rebuild:
- delete_virtualenv_if_exist()
+ delete_virtualenv_if_exist(venv_dir)
# Complete as many bootstrap steps as possible (see file comment for the steps).
- setup_virtualenv_if_not_exists()
- install_kudu_client_if_possible()
- install_adls_deps()
+ setup_virtualenv_if_not_exists(venv_dir, options.python3)
+ install_kudu_client_if_possible(venv_dir, options.python3)
+ install_adls_deps(venv_dir, options.python3)
+ install_py_version_deps(venv_dir, options.python3)
diff --git a/infra/python/deps/pip_download.py b/infra/python/deps/pip_download.py
index 9c41135d8..03713f927 100755
--- a/infra/python/deps/pip_download.py
+++ b/infra/python/deps/pip_download.py
@@ -38,7 +38,8 @@ PYPI_MIRROR = os.environ.get('PYPI_MIRROR', 'https://pypi.python.org')
# The requirement files that list all of the required packages and versions.
REQUIREMENTS_FILES = ['requirements.txt', 'setuptools-requirements.txt',
- 'kudu-requirements.txt', 'adls-requirements.txt']
+ 'kudu-requirements.txt', 'adls-requirements.txt',
+ 'py2-requirements.txt', 'py3-requirements.txt']
def check_digest(filename, algorithm, expected_digest):
diff --git a/infra/python/deps/setuptools-requirements.txt b/infra/python/deps/py2-requirements.txt
similarity index 65%
copy from infra/python/deps/setuptools-requirements.txt
copy to infra/python/deps/py2-requirements.txt
index 071f9fc54..122b5ab68 100644
--- a/infra/python/deps/setuptools-requirements.txt
+++ b/infra/python/deps/py2-requirements.txt
@@ -15,7 +15,20 @@
# specific language governing permissions and limitations
# under the License.
-# Newer versions of setuptools don't support Python 2.7
-setuptools == 44.1.1
- wheel == 0.35.1
-setuptools-scm == 4.1.2
+# Python2-only requirements
+
+cm-api == 10.0.0
+ # Already available as part of python on Linux.
+ readline == 6.2.4.1; sys_platform == 'darwin'
+flake8 == 3.9.2
+ mccabe == 0.6.1
+ pycodestyle == 2.7.0
+ pyflakes == 2.3.1
+ enum34 == 1.1.10
+ typing == 3.10.0.0
+ configparser == 4.0.2
+ functools32 == 3.2.3-2
+ importlib-metadata == 2.1.3
+ contextlib2 == 0.6.0
+ pathlib2 == 2.3.7.post1
+ zipp == 1.2.0
diff --git a/infra/python/deps/setuptools-requirements.txt b/infra/python/deps/py3-requirements.txt
similarity index 70%
copy from infra/python/deps/setuptools-requirements.txt
copy to infra/python/deps/py3-requirements.txt
index 071f9fc54..d6195a1e8 100644
--- a/infra/python/deps/setuptools-requirements.txt
+++ b/infra/python/deps/py3-requirements.txt
@@ -15,7 +15,17 @@
# specific language governing permissions and limitations
# under the License.
-# Newer versions of setuptools don't support Python 2.7
-setuptools == 44.1.1
- wheel == 0.35.1
-setuptools-scm == 4.1.2
+# Python3-only requirements
+
+pylint == 2.10.2
+ astroid == 2.7.3
+ lazy-object-proxy == 1.6.0
+ wrapt == 1.12.1
+ typed-ast == 1.4.3
+ configparser == 4.0.2
+ isort == 4.3.21
+ futures == 3.3.0; python_version == "2.7"
+ singledispatch == 3.6.1
+ toml == 0.10.2
+ platformdirs == 2.4.1
+ typing-extensions == 3.10.0.2
diff --git a/infra/python/deps/requirements.txt b/infra/python/deps/requirements.txt
index 271ddeadc..fe61de14b 100644
--- a/infra/python/deps/requirements.txt
+++ b/infra/python/deps/requirements.txt
@@ -20,23 +20,8 @@
# Dependents are indented. Dependents that have multiple parents are not listed
# multiple times (though maybe they could be).
-allpairs == 2.0.1
+allpairspy == 2.5.0
argparse == 1.4.0
-cm-api == 10.0.0
- # Already available as part of python on Linux.
- readline == 6.2.4.1; sys_platform == 'darwin'
-flake8 == 3.9.2
- mccabe == 0.6.1
- pycodestyle == 2.7.0
- pyflakes == 2.3.1
- enum34 == 1.1.10
- typing == 3.10.0.0
- configparser == 4.0.2
- functools32 == 3.2.3-2
- importlib-metadata == 2.1.3
- contextlib2 == 0.6.0
- pathlib2 == 2.3.7.post1
- zipp == 1.2.0
future == 0.18.3
gcovr == 4.2
Jinja2 == 2.11.3
@@ -61,14 +46,13 @@ pyparsing == 2.0.3
pytest == 2.9.2
py == 1.4.32
pytest-forked == 0.2
- pytest-random == 0.02
pytest-runner == 4.2
pytest-xdist == 1.17.1
pytest-timeout == 1.2.1
python-magic == 0.4.11
pywebhdfs == 0.3.2
pbr == 3.1.1
-requests == 2.20.0
+requests == 2.21.0
chardet == 3.0.4
idna == 2.8
urllib3 == 1.24.2
diff --git a/infra/python/deps/setuptools-requirements.txt b/infra/python/deps/setuptools-requirements.txt
index 071f9fc54..713bfa0af 100644
--- a/infra/python/deps/setuptools-requirements.txt
+++ b/infra/python/deps/setuptools-requirements.txt
@@ -18,4 +18,4 @@
# Newer versions of setuptools don't support Python 2.7
setuptools == 44.1.1
wheel == 0.35.1
-setuptools-scm == 4.1.2
+setuptools-scm == 5.0.2
diff --git a/testdata/bin/generate-test-vectors.py b/testdata/bin/generate-test-vectors.py
index 4998a8caa..c7d288bd8 100755
--- a/testdata/bin/generate-test-vectors.py
+++ b/testdata/bin/generate-test-vectors.py
@@ -48,8 +48,7 @@ import os
import sys
from itertools import product
from optparse import OptionParser
-import metacomm.combinatorics.all_pairs2
-all_pairs = metacomm.combinatorics.all_pairs2.all_pairs2
+from allpairspy import AllPairs as all_pairs
parser = OptionParser()
parser.add_option("-w", "--workload", dest="workload",
diff --git a/tests/common/test_vector.py b/tests/common/test_vector.py
index 005c35adb..8fcac5a79 100644
--- a/tests/common/test_vector.py
+++ b/tests/common/test_vector.py
@@ -136,8 +136,8 @@ class ImpalaTestMatrix(object):
if self.is_valid(vec)]
def __generate_pairwise_combinations(self):
- import metacomm.combinatorics.all_pairs2
- all_pairs = metacomm.combinatorics.all_pairs2.all_pairs2
+ from allpairspy import AllPairs
+ all_pairs = AllPairs
# Pairwise fails if the number of inputs == 1. Use exhaustive in this case the
# results will be the same.
diff --git a/tests/custom_cluster/test_hs2_fault_injection.py b/tests/custom_cluster/test_hs2_fault_injection.py
index 0b3f8e7b4..27e536fdd 100644
--- a/tests/custom_cluster/test_hs2_fault_injection.py
+++ b/tests/custom_cluster/test_hs2_fault_injection.py
@@ -1,5 +1,3 @@
-#!/usr/bin/env python
-#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
diff --git a/tests/query_test/test_decimal_casting.py b/tests/query_test/test_decimal_casting.py
index 752e3ad64..f487e8bc3 100644
--- a/tests/query_test/test_decimal_casting.py
+++ b/tests/query_test/test_decimal_casting.py
@@ -19,7 +19,7 @@
#
import pytest
from decimal import Decimal, getcontext, ROUND_DOWN, ROUND_HALF_UP
-from metacomm.combinatorics.all_pairs2 import all_pairs2 as all_pairs
+from allpairspy import AllPairs as all_pairs
from random import randint
from tests.common.impala_test_suite import ImpalaTestSuite
[impala] 06/06: IMPALA-11977: Fix Python 3 broken imports and object model differences
Posted by jo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
joemcdonnell pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 0c7c6a335ef06dcd123804803f4d34fafcf4c9a8
Author: Joe McDonnell <jo...@cloudera.com>
AuthorDate: Sat Mar 4 15:17:46 2023 -0800
IMPALA-11977: Fix Python 3 broken imports and object model differences
Python 3 changed some object model methods:
- __nonzero__ was removed in favor of __bool__
- func_dict / func_name were removed in favor of __dict__ / __name__
- The next() function was deprecated in favor of __next__
(Code locations should use next(iter) rather than iter.next())
- metaclasses are specified a different way
- Locations that specify __eq__ should also specify __hash__
Python 3 also moved some packages around (urllib2, Queue, httplib,
etc), and this adapts the code to use the new locations (usually
handled on Python 2 via future). This also fixes the code to
avoid referencing exception variables outside the exception block
and variables outside of a comprehension. Several of these seem
like false positives, but it is better to avoid the warning.
This fixes these pylint warnings:
bad-python3-import
eq-without-hash
metaclass-assignment
next-method-called
nonzero-method
exception-escape
comprehension-escape
Testing:
- Ran core tests
- Ran release exhaustive tests
Change-Id: I988ae6c139142678b0d40f1f4170b892eabf25ee
Reviewed-on: http://gerrit.cloudera.org:8080/19592
Reviewed-by: Joe McDonnell <jo...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
bin/banned_py3k_warnings.txt | 7 ++
testdata/bin/generate-schema-statements.py | 3 +-
testdata/common/widetable.py | 8 +--
tests/benchmark/report_benchmark_results.py | 13 ++--
tests/common/custom_cluster_test_suite.py | 80 +++++++++++------------
tests/common/impala_connection.py | 4 +-
tests/common/test_result_verifier.py | 15 +++++
tests/comparison/cluster.py | 18 ++---
tests/comparison/common.py | 6 ++
tests/comparison/data_generator_mapred_common.py | 4 +-
tests/comparison/db_connection.py | 5 +-
tests/comparison/db_types.py | 5 +-
tests/comparison/query.py | 15 ++---
tests/conftest.py | 2 +-
tests/custom_cluster/test_admission_controller.py | 20 +++---
tests/custom_cluster/test_executor_groups.py | 8 +--
tests/custom_cluster/test_local_catalog.py | 6 +-
tests/custom_cluster/test_saml2_sso.py | 32 +++++----
tests/custom_cluster/test_udf_concurrency.py | 4 +-
tests/hs2/test_hs2.py | 5 +-
tests/hs2/test_json_endpoints.py | 5 +-
tests/performance/query.py | 3 +
tests/query_test/test_insert_parquet.py | 9 +++
tests/shell/test_shell_commandline.py | 2 +-
tests/shell/test_shell_interactive.py | 32 +++++----
tests/statestore/test_statestore.py | 8 ++-
tests/stress/concurrent_select.py | 11 ++--
tests/stress/query_retries_stress_runner.py | 4 +-
tests/util/concurrent_workload.py | 10 +--
tests/util/filesystem_base.py | 5 +-
tests/util/hdfs_util.py | 10 +--
tests/util/ssh_util.py | 6 +-
32 files changed, 211 insertions(+), 154 deletions(-)
diff --git a/bin/banned_py3k_warnings.txt b/bin/banned_py3k_warnings.txt
index 652d91075..ae4bc7c77 100644
--- a/bin/banned_py3k_warnings.txt
+++ b/bin/banned_py3k_warnings.txt
@@ -17,3 +17,10 @@ round-builtin
deprecated-string-function
sys-max-int
exception-message-attribute
+bad-python3-import
+eq-without-hash
+metaclass-assignment
+next-method-called
+nonzero-method
+comprehension-escape
+exception-escape
diff --git a/testdata/bin/generate-schema-statements.py b/testdata/bin/generate-schema-statements.py
index 232db516b..750fb3a8d 100755
--- a/testdata/bin/generate-schema-statements.py
+++ b/testdata/bin/generate-schema-statements.py
@@ -95,6 +95,7 @@
# serially.
#
from __future__ import absolute_import, division, print_function
+from builtins import object
import collections
import csv
import glob
@@ -592,7 +593,7 @@ class Statements(object):
with open(filename, 'w') as f:
f.write('\n\n'.join(output))
- def __nonzero__(self):
+ def __bool__(self):
return bool(self.create or self.load or self.load_base)
def eval_section(section_str):
diff --git a/testdata/common/widetable.py b/testdata/common/widetable.py
index 405bdedf3..6bb8cd354 100755
--- a/testdata/common/widetable.py
+++ b/testdata/common/widetable.py
@@ -52,7 +52,7 @@ def get_columns(num_cols):
iter = itertools.cycle(templates)
# Produces [bool_col1, tinyint_col1, ..., bool_col2, tinyint_col2, ...]
# The final list has 'num_cols' elements.
- return [iter.next() % (i // len(templates) + 1) for i in range(num_cols)]
+ return [next(iter) % (i // len(templates) + 1) for i in range(num_cols)]
# Data generators for different types. Each generator yields an infinite number of
# value strings suitable for writing to a CSV file.
@@ -83,7 +83,7 @@ def quote(iter_fn):
def new_iter_fn():
iter = iter_fn()
while True:
- yield "'%s'" % iter.next()
+ yield "'%s'" % next(iter)
return new_iter_fn
def get_data(num_cols, num_rows, delimiter=',', quote_strings=False):
@@ -101,12 +101,12 @@ def get_data(num_cols, num_rows, delimiter=',', quote_strings=False):
]
# Create a generator instance for each column, cycling through the different types
iter = itertools.cycle(generators)
- column_generators = [iter.next()() for i in range(num_cols)]
+ column_generators = [next(iter)() for i in range(num_cols)]
# Populate each row using column_generators
rows = []
for i in range(num_rows):
- vals = [gen.next() for gen in column_generators]
+ vals = [next(gen) for gen in column_generators]
rows.append(delimiter.join(vals))
return rows
diff --git a/tests/benchmark/report_benchmark_results.py b/tests/benchmark/report_benchmark_results.py
index b7b1c50a5..6a447d5ed 100755
--- a/tests/benchmark/report_benchmark_results.py
+++ b/tests/benchmark/report_benchmark_results.py
@@ -238,12 +238,15 @@ def all_query_results(grouped):
def get_commit_date(commit_sha):
- import urllib2
+ try:
+ from urllib.request import Request, urlopen
+ except ImportError:
+ from urllib2 import Request, urlopen
url = 'https://api.github.com/repos/apache/impala/commits/' + commit_sha
try:
- request = urllib2.Request(url)
- response = urllib2.urlopen(request).read()
+ request = Request(url)
+ response = urlopen(request).read()
data = json.loads(response.decode('utf8'))
return data['commit']['committer']['date'][:10]
except:
@@ -251,7 +254,7 @@ def get_commit_date(commit_sha):
def get_impala_version(grouped):
"""Figure out Impala version by looking at query profile."""
- first_result = all_query_results(grouped).next()
+ first_result = next(all_query_results(grouped))
profile = first_result['result_list'][0]['runtime_profile']
match = re.search('Impala Version:\s(.*)\s\(build\s(.*)\)', profile)
version = match.group(1)
@@ -915,7 +918,7 @@ class ExecSummaryComparison(object):
table.align = 'l'
table.float_format = '.2'
table_contains_at_least_one_row = False
- for row in [row for row in self.rows if is_significant(row)]:
+ for row in [r for r in self.rows if is_significant(r)]:
table_row = [row[OPERATOR],
'{0:.2%}'.format(row[PERCENT_OF_QUERY]),
'{0:.2%}'.format(row[RSTD]),
diff --git a/tests/common/custom_cluster_test_suite.py b/tests/common/custom_cluster_test_suite.py
index fb951ff58..e56f59124 100644
--- a/tests/common/custom_cluster_test_suite.py
+++ b/tests/common/custom_cluster_test_suite.py
@@ -115,67 +115,67 @@ class CustomClusterTestSuite(ImpalaTestSuite):
method's func_dict"""
def decorate(func):
if impalad_args is not None:
- func.func_dict[IMPALAD_ARGS] = impalad_args
- func.func_dict[STATESTORED_ARGS] = statestored_args
+ func.__dict__[IMPALAD_ARGS] = impalad_args
+ func.__dict__[STATESTORED_ARGS] = statestored_args
if catalogd_args is not None:
- func.func_dict[CATALOGD_ARGS] = catalogd_args
+ func.__dict__[CATALOGD_ARGS] = catalogd_args
if start_args is not None:
- func.func_dict[START_ARGS] = start_args.split()
+ func.__dict__[START_ARGS] = start_args.split()
if jvm_args is not None:
- func.func_dict[JVM_ARGS] = jvm_args
+ func.__dict__[JVM_ARGS] = jvm_args
if hive_conf_dir is not None:
- func.func_dict[HIVE_CONF_DIR] = hive_conf_dir
+ func.__dict__[HIVE_CONF_DIR] = hive_conf_dir
if kudu_args is not None:
- func.func_dict[KUDU_ARGS] = kudu_args
+ func.__dict__[KUDU_ARGS] = kudu_args
if default_query_options is not None:
- func.func_dict[DEFAULT_QUERY_OPTIONS] = default_query_options
+ func.__dict__[DEFAULT_QUERY_OPTIONS] = default_query_options
if impala_log_dir is not None:
- func.func_dict[IMPALA_LOG_DIR] = impala_log_dir
+ func.__dict__[IMPALA_LOG_DIR] = impala_log_dir
if cluster_size is not None:
- func.func_dict[CLUSTER_SIZE] = cluster_size
+ func.__dict__[CLUSTER_SIZE] = cluster_size
if num_exclusive_coordinators is not None:
- func.func_dict[NUM_EXCLUSIVE_COORDINATORS] = num_exclusive_coordinators
+ func.__dict__[NUM_EXCLUSIVE_COORDINATORS] = num_exclusive_coordinators
if statestored_timeout_s is not None:
- func.func_dict[STATESTORED_TIMEOUT_S] = statestored_timeout_s
+ func.__dict__[STATESTORED_TIMEOUT_S] = statestored_timeout_s
if impalad_timeout_s is not None:
- func.func_dict[IMPALAD_TIMEOUT_S] = impalad_timeout_s
+ func.__dict__[IMPALAD_TIMEOUT_S] = impalad_timeout_s
if expect_cores is not None:
- func.func_dict[EXPECT_CORES] = expect_cores
+ func.__dict__[EXPECT_CORES] = expect_cores
if reset_ranger is not False:
- func.func_dict[RESET_RANGER] = True
+ func.__dict__[RESET_RANGER] = True
return func
return decorate
def setup_method(self, method):
cluster_args = list()
for arg in [IMPALAD_ARGS, STATESTORED_ARGS, CATALOGD_ARGS, ADMISSIOND_ARGS, JVM_ARGS]:
- if arg in method.func_dict:
- cluster_args.append("--%s=%s " % (arg, method.func_dict[arg]))
- if START_ARGS in method.func_dict:
- cluster_args.extend(method.func_dict[START_ARGS])
+ if arg in method.__dict__:
+ cluster_args.append("--%s=%s " % (arg, method.__dict__[arg]))
+ if START_ARGS in method.__dict__:
+ cluster_args.extend(method.__dict__[START_ARGS])
- if HIVE_CONF_DIR in method.func_dict:
- self._start_hive_service(method.func_dict[HIVE_CONF_DIR])
+ if HIVE_CONF_DIR in method.__dict__:
+ self._start_hive_service(method.__dict__[HIVE_CONF_DIR])
# Should let Impala adopt the same hive-site.xml. The only way is to add it in the
# beginning of the CLASSPATH. Because there's already a hive-site.xml in the
# default CLASSPATH (see bin/set-classpath.sh).
cluster_args.append(
- '--env_vars=CUSTOM_CLASSPATH=%s ' % method.func_dict[HIVE_CONF_DIR])
+ '--env_vars=CUSTOM_CLASSPATH=%s ' % method.__dict__[HIVE_CONF_DIR])
- if KUDU_ARGS in method.func_dict:
- self._restart_kudu_service(method.func_dict[KUDU_ARGS])
+ if KUDU_ARGS in method.__dict__:
+ self._restart_kudu_service(method.__dict__[KUDU_ARGS])
- if RESET_RANGER in method.func_dict:
+ if RESET_RANGER in method.__dict__:
self._reset_ranger_policy_repository()
cluster_size = DEFAULT_CLUSTER_SIZE
- if CLUSTER_SIZE in method.func_dict:
- cluster_size = method.func_dict[CLUSTER_SIZE]
+ if CLUSTER_SIZE in method.__dict__:
+ cluster_size = method.__dict__[CLUSTER_SIZE]
use_exclusive_coordinators = False
num_coordinators = cluster_size
- if NUM_EXCLUSIVE_COORDINATORS in method.func_dict:
- num_coordinators = method.func_dict[NUM_EXCLUSIVE_COORDINATORS]
+ if NUM_EXCLUSIVE_COORDINATORS in method.__dict__:
+ num_coordinators = method.__dict__[NUM_EXCLUSIVE_COORDINATORS]
use_exclusive_coordinators = True
# Start a clean new cluster before each test
@@ -183,17 +183,17 @@ class CustomClusterTestSuite(ImpalaTestSuite):
"cluster_size": cluster_size,
"num_coordinators": num_coordinators,
"expected_num_impalads": cluster_size,
- "default_query_options": method.func_dict.get(DEFAULT_QUERY_OPTIONS),
+ "default_query_options": method.__dict__.get(DEFAULT_QUERY_OPTIONS),
"use_exclusive_coordinators": use_exclusive_coordinators
}
- if IMPALA_LOG_DIR in method.func_dict:
- kwargs["impala_log_dir"] = method.func_dict[IMPALA_LOG_DIR]
- if STATESTORED_TIMEOUT_S in method.func_dict:
- kwargs["statestored_timeout_s"] = method.func_dict[STATESTORED_TIMEOUT_S]
- if IMPALAD_TIMEOUT_S in method.func_dict:
- kwargs["impalad_timeout_s"] = method.func_dict[IMPALAD_TIMEOUT_S]
-
- if method.func_dict.get(EXPECT_CORES, False):
+ if IMPALA_LOG_DIR in method.__dict__:
+ kwargs["impala_log_dir"] = method.__dict__[IMPALA_LOG_DIR]
+ if STATESTORED_TIMEOUT_S in method.__dict__:
+ kwargs["statestored_timeout_s"] = method.__dict__[STATESTORED_TIMEOUT_S]
+ if IMPALAD_TIMEOUT_S in method.__dict__:
+ kwargs["impalad_timeout_s"] = method.__dict__[IMPALAD_TIMEOUT_S]
+
+ if method.__dict__.get(EXPECT_CORES, False):
# Make a note of any core files that already exist
possible_cores = find_all_files('*core*')
self.pre_test_cores = set([f for f in possible_cores if is_core_dump(f)])
@@ -209,10 +209,10 @@ class CustomClusterTestSuite(ImpalaTestSuite):
super(CustomClusterTestSuite, self).setup_class()
def teardown_method(self, method):
- if HIVE_CONF_DIR in method.func_dict:
+ if HIVE_CONF_DIR in method.__dict__:
self._start_hive_service(None) # Restart Hive Service using default configs
- if method.func_dict.get(EXPECT_CORES, False):
+ if method.__dict__.get(EXPECT_CORES, False):
# The core dumps expected to be generated by this test should be cleaned up
possible_cores = find_all_files('*core*')
post_test_cores = set([f for f in possible_cores if is_core_dump(f)])
diff --git a/tests/common/impala_connection.py b/tests/common/impala_connection.py
index 54c706d2b..f3832beda 100644
--- a/tests/common/impala_connection.py
+++ b/tests/common/impala_connection.py
@@ -22,6 +22,7 @@
from __future__ import absolute_import, division, print_function
import abc
import codecs
+from future.utils import with_metaclass
import logging
import re
@@ -77,8 +78,7 @@ class OperationHandle(object):
# Represents an Impala connection.
-class ImpalaConnection(object):
- __metaclass__ = abc.ABCMeta
+class ImpalaConnection(with_metaclass(abc.ABCMeta, object)):
def __enter__(self):
return self
diff --git a/tests/common/test_result_verifier.py b/tests/common/test_result_verifier.py
index e3b90aad9..d71fe6349 100644
--- a/tests/common/test_result_verifier.py
+++ b/tests/common/test_result_verifier.py
@@ -65,6 +65,11 @@ class QueryTestResult(object):
return False
return self.column_types == other.column_types and self.rows == other.rows
+ def __hash__(self):
+ # This is not intended to be hashed. If that is happening, then something is wrong.
+ # The regexes in ResultRow make it difficult to implement this correctly.
+ assert False
+
def __ne__(self, other):
return not self.__eq__(other)
@@ -158,6 +163,11 @@ class ResultRow(object):
return other.regex.match(self.row_string)
return self.columns == other.columns
+ def __hash__(self):
+ # This is not intended to be hashed. If that is happening, then something is wrong.
+ # The regexes make it difficult to implement this correctly.
+ assert False
+
def __ne__(self, other):
return not self.__eq__(other)
@@ -225,6 +235,11 @@ class ResultColumn(object):
else:
return self.value == other.value
+ def __hash__(self):
+ # This is not intended to be hashed. If that is happening, then something is wrong.
+ # The regexes make it difficult to implement this correctly.
+ assert False
+
def __ne__(self, other):
return not self.__eq__(other)
diff --git a/tests/comparison/cluster.py b/tests/comparison/cluster.py
index 517df3f94..dc1daf8d6 100644
--- a/tests/comparison/cluster.py
+++ b/tests/comparison/cluster.py
@@ -22,6 +22,7 @@
from __future__ import absolute_import, division, print_function
from builtins import int, range, zip
+from future.utils import with_metaclass
import hdfs
import logging
import os
@@ -34,17 +35,20 @@ from collections import defaultdict
from collections import OrderedDict
from contextlib import contextmanager
from getpass import getuser
+from io import BytesIO
from multiprocessing.pool import ThreadPool
from random import choice
-from StringIO import StringIO
from sys import maxsize
from tempfile import mkdtemp
from threading import Lock
from time import mktime, strptime
-from urlparse import urlparse
from xml.etree.ElementTree import parse as parse_xml
from zipfile import ZipFile
+try:
+ from urllib.parse import urlparse
+except ImportError:
+ from urlparse import urlparse
from tests.comparison.db_connection import HiveConnection, ImpalaConnection
from tests.common.environ import HIVE_MAJOR_VERSION
@@ -65,14 +69,12 @@ CM_CLEAR_PORT = 7180
CM_TLS_PORT = 7183
-class Cluster(object):
+class Cluster(with_metaclass(ABCMeta, object)):
"""This is a base class for clusters. Cluster classes provide various methods for
interacting with a cluster. Ideally the various cluster implementations provide
the same set of methods so any cluster implementation can be chosen at runtime.
"""
- __metaclass__ = ABCMeta
-
def __init__(self):
self._hadoop_configs = None
self._local_hadoop_conf_dir = None
@@ -323,7 +325,7 @@ class CmCluster(Cluster):
def _init_local_hadoop_conf_dir(self):
self._local_hadoop_conf_dir = mkdtemp()
- data = StringIO(self.cm.get("/clusters/%s/services/%s/clientConfig"
+ data = BytesIO(self.cm.get("/clusters/%s/services/%s/clientConfig"
% (self.cm_cluster.name, self._find_service("HIVE").name)))
zip_file = ZipFile(data)
for name in zip_file.namelist():
@@ -655,9 +657,7 @@ class CmImpala(Impala):
raise Exception("Failed to restart Impala: %s" % command.resultMessage)
-class Impalad(object):
-
- __metaclass__ = ABCMeta
+class Impalad(with_metaclass(ABCMeta, object)):
def __init__(self):
self.impala = None
diff --git a/tests/comparison/common.py b/tests/comparison/common.py
index 5b2e71dcd..fb768f310 100644
--- a/tests/comparison/common.py
+++ b/tests/comparison/common.py
@@ -451,6 +451,9 @@ class ArrayColumn(CollectionColumn):
return True
return self.name == other.name and self.owner.identifier == other.owner.identifier
+ def __hash__(self):
+ return hash((self.name, self.owner.identifier))
+
def __deepcopy__(self, memo):
other = ArrayColumn(
owner=self.owner,
@@ -480,6 +483,9 @@ class MapColumn(CollectionColumn):
return True
return self.name == other.name and self.owner.identifier == other.owner.identifier
+ def __hash__(self):
+ return hash((self.name, self.owner.identifier))
+
def __deepcopy__(self, memo):
other = MapColumn(
owner=self.owner,
diff --git a/tests/comparison/data_generator_mapred_common.py b/tests/comparison/data_generator_mapred_common.py
index 78d7d8d38..2367a0a3f 100644
--- a/tests/comparison/data_generator_mapred_common.py
+++ b/tests/comparison/data_generator_mapred_common.py
@@ -27,7 +27,7 @@ from __future__ import absolute_import, division, print_function
from base import range
import base64
import pickle
-import StringIO
+from io import BytesIO
from tests.comparison.db_types import Decimal
from tests.comparison.random_val_generator import RandomValGenerator
@@ -94,7 +94,7 @@ def estimate_bytes_per_row(table_data_generator, row_count):
original_row_count = table_data_generator.row_count
original_output_file = table_data_generator.output_file
table_data_generator.row_count = row_count
- table_data_generator.output_file = StringIO.StringIO()
+ table_data_generator.output_file = BytesIO()
table_data_generator.populate_output_file()
table_data_generator.output_file.flush()
bytes_per_row = len(table_data_generator.output_file.getvalue()) / float(row_count)
diff --git a/tests/comparison/db_connection.py b/tests/comparison/db_connection.py
index 0177deb6a..8705024eb 100644
--- a/tests/comparison/db_connection.py
+++ b/tests/comparison/db_connection.py
@@ -23,6 +23,7 @@
'''
from __future__ import absolute_import, division, print_function
from builtins import filter, map, range, zip
+from future.utils import with_metaclass
import hashlib
import impala.dbapi
import re
@@ -572,9 +573,7 @@ class DbCursor(object):
return ()
-class DbConnection(object):
-
- __metaclass__ = ABCMeta
+class DbConnection(with_metaclass(ABCMeta, object)):
LOCK = Lock()
diff --git a/tests/comparison/db_types.py b/tests/comparison/db_types.py
index 5f78d4e84..27e2a6025 100644
--- a/tests/comparison/db_types.py
+++ b/tests/comparison/db_types.py
@@ -16,6 +16,7 @@
# under the License.
from __future__ import absolute_import, division, print_function
+from future.utils import with_metaclass
import re
import sys
@@ -43,15 +44,13 @@ class DataTypeMetaclass(type):
getattr(other, 'CMP_VALUE', other.__name__))
-class DataType(ValExpr):
+class DataType(with_metaclass(DataTypeMetaclass, ValExpr)):
'''Base class for data types.
Data types are represented as classes so inheritance can be used.
'''
- __metaclass__ = DataTypeMetaclass
-
@staticmethod
def group_by_type(vals):
'''Group cols by their data type and return a dict of the results.'''
diff --git a/tests/comparison/query.py b/tests/comparison/query.py
index dd73f895e..1d8004abd 100644
--- a/tests/comparison/query.py
+++ b/tests/comparison/query.py
@@ -16,7 +16,8 @@
# under the License.
from __future__ import absolute_import, division, print_function
-from builtins import range
+from builtins import object, range
+from future.utils import with_metaclass
from abc import ABCMeta, abstractproperty
from copy import deepcopy
from logging import getLogger
@@ -47,13 +48,11 @@ class StatementExecutionMode(object):
) = range(5)
-class AbstractStatement(object):
+class AbstractStatement(with_metaclass(ABCMeta, object)):
"""
Abstract query representation
"""
- __metaclass__ = ABCMeta
-
def __init__(self):
# reference to statement's parent. For example the right side of a UNION clause
# SELECT will have a parent as the SELECT on the left, which for the query
@@ -236,9 +235,9 @@ class SelectItemSubList(object):
def __len__(self):
return sum(1 for _ in self)
- def __nonzero__(self):
+ def __bool__(self):
try:
- iter(self).next()
+ next(iter(self))
return True
except StopIteration:
return False
@@ -260,7 +259,7 @@ class SelectItemSubList(object):
items = list()
while start < stop:
try:
- idx, item = self_iter.next()
+ idx, item = next(self_iter)
except StopIteration:
break
if idx < start:
@@ -298,7 +297,7 @@ class SelectItemSubList(object):
filtered_idx = 0
while start < stop:
try:
- idx, item = self_iter.next()
+ idx, item = next(self_iter)
except StopIteration:
break
if not self.filter(item):
diff --git a/tests/conftest.py b/tests/conftest.py
index d7bfd4372..eca1471c2 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -230,7 +230,7 @@ def pytest_generate_tests(metafunc):
if len(vectors) == 0:
LOG.warning("No test vectors generated for test '%s'. Check constraints and "
- "input vectors" % metafunc.function.func_name)
+ "input vectors" % metafunc.function.__name__)
vector_names = list(map(str, vectors))
# In the case this is a test result update or sanity run, select a single test vector
diff --git a/tests/custom_cluster/test_admission_controller.py b/tests/custom_cluster/test_admission_controller.py
index c25719248..ab8068412 100644
--- a/tests/custom_cluster/test_admission_controller.py
+++ b/tests/custom_cluster/test_admission_controller.py
@@ -1538,11 +1538,11 @@ class TestAdmissionControllerWithACService(TestAdmissionController):
if self.exploration_strategy() != 'exhaustive':
pytest.skip('runs only in exhaustive')
- if 'start_args' not in method.func_dict:
- method.func_dict['start_args'] = list()
- method.func_dict["start_args"].append("--enable_admission_service")
- if "impalad_args" in method.func_dict:
- method.func_dict["admissiond_args"] = method.func_dict["impalad_args"]
+ if 'start_args' not in method.__dict__:
+ method.__dict__['start_args'] = list()
+ method.__dict__["start_args"].append("--enable_admission_service")
+ if "impalad_args" in method.__dict__:
+ method.__dict__["admissiond_args"] = method.__dict__["impalad_args"]
super(TestAdmissionController, self).setup_method(method)
@SkipIfNotHdfsMinicluster.tuned_for_minicluster
@@ -2284,9 +2284,9 @@ class TestAdmissionControllerStressWithACService(TestAdmissionControllerStress):
def setup_method(self, method):
if self.exploration_strategy() != 'exhaustive':
pytest.skip('runs only in exhaustive')
- if 'start_args' not in method.func_dict:
- method.func_dict['start_args'] = list()
- method.func_dict["start_args"].append("--enable_admission_service")
- if "impalad_args" in method.func_dict:
- method.func_dict["admissiond_args"] = method.func_dict["impalad_args"]
+ if 'start_args' not in method.__dict__:
+ method.__dict__['start_args'] = list()
+ method.__dict__["start_args"].append("--enable_admission_service")
+ if "impalad_args" in method.__dict__:
+ method.__dict__["admissiond_args"] = method.__dict__["impalad_args"]
super(TestAdmissionControllerStress, self).setup_method(method)
diff --git a/tests/custom_cluster/test_executor_groups.py b/tests/custom_cluster/test_executor_groups.py
index 5d624d969..79d6d5c6d 100644
--- a/tests/custom_cluster/test_executor_groups.py
+++ b/tests/custom_cluster/test_executor_groups.py
@@ -44,10 +44,10 @@ class TestExecutorGroups(CustomClusterTestSuite):
def setup_method(self, method):
# Always start the base cluster with the coordinator in its own executor group.
- existing_args = method.func_dict.get("impalad_args", "")
- method.func_dict["impalad_args"] = "%s -executor_groups=coordinator" % existing_args
- method.func_dict["cluster_size"] = 1
- method.func_dict["num_exclusive_coordinators"] = 1
+ existing_args = method.__dict__.get("impalad_args", "")
+ method.__dict__["impalad_args"] = "%s -executor_groups=coordinator" % existing_args
+ method.__dict__["cluster_size"] = 1
+ method.__dict__["num_exclusive_coordinators"] = 1
self.num_groups = 1
self.num_impalads = 1
super(TestExecutorGroups, self).setup_method(method)
diff --git a/tests/custom_cluster/test_local_catalog.py b/tests/custom_cluster/test_local_catalog.py
index 25e0c9f62..89f19e075 100644
--- a/tests/custom_cluster/test_local_catalog.py
+++ b/tests/custom_cluster/test_local_catalog.py
@@ -20,7 +20,7 @@
from __future__ import absolute_import, division, print_function
from builtins import range
import pytest
-import Queue
+import queue
import random
import re
import threading
@@ -261,7 +261,7 @@ class TestLocalCatalogRetries(CustomClusterTestSuite):
inconsistent_seen = [0]
inconsistent_seen_lock = threading.Lock()
# Tracks query failures for all other reasons.
- failed_queries = Queue.Queue()
+ failed_queries = queue.Queue()
try:
client1 = self.cluster.impalads[0].service.create_beeswax_client()
client2 = self.cluster.impalads[1].service.create_beeswax_client()
@@ -378,7 +378,7 @@ class TestLocalCatalogRetries(CustomClusterTestSuite):
replans_seen_lock = threading.Lock()
# Queue to propagate exceptions from failed queries, if any.
- failed_queries = Queue.Queue()
+ failed_queries = queue.Queue()
def stress_thread(client):
while replans_seen[0] == 0:
diff --git a/tests/custom_cluster/test_saml2_sso.py b/tests/custom_cluster/test_saml2_sso.py
index 2cf4babce..3df48b183 100644
--- a/tests/custom_cluster/test_saml2_sso.py
+++ b/tests/custom_cluster/test_saml2_sso.py
@@ -22,17 +22,23 @@ import datetime
import os
import pytest
import uuid
-import urllib2
-import urlparse
import xml.etree.ElementTree as ET
import zlib
+try:
+ from urllib.parse import parse_qs, urlparse
+ from urllib.request import HTTPErrorProcessor, build_opener, Request
+except ImportError:
+ from urllib2 import HTTPErrorProcessor, build_opener, Request
+ from urlparse import parse_qs, urlparse
+
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
from tests.common.test_vector import ImpalaTestVector
from tests.common.test_dimensions import create_client_protocol_dimension
from tests.shell.util import run_impala_shell_cmd
-class NoRedirection(urllib2.HTTPErrorProcessor):
+
+class NoRedirection(HTTPErrorProcessor):
"""Allows inspecting http redirection responses. """
def http_response(self, request, response):
return response
@@ -150,8 +156,8 @@ class TestClientSaml(CustomClusterTestSuite):
def _request_resource(self):
""" Initial POST request to hs2-http port, response should be redirected
to IDP and contain the authnrequest. """
- opener = urllib2.build_opener(NoRedirection)
- req = urllib2.Request("http://localhost:%s" % TestClientSaml.HOST_PORT, " ")
+ opener = build_opener(NoRedirection)
+ req = Request("http://localhost:%s" % TestClientSaml.HOST_PORT, " ")
req.add_header('X-Hive-Token-Response-Port', TestClientSaml.CLIENT_PORT)
response = opener.open(req)
relay_state, client_id, saml_req_xml = \
@@ -161,11 +167,11 @@ class TestClientSaml(CustomClusterTestSuite):
def _parse_redirection_response(self, response):
assert response.getcode() == 302
- client_id = response.info().getheader("X-Hive-Client-Identifier")
+ client_id = response.info().get("X-Hive-Client-Identifier", None)
assert client_id is not None
- new_url = response.info().getheader("location")
+ new_url = response.info()["location"]
assert new_url.startswith(TestClientSaml.IDP_URL)
- query = urlparse.parse_qs(urlparse.urlparse(new_url).query.encode('ASCII'))
+ query = parse_qs(urlparse(new_url).query.encode('ASCII'))
relay_state = query["RelayState"][0]
assert relay_state is not None
saml_req = query["SAMLRequest"][0]
@@ -181,15 +187,15 @@ class TestClientSaml(CustomClusterTestSuite):
def _request_resource_with_bearer(self, client_id, bearer_token):
""" Send POST request to hs2-http port again, this time with bearer tokan.
The response should contain a security cookie if the validation succeeded """
- req = urllib2.Request("http://localhost:%s" % TestClientSaml.HOST_PORT, " ")
+ req = Request("http://localhost:%s" % TestClientSaml.HOST_PORT, " ")
req.add_header('X-Hive-Client-Identifier', client_id)
req.add_header('Authorization', "Bearer " + bearer_token)
- opener = urllib2.build_opener(NoRedirection)
+ opener = build_opener(NoRedirection)
response = opener.open(req)
# saml2_ee_test_mode=true leads to returning 401 unauthorized - otherwise the
# call would hang if there is no Thrift message.
assert response.getcode() == 401
- cookies = response.info().getheader('Set-Cookie')
+ cookies = response.info()['Set-Cookie']
assert cookies.startswith("impala.auth=")
def _send_authn_response(self, request_id, relay_state,
@@ -201,8 +207,8 @@ class TestClientSaml(CustomClusterTestSuite):
authn_resp = self._generate_authn_response(request_id, attributes_xml)
encoded_authn_resp = base64.urlsafe_b64encode(authn_resp)
body = "SAMLResponse=%s&RelayState=%s" % (encoded_authn_resp, relay_state)
- opener = urllib2.build_opener(NoRedirection)
- req = urllib2.Request(TestClientSaml.SP_CALLBACK_URL, body)
+ opener = build_opener(NoRedirection)
+ req = Request(TestClientSaml.SP_CALLBACK_URL, body)
response = opener.open(req)
bearer_token = self._parse_xhtml_form(response, expect_success)
return bearer_token
diff --git a/tests/custom_cluster/test_udf_concurrency.py b/tests/custom_cluster/test_udf_concurrency.py
index aa35eac39..3fd9aa837 100644
--- a/tests/custom_cluster/test_udf_concurrency.py
+++ b/tests/custom_cluster/test_udf_concurrency.py
@@ -128,7 +128,7 @@ class TestUdfConcurrency(CustomClusterTestSuite):
# join all threads.
for t in runner_threads: t.join()
- for e in errors: print(e)
+ for err in errors: print(err)
# Checks that no impalad has crashed.
assert cluster.num_responsive_coordinators() == exp_num_coordinators
@@ -210,5 +210,5 @@ class TestUdfConcurrency(CustomClusterTestSuite):
for t in runner_threads: t.join()
# Check for any errors.
- for e in errors: print(e)
+ for err in errors: print(err)
assert len(errors) == 0
diff --git a/tests/hs2/test_hs2.py b/tests/hs2/test_hs2.py
index 6353fbf95..70191f032 100644
--- a/tests/hs2/test_hs2.py
+++ b/tests/hs2/test_hs2.py
@@ -29,7 +29,10 @@ import threading
import time
import uuid
-from urllib2 import urlopen
+try:
+ from urllib.request import urlopen
+except ImportError:
+ from urllib2 import urlopen
from ImpalaService import ImpalaHiveServer2Service
from tests.common.environ import ImpalaTestClusterProperties
diff --git a/tests/hs2/test_json_endpoints.py b/tests/hs2/test_json_endpoints.py
index 64bce618c..b531fcc45 100644
--- a/tests/hs2/test_json_endpoints.py
+++ b/tests/hs2/test_json_endpoints.py
@@ -22,7 +22,10 @@ import json
import pytest
from time import time
-from urllib2 import urlopen
+try:
+ from urllib.request import urlopen
+except ImportError:
+ from urllib2 import urlopen
from tests.common.environ import IS_DOCKERIZED_TEST_CLUSTER
from tests.common.impala_cluster import ImpalaCluster
diff --git a/tests/performance/query.py b/tests/performance/query.py
index 5832e5dfa..23e92686a 100644
--- a/tests/performance/query.py
+++ b/tests/performance/query.py
@@ -54,6 +54,9 @@ class Query(object):
self.workload_name == other.workload_name and
self.db == other.db)
+ def __hash__(self):
+ return hash((self.query_str, self.name, self.scale_factor, self.test_vector, self.db))
+
def _build_query(self):
"""Populates db, query_str, table_format_str"""
self.db = QueryTestSectionReader.get_db_name(self.test_vector, self.scale_factor)
diff --git a/tests/query_test/test_insert_parquet.py b/tests/query_test/test_insert_parquet.py
index 9264643ab..ed9d819eb 100644
--- a/tests/query_test/test_insert_parquet.py
+++ b/tests/query_test/test_insert_parquet.py
@@ -54,6 +54,9 @@ class RoundFloat():
"""Compares this objects's value to a numeral after rounding it."""
return round(self.value, self.num_digits) == round(numeral, self.num_digits)
+ def __hash__(self):
+ return hash(round(self.value, self.num_digits))
+
class TimeStamp():
"""Class to construct timestamps with a default format specifier."""
@@ -68,6 +71,9 @@ class TimeStamp():
"""Compares this objects's value to another timetuple."""
return self.timetuple == other_timetuple
+ def __hash__(self):
+ return hash(self.timetuple)
+
class Date():
"""Class to compare dates specified as year-month-day to dates specified as days since
@@ -79,6 +85,9 @@ class Date():
def __eq__(self, other_days_since_eopch):
return self.days_since_epoch == other_days_since_eopch
+ def __hash__(self):
+ return hash(self.days_since_epoch)
+
ColumnStats = namedtuple('ColumnStats', ['name', 'min', 'max', 'null_count'])
diff --git a/tests/shell/test_shell_commandline.py b/tests/shell/test_shell_commandline.py
index f02adba18..f469d2be3 100644
--- a/tests/shell/test_shell_commandline.py
+++ b/tests/shell/test_shell_commandline.py
@@ -80,7 +80,7 @@ def empty_table(unique_database, request):
Returns:
fq_table_name (str): the fully qualified name of the table: : dbname.table_name
"""
- table_name = request.node.function.func_name
+ table_name = request.node.function.__name__
fq_table_name = '.'.join([unique_database, table_name])
stmt = "CREATE TABLE %s (i integer, s string)" % fq_table_name
request.instance.execute_query_expect_success(request.instance.client, stmt,
diff --git a/tests/shell/test_shell_interactive.py b/tests/shell/test_shell_interactive.py
index 101793613..d0920cdcf 100755
--- a/tests/shell/test_shell_interactive.py
+++ b/tests/shell/test_shell_interactive.py
@@ -19,7 +19,8 @@
# under the License.
from __future__ import absolute_import, division, print_function
-import httplib
+import http.client
+import http.server
import logging
import os
import pexpect
@@ -27,6 +28,7 @@ import pytest
import re
import signal
import socket
+import socketserver
import sys
import threading
from time import sleep
@@ -47,8 +49,6 @@ from tests.common.test_dimensions import (
from tests.shell.util import (assert_var_substitution, ImpalaShell, get_impalad_port,
get_shell_cmd, get_open_sessions_metric, spawn_shell, get_unused_port,
create_impala_shell_executable_dimension, get_impala_shell_executable)
-import SimpleHTTPServer
-import SocketServer
QUERY_FILE_PATH = os.path.join(os.environ['IMPALA_HOME'], 'tests', 'shell')
@@ -79,32 +79,30 @@ def tmp_history_file(request):
return tmp.name
-class RequestHandler503(SimpleHTTPServer.SimpleHTTPRequestHandler):
+class RequestHandler503(http.server.SimpleHTTPRequestHandler):
"""A custom http handler that checks for duplicate 'Host' headers from the most
recent http request, and always returns a 503 http code."""
def __init__(self, request, client_address, server):
- SimpleHTTPServer.SimpleHTTPRequestHandler.__init__(self, request, client_address,
- server)
+ http.server.SimpleHTTPRequestHandler.__init__(self, request, client_address,
+ server)
def should_send_body_text(self):
# in RequestHandler503 we do not send any body text
return False
def do_POST(self):
- # The unfortunately named self.headers here is an instance of mimetools.Message that
- # contains the request headers.
- request_headers = self.headers.headers
-
- # Ensure that only one 'Host' header is contained in the request before responding.
- host_hdr_count = sum([header.startswith('Host:') for header in request_headers])
- assert host_hdr_count == 1, "duplicate 'Host:' headers in %s" % request_headers
+ # Ensure that a 'Host' header is contained in the request before responding.
+ assert "Host" in self.headers
# Respond with 503.
- self.send_response(code=httplib.SERVICE_UNAVAILABLE, message="Service Unavailable")
+ self.send_response(code=http.client.SERVICE_UNAVAILABLE,
+ message="Service Unavailable")
+ # The Python 3 version of SimpleHTTPRequestHandler requires this to be called
+ # explicitly
+ self.end_headers()
if self.should_send_body_text():
- # Optionally send ody text with 503 message.
- self.end_headers()
+ # Optionally send body text with 503 message.
self.wfile.write("EXTRA")
@@ -123,7 +121,7 @@ class TestHTTPServer503(object):
def __init__(self, clazz):
self.HOST = "localhost"
self.PORT = get_unused_port()
- self.httpd = SocketServer.TCPServer((self.HOST, self.PORT), clazz)
+ self.httpd = socketserver.TCPServer((self.HOST, self.PORT), clazz)
self.http_server_thread = threading.Thread(target=self.httpd.serve_forever)
self.http_server_thread.start()
diff --git a/tests/statestore/test_statestore.py b/tests/statestore/test_statestore.py
index 7ca3d713c..ba7605ae4 100644
--- a/tests/statestore/test_statestore.py
+++ b/tests/statestore/test_statestore.py
@@ -25,9 +25,13 @@ import socket
import threading
import traceback
import time
-import urllib2
import uuid
+try:
+ from urllib.request import urlopen
+except ImportError:
+ from urllib2 import urlopen
+
from Types.ttypes import TNetworkAddress
from thrift.protocol import TBinaryProtocol
from thrift.server.TServer import TServer
@@ -63,7 +67,7 @@ LOG = logging.getLogger('test_statestore')
# Test that topic deletions take effect correctly.
def get_statestore_subscribers(host='localhost', port=25010):
- response = urllib2.urlopen("http://{0}:{1}/subscribers?json".format(host, port))
+ response = urlopen("http://{0}:{1}/subscribers?json".format(host, port))
page = response.read()
return json.loads(page)
diff --git a/tests/stress/concurrent_select.py b/tests/stress/concurrent_select.py
index 96b8978f7..9503e530c 100755
--- a/tests/stress/concurrent_select.py
+++ b/tests/stress/concurrent_select.py
@@ -64,7 +64,7 @@ import re
import signal
import sys
import threading
-from Queue import Empty # Must be before Queue below
+from queue import Empty # Must be before Queue below
from argparse import ArgumentDefaultsHelpFormatter, ArgumentParser, Namespace, SUPPRESS
from collections import defaultdict
from copy import copy
@@ -196,10 +196,11 @@ def print_crash_info_if_exists(impala, start_time):
LOG.info(
"Timeout checking if impalads crashed: %s."
% e + (" Will retry." if remaining_attempts else ""))
- else:
- LOG.error(
- "Aborting after %s failed attempts to check if impalads crashed", max_attempts)
- raise e
+ if not remaining_attempts:
+ LOG.error(
+ "Aborting after %s failed attempts to check if impalads crashed",
+ max_attempts)
+ raise e
for message in crashed_impalads.values():
print(message, file=sys.stderr)
return crashed_impalads
diff --git a/tests/stress/query_retries_stress_runner.py b/tests/stress/query_retries_stress_runner.py
index 87560d3e2..cd221a8d1 100755
--- a/tests/stress/query_retries_stress_runner.py
+++ b/tests/stress/query_retries_stress_runner.py
@@ -34,7 +34,7 @@ import subprocess
import sys
import threading
import traceback
-import Queue
+import queue
from argparse import ArgumentParser
from argparse import RawDescriptionHelpFormatter
@@ -113,7 +113,7 @@ def run_concurrent_workloads(concurrency, coordinator, database, queries):
# The exception queue is used to pass errors from the workload threads back to the main
# thread.
- exception_queue = Queue.Queue()
+ exception_queue = queue.Queue()
# The main method for the workload runner threads.
def __run_workload(stream_id):
diff --git a/tests/util/concurrent_workload.py b/tests/util/concurrent_workload.py
index 2ceb81574..d44ebcd1b 100755
--- a/tests/util/concurrent_workload.py
+++ b/tests/util/concurrent_workload.py
@@ -28,7 +28,7 @@ import logging
import _strptime # noqa: F401
import sys
import time
-from Queue import Queue
+from queue import Queue
from threading import current_thread, Event, Thread
from tests.common.impala_cluster import ImpalaCluster
@@ -78,17 +78,17 @@ class ConcurrentWorkload(object):
logging.exception("Caught error, stopping")
logging.info("%s exiting" % current_thread().name)
- def compute_query_rate(self, queue, stop_ev):
+ def compute_query_rate(self, queue_obj, stop_ev):
"""Computes the query throughput rate in queries per second averaged over the last 5
seconds. This method only returns when 'stop_ev' is set by the caller."""
AVG_WINDOW_S = 5
times = []
while not stop_ev.is_set():
# Don't block to check for stop_ev
- if queue.empty():
+ if queue_obj.empty():
time.sleep(0.1)
continue
- queue.get()
+ queue_obj.get()
now = time.time()
times.append(now)
# Keep only timestamps within the averaging window
@@ -118,7 +118,7 @@ class ConcurrentWorkload(object):
self.stop()
assert self.stop_ev.is_set(), "Stop event expected to be set but it isn't"
- def _print_query_rate(self, queue, stop_ev):
+ def _print_query_rate(self, queue_obj, stop_ev):
"""Prints the query throughput rate until 'stop_ev' is set by the caller."""
PERIOD_S = 1
diff --git a/tests/util/filesystem_base.py b/tests/util/filesystem_base.py
index 8f479bcb6..2f304bbaa 100644
--- a/tests/util/filesystem_base.py
+++ b/tests/util/filesystem_base.py
@@ -19,9 +19,10 @@
from __future__ import absolute_import, division, print_function
from abc import ABCMeta, abstractmethod
+from future.utils import with_metaclass
-class BaseFilesystem(object):
- __metaclass__ = ABCMeta
+
+class BaseFilesystem(with_metaclass(ABCMeta, object)):
@abstractmethod
def create_file(self, path, file_data, overwrite):
diff --git a/tests/util/hdfs_util.py b/tests/util/hdfs_util.py
index ef5a0c38c..2a3cec858 100644
--- a/tests/util/hdfs_util.py
+++ b/tests/util/hdfs_util.py
@@ -19,7 +19,7 @@
from __future__ import absolute_import, division, print_function
import getpass
-import httplib
+import http.client
import os.path
import re
import requests
@@ -130,7 +130,7 @@ class PyWebHdfsClientWithChmod(PyWebHdfsClient):
'775'"""
uri = self._create_uri(path, "SETPERMISSION", permission=permission)
response = requests.put(uri, allow_redirects=True)
- if not response.status_code == httplib.OK:
+ if not response.status_code == http.client.OK:
_raise_pywebhdfs_exception(response.status_code, response.text)
return True
@@ -138,21 +138,21 @@ class PyWebHdfsClientWithChmod(PyWebHdfsClient):
"""Sets the owner and the group of 'path to 'user' / 'group'"""
uri = self._create_uri(path, "SETOWNER", owner=user, group=group)
response = requests.put(uri, allow_redirects=True)
- if not response.status_code == httplib.OK:
+ if not response.status_code == http.client.OK:
_raise_pywebhdfs_exception(response.status_code, response.text)
return True
def setacl(self, path, acls):
uri = self._create_uri(path, "SETACL", aclspec=acls)
response = requests.put(uri, allow_redirects=True)
- if not response.status_code == httplib.OK:
+ if not response.status_code == http.client.OK:
_raise_pywebhdfs_exception(response.status_code, response.text)
return True
def getacl(self, path):
uri = self._create_uri(path, "GETACLSTATUS")
response = requests.get(uri, allow_redirects=True)
- if not response.status_code == httplib.OK:
+ if not response.status_code == http.client.OK:
_raise_pywebhdfs_exception(response.status_code, response.text)
return response.json()
diff --git a/tests/util/ssh_util.py b/tests/util/ssh_util.py
index 61b249bf8..69aa13f97 100644
--- a/tests/util/ssh_util.py
+++ b/tests/util/ssh_util.py
@@ -72,9 +72,9 @@ class SshClient(paramiko.SSHClient):
raise
except Exception as e:
LOG.warn("Error connecting to %s" % host_name, exc_info=True)
- else:
- LOG.error("Failed to ssh to %s" % host_name)
- raise e
+ if retry >= retries - 1:
+ LOG.error("Failed to ssh to %s" % host_name)
+ raise e
self.get_transport().set_keepalive(10)
[impala] 02/06: IMPALA-11973: Add absolute_import, division to all eligible Python files
Posted by jo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
joemcdonnell pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 82bd087fb1dcd2685ad1994364dd41182dcf840f
Author: Joe McDonnell <jo...@cloudera.com>
AuthorDate: Wed Mar 1 16:03:34 2023 -0800
IMPALA-11973: Add absolute_import, division to all eligible Python files
This takes steps to make Python 2 behave like Python 3 as
a way to flush out issues with running on Python 3. Specifically,
it handles two main differences:
1. Python 3 requires absolute imports within packages. This
can be emulated via "from __future__ import absolute_import"
2. Python 3 changed division to "true" division that doesn't
round to an integer. This can be emulated via
"from __future__ import division"
This changes all Python files to add imports for absolute_import
and division. For completeness, this also includes print_function in the
import.
I scrutinized each old-division location and converted some locations
to use the integer division '//' operator if it needed an integer
result (e.g. for indices, counts of records, etc). Some code was also using
relative imports and needed to be adjusted to handle absolute_import.
This fixes all Pylint warnings about no-absolute-import and old-division,
and these warnings are now banned.
Testing:
- Ran core tests
Change-Id: Idb0fcbd11f3e8791f5951c4944be44fb580e576b
Reviewed-on: http://gerrit.cloudera.org:8080/19588
Reviewed-by: Joe McDonnell <jo...@cloudera.com>
Tested-by: Joe McDonnell <jo...@cloudera.com>
---
bin/banned_py3k_warnings.txt | 2 ++
bin/dump_breakpad_symbols.py | 1 +
bin/generate_minidump_collection_testdata.py | 1 +
bin/get_code_size.py | 2 +-
bin/inline_pom.py | 2 +-
bin/load-data.py | 2 +-
bin/parse-thrift-profile.py | 2 +-
bin/run-workload.py | 2 +-
bin/single_node_perf_run.py | 2 +-
bin/start-impala-cluster.py | 3 ++-
docker/setup_build_context.py | 1 +
infra/python/bootstrap_virtualenv.py | 2 +-
lib/python/impala_py_lib/gdb/impala-gdb.py | 2 +-
lib/python/impala_py_lib/helpers.py | 1 +
lib/python/impala_py_lib/profiles.py | 2 +-
testdata/bin/check-hbase-nodes.py | 3 ++-
testdata/bin/generate-schema-statements.py | 2 +-
testdata/bin/generate-test-vectors.py | 2 +-
testdata/bin/load-tpc-kudu.py | 2 +-
testdata/bin/load_nested.py | 1 +
testdata/bin/random_avro_schema.py | 1 +
testdata/bin/rewrite-iceberg-metadata.py | 2 +-
testdata/bin/wait-for-hiveserver2.py | 2 +-
testdata/bin/wait-for-metastore.py | 2 +-
testdata/common/cgroups.py | 2 +-
testdata/common/text_delims_table.py | 4 ++--
testdata/common/widetable.py | 4 ++--
tests/authorization/test_authorization.py | 1 +
tests/authorization/test_authorized_proxy.py | 1 +
tests/authorization/test_provider.py | 1 +
tests/authorization/test_ranger.py | 1 +
tests/beeswax/impala_beeswax.py | 3 ++-
tests/benchmark/plugins/clear_buffer_cache.py | 1 +
tests/benchmark/plugins/vtune_plugin.py | 1 +
tests/benchmark/report_benchmark_results.py | 3 +--
tests/catalog_service/test_catalog_service_client.py | 1 +
tests/catalog_service/test_large_num_partitions.py | 1 +
tests/common/base_test_suite.py | 1 +
tests/common/custom_cluster_test_suite.py | 1 +
tests/common/environ.py | 1 +
tests/common/file_utils.py | 1 +
tests/common/iceberg_test_suite.py | 1 +
tests/common/impala_cluster.py | 2 +-
tests/common/impala_connection.py | 1 +
tests/common/impala_service.py | 1 +
tests/common/impala_test_suite.py | 2 +-
tests/common/kudu_test_suite.py | 1 +
tests/common/network.py | 1 +
tests/common/parametrize.py | 1 +
tests/common/patterns.py | 1 +
tests/common/resource_pool_config.py | 2 +-
tests/common/skip.py | 1 +
tests/common/test_dimensions.py | 1 +
tests/common/test_result_verifier.py | 1 +
tests/common/test_vector.py | 1 +
tests/comparison/cli_options.py | 1 +
tests/comparison/cluster.py | 10 +++++-----
tests/comparison/common.py | 1 +
tests/comparison/compat.py | 1 +
tests/comparison/data_generator.py | 5 +++--
tests/comparison/data_generator_mapred_common.py | 5 +++--
tests/comparison/db_connection.py | 2 +-
tests/comparison/db_types.py | 3 ++-
tests/comparison/discrepancy_searcher.py | 8 ++++----
tests/comparison/funcs.py | 1 +
tests/comparison/leopard/controller.py | 3 ++-
tests/comparison/leopard/front_end.py | 5 +++--
tests/comparison/leopard/impala_docker_env.py | 2 +-
tests/comparison/leopard/job.py | 4 ++--
tests/comparison/leopard/report.py | 1 +
tests/comparison/leopard/schedule_item.py | 1 +
tests/comparison/model_translator.py | 1 +
tests/comparison/query.py | 1 +
tests/comparison/query_flattener.py | 1 +
tests/comparison/query_generator.py | 2 +-
tests/comparison/query_profile.py | 1 +
tests/comparison/random_val_generator.py | 1 +
tests/comparison/statement_generator.py | 1 +
tests/comparison/tests/conftest.py | 1 +
tests/comparison/tests/fake_query.py | 1 +
.../tests/hive/test_hive_create_agg_or_analytic_tree.py | 1 +
.../tests/hive/test_hive_create_relational_join_condition.py | 1 +
tests/comparison/tests/query_object_testdata.py | 1 +
tests/comparison/tests/test_cluster.py | 1 +
tests/comparison/tests/test_cursor.py | 1 +
tests/comparison/tests/test_query_generator.py | 3 ++-
tests/comparison/tests/test_query_objects.py | 1 +
tests/comparison/tests/test_use_nested_with.py | 1 +
tests/comparison/util/verify-oracle-connection.py | 2 +-
tests/conftest.py | 4 ++--
tests/custom_cluster/test_admission_controller.py | 5 +++--
tests/custom_cluster/test_alloc_fail.py | 1 +
tests/custom_cluster/test_always_false_filter.py | 1 +
tests/custom_cluster/test_auto_scaling.py | 1 +
tests/custom_cluster/test_automatic_invalidation.py | 1 +
tests/custom_cluster/test_blacklist.py | 2 +-
tests/custom_cluster/test_blacklisted_dbs_and_tables.py | 1 +
tests/custom_cluster/test_breakpad.py | 2 +-
tests/custom_cluster/test_catalog_hms_failures.py | 2 +-
tests/custom_cluster/test_catalog_wait.py | 1 +
tests/custom_cluster/test_client_ssl.py | 2 +-
tests/custom_cluster/test_codegen_cache.py | 1 +
tests/custom_cluster/test_compact_catalog_updates.py | 1 +
tests/custom_cluster/test_concurrent_ddls.py | 1 +
tests/custom_cluster/test_concurrent_kudu_create.py | 1 +
tests/custom_cluster/test_coordinators.py | 1 +
tests/custom_cluster/test_custom_hive_configs.py | 1 +
tests/custom_cluster/test_custom_statestore.py | 1 +
tests/custom_cluster/test_data_cache.py | 1 +
tests/custom_cluster/test_delegation.py | 1 +
tests/custom_cluster/test_disable_catalog_data_ops.py | 1 +
tests/custom_cluster/test_disable_features.py | 1 +
tests/custom_cluster/test_disk_spill_configurations.py | 1 +
tests/custom_cluster/test_events_custom_configs.py | 2 +-
tests/custom_cluster/test_exchange_deferred_batches.py | 1 +
tests/custom_cluster/test_exchange_delays.py | 1 +
tests/custom_cluster/test_exchange_eos.py | 1 +
tests/custom_cluster/test_executor_groups.py | 1 +
tests/custom_cluster/test_frontend_connection_limit.py | 1 +
tests/custom_cluster/test_geospatial_library.py | 2 ++
tests/custom_cluster/test_hbase_hms_column_order.py | 1 +
tests/custom_cluster/test_hdfs_fd_caching.py | 1 +
tests/custom_cluster/test_hdfs_timeout.py | 1 +
tests/custom_cluster/test_hedged_reads.py | 1 +
tests/custom_cluster/test_hive_parquet_codec_interop.py | 1 +
tests/custom_cluster/test_hive_parquet_timestamp_conversion.py | 1 +
tests/custom_cluster/test_hive_text_codec_interop.py | 1 +
tests/custom_cluster/test_hs2.py | 1 +
tests/custom_cluster/test_hs2_fault_injection.py | 1 +
tests/custom_cluster/test_incremental_metadata_updates.py | 1 +
tests/custom_cluster/test_insert_behaviour.py | 1 +
tests/custom_cluster/test_jvm_mem_tracking.py | 1 +
tests/custom_cluster/test_krpc_mem_usage.py | 1 +
tests/custom_cluster/test_krpc_metrics.py | 1 +
tests/custom_cluster/test_krpc_options.py | 1 +
tests/custom_cluster/test_krpc_socket.py | 1 +
tests/custom_cluster/test_kudu.py | 1 +
tests/custom_cluster/test_kudu_not_available.py | 1 +
tests/custom_cluster/test_kudu_table_create_without_hms.py | 1 +
tests/custom_cluster/test_lineage.py | 1 +
tests/custom_cluster/test_local_catalog.py | 2 +-
tests/custom_cluster/test_local_tz_conversion.py | 1 +
tests/custom_cluster/test_logging.py | 1 +
tests/custom_cluster/test_mem_reservations.py | 1 +
tests/custom_cluster/test_metadata_no_events_processing.py | 1 +
tests/custom_cluster/test_metadata_replicas.py | 1 +
tests/custom_cluster/test_metastore_events_cleanup.py | 1 +
tests/custom_cluster/test_metastore_service.py | 1 +
tests/custom_cluster/test_mt_dop.py | 1 +
tests/custom_cluster/test_observability.py | 1 +
tests/custom_cluster/test_parquet_max_page_header.py | 1 +
tests/custom_cluster/test_partition.py | 1 +
tests/custom_cluster/test_pause_monitor.py | 1 +
tests/custom_cluster/test_permanent_udfs.py | 1 +
tests/custom_cluster/test_preload_table_types.py | 1 +
tests/custom_cluster/test_process_failures.py | 1 +
tests/custom_cluster/test_query_concurrency.py | 1 +
tests/custom_cluster/test_query_event_hooks.py | 1 +
tests/custom_cluster/test_query_expiration.py | 1 +
tests/custom_cluster/test_query_retries.py | 2 +-
tests/custom_cluster/test_re2_max_mem.py | 1 +
tests/custom_cluster/test_redaction.py | 1 +
tests/custom_cluster/test_reserved_words_version.py | 1 +
tests/custom_cluster/test_restart_services.py | 6 +++---
tests/custom_cluster/test_result_spooling.py | 1 +
tests/custom_cluster/test_rpc_exception.py | 1 +
tests/custom_cluster/test_rpc_timeout.py | 1 +
tests/custom_cluster/test_runtime_profile.py | 1 +
tests/custom_cluster/test_s3a_access.py | 1 +
tests/custom_cluster/test_saml2_sso.py | 1 +
tests/custom_cluster/test_scheduler_locality.py | 1 +
tests/custom_cluster/test_scratch_disk.py | 2 +-
tests/custom_cluster/test_seq_file_filtering.py | 1 +
tests/custom_cluster/test_services_rpc_errors.py | 1 +
tests/custom_cluster/test_session_expiration.py | 1 +
tests/custom_cluster/test_set_and_unset.py | 1 +
tests/custom_cluster/test_shared_tzdb.py | 1 +
tests/custom_cluster/test_shell_commandline.py | 1 +
tests/custom_cluster/test_shell_interactive.py | 1 +
tests/custom_cluster/test_shell_interactive_reconnect.py | 1 +
tests/custom_cluster/test_startup_filesystem_checks.py | 1 +
tests/custom_cluster/test_stats_extrapolation.py | 1 +
tests/custom_cluster/test_thrift_debug_string_exception.py | 1 +
tests/custom_cluster/test_thrift_socket.py | 1 +
tests/custom_cluster/test_topic_update_frequency.py | 2 +-
tests/custom_cluster/test_udf_concurrency.py | 2 +-
tests/custom_cluster/test_web_pages.py | 2 +-
tests/custom_cluster/test_wide_table_operations.py | 1 +
tests/data_errors/test_data_errors.py | 1 +
tests/experiments/test_targeted_perf.py | 1 +
tests/failure/test_failpoints.py | 1 +
tests/hs2/hs2_test_suite.py | 3 ++-
tests/hs2/test_fetch.py | 1 +
tests/hs2/test_fetch_first.py | 1 +
tests/hs2/test_fetch_timeout.py | 1 +
tests/hs2/test_hs2.py | 1 +
tests/hs2/test_json_endpoints.py | 1 +
tests/infra/test_perf_infra.py | 1 +
tests/infra/test_stress_infra.py | 1 +
tests/infra/test_utils.py | 1 +
tests/metadata/test_catalogd_debug_actions.py | 1 +
tests/metadata/test_compute_stats.py | 1 +
tests/metadata/test_ddl.py | 5 +++--
tests/metadata/test_ddl_base.py | 1 +
tests/metadata/test_event_processing.py | 1 +
tests/metadata/test_explain.py | 1 +
tests/metadata/test_hdfs_encryption.py | 1 +
tests/metadata/test_hdfs_permissions.py | 1 +
tests/metadata/test_hidden_files.py | 1 +
tests/metadata/test_hms_integration.py | 2 +-
tests/metadata/test_last_ddl_time_update.py | 1 +
tests/metadata/test_load.py | 1 +
tests/metadata/test_metadata_query_statements.py | 1 +
tests/metadata/test_partition_metadata.py | 1 +
tests/metadata/test_recover_partitions.py | 1 +
tests/metadata/test_recursive_listing.py | 1 +
tests/metadata/test_refresh_partition.py | 1 +
tests/metadata/test_reset_metadata.py | 1 +
tests/metadata/test_reuse_partitions.py | 1 +
tests/metadata/test_set.py | 1 +
tests/metadata/test_show_create_table.py | 1 +
tests/metadata/test_stale_metadata.py | 1 +
tests/metadata/test_stats_extrapolation.py | 1 +
tests/metadata/test_testcase_builder.py | 1 +
tests/metadata/test_views_compatibility.py | 1 +
tests/observability/test_jvm_metrics.py | 1 +
tests/observability/test_log_fragments.py | 1 +
tests/observability/test_profile_tool.py | 1 +
tests/performance/query.py | 1 +
tests/performance/query_exec_functions.py | 1 +
tests/performance/query_executor.py | 1 +
tests/performance/scheduler.py | 1 +
tests/performance/workload.py | 1 +
tests/performance/workload_runner.py | 1 +
tests/query_test/test_acid.py | 1 +
tests/query_test/test_acid_row_validation.py | 1 +
tests/query_test/test_aggregation.py | 2 +-
tests/query_test/test_analytic_tpcds.py | 1 +
tests/query_test/test_async_codegen.py | 1 +
tests/query_test/test_avro_schema_resolution.py | 1 +
tests/query_test/test_beeswax.py | 1 +
tests/query_test/test_cancellation.py | 1 +
tests/query_test/test_cast_with_format.py | 1 +
tests/query_test/test_chars.py | 1 +
tests/query_test/test_codegen.py | 1 +
tests/query_test/test_compressed_formats.py | 4 ++--
tests/query_test/test_datasketches.py | 1 +
tests/query_test/test_datastream_sender.py | 1 +
tests/query_test/test_date_queries.py | 1 +
tests/query_test/test_decimal_casting.py | 1 +
tests/query_test/test_decimal_fuzz.py | 3 ++-
tests/query_test/test_decimal_queries.py | 1 +
tests/query_test/test_delimited_text.py | 1 +
tests/query_test/test_errorlog.py | 1 +
tests/query_test/test_exprs.py | 3 ++-
tests/query_test/test_fetch.py | 1 +
tests/query_test/test_geospatial_functions.py | 1 +
tests/query_test/test_hash_join_timer.py | 5 +++--
tests/query_test/test_hbase_queries.py | 1 +
tests/query_test/test_hdfs_caching.py | 2 +-
tests/query_test/test_hdfs_file_mods.py | 1 +
tests/query_test/test_iceberg.py | 1 +
tests/query_test/test_insert.py | 1 +
tests/query_test/test_insert_behaviour.py | 1 +
tests/query_test/test_insert_parquet.py | 1 +
tests/query_test/test_insert_permutation.py | 1 +
tests/query_test/test_invalid_test_header.py | 1 +
tests/query_test/test_io_metrics.py | 1 +
tests/query_test/test_join_queries.py | 1 +
tests/query_test/test_kudu.py | 2 +-
tests/query_test/test_lifecycle.py | 1 +
tests/query_test/test_limit.py | 2 +-
tests/query_test/test_limit_pushdown_analytic.py | 1 +
tests/query_test/test_local_fs.py | 1 +
tests/query_test/test_mem_usage_scaling.py | 2 ++
tests/query_test/test_mt_dop.py | 1 +
tests/query_test/test_multiple_filesystems.py | 1 +
tests/query_test/test_nested_types.py | 1 +
tests/query_test/test_observability.py | 1 +
tests/query_test/test_orc_stats.py | 1 +
tests/query_test/test_parquet_bloom_filter.py | 1 +
tests/query_test/test_parquet_late_materialization.py | 1 +
tests/query_test/test_parquet_page_index.py | 1 +
tests/query_test/test_parquet_stats.py | 1 +
tests/query_test/test_partitioning.py | 1 +
tests/query_test/test_queries.py | 1 +
tests/query_test/test_query_compilation.py | 1 +
tests/query_test/test_query_mem_limit.py | 5 +++--
tests/query_test/test_query_opts.py | 1 +
tests/query_test/test_resource_limits.py | 1 +
tests/query_test/test_result_spooling.py | 1 +
tests/query_test/test_rows_availability.py | 1 +
tests/query_test/test_runtime_filters.py | 1 +
tests/query_test/test_scanners.py | 2 +-
tests/query_test/test_scanners_fuzz.py | 1 +
tests/query_test/test_scratch_limit.py | 1 +
tests/query_test/test_sfs.py | 1 +
tests/query_test/test_sort.py | 1 +
tests/query_test/test_spilling.py | 1 +
tests/query_test/test_tablesample.py | 1 +
tests/query_test/test_tpcds_queries.py | 1 +
tests/query_test/test_tpch_nested_queries.py | 1 +
tests/query_test/test_tpch_queries.py | 1 +
tests/query_test/test_udfs.py | 1 +
tests/query_test/test_utf8_strings.py | 1 +
tests/run-tests.py | 2 +-
tests/shell/test_cookie_util.py | 1 +
tests/shell/test_shell_client.py | 7 ++++---
tests/shell/test_shell_commandline.py | 7 ++++---
tests/shell/test_shell_interactive.py | 8 ++++----
tests/shell/util.py | 2 +-
tests/statestore/test_statestore.py | 2 +-
tests/stress/concurrent_select.py | 8 ++++----
tests/stress/extract_min_mem.py | 2 +-
tests/stress/mem_broker.py | 1 +
tests/stress/queries.py | 1 +
tests/stress/query_retries_stress_runner.py | 1 +
tests/stress/query_runner.py | 1 +
tests/stress/runtime_info.py | 2 +-
tests/stress/stress_util.py | 1 +
tests/stress/test_acid_stress.py | 1 +
tests/stress/test_ddl_stress.py | 1 +
tests/stress/test_insert_stress.py | 1 +
tests/stress/util.py | 2 +-
tests/unittests/test_command.py | 1 +
tests/unittests/test_file_parser.py | 2 +-
tests/unittests/test_result_verifier.py | 1 +
tests/util/acid_txn.py | 2 +-
tests/util/adls_util.py | 1 +
tests/util/auto_scaler.py | 3 ++-
tests/util/calculation_util.py | 5 +++--
tests/util/cancel_util.py | 1 +
tests/util/cluster_controller.py | 1 +
tests/util/compute_table_stats.py | 1 +
tests/util/concurrent_workload.py | 1 +
tests/util/event_processor_utils.py | 1 +
tests/util/failpoints_util.py | 1 +
tests/util/filesystem_base.py | 1 +
tests/util/filesystem_utils.py | 1 +
tests/util/get_parquet_metadata.py | 1 +
tests/util/hdfs_util.py | 1 +
tests/util/iceberg_util.py | 1 +
tests/util/parse_util.py | 1 +
tests/util/plugin_runner.py | 1 +
tests/util/run_impyla_http_query.py | 2 +-
tests/util/shell_util.py | 1 +
tests/util/ssh_util.py | 1 +
tests/util/test_file_parser.py | 2 +-
tests/util/thrift_util.py | 1 +
tests/util/web_pages_util.py | 1 +
tests/verifiers/mem_usage_verifier.py | 1 +
tests/verifiers/metric_verifier.py | 1 +
tests/verifiers/test_verify_metrics.py | 1 +
tests/webserver/test_web_pages.py | 1 +
354 files changed, 409 insertions(+), 114 deletions(-)
diff --git a/bin/banned_py3k_warnings.txt b/bin/banned_py3k_warnings.txt
index e69de29bb..c5e7bdc12 100644
--- a/bin/banned_py3k_warnings.txt
+++ b/bin/banned_py3k_warnings.txt
@@ -0,0 +1,2 @@
+no-absolute-import
+old-division
diff --git a/bin/dump_breakpad_symbols.py b/bin/dump_breakpad_symbols.py
index b4edff494..81bf00d54 100755
--- a/bin/dump_breakpad_symbols.py
+++ b/bin/dump_breakpad_symbols.py
@@ -53,6 +53,7 @@
# $IMPALA_TOOLCHAIN_PACKAGES_HOME/breakpad-*/bin/minidump_stackwalk \
# /tmp/impala-minidumps/impalad/03c0ee26-bfd1-cf3e-43fa49ca-1a6aae25.dmp /tmp/syms
+from __future__ import absolute_import, division, print_function
import errno
import logging
import glob
diff --git a/bin/generate_minidump_collection_testdata.py b/bin/generate_minidump_collection_testdata.py
index 2cee8b9af..09341f539 100755
--- a/bin/generate_minidump_collection_testdata.py
+++ b/bin/generate_minidump_collection_testdata.py
@@ -27,6 +27,7 @@
# create the files in the interval [now - duration, now]. Minidumps are simulated by
# making the files easily compressible by having some repeated data.
+from __future__ import absolute_import, division, print_function
import errno
import os
import random
diff --git a/bin/get_code_size.py b/bin/get_code_size.py
index b7d6eda6f..44afba04f 100755
--- a/bin/get_code_size.py
+++ b/bin/get_code_size.py
@@ -19,7 +19,7 @@
# This tool walks the build directory (release by default) and will print the text, data,
# and bss section sizes of the archives.
-from __future__ import print_function
+from __future__ import absolute_import, division, print_function
import fnmatch
import os
import re
diff --git a/bin/inline_pom.py b/bin/inline_pom.py
index 1b021683a..3b99560d1 100755
--- a/bin/inline_pom.py
+++ b/bin/inline_pom.py
@@ -21,7 +21,7 @@
#
# Usage: inline_pom.py <pom.xml>...
-from __future__ import print_function
+from __future__ import absolute_import, division, print_function
import re
import sys
from tempfile import mkstemp
diff --git a/bin/load-data.py b/bin/load-data.py
index 362ec8ee3..3c35cfc59 100755
--- a/bin/load-data.py
+++ b/bin/load-data.py
@@ -20,7 +20,7 @@
# This script is used to load the proper datasets for the specified workloads. It loads
# all data via Hive except for parquet data which needs to be loaded via Impala.
# Most ddl commands are executed by Impala.
-from __future__ import print_function
+from __future__ import absolute_import, division, print_function
import collections
import getpass
import logging
diff --git a/bin/parse-thrift-profile.py b/bin/parse-thrift-profile.py
index 7661e1ff0..8e16aa079 100755
--- a/bin/parse-thrift-profile.py
+++ b/bin/parse-thrift-profile.py
@@ -39,7 +39,7 @@
# 2018-04-13T15:06:34.144000 e44af7f93edb8cd6:1b1f801600000000 TRuntimeProfileTree(nodes=[TRuntimeProf...
-from __future__ import print_function
+from __future__ import absolute_import, division, print_function
from impala_py_lib import profiles
import sys
diff --git a/bin/run-workload.py b/bin/run-workload.py
index abe95ceb1..1da5dd0f5 100755
--- a/bin/run-workload.py
+++ b/bin/run-workload.py
@@ -27,7 +27,7 @@
# - Stores the execution details in JSON format.
#
-from __future__ import print_function
+from __future__ import absolute_import, division, print_function
import getpass
import json
import logging
diff --git a/bin/single_node_perf_run.py b/bin/single_node_perf_run.py
index e1b74de64..f64d94e6c 100755
--- a/bin/single_node_perf_run.py
+++ b/bin/single_node_perf_run.py
@@ -69,7 +69,7 @@
# --start_minicluster start a new Hadoop minicluster
# --ninja use ninja, rather than Make, as the build tool
-from __future__ import print_function
+from __future__ import absolute_import, division, print_function
from optparse import OptionParser
from tempfile import mkdtemp
diff --git a/bin/start-impala-cluster.py b/bin/start-impala-cluster.py
index b1e94545e..6c5fa07d4 100755
--- a/bin/start-impala-cluster.py
+++ b/bin/start-impala-cluster.py
@@ -20,6 +20,7 @@
# Starts up an Impala cluster (ImpalaD + State Store) with the specified number of
# ImpalaD instances. Each ImpalaD runs on a different port allowing this to be run
# on a single machine.
+from __future__ import absolute_import, division, print_function
import getpass
import itertools
import json
@@ -473,7 +474,7 @@ def compute_impalad_mem_limit(cluster_size):
# memory choice here to max out at 12GB. This should be sufficient for tests.
#
# Beware that ASAN builds use more memory than regular builds.
- physical_mem_gb = psutil.virtual_memory().total / 1024 / 1024 / 1024
+ physical_mem_gb = psutil.virtual_memory().total // 1024 // 1024 // 1024
available_mem = int(os.getenv("IMPALA_CLUSTER_MAX_MEM_GB", str(physical_mem_gb)))
mem_limit = int(0.7 * available_mem * 1024 * 1024 * 1024 / cluster_size)
return min(12 * 1024 * 1024 * 1024, mem_limit)
diff --git a/docker/setup_build_context.py b/docker/setup_build_context.py
index cd2955d97..9d32ab1d6 100755
--- a/docker/setup_build_context.py
+++ b/docker/setup_build_context.py
@@ -20,6 +20,7 @@
# Most artifacts are symlinked so need to be dereferenced (e.g. with tar -h) before
# being used as a build context.
+from __future__ import absolute_import, division, print_function
import argparse
import glob
import os
diff --git a/infra/python/bootstrap_virtualenv.py b/infra/python/bootstrap_virtualenv.py
index bd9c08144..5a96ad3c6 100644
--- a/infra/python/bootstrap_virtualenv.py
+++ b/infra/python/bootstrap_virtualenv.py
@@ -32,7 +32,7 @@
# This module can be run with python >= 2.7. It makes no guarantees about usage on
# python < 2.7.
-from __future__ import print_function
+from __future__ import absolute_import, division, print_function
import glob
import logging
import optparse
diff --git a/lib/python/impala_py_lib/gdb/impala-gdb.py b/lib/python/impala_py_lib/gdb/impala-gdb.py
index 1bf81eabc..c5aa93d21 100644
--- a/lib/python/impala_py_lib/gdb/impala-gdb.py
+++ b/lib/python/impala_py_lib/gdb/impala-gdb.py
@@ -19,7 +19,7 @@
# A collection of useful Python GDB modules and commands for
# debugging Impala core dumps.
#
-from __future__ import print_function
+from __future__ import absolute_import, division, print_function
import gdb
from collections import defaultdict
diff --git a/lib/python/impala_py_lib/helpers.py b/lib/python/impala_py_lib/helpers.py
index 9631ba75b..4fc2f32a8 100644
--- a/lib/python/impala_py_lib/helpers.py
+++ b/lib/python/impala_py_lib/helpers.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import fnmatch
import logging
import os
diff --git a/lib/python/impala_py_lib/profiles.py b/lib/python/impala_py_lib/profiles.py
index fc7a37298..4d5ecb731 100644
--- a/lib/python/impala_py_lib/profiles.py
+++ b/lib/python/impala_py_lib/profiles.py
@@ -18,7 +18,7 @@
# This file contains library functions to decode and access Impala query profiles.
-from __future__ import print_function
+from __future__ import absolute_import, division, print_function
import base64
import datetime
import zlib
diff --git a/testdata/bin/check-hbase-nodes.py b/testdata/bin/check-hbase-nodes.py
index ffe7a7c23..876955de5 100755
--- a/testdata/bin/check-hbase-nodes.py
+++ b/testdata/bin/check-hbase-nodes.py
@@ -20,6 +20,7 @@
"""Given a series of hosts and Zookeeper nodes, make sure that each node is accessible.
"""
+from __future__ import absolute_import, division, print_function
import argparse
import hdfs
import logging
@@ -191,4 +192,4 @@ if __name__ == "__main__":
LOGGER.error(msg)
sys.exit(errors)
else:
- sys.exit(1)
\ No newline at end of file
+ sys.exit(1)
diff --git a/testdata/bin/generate-schema-statements.py b/testdata/bin/generate-schema-statements.py
index acd369db4..23cb88556 100755
--- a/testdata/bin/generate-schema-statements.py
+++ b/testdata/bin/generate-schema-statements.py
@@ -94,7 +94,7 @@
# This should be used sparingly, because these commands are executed
# serially.
#
-from __future__ import print_function
+from __future__ import absolute_import, division, print_function
import collections
import csv
import glob
diff --git a/testdata/bin/generate-test-vectors.py b/testdata/bin/generate-test-vectors.py
index c7d288bd8..00e7228e8 100755
--- a/testdata/bin/generate-test-vectors.py
+++ b/testdata/bin/generate-test-vectors.py
@@ -40,7 +40,7 @@
# The pairwise generation is done using the Python 'AllPairs' module. This module can be
# downloaded from http://pypi.python.org/pypi/AllPairs/2.0.1
#
-from __future__ import print_function
+from __future__ import absolute_import, division, print_function
import collections
import csv
import math
diff --git a/testdata/bin/load-tpc-kudu.py b/testdata/bin/load-tpc-kudu.py
index 422d064ba..6f0610ffa 100755
--- a/testdata/bin/load-tpc-kudu.py
+++ b/testdata/bin/load-tpc-kudu.py
@@ -22,7 +22,7 @@
# Kudu tables are created in the specified 'target-db' using the existing HDFS tables
# from 'source-db'.
-from __future__ import print_function
+from __future__ import absolute_import, division, print_function
import logging
import os
import sqlparse
diff --git a/testdata/bin/load_nested.py b/testdata/bin/load_nested.py
index a4ae42646..7a4faceed 100755
--- a/testdata/bin/load_nested.py
+++ b/testdata/bin/load_nested.py
@@ -20,6 +20,7 @@
'''This script creates a nested version of TPC-H. Non-nested TPC-H must already be
loaded.
'''
+from __future__ import absolute_import, division, print_function
import logging
import os
diff --git a/testdata/bin/random_avro_schema.py b/testdata/bin/random_avro_schema.py
index ccdf25c05..e065b1dcc 100755
--- a/testdata/bin/random_avro_schema.py
+++ b/testdata/bin/random_avro_schema.py
@@ -17,6 +17,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
from random import choice, randint, random, shuffle
from os.path import join as join_path
from optparse import OptionParser
diff --git a/testdata/bin/rewrite-iceberg-metadata.py b/testdata/bin/rewrite-iceberg-metadata.py
index 26997345b..2f8e22e32 100755
--- a/testdata/bin/rewrite-iceberg-metadata.py
+++ b/testdata/bin/rewrite-iceberg-metadata.py
@@ -17,7 +17,7 @@
# specific language governing permissions and limitations
# under the License.
-from __future__ import print_function
+from __future__ import absolute_import, division, print_function
import glob
import json
import os
diff --git a/testdata/bin/wait-for-hiveserver2.py b/testdata/bin/wait-for-hiveserver2.py
index 6e99fa5f3..6ba00f012 100755
--- a/testdata/bin/wait-for-hiveserver2.py
+++ b/testdata/bin/wait-for-hiveserver2.py
@@ -22,7 +22,7 @@
# TODO: Consider combining this with wait-for-metastore.py. A TCLIService client
# can perhaps also talk to the metastore.
-from __future__ import print_function
+from __future__ import absolute_import, division, print_function
import os
import time
import getpass
diff --git a/testdata/bin/wait-for-metastore.py b/testdata/bin/wait-for-metastore.py
index 0707cc16b..6b62487c0 100755
--- a/testdata/bin/wait-for-metastore.py
+++ b/testdata/bin/wait-for-metastore.py
@@ -21,7 +21,7 @@
# to execute the get_database("default") Thrift RPC until the call succeeds,
# or a timeout is reached.
-from __future__ import print_function
+from __future__ import absolute_import, division, print_function
import os
import time
from optparse import OptionParser
diff --git a/testdata/common/cgroups.py b/testdata/common/cgroups.py
index 5e1f6f73e..36fe5b75f 100755
--- a/testdata/common/cgroups.py
+++ b/testdata/common/cgroups.py
@@ -19,7 +19,7 @@
# Utility code for creating cgroups for the Impala development environment.
# May be used as a library or as a command-line utility for manual testing.
-from __future__ import print_function
+from __future__ import absolute_import, division, print_function
import os
import sys
import errno
diff --git a/testdata/common/text_delims_table.py b/testdata/common/text_delims_table.py
index 92b2d01a4..a5b06acf0 100755
--- a/testdata/common/text_delims_table.py
+++ b/testdata/common/text_delims_table.py
@@ -22,7 +22,7 @@
# command line, will generate data files in the specified directory and a
# print a SQL load statement to incorporate into dataload SQL script generation.
-from __future__ import print_function
+from __future__ import absolute_import, division, print_function
from shutil import rmtree
from optparse import OptionParser
from contextlib import contextmanager
@@ -35,7 +35,7 @@ parser.add_option("--only_newline", dest="only_newline", default=False, action="
parser.add_option("--file_len", dest="file_len", type="int")
def generate_testescape_files(table_location, only_newline, file_len):
- data = ''.join(["1234567890" for _ in xrange(1 + file_len / 10)])
+ data = ''.join(["1234567890" for _ in xrange(1 + file_len // 10)])
suffix_list = ["\\", ",", "a"]
if only_newline:
diff --git a/testdata/common/widetable.py b/testdata/common/widetable.py
index dcd31de49..f04b5cc69 100755
--- a/testdata/common/widetable.py
+++ b/testdata/common/widetable.py
@@ -22,7 +22,7 @@
# generate a CSV data file and prints a SQL load statement to incorporate
# into dataload SQL script generation.
-from __future__ import print_function
+from __future__ import absolute_import, division, print_function
from datetime import datetime, timedelta
import itertools
import optparse
@@ -51,7 +51,7 @@ def get_columns(num_cols):
iter = itertools.cycle(templates)
# Produces [bool_col1, tinyint_col1, ..., bool_col2, tinyint_col2, ...]
# The final list has 'num_cols' elements.
- return [iter.next() % (i / len(templates) + 1) for i in xrange(num_cols)]
+ return [iter.next() % (i // len(templates) + 1) for i in xrange(num_cols)]
# Data generators for different types. Each generator yields an infinite number of
# value strings suitable for writing to a CSV file.
diff --git a/tests/authorization/test_authorization.py b/tests/authorization/test_authorization.py
index 6f52ccc4d..059008370 100644
--- a/tests/authorization/test_authorization.py
+++ b/tests/authorization/test_authorization.py
@@ -17,6 +17,7 @@
#
# Client tests for SQL statement authorization
+from __future__ import absolute_import, division, print_function
import os
import pytest
import tempfile
diff --git a/tests/authorization/test_authorized_proxy.py b/tests/authorization/test_authorized_proxy.py
index 832a8505c..68053a840 100644
--- a/tests/authorization/test_authorized_proxy.py
+++ b/tests/authorization/test_authorized_proxy.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import pytest
import os
import grp
diff --git a/tests/authorization/test_provider.py b/tests/authorization/test_provider.py
index 362ca7ae9..f7e71ac61 100644
--- a/tests/authorization/test_provider.py
+++ b/tests/authorization/test_provider.py
@@ -17,6 +17,7 @@
#
# Client tests for SQL statement authorization
+from __future__ import absolute_import, division, print_function
import pytest
import os
import tempfile
diff --git a/tests/authorization/test_ranger.py b/tests/authorization/test_ranger.py
index e90e27055..03669f081 100644
--- a/tests/authorization/test_ranger.py
+++ b/tests/authorization/test_ranger.py
@@ -17,6 +17,7 @@
#
# Client tests for SQL statement authorization
+from __future__ import absolute_import, division, print_function
import os
import grp
import json
diff --git a/tests/beeswax/impala_beeswax.py b/tests/beeswax/impala_beeswax.py
index 201532dc9..f7c2eb3f3 100644
--- a/tests/beeswax/impala_beeswax.py
+++ b/tests/beeswax/impala_beeswax.py
@@ -25,6 +25,7 @@
# client.connect()
# result = client.execute(query_string)
# where result is an object of the class ImpalaBeeswaxResult.
+from __future__ import absolute_import, division, print_function
import logging
import time
import shlex
@@ -282,7 +283,7 @@ class ImpalaBeeswaxClient(object):
setattr(max_stats, attr, max(getattr(max_stats, attr), val))
if len(node.exec_stats) > 0:
- avg_time = agg_stats.latency_ns / len(node.exec_stats)
+ avg_time = agg_stats.latency_ns // len(node.exec_stats)
else:
avg_time = 0
diff --git a/tests/benchmark/plugins/clear_buffer_cache.py b/tests/benchmark/plugins/clear_buffer_cache.py
index 8fada8d70..6c35ff96a 100644
--- a/tests/benchmark/plugins/clear_buffer_cache.py
+++ b/tests/benchmark/plugins/clear_buffer_cache.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
from tests.util.cluster_controller import ClusterController
from tests.benchmark.plugins import Plugin
diff --git a/tests/benchmark/plugins/vtune_plugin.py b/tests/benchmark/plugins/vtune_plugin.py
index 61c4fe044..ada5e1069 100644
--- a/tests/benchmark/plugins/vtune_plugin.py
+++ b/tests/benchmark/plugins/vtune_plugin.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
from os import environ
from tests.util.cluster_controller import ClusterController
from tests.benchmark.plugins import Plugin
diff --git a/tests/benchmark/report_benchmark_results.py b/tests/benchmark/report_benchmark_results.py
index 7ce8ba688..680a871f9 100755
--- a/tests/benchmark/report_benchmark_results.py
+++ b/tests/benchmark/report_benchmark_results.py
@@ -28,8 +28,7 @@
# be an int (2). The following line changes this behavior so that float will be returned
# if necessary (2.5).
-from __future__ import division
-from __future__ import print_function
+from __future__ import absolute_import, division, print_function
import difflib
import json
import logging
diff --git a/tests/catalog_service/test_catalog_service_client.py b/tests/catalog_service/test_catalog_service_client.py
index 939c4a255..8d89d9a19 100644
--- a/tests/catalog_service/test_catalog_service_client.py
+++ b/tests/catalog_service/test_catalog_service_client.py
@@ -17,6 +17,7 @@
#
# Tests to validate the Catalog Service client APIs.
+from __future__ import absolute_import, division, print_function
import logging
import pytest
diff --git a/tests/catalog_service/test_large_num_partitions.py b/tests/catalog_service/test_large_num_partitions.py
index 548723ba9..e72d1720b 100644
--- a/tests/catalog_service/test_large_num_partitions.py
+++ b/tests/catalog_service/test_large_num_partitions.py
@@ -18,6 +18,7 @@
# Tests to validate the Catalog Service works properly when partitions
# need to be fetched in multiple batches.
+from __future__ import absolute_import, division, print_function
from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.test_dimensions import create_single_exec_option_dimension
diff --git a/tests/common/base_test_suite.py b/tests/common/base_test_suite.py
index e4bb0670b..bd9cb1882 100644
--- a/tests/common/base_test_suite.py
+++ b/tests/common/base_test_suite.py
@@ -16,6 +16,7 @@
# under the License.
# The base class that should be used for tests.
+from __future__ import absolute_import, division, print_function
import logging
from tests.common.test_vector import ImpalaTestMatrix
diff --git a/tests/common/custom_cluster_test_suite.py b/tests/common/custom_cluster_test_suite.py
index bc940cd0b..fb951ff58 100644
--- a/tests/common/custom_cluster_test_suite.py
+++ b/tests/common/custom_cluster_test_suite.py
@@ -18,6 +18,7 @@
# Superclass for all tests that need a custom cluster.
# TODO: Configure cluster size and other parameters.
+from __future__ import absolute_import, division, print_function
import logging
import os
import os.path
diff --git a/tests/common/environ.py b/tests/common/environ.py
index dfb9c2d98..80c2750ac 100644
--- a/tests/common/environ.py
+++ b/tests/common/environ.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import json
import logging
import os
diff --git a/tests/common/file_utils.py b/tests/common/file_utils.py
index fe10a5230..9850c3270 100644
--- a/tests/common/file_utils.py
+++ b/tests/common/file_utils.py
@@ -19,6 +19,7 @@
# and other functions used for checking for strings in files and
# directories.
+from __future__ import absolute_import, division, print_function
import os
import re
import tempfile
diff --git a/tests/common/iceberg_test_suite.py b/tests/common/iceberg_test_suite.py
index f4ac06622..f7faba902 100644
--- a/tests/common/iceberg_test_suite.py
+++ b/tests/common/iceberg_test_suite.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import datetime
from tests.common.impala_test_suite import ImpalaTestSuite
diff --git a/tests/common/impala_cluster.py b/tests/common/impala_cluster.py
index 4a0ccb940..0b43189eb 100644
--- a/tests/common/impala_cluster.py
+++ b/tests/common/impala_cluster.py
@@ -17,7 +17,7 @@
#
# Basic object model of an Impala cluster (set of Impala processes).
-from __future__ import print_function
+from __future__ import absolute_import, division, print_function
import json
import logging
import os
diff --git a/tests/common/impala_connection.py b/tests/common/impala_connection.py
index 7fbc7597b..21128db5d 100644
--- a/tests/common/impala_connection.py
+++ b/tests/common/impala_connection.py
@@ -19,6 +19,7 @@
# in the future will support HS2 connections. Provides tracing around all
# operations.
+from __future__ import absolute_import, division, print_function
import abc
import logging
import re
diff --git a/tests/common/impala_service.py b/tests/common/impala_service.py
index 822e361f6..84e008b8a 100644
--- a/tests/common/impala_service.py
+++ b/tests/common/impala_service.py
@@ -19,6 +19,7 @@
# programatically interact with the services and perform operations such as querying
# the debug webpage, getting metric values, or creating client connections.
+from __future__ import absolute_import, division, print_function
from collections import defaultdict
import json
import logging
diff --git a/tests/common/impala_test_suite.py b/tests/common/impala_test_suite.py
index 0245b9330..605d9894a 100644
--- a/tests/common/impala_test_suite.py
+++ b/tests/common/impala_test_suite.py
@@ -17,7 +17,7 @@
#
# The base class that should be used for almost all Impala tests
-from __future__ import print_function
+from __future__ import absolute_import, division, print_function
import glob
import grp
import json
diff --git a/tests/common/kudu_test_suite.py b/tests/common/kudu_test_suite.py
index 1691addb8..5292b31eb 100644
--- a/tests/common/kudu_test_suite.py
+++ b/tests/common/kudu_test_suite.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import os
import pytest
import requests
diff --git a/tests/common/network.py b/tests/common/network.py
index ea9b739c2..55502b4eb 100644
--- a/tests/common/network.py
+++ b/tests/common/network.py
@@ -17,6 +17,7 @@
# Tools for identifying network characteristics.
+from __future__ import absolute_import, division, print_function
import socket
diff --git a/tests/common/parametrize.py b/tests/common/parametrize.py
index 0ba42057a..fc5b2b8c0 100644
--- a/tests/common/parametrize.py
+++ b/tests/common/parametrize.py
@@ -17,6 +17,7 @@
# Fixture parametrizations should go here, not in conftest.py.
+from __future__ import absolute_import, division, print_function
import pytest
from tests.common.patterns import is_valid_impala_identifier
diff --git a/tests/common/patterns.py b/tests/common/patterns.py
index b8d832588..1214e5e37 100644
--- a/tests/common/patterns.py
+++ b/tests/common/patterns.py
@@ -18,6 +18,7 @@
# Common patterns that ought to be the same throughout the framework should be placed
# here.
+from __future__ import absolute_import, division, print_function
import re
# http://www.cloudera.com/content/www/en-us/documentation/enterprise/latest/topics/impala_identifiers.html
diff --git a/tests/common/resource_pool_config.py b/tests/common/resource_pool_config.py
index 88f9b55e4..934107f7a 100644
--- a/tests/common/resource_pool_config.py
+++ b/tests/common/resource_pool_config.py
@@ -20,7 +20,7 @@
# the tests it is used for. However, it is generic enough that it can be extended if
# more functionality is required for adding tests.
-from __future__ import print_function
+from __future__ import absolute_import, division, print_function
import os
from time import sleep, time
import xml.etree.ElementTree as ET
diff --git a/tests/common/skip.py b/tests/common/skip.py
index b908ea9e8..c79f80ca9 100644
--- a/tests/common/skip.py
+++ b/tests/common/skip.py
@@ -20,6 +20,7 @@
# annotate the class or test routine with the marker.
#
+from __future__ import absolute_import, division, print_function
import pytest
from functools import partial
diff --git a/tests/common/test_dimensions.py b/tests/common/test_dimensions.py
index 869179402..48d147541 100644
--- a/tests/common/test_dimensions.py
+++ b/tests/common/test_dimensions.py
@@ -17,6 +17,7 @@
# Common test dimensions and associated utility functions.
+from __future__ import absolute_import, division, print_function
import copy
import os
from itertools import product
diff --git a/tests/common/test_result_verifier.py b/tests/common/test_result_verifier.py
index 1532712c4..59f637d9a 100644
--- a/tests/common/test_result_verifier.py
+++ b/tests/common/test_result_verifier.py
@@ -17,6 +17,7 @@
# This modules contians utility functions used to help verify query test results.
+from __future__ import absolute_import, division, print_function
import logging
import math
import re
diff --git a/tests/common/test_vector.py b/tests/common/test_vector.py
index 8fcac5a79..d1b37bb59 100644
--- a/tests/common/test_vector.py
+++ b/tests/common/test_vector.py
@@ -56,6 +56,7 @@
#
# Additional examples of usage can be found within the test suites.
+from __future__ import absolute_import, division, print_function
from itertools import product
# A list of test dimension values.
diff --git a/tests/comparison/cli_options.py b/tests/comparison/cli_options.py
index 1d737cfaa..339faff49 100644
--- a/tests/comparison/cli_options.py
+++ b/tests/comparison/cli_options.py
@@ -17,6 +17,7 @@
'''Helpers for parsing command line options'''
+from __future__ import absolute_import, division, print_function
import logging
import os
import sys
diff --git a/tests/comparison/cluster.py b/tests/comparison/cluster.py
index 81fe672b1..c8bb416d8 100644
--- a/tests/comparison/cluster.py
+++ b/tests/comparison/cluster.py
@@ -16,11 +16,11 @@
# under the License.
"""This module provides utilities for interacting with a cluster."""
-from __future__ import print_function
# This should be moved into the test/util folder eventually. The problem is this
# module depends on db_connection which use some query generator classes.
+from __future__ import absolute_import, division, print_function
import hdfs
import logging
import os
@@ -46,7 +46,7 @@ from xml.etree.ElementTree import parse as parse_xml
from zipfile import ZipFile
-from db_connection import HiveConnection, ImpalaConnection
+from tests.comparison.db_connection import HiveConnection, ImpalaConnection
from tests.common.environ import HIVE_MAJOR_VERSION
from tests.common.errors import Timeout
from tests.util.shell_util import shell as local_shell
@@ -795,7 +795,7 @@ class Impalad(object):
if not pid:
raise Exception("Impalad at %s is not running" % self.label)
mem_kb = self.shell("ps --no-header -o rss -p %s" % pid)
- return int(mem_kb) / 1024
+ return int(mem_kb) // 1024
def _read_web_page(self, relative_url, params={}, timeout_secs=DEFAULT_TIMEOUT):
if "json" not in params:
@@ -874,7 +874,7 @@ class MiniClusterImpalad(Impalad):
return int(pid)
def find_process_mem_mb_limit(self):
- return long(self.get_metric("mem-tracker.process.limit")["value"]) / 1024 ** 2
+ return long(self.get_metric("mem-tracker.process.limit")["value"]) // 1024 ** 2
def find_core_dump_dir(self):
raise NotImplementedError()
@@ -916,7 +916,7 @@ class CmImpalad(Impalad):
return int(pid)
def find_process_mem_mb_limit(self):
- return self._get_cm_config("impalad_memory_limit", value_type=int) / 1024 ** 2
+ return self._get_cm_config("impalad_memory_limit", value_type=int) // 1024 ** 2
def find_core_dump_dir(self):
return self._get_cm_config("core_dump_dir")
diff --git a/tests/comparison/common.py b/tests/comparison/common.py
index 2b3217f7b..1025e7c79 100644
--- a/tests/comparison/common.py
+++ b/tests/comparison/common.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import json
from collections import defaultdict
diff --git a/tests/comparison/compat.py b/tests/comparison/compat.py
index 80e8af752..aa63f8ab9 100644
--- a/tests/comparison/compat.py
+++ b/tests/comparison/compat.py
@@ -20,6 +20,7 @@
# differences based on DB API2 libraries (like Impyla vs. psycopg2). Putting the
# handling in a single place will make it easier to track what workarounds exist.
+from __future__ import absolute_import, division, print_function
from tests.comparison import db_connection
diff --git a/tests/comparison/data_generator.py b/tests/comparison/data_generator.py
index a52ce6ffb..45dd611b7 100755
--- a/tests/comparison/data_generator.py
+++ b/tests/comparison/data_generator.py
@@ -26,6 +26,7 @@
'''
+from __future__ import absolute_import, division, print_function
import os
from copy import deepcopy
from logging import getLogger
@@ -217,8 +218,8 @@ class DbPopulator(object):
reducer_count = 0
mapper_input_data = list()
for table_data_generator in table_data_generators:
- reducer_count += (table_data_generator.row_count /
- estimate_rows_per_reducer(table_data_generator, MB_PER_REDUCER)) + 1
+ reducer_count += (table_data_generator.row_count
+ // estimate_rows_per_reducer(table_data_generator, MB_PER_REDUCER)) + 1
mapper_input_data.append(serialize(table_data_generator))
hdfs.write(mapper_input_file, data='\n'.join(mapper_input_data))
diff --git a/tests/comparison/data_generator_mapred_common.py b/tests/comparison/data_generator_mapred_common.py
index df6c087b5..dfc811147 100644
--- a/tests/comparison/data_generator_mapred_common.py
+++ b/tests/comparison/data_generator_mapred_common.py
@@ -23,6 +23,7 @@
'''
+from __future__ import absolute_import, division, print_function
import base64
import pickle
import StringIO
@@ -107,7 +108,7 @@ def estimate_rows_per_reducer(table_data_generator, mb_per_reducer):
bytes_per_row = estimate_bytes_per_row(table_data_generator, 1)
if bytes_per_row >= bytes_per_reducer:
return 1
- rows_per_reducer = bytes_per_reducer / bytes_per_row
+ rows_per_reducer = bytes_per_reducer // bytes_per_row
bytes_per_row = estimate_bytes_per_row(table_data_generator,
max(int(rows_per_reducer * 0.001), 1))
- return max(bytes_per_reducer / bytes_per_row, 1)
+ return max(bytes_per_reducer // bytes_per_row, 1)
diff --git a/tests/comparison/db_connection.py b/tests/comparison/db_connection.py
index a70a896c3..e641b15c9 100644
--- a/tests/comparison/db_connection.py
+++ b/tests/comparison/db_connection.py
@@ -21,7 +21,7 @@
connection.
'''
-from __future__ import print_function
+from __future__ import absolute_import, division, print_function
import hashlib
import impala.dbapi
import re
diff --git a/tests/comparison/db_types.py b/tests/comparison/db_types.py
index 1684a35a1..5f78d4e84 100644
--- a/tests/comparison/db_types.py
+++ b/tests/comparison/db_types.py
@@ -15,12 +15,13 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import re
import sys
from collections import defaultdict
-from common import ValExpr, ValExprList
+from tests.comparison.common import ValExpr, ValExprList
module_contents = dict()
diff --git a/tests/comparison/discrepancy_searcher.py b/tests/comparison/discrepancy_searcher.py
index 7a223db8f..ce56d7961 100755
--- a/tests/comparison/discrepancy_searcher.py
+++ b/tests/comparison/discrepancy_searcher.py
@@ -22,9 +22,9 @@
results.
'''
-from __future__ import print_function
-# TODO: IMPALA-4600: refactor this module
+# TODO: IMPALA-4600: refactor this module
+from __future__ import absolute_import, division, print_function
from copy import deepcopy
from decimal import Decimal
from itertools import izip
@@ -455,8 +455,8 @@ class QueryExecutor(object):
log_file.flush()
cursor.execute(query_sql)
col_count = len(cursor.description)
- batch_size = max(10000 / col_count, 1)
- row_limit = self.TOO_MUCH_DATA / col_count
+ batch_size = max(10000 // col_count, 1)
+ row_limit = self.TOO_MUCH_DATA // col_count
data_set = list()
current_thread().data_set = data_set
current_thread().cursor_description = cursor.description
diff --git a/tests/comparison/funcs.py b/tests/comparison/funcs.py
index 98670791c..7a0f37272 100644
--- a/tests/comparison/funcs.py
+++ b/tests/comparison/funcs.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
from copy import deepcopy
from itertools import ifilter
diff --git a/tests/comparison/leopard/controller.py b/tests/comparison/leopard/controller.py
index 24e18ce96..16abd1ed8 100755
--- a/tests/comparison/leopard/controller.py
+++ b/tests/comparison/leopard/controller.py
@@ -17,9 +17,10 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
from time import sleep, localtime, strftime
from tests.comparison.query_profile import DefaultProfile, ImpalaNestedTypesProfile
-from schedule_item import ScheduleItem
+from tests.comparison.leopard.schedule_item import ScheduleItem
from threading import Thread
import os
import pickle
diff --git a/tests/comparison/leopard/front_end.py b/tests/comparison/leopard/front_end.py
index ada5bd130..5f344ec3f 100755
--- a/tests/comparison/leopard/front_end.py
+++ b/tests/comparison/leopard/front_end.py
@@ -17,6 +17,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import logging
import os
import pickle
@@ -29,8 +30,8 @@ except ImportError as e:
raise Exception(
"Please run impala-pip install -r $IMPALA_HOME/infra/python/deps/extended-test-"
"requirements.txt:\n{0}".format(str(e)))
-from schedule_item import ScheduleItem
-from controller import PATH_TO_REPORTS, PATH_TO_SCHEDULE
+from tests.comparison.leopard.schedule_item import ScheduleItem
+from tests.comparison.leopard.controller import PATH_TO_REPORTS, PATH_TO_SCHEDULE
from threading import Thread
from tests.comparison.query_profile import DefaultProfile
from tests.comparison.db_types import (
diff --git a/tests/comparison/leopard/impala_docker_env.py b/tests/comparison/leopard/impala_docker_env.py
index 63fc33199..af3a09d48 100755
--- a/tests/comparison/leopard/impala_docker_env.py
+++ b/tests/comparison/leopard/impala_docker_env.py
@@ -17,7 +17,7 @@
'''This module generates a docker environment for a job'''
-from __future__ import division
+from __future__ import absolute_import, division, print_function
try:
from fabric.api import sudo, run, settings
except ImportError as e:
diff --git a/tests/comparison/leopard/job.py b/tests/comparison/leopard/job.py
index b526d4c1b..4f103dca3 100755
--- a/tests/comparison/leopard/job.py
+++ b/tests/comparison/leopard/job.py
@@ -15,7 +15,7 @@
# specific language governing permissions and limitations
# under the License.
-from __future__ import division
+from __future__ import absolute_import, division, print_function
from os.path import join as join_path
from tests.comparison.query_generator import QueryGenerator
from time import time
@@ -36,7 +36,7 @@ from tests.comparison.leopard.controller import (
from tests.comparison.discrepancy_searcher import QueryResultComparator
from tests.comparison.query_profile import DefaultProfile, ImpalaNestedTypesProfile
from threading import Thread
-from impala_docker_env import ImpalaDockerEnv
+from tests.comparison.leopard.impala_docker_env import ImpalaDockerEnv
import logging
import os
diff --git a/tests/comparison/leopard/report.py b/tests/comparison/leopard/report.py
index db463c993..740128074 100644
--- a/tests/comparison/leopard/report.py
+++ b/tests/comparison/leopard/report.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import pickle
import re
import os
diff --git a/tests/comparison/leopard/schedule_item.py b/tests/comparison/leopard/schedule_item.py
index b08bc42a6..595543261 100644
--- a/tests/comparison/leopard/schedule_item.py
+++ b/tests/comparison/leopard/schedule_item.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import random
import string
import pickle
diff --git a/tests/comparison/model_translator.py b/tests/comparison/model_translator.py
index 7064d5cbd..653a44b39 100644
--- a/tests/comparison/model_translator.py
+++ b/tests/comparison/model_translator.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
from inspect import getmro
from logging import getLogger
from re import sub
diff --git a/tests/comparison/query.py b/tests/comparison/query.py
index f7bd59304..c3e80352f 100644
--- a/tests/comparison/query.py
+++ b/tests/comparison/query.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
from abc import ABCMeta, abstractproperty
from copy import deepcopy
from logging import getLogger
diff --git a/tests/comparison/query_flattener.py b/tests/comparison/query_flattener.py
index 2a119131e..50de47159 100644
--- a/tests/comparison/query_flattener.py
+++ b/tests/comparison/query_flattener.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
from copy import deepcopy
from logging import getLogger
diff --git a/tests/comparison/query_generator.py b/tests/comparison/query_generator.py
index 361c1cb46..f4150fc86 100644
--- a/tests/comparison/query_generator.py
+++ b/tests/comparison/query_generator.py
@@ -15,7 +15,7 @@
# specific language governing permissions and limitations
# under the License.
-from __future__ import print_function
+from __future__ import absolute_import, division, print_function
from collections import defaultdict
from copy import deepcopy
from itertools import ifilter
diff --git a/tests/comparison/query_profile.py b/tests/comparison/query_profile.py
index 553010485..f5003732a 100644
--- a/tests/comparison/query_profile.py
+++ b/tests/comparison/query_profile.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
from logging import getLogger
from random import choice, randint, random, shuffle
diff --git a/tests/comparison/random_val_generator.py b/tests/comparison/random_val_generator.py
index 340a18c32..45d14756e 100644
--- a/tests/comparison/random_val_generator.py
+++ b/tests/comparison/random_val_generator.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
from datetime import datetime, timedelta
from decimal import Decimal as PyDecimal
from random import randint, random, uniform
diff --git a/tests/comparison/statement_generator.py b/tests/comparison/statement_generator.py
index 37842572b..3cfa713c4 100644
--- a/tests/comparison/statement_generator.py
+++ b/tests/comparison/statement_generator.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
from copy import deepcopy
from tests.comparison.common import Table
diff --git a/tests/comparison/tests/conftest.py b/tests/comparison/tests/conftest.py
index dd39fdebf..69e4f7257 100644
--- a/tests/comparison/tests/conftest.py
+++ b/tests/comparison/tests/conftest.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import pytest
from tests.comparison import cli_options
diff --git a/tests/comparison/tests/fake_query.py b/tests/comparison/tests/fake_query.py
index 3bf141011..e1cf46ed0 100644
--- a/tests/comparison/tests/fake_query.py
+++ b/tests/comparison/tests/fake_query.py
@@ -32,6 +32,7 @@
# gain confidence, we can modify them to be more testable, and we can remove items from
# here.
+from __future__ import absolute_import, division, print_function
from tests.comparison.common import Column, Table
from tests.comparison.funcs import AnalyticFirstValue
from tests.comparison.query import Query, SelectClause, SelectItem
diff --git a/tests/comparison/tests/hive/test_hive_create_agg_or_analytic_tree.py b/tests/comparison/tests/hive/test_hive_create_agg_or_analytic_tree.py
index 45f0de1e6..26c0427e1 100644
--- a/tests/comparison/tests/hive/test_hive_create_agg_or_analytic_tree.py
+++ b/tests/comparison/tests/hive/test_hive_create_agg_or_analytic_tree.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import pytest
from tests.comparison.db_types import Int
diff --git a/tests/comparison/tests/hive/test_hive_create_relational_join_condition.py b/tests/comparison/tests/hive/test_hive_create_relational_join_condition.py
index 4d4eea13f..49b4b7b3a 100644
--- a/tests/comparison/tests/hive/test_hive_create_relational_join_condition.py
+++ b/tests/comparison/tests/hive/test_hive_create_relational_join_condition.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
from tests.comparison.common import TableExprList, Column, Table
from tests.comparison.db_types import Int
from tests.comparison.funcs import Equals
diff --git a/tests/comparison/tests/query_object_testdata.py b/tests/comparison/tests/query_object_testdata.py
index 154c8b104..ad3c8db30 100644
--- a/tests/comparison/tests/query_object_testdata.py
+++ b/tests/comparison/tests/query_object_testdata.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
from collections import namedtuple
from fake_query import (
diff --git a/tests/comparison/tests/test_cluster.py b/tests/comparison/tests/test_cluster.py
index 2e3bc5416..ece822b15 100644
--- a/tests/comparison/tests/test_cluster.py
+++ b/tests/comparison/tests/test_cluster.py
@@ -17,6 +17,7 @@
# These are unit tests for cluster.py.
+from __future__ import absolute_import, division, print_function
from time import time
from tests.common.errors import Timeout
diff --git a/tests/comparison/tests/test_cursor.py b/tests/comparison/tests/test_cursor.py
index 38c296c1d..ee66e3fb3 100644
--- a/tests/comparison/tests/test_cursor.py
+++ b/tests/comparison/tests/test_cursor.py
@@ -16,6 +16,7 @@
# under the License.
+from __future__ import absolute_import, division, print_function
import pytest
from tests.comparison.db_connection import ImpalaConnection, PostgresqlConnection
diff --git a/tests/comparison/tests/test_query_generator.py b/tests/comparison/tests/test_query_generator.py
index d64fc3e53..6f7e23658 100644
--- a/tests/comparison/tests/test_query_generator.py
+++ b/tests/comparison/tests/test_query_generator.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
from tests.comparison.db_types import Boolean
from tests.comparison.funcs import And, Equals, Or
from tests.comparison.query_generator import QueryGenerator
@@ -41,4 +42,4 @@ def test_func_tree_contains_funcs():
and_func.parent = None
assert qgen._func_tree_contains_funcs(equals_func, [And])
assert qgen._func_tree_contains_funcs(equals_func, [Equals])
- assert not qgen._func_tree_contains_funcs(equals_func, [Or])
\ No newline at end of file
+ assert not qgen._func_tree_contains_funcs(equals_func, [Or])
diff --git a/tests/comparison/tests/test_query_objects.py b/tests/comparison/tests/test_query_objects.py
index 029f524bc..177b98e79 100644
--- a/tests/comparison/tests/test_query_objects.py
+++ b/tests/comparison/tests/test_query_objects.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import pytest
from tests.comparison.model_translator import (
diff --git a/tests/comparison/tests/test_use_nested_with.py b/tests/comparison/tests/test_use_nested_with.py
index ad9b23690..b20667056 100644
--- a/tests/comparison/tests/test_use_nested_with.py
+++ b/tests/comparison/tests/test_use_nested_with.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
from tests.comparison.common import TableExprList, Column, Table
from tests.comparison.db_types import Int
from tests.comparison.query_generator import QueryGenerator
diff --git a/tests/comparison/util/verify-oracle-connection.py b/tests/comparison/util/verify-oracle-connection.py
index f4435b40f..cbc14a757 100755
--- a/tests/comparison/util/verify-oracle-connection.py
+++ b/tests/comparison/util/verify-oracle-connection.py
@@ -31,7 +31,7 @@
# Importing the whole module instead of doing selective import seems to help find linker
# errors.
-from __future__ import print_function
+from __future__ import absolute_import, division, print_function
import cx_Oracle
# Host on which Oracle Database lies.
diff --git a/tests/conftest.py b/tests/conftest.py
index 0616e9110..ae8fc0a3c 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -17,7 +17,7 @@
# py.test configuration module
#
-from __future__ import print_function
+from __future__ import absolute_import, division, print_function
from impala.dbapi import connect as impala_connect
from kudu import connect as kudu_connect
from random import choice, sample
@@ -32,7 +32,7 @@ import sys
import tests.common
from impala_py_lib.helpers import find_all_files, is_core_dump
from tests.common.environ import build_flavor_timeout
-from common.test_result_verifier import QueryTestResult
+from tests.common.test_result_verifier import QueryTestResult
from tests.common.patterns import is_valid_impala_identifier
from tests.comparison.db_connection import ImpalaConnection
from tests.util.filesystem_utils import FILESYSTEM, ISILON_WEBHDFS_PORT, WAREHOUSE
diff --git a/tests/custom_cluster/test_admission_controller.py b/tests/custom_cluster/test_admission_controller.py
index 445e0d208..b10f8e4ef 100644
--- a/tests/custom_cluster/test_admission_controller.py
+++ b/tests/custom_cluster/test_admission_controller.py
@@ -17,6 +17,7 @@
# Tests admission control
+from __future__ import absolute_import, division, print_function
import itertools
import logging
import os
@@ -75,7 +76,7 @@ STATESTORE_RPC_FREQUENCY_MS = 100
# the time the next query is submitted. Otherwise the different impalads will see stale
# state for some admission decisions.
SUBMISSION_DELAY_MS = \
- [0, STATESTORE_RPC_FREQUENCY_MS / 2, STATESTORE_RPC_FREQUENCY_MS * 3 / 2]
+ [0, STATESTORE_RPC_FREQUENCY_MS // 2, STATESTORE_RPC_FREQUENCY_MS * 3 // 2]
# Whether we will submit queries to all available impalads (in a round-robin fashion)
ROUND_ROBIN_SUBMISSION = [True, False]
@@ -2263,7 +2264,7 @@ class TestAdmissionControllerStress(TestAdmissionControllerBase):
# of running requests is very high so that requests are only queued/rejected due to
# the mem limit.
num_impalads = len(self.cluster.impalads)
- query_mem_limit = (proc_limit / MAX_NUM_CONCURRENT_QUERIES / num_impalads) - 1
+ query_mem_limit = (proc_limit // MAX_NUM_CONCURRENT_QUERIES // num_impalads) - 1
self.run_admission_test(vector,
{'request_pool': self.pool_name, 'mem_limit': query_mem_limit})
diff --git a/tests/custom_cluster/test_alloc_fail.py b/tests/custom_cluster/test_alloc_fail.py
index 888dd9a77..507ea8af7 100644
--- a/tests/custom_cluster/test_alloc_fail.py
+++ b/tests/custom_cluster/test_alloc_fail.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import pytest
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
diff --git a/tests/custom_cluster/test_always_false_filter.py b/tests/custom_cluster/test_always_false_filter.py
index 1bc8a2dfc..ead7a4ee1 100644
--- a/tests/custom_cluster/test_always_false_filter.py
+++ b/tests/custom_cluster/test_always_false_filter.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import pytest
import re
diff --git a/tests/custom_cluster/test_auto_scaling.py b/tests/custom_cluster/test_auto_scaling.py
index eb7832660..86dda7687 100644
--- a/tests/custom_cluster/test_auto_scaling.py
+++ b/tests/custom_cluster/test_auto_scaling.py
@@ -17,6 +17,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import logging
import pytest
from time import sleep, time
diff --git a/tests/custom_cluster/test_automatic_invalidation.py b/tests/custom_cluster/test_automatic_invalidation.py
index eda202177..321add208 100644
--- a/tests/custom_cluster/test_automatic_invalidation.py
+++ b/tests/custom_cluster/test_automatic_invalidation.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import os
import pytest
import time
diff --git a/tests/custom_cluster/test_blacklist.py b/tests/custom_cluster/test_blacklist.py
index 20b18b508..d98cd8221 100644
--- a/tests/custom_cluster/test_blacklist.py
+++ b/tests/custom_cluster/test_blacklist.py
@@ -15,7 +15,7 @@
# specific language governing permissions and limitations
# under the License.
-from __future__ import print_function
+from __future__ import absolute_import, division, print_function
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
import pytest
diff --git a/tests/custom_cluster/test_blacklisted_dbs_and_tables.py b/tests/custom_cluster/test_blacklisted_dbs_and_tables.py
index bdcb33de5..8ce0e5969 100644
--- a/tests/custom_cluster/test_blacklisted_dbs_and_tables.py
+++ b/tests/custom_cluster/test_blacklisted_dbs_and_tables.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import pytest
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
diff --git a/tests/custom_cluster/test_breakpad.py b/tests/custom_cluster/test_breakpad.py
index 1d17f8570..c44d3259b 100644
--- a/tests/custom_cluster/test_breakpad.py
+++ b/tests/custom_cluster/test_breakpad.py
@@ -15,7 +15,7 @@
# specific language governing permissions and limitations
# under the License.
-from __future__ import print_function
+from __future__ import absolute_import, division, print_function
import glob
import os
import psutil
diff --git a/tests/custom_cluster/test_catalog_hms_failures.py b/tests/custom_cluster/test_catalog_hms_failures.py
index 78c52f42b..09e26a433 100644
--- a/tests/custom_cluster/test_catalog_hms_failures.py
+++ b/tests/custom_cluster/test_catalog_hms_failures.py
@@ -14,7 +14,7 @@
# specific language governing permissions and limitations
# under the License.
-from __future__ import print_function
+from __future__ import absolute_import, division, print_function
import pytest
import os
import time
diff --git a/tests/custom_cluster/test_catalog_wait.py b/tests/custom_cluster/test_catalog_wait.py
index 11b90d15a..eb1cc542e 100644
--- a/tests/custom_cluster/test_catalog_wait.py
+++ b/tests/custom_cluster/test_catalog_wait.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import pytest
from time import sleep
diff --git a/tests/custom_cluster/test_client_ssl.py b/tests/custom_cluster/test_client_ssl.py
index b02a688aa..99109cd00 100644
--- a/tests/custom_cluster/test_client_ssl.py
+++ b/tests/custom_cluster/test_client_ssl.py
@@ -16,7 +16,7 @@
# under the License.
#
-from __future__ import print_function
+from __future__ import absolute_import, division, print_function
import json
import logging
import os
diff --git a/tests/custom_cluster/test_codegen_cache.py b/tests/custom_cluster/test_codegen_cache.py
index bdd69c614..72b8e28d6 100644
--- a/tests/custom_cluster/test_codegen_cache.py
+++ b/tests/custom_cluster/test_codegen_cache.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import pytest
from copy import copy
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
diff --git a/tests/custom_cluster/test_compact_catalog_updates.py b/tests/custom_cluster/test_compact_catalog_updates.py
index 4440e3bb9..9ca1dc92a 100644
--- a/tests/custom_cluster/test_compact_catalog_updates.py
+++ b/tests/custom_cluster/test_compact_catalog_updates.py
@@ -17,6 +17,7 @@
#
# Test Catalog behavior when --compact_catalog_topic is false.
+from __future__ import absolute_import, division, print_function
import pytest
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
diff --git a/tests/custom_cluster/test_concurrent_ddls.py b/tests/custom_cluster/test_concurrent_ddls.py
index 7ec6153ce..8d61e2735 100644
--- a/tests/custom_cluster/test_concurrent_ddls.py
+++ b/tests/custom_cluster/test_concurrent_ddls.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import pytest
import threading
diff --git a/tests/custom_cluster/test_concurrent_kudu_create.py b/tests/custom_cluster/test_concurrent_kudu_create.py
index d5d285702..ad86ee925 100644
--- a/tests/custom_cluster/test_concurrent_kudu_create.py
+++ b/tests/custom_cluster/test_concurrent_kudu_create.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import pytest
import threading
import time
diff --git a/tests/custom_cluster/test_coordinators.py b/tests/custom_cluster/test_coordinators.py
index 635892da5..436e26572 100644
--- a/tests/custom_cluster/test_coordinators.py
+++ b/tests/custom_cluster/test_coordinators.py
@@ -17,6 +17,7 @@
#
# The base class that should be used for almost all Impala tests
+from __future__ import absolute_import, division, print_function
import logging
import pytest
import os
diff --git a/tests/custom_cluster/test_custom_hive_configs.py b/tests/custom_cluster/test_custom_hive_configs.py
index 933a114af..4fcb88e72 100644
--- a/tests/custom_cluster/test_custom_hive_configs.py
+++ b/tests/custom_cluster/test_custom_hive_configs.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import pytest
from os import getenv
diff --git a/tests/custom_cluster/test_custom_statestore.py b/tests/custom_cluster/test_custom_statestore.py
index 742ba19e0..185e3f049 100644
--- a/tests/custom_cluster/test_custom_statestore.py
+++ b/tests/custom_cluster/test_custom_statestore.py
@@ -18,6 +18,7 @@
# Tests statestore with non-default startup options
+from __future__ import absolute_import, division, print_function
import logging
import os
import pytest
diff --git a/tests/custom_cluster/test_data_cache.py b/tests/custom_cluster/test_data_cache.py
index c5c2555d8..988c08dfc 100644
--- a/tests/custom_cluster/test_data_cache.py
+++ b/tests/custom_cluster/test_data_cache.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import pytest
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
diff --git a/tests/custom_cluster/test_delegation.py b/tests/custom_cluster/test_delegation.py
index f63ffc885..72660b394 100644
--- a/tests/custom_cluster/test_delegation.py
+++ b/tests/custom_cluster/test_delegation.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
#
+from __future__ import absolute_import, division, print_function
import getpass
import pytest
from tests.hs2.hs2_test_suite import HS2TestSuite, needs_session
diff --git a/tests/custom_cluster/test_disable_catalog_data_ops.py b/tests/custom_cluster/test_disable_catalog_data_ops.py
index 062de847b..181afbe5a 100644
--- a/tests/custom_cluster/test_disable_catalog_data_ops.py
+++ b/tests/custom_cluster/test_disable_catalog_data_ops.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import pytest
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
diff --git a/tests/custom_cluster/test_disable_features.py b/tests/custom_cluster/test_disable_features.py
index 632322301..83d0d4abb 100644
--- a/tests/custom_cluster/test_disable_features.py
+++ b/tests/custom_cluster/test_disable_features.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import pytest
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
diff --git a/tests/custom_cluster/test_disk_spill_configurations.py b/tests/custom_cluster/test_disk_spill_configurations.py
index efddd231b..f73fd15fc 100644
--- a/tests/custom_cluster/test_disk_spill_configurations.py
+++ b/tests/custom_cluster/test_disk_spill_configurations.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import pytest
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
diff --git a/tests/custom_cluster/test_events_custom_configs.py b/tests/custom_cluster/test_events_custom_configs.py
index 95fb83e6c..703f11b61 100644
--- a/tests/custom_cluster/test_events_custom_configs.py
+++ b/tests/custom_cluster/test_events_custom_configs.py
@@ -14,7 +14,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-from __future__ import print_function
+from __future__ import absolute_import, division, print_function
import logging
import pytest
diff --git a/tests/custom_cluster/test_exchange_deferred_batches.py b/tests/custom_cluster/test_exchange_deferred_batches.py
index 31cd78e56..52a70a074 100644
--- a/tests/custom_cluster/test_exchange_deferred_batches.py
+++ b/tests/custom_cluster/test_exchange_deferred_batches.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import pytest
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
from tests.common.skip import SkipIfBuildType
diff --git a/tests/custom_cluster/test_exchange_delays.py b/tests/custom_cluster/test_exchange_delays.py
index b6aa044c5..44d79ea2a 100644
--- a/tests/custom_cluster/test_exchange_delays.py
+++ b/tests/custom_cluster/test_exchange_delays.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import pytest
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
from tests.common.environ import build_flavor_timeout
diff --git a/tests/custom_cluster/test_exchange_eos.py b/tests/custom_cluster/test_exchange_eos.py
index b26739fdd..c6e2999dd 100644
--- a/tests/custom_cluster/test_exchange_eos.py
+++ b/tests/custom_cluster/test_exchange_eos.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import pytest
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
diff --git a/tests/custom_cluster/test_executor_groups.py b/tests/custom_cluster/test_executor_groups.py
index 3044b6232..f0d3c3293 100644
--- a/tests/custom_cluster/test_executor_groups.py
+++ b/tests/custom_cluster/test_executor_groups.py
@@ -17,6 +17,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
from tests.util.concurrent_workload import ConcurrentWorkload
diff --git a/tests/custom_cluster/test_frontend_connection_limit.py b/tests/custom_cluster/test_frontend_connection_limit.py
index 54d1a7f68..911b808c9 100644
--- a/tests/custom_cluster/test_frontend_connection_limit.py
+++ b/tests/custom_cluster/test_frontend_connection_limit.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import pytest
from threading import Thread
diff --git a/tests/custom_cluster/test_geospatial_library.py b/tests/custom_cluster/test_geospatial_library.py
index 9d5ff6001..5ce0d9730 100644
--- a/tests/custom_cluster/test_geospatial_library.py
+++ b/tests/custom_cluster/test_geospatial_library.py
@@ -14,6 +14,8 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+
+from __future__ import absolute_import, division, print_function
import pytest
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
diff --git a/tests/custom_cluster/test_hbase_hms_column_order.py b/tests/custom_cluster/test_hbase_hms_column_order.py
index 869a75c4b..447dd071d 100644
--- a/tests/custom_cluster/test_hbase_hms_column_order.py
+++ b/tests/custom_cluster/test_hbase_hms_column_order.py
@@ -17,6 +17,7 @@
#
# Tests for IMPALA-1658
+from __future__ import absolute_import, division, print_function
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
diff --git a/tests/custom_cluster/test_hdfs_fd_caching.py b/tests/custom_cluster/test_hdfs_fd_caching.py
index fd9b04f82..c2c66be49 100644
--- a/tests/custom_cluster/test_hdfs_fd_caching.py
+++ b/tests/custom_cluster/test_hdfs_fd_caching.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import pytest
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
diff --git a/tests/custom_cluster/test_hdfs_timeout.py b/tests/custom_cluster/test_hdfs_timeout.py
index fb2d3e38e..629b63de2 100644
--- a/tests/custom_cluster/test_hdfs_timeout.py
+++ b/tests/custom_cluster/test_hdfs_timeout.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import pytest
import re
import time
diff --git a/tests/custom_cluster/test_hedged_reads.py b/tests/custom_cluster/test_hedged_reads.py
index e1d36e73b..b82181743 100644
--- a/tests/custom_cluster/test_hedged_reads.py
+++ b/tests/custom_cluster/test_hedged_reads.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import pytest
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
from tests.common.skip import SkipIf
diff --git a/tests/custom_cluster/test_hive_parquet_codec_interop.py b/tests/custom_cluster/test_hive_parquet_codec_interop.py
index c9601d72d..a020a2be7 100644
--- a/tests/custom_cluster/test_hive_parquet_codec_interop.py
+++ b/tests/custom_cluster/test_hive_parquet_codec_interop.py
@@ -17,6 +17,7 @@
#
# Tests for Hive-IMPALA parquet compression codec interoperability
+from __future__ import absolute_import, division, print_function
import pytest
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
diff --git a/tests/custom_cluster/test_hive_parquet_timestamp_conversion.py b/tests/custom_cluster/test_hive_parquet_timestamp_conversion.py
index c93a0d0f4..3fe286d5f 100644
--- a/tests/custom_cluster/test_hive_parquet_timestamp_conversion.py
+++ b/tests/custom_cluster/test_hive_parquet_timestamp_conversion.py
@@ -17,6 +17,7 @@
#
# Tests for IMPALA-1658
+from __future__ import absolute_import, division, print_function
import os
import pytest
from subprocess import check_call
diff --git a/tests/custom_cluster/test_hive_text_codec_interop.py b/tests/custom_cluster/test_hive_text_codec_interop.py
index a23d88102..e5223be3a 100644
--- a/tests/custom_cluster/test_hive_text_codec_interop.py
+++ b/tests/custom_cluster/test_hive_text_codec_interop.py
@@ -17,6 +17,7 @@
#
# Tests for Hive-IMPALA text compression codec interoperability
+from __future__ import absolute_import, division, print_function
import pytest
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
diff --git a/tests/custom_cluster/test_hs2.py b/tests/custom_cluster/test_hs2.py
index 62185118f..8a38bef38 100644
--- a/tests/custom_cluster/test_hs2.py
+++ b/tests/custom_cluster/test_hs2.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
import pytest
diff --git a/tests/custom_cluster/test_hs2_fault_injection.py b/tests/custom_cluster/test_hs2_fault_injection.py
index 27e536fdd..cb9866044 100644
--- a/tests/custom_cluster/test_hs2_fault_injection.py
+++ b/tests/custom_cluster/test_hs2_fault_injection.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import pytest
import requests
diff --git a/tests/custom_cluster/test_incremental_metadata_updates.py b/tests/custom_cluster/test_incremental_metadata_updates.py
index 0029cf3a3..82334f080 100755
--- a/tests/custom_cluster/test_incremental_metadata_updates.py
+++ b/tests/custom_cluster/test_incremental_metadata_updates.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import pytest
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
diff --git a/tests/custom_cluster/test_insert_behaviour.py b/tests/custom_cluster/test_insert_behaviour.py
index 89d06f5cc..44076a2a6 100644
--- a/tests/custom_cluster/test_insert_behaviour.py
+++ b/tests/custom_cluster/test_insert_behaviour.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import pytest
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
diff --git a/tests/custom_cluster/test_jvm_mem_tracking.py b/tests/custom_cluster/test_jvm_mem_tracking.py
index e4b9a07e0..3999a0cb2 100644
--- a/tests/custom_cluster/test_jvm_mem_tracking.py
+++ b/tests/custom_cluster/test_jvm_mem_tracking.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import logging
import json
import pytest
diff --git a/tests/custom_cluster/test_krpc_mem_usage.py b/tests/custom_cluster/test_krpc_mem_usage.py
index 0d76ca866..ffcf44957 100644
--- a/tests/custom_cluster/test_krpc_mem_usage.py
+++ b/tests/custom_cluster/test_krpc_mem_usage.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import pytest
import time
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
diff --git a/tests/custom_cluster/test_krpc_metrics.py b/tests/custom_cluster/test_krpc_metrics.py
index 0acc72eb2..7d5a4e531 100644
--- a/tests/custom_cluster/test_krpc_metrics.py
+++ b/tests/custom_cluster/test_krpc_metrics.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import pytest
import time
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
diff --git a/tests/custom_cluster/test_krpc_options.py b/tests/custom_cluster/test_krpc_options.py
index b949621b7..3958eb512 100644
--- a/tests/custom_cluster/test_krpc_options.py
+++ b/tests/custom_cluster/test_krpc_options.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import pytest
import socket
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
diff --git a/tests/custom_cluster/test_krpc_socket.py b/tests/custom_cluster/test_krpc_socket.py
index 1845102cf..009547438 100644
--- a/tests/custom_cluster/test_krpc_socket.py
+++ b/tests/custom_cluster/test_krpc_socket.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import pytest
import socket
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
diff --git a/tests/custom_cluster/test_kudu.py b/tests/custom_cluster/test_kudu.py
index 74155da95..1e3d39a8b 100644
--- a/tests/custom_cluster/test_kudu.py
+++ b/tests/custom_cluster/test_kudu.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import logging
import os
import pytest
diff --git a/tests/custom_cluster/test_kudu_not_available.py b/tests/custom_cluster/test_kudu_not_available.py
index 090854d92..23b09c026 100644
--- a/tests/custom_cluster/test_kudu_not_available.py
+++ b/tests/custom_cluster/test_kudu_not_available.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
from impala.dbapi import connect
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
diff --git a/tests/custom_cluster/test_kudu_table_create_without_hms.py b/tests/custom_cluster/test_kudu_table_create_without_hms.py
index cfe957ef4..fecbd1e4c 100644
--- a/tests/custom_cluster/test_kudu_table_create_without_hms.py
+++ b/tests/custom_cluster/test_kudu_table_create_without_hms.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import pytest
import os
diff --git a/tests/custom_cluster/test_lineage.py b/tests/custom_cluster/test_lineage.py
index 71668b0db..6551d545e 100644
--- a/tests/custom_cluster/test_lineage.py
+++ b/tests/custom_cluster/test_lineage.py
@@ -17,6 +17,7 @@
#
# Tests for column lineage.
+from __future__ import absolute_import, division, print_function
import json
import logging
import os
diff --git a/tests/custom_cluster/test_local_catalog.py b/tests/custom_cluster/test_local_catalog.py
index 5f7a25332..baeb453bc 100644
--- a/tests/custom_cluster/test_local_catalog.py
+++ b/tests/custom_cluster/test_local_catalog.py
@@ -17,7 +17,7 @@
# Test behaviors specific to --use_local_catalog being enabled.
-from __future__ import print_function
+from __future__ import absolute_import, division, print_function
import pytest
import Queue
import random
diff --git a/tests/custom_cluster/test_local_tz_conversion.py b/tests/custom_cluster/test_local_tz_conversion.py
index ca1a720d7..35b4e809e 100644
--- a/tests/custom_cluster/test_local_tz_conversion.py
+++ b/tests/custom_cluster/test_local_tz_conversion.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import pytest
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
from tests.common.test_vector import ImpalaTestDimension
diff --git a/tests/custom_cluster/test_logging.py b/tests/custom_cluster/test_logging.py
index 34cf2578e..13f6595c2 100644
--- a/tests/custom_cluster/test_logging.py
+++ b/tests/custom_cluster/test_logging.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import pytest
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
diff --git a/tests/custom_cluster/test_mem_reservations.py b/tests/custom_cluster/test_mem_reservations.py
index 6365ec126..f6e66ae81 100644
--- a/tests/custom_cluster/test_mem_reservations.py
+++ b/tests/custom_cluster/test_mem_reservations.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import pytest
import threading
diff --git a/tests/custom_cluster/test_metadata_no_events_processing.py b/tests/custom_cluster/test_metadata_no_events_processing.py
index 9d380cf4b..ee6af0821 100644
--- a/tests/custom_cluster/test_metadata_no_events_processing.py
+++ b/tests/custom_cluster/test_metadata_no_events_processing.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
from tests.common.skip import SkipIfFS
diff --git a/tests/custom_cluster/test_metadata_replicas.py b/tests/custom_cluster/test_metadata_replicas.py
index a458b23aa..4afa4249f 100644
--- a/tests/custom_cluster/test_metadata_replicas.py
+++ b/tests/custom_cluster/test_metadata_replicas.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import pytest
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
from tests.common.skip import SkipIfFS
diff --git a/tests/custom_cluster/test_metastore_events_cleanup.py b/tests/custom_cluster/test_metastore_events_cleanup.py
index 79662680e..a2f578422 100644
--- a/tests/custom_cluster/test_metastore_events_cleanup.py
+++ b/tests/custom_cluster/test_metastore_events_cleanup.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import pytest
import os
diff --git a/tests/custom_cluster/test_metastore_service.py b/tests/custom_cluster/test_metastore_service.py
index 62a621a73..c74ab63fd 100644
--- a/tests/custom_cluster/test_metastore_service.py
+++ b/tests/custom_cluster/test_metastore_service.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import pytest
from hive_metastore.ttypes import Database
from hive_metastore.ttypes import FieldSchema
diff --git a/tests/custom_cluster/test_mt_dop.py b/tests/custom_cluster/test_mt_dop.py
index e832a26f2..38878c086 100644
--- a/tests/custom_cluster/test_mt_dop.py
+++ b/tests/custom_cluster/test_mt_dop.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import os
import pytest
from copy import deepcopy
diff --git a/tests/custom_cluster/test_observability.py b/tests/custom_cluster/test_observability.py
index 4a4a5a976..c5fd9e2b2 100644
--- a/tests/custom_cluster/test_observability.py
+++ b/tests/custom_cluster/test_observability.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import pytest
import re
diff --git a/tests/custom_cluster/test_parquet_max_page_header.py b/tests/custom_cluster/test_parquet_max_page_header.py
index 48cfa0fce..73496487b 100644
--- a/tests/custom_cluster/test_parquet_max_page_header.py
+++ b/tests/custom_cluster/test_parquet_max_page_header.py
@@ -17,6 +17,7 @@
#
# Tests for IMPALA-2273
+from __future__ import absolute_import, division, print_function
import os
import pytest
import random
diff --git a/tests/custom_cluster/test_partition.py b/tests/custom_cluster/test_partition.py
index aa996a5ce..89d80f2c1 100644
--- a/tests/custom_cluster/test_partition.py
+++ b/tests/custom_cluster/test_partition.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import logging
import pytest
import shutil
diff --git a/tests/custom_cluster/test_pause_monitor.py b/tests/custom_cluster/test_pause_monitor.py
index bf471901f..165b11dc5 100644
--- a/tests/custom_cluster/test_pause_monitor.py
+++ b/tests/custom_cluster/test_pause_monitor.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import signal
import time
diff --git a/tests/custom_cluster/test_permanent_udfs.py b/tests/custom_cluster/test_permanent_udfs.py
index 31f34094a..c84ac6667 100644
--- a/tests/custom_cluster/test_permanent_udfs.py
+++ b/tests/custom_cluster/test_permanent_udfs.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import glob
import os
import pytest
diff --git a/tests/custom_cluster/test_preload_table_types.py b/tests/custom_cluster/test_preload_table_types.py
index ae5f16f2d..f86427eac 100644
--- a/tests/custom_cluster/test_preload_table_types.py
+++ b/tests/custom_cluster/test_preload_table_types.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
diff --git a/tests/custom_cluster/test_process_failures.py b/tests/custom_cluster/test_process_failures.py
index 0bf5b2ab3..21464ed3f 100644
--- a/tests/custom_cluster/test_process_failures.py
+++ b/tests/custom_cluster/test_process_failures.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import pytest
from beeswaxd.BeeswaxService import QueryState
diff --git a/tests/custom_cluster/test_query_concurrency.py b/tests/custom_cluster/test_query_concurrency.py
index 703ffd646..49a9ca246 100644
--- a/tests/custom_cluster/test_query_concurrency.py
+++ b/tests/custom_cluster/test_query_concurrency.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import pytest
import time
from threading import Thread
diff --git a/tests/custom_cluster/test_query_event_hooks.py b/tests/custom_cluster/test_query_event_hooks.py
index 36fe36047..e709f83d3 100644
--- a/tests/custom_cluster/test_query_event_hooks.py
+++ b/tests/custom_cluster/test_query_event_hooks.py
@@ -17,6 +17,7 @@
#
# Client tests for Query Event Hooks
+from __future__ import absolute_import, division, print_function
import os
import pytest
import tempfile
diff --git a/tests/custom_cluster/test_query_expiration.py b/tests/custom_cluster/test_query_expiration.py
index d6b0011fd..d7dd5a0c3 100644
--- a/tests/custom_cluster/test_query_expiration.py
+++ b/tests/custom_cluster/test_query_expiration.py
@@ -17,6 +17,7 @@
#
# Tests for query expiration.
+from __future__ import absolute_import, division, print_function
import pytest
import re
import threading
diff --git a/tests/custom_cluster/test_query_retries.py b/tests/custom_cluster/test_query_retries.py
index d7e8e4fb7..5d321ceed 100644
--- a/tests/custom_cluster/test_query_retries.py
+++ b/tests/custom_cluster/test_query_retries.py
@@ -21,7 +21,7 @@
# TODO: Re-factor tests into multiple classes.
# TODO: Add a test that cancels queries while a retry is running
-from __future__ import print_function
+from __future__ import absolute_import, division, print_function
import pytest
import re
import shutil
diff --git a/tests/custom_cluster/test_re2_max_mem.py b/tests/custom_cluster/test_re2_max_mem.py
index 9348bcb1a..ded88bd2d 100755
--- a/tests/custom_cluster/test_re2_max_mem.py
+++ b/tests/custom_cluster/test_re2_max_mem.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import pytest
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
diff --git a/tests/custom_cluster/test_redaction.py b/tests/custom_cluster/test_redaction.py
index fd95a1657..607502723 100644
--- a/tests/custom_cluster/test_redaction.py
+++ b/tests/custom_cluster/test_redaction.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import getpass
import logging
import os
diff --git a/tests/custom_cluster/test_reserved_words_version.py b/tests/custom_cluster/test_reserved_words_version.py
index 0c3b80075..b317f93cb 100644
--- a/tests/custom_cluster/test_reserved_words_version.py
+++ b/tests/custom_cluster/test_reserved_words_version.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import pytest
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
diff --git a/tests/custom_cluster/test_restart_services.py b/tests/custom_cluster/test_restart_services.py
index 9740974bb..7d9c003d1 100644
--- a/tests/custom_cluster/test_restart_services.py
+++ b/tests/custom_cluster/test_restart_services.py
@@ -15,7 +15,7 @@
# specific language governing permissions and limitations
# under the License.
-from __future__ import print_function
+from __future__ import absolute_import, division, print_function
import logging
import os
import pytest
@@ -612,7 +612,7 @@ class TestGracefulShutdown(CustomClusterTestSuite, HS2TestSuite):
self.client, SHUTDOWN_EXEC3.format(HIGH_DEADLINE))
grace, deadline, _, _ = parse_shutdown_result(result)
assert grace == "{0}s000ms".format(self.EXEC_SHUTDOWN_GRACE_PERIOD_S)
- assert deadline == "{0}m{1}s".format(HIGH_DEADLINE / 60, HIGH_DEADLINE % 60)
+ assert deadline == "{0}m{1}s".format(HIGH_DEADLINE // 60, HIGH_DEADLINE % 60)
result = self.execute_query_expect_success(
self.client, SHUTDOWN_EXEC3.format(VERY_HIGH_DEADLINE))
@@ -660,7 +660,7 @@ class TestGracefulShutdown(CustomClusterTestSuite, HS2TestSuite):
result = self.execute_query_expect_success(self.client, SHUTDOWN)
grace, deadline, registered, _ = parse_shutdown_result(result)
assert grace == "{0}s000ms".format(self.COORD_SHUTDOWN_GRACE_PERIOD_S)
- assert deadline == "{0}m".format(self.COORD_SHUTDOWN_DEADLINE_S / 60), "4"
+ assert deadline == "{0}m".format(self.COORD_SHUTDOWN_DEADLINE_S // 60), "4"
assert registered == "3"
# Expect that the beeswax shutdown error occurs when calling fn()
diff --git a/tests/custom_cluster/test_result_spooling.py b/tests/custom_cluster/test_result_spooling.py
index 03f0a1709..795c38c5a 100644
--- a/tests/custom_cluster/test_result_spooling.py
+++ b/tests/custom_cluster/test_result_spooling.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import pytest
import re
diff --git a/tests/custom_cluster/test_rpc_exception.py b/tests/custom_cluster/test_rpc_exception.py
index 4921ca926..dcbdde595 100644
--- a/tests/custom_cluster/test_rpc_exception.py
+++ b/tests/custom_cluster/test_rpc_exception.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import pytest
import time
from tests.beeswax.impala_beeswax import ImpalaBeeswaxException
diff --git a/tests/custom_cluster/test_rpc_timeout.py b/tests/custom_cluster/test_rpc_timeout.py
index 8b266629f..410e22e9e 100644
--- a/tests/custom_cluster/test_rpc_timeout.py
+++ b/tests/custom_cluster/test_rpc_timeout.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import pytest
from tests.beeswax.impala_beeswax import ImpalaBeeswaxException
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
diff --git a/tests/custom_cluster/test_runtime_profile.py b/tests/custom_cluster/test_runtime_profile.py
index cb5e2963c..5b7841ea7 100644
--- a/tests/custom_cluster/test_runtime_profile.py
+++ b/tests/custom_cluster/test_runtime_profile.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import pytest
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
from tests.common.skip import SkipIfFS
diff --git a/tests/custom_cluster/test_s3a_access.py b/tests/custom_cluster/test_s3a_access.py
index 62160d686..6bc876df1 100644
--- a/tests/custom_cluster/test_s3a_access.py
+++ b/tests/custom_cluster/test_s3a_access.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
#
+from __future__ import absolute_import, division, print_function
import os
import pytest
import stat
diff --git a/tests/custom_cluster/test_saml2_sso.py b/tests/custom_cluster/test_saml2_sso.py
index 476e0613f..2cf4babce 100644
--- a/tests/custom_cluster/test_saml2_sso.py
+++ b/tests/custom_cluster/test_saml2_sso.py
@@ -16,6 +16,7 @@
# under the License.
#
+from __future__ import absolute_import, division, print_function
import base64
import datetime
import os
diff --git a/tests/custom_cluster/test_scheduler_locality.py b/tests/custom_cluster/test_scheduler_locality.py
index adfbcd581..9a156cc3e 100644
--- a/tests/custom_cluster/test_scheduler_locality.py
+++ b/tests/custom_cluster/test_scheduler_locality.py
@@ -17,6 +17,7 @@
#
# Tests for local and remote disk scheduling.
+from __future__ import absolute_import, division, print_function
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
from tests.common.network import get_external_ip
from tests.common.skip import SkipIfNotHdfsMinicluster
diff --git a/tests/custom_cluster/test_scratch_disk.py b/tests/custom_cluster/test_scratch_disk.py
index 4ed4ebba5..dc3797b8e 100644
--- a/tests/custom_cluster/test_scratch_disk.py
+++ b/tests/custom_cluster/test_scratch_disk.py
@@ -17,7 +17,7 @@
#
# Tests for query expiration.
-from __future__ import print_function
+from __future__ import absolute_import, division, print_function
import os
import pytest
import re
diff --git a/tests/custom_cluster/test_seq_file_filtering.py b/tests/custom_cluster/test_seq_file_filtering.py
index 126c7c452..fa731b7c5 100644
--- a/tests/custom_cluster/test_seq_file_filtering.py
+++ b/tests/custom_cluster/test_seq_file_filtering.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import pytest
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
diff --git a/tests/custom_cluster/test_services_rpc_errors.py b/tests/custom_cluster/test_services_rpc_errors.py
index 0789d262e..3efcb393b 100644
--- a/tests/custom_cluster/test_services_rpc_errors.py
+++ b/tests/custom_cluster/test_services_rpc_errors.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import pytest
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
diff --git a/tests/custom_cluster/test_session_expiration.py b/tests/custom_cluster/test_session_expiration.py
index fb9771c83..fccf35f34 100644
--- a/tests/custom_cluster/test_session_expiration.py
+++ b/tests/custom_cluster/test_session_expiration.py
@@ -17,6 +17,7 @@
#
# Tests for query expiration.
+from __future__ import absolute_import, division, print_function
import pytest
import socket
from time import sleep
diff --git a/tests/custom_cluster/test_set_and_unset.py b/tests/custom_cluster/test_set_and_unset.py
index 3d3c6ab5b..3425a5895 100644
--- a/tests/custom_cluster/test_set_and_unset.py
+++ b/tests/custom_cluster/test_set_and_unset.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import pytest
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
diff --git a/tests/custom_cluster/test_shared_tzdb.py b/tests/custom_cluster/test_shared_tzdb.py
index e0fd2f889..85b6d31c1 100644
--- a/tests/custom_cluster/test_shared_tzdb.py
+++ b/tests/custom_cluster/test_shared_tzdb.py
@@ -17,6 +17,7 @@
#
# Tests for IMPALA-3307
+from __future__ import absolute_import, division, print_function
import pytest
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
diff --git a/tests/custom_cluster/test_shell_commandline.py b/tests/custom_cluster/test_shell_commandline.py
index c7379a93d..9bdf4fba2 100644
--- a/tests/custom_cluster/test_shell_commandline.py
+++ b/tests/custom_cluster/test_shell_commandline.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import os
import pytest
import re
diff --git a/tests/custom_cluster/test_shell_interactive.py b/tests/custom_cluster/test_shell_interactive.py
index f9ed9c8bb..b14bcd65a 100644
--- a/tests/custom_cluster/test_shell_interactive.py
+++ b/tests/custom_cluster/test_shell_interactive.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import pytest
from multiprocessing.pool import ThreadPool
diff --git a/tests/custom_cluster/test_shell_interactive_reconnect.py b/tests/custom_cluster/test_shell_interactive_reconnect.py
index dde6deb19..20fcdc581 100644
--- a/tests/custom_cluster/test_shell_interactive_reconnect.py
+++ b/tests/custom_cluster/test_shell_interactive_reconnect.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import pytest
import tempfile
import socket
diff --git a/tests/custom_cluster/test_startup_filesystem_checks.py b/tests/custom_cluster/test_startup_filesystem_checks.py
index 4d2da9dfe..7b2e81a76 100644
--- a/tests/custom_cluster/test_startup_filesystem_checks.py
+++ b/tests/custom_cluster/test_startup_filesystem_checks.py
@@ -17,6 +17,7 @@
#
# Tests for the behavior of startup_filesystem_check_directories
+from __future__ import absolute_import, division, print_function
import logging
import pytest
import os
diff --git a/tests/custom_cluster/test_stats_extrapolation.py b/tests/custom_cluster/test_stats_extrapolation.py
index 9b219210e..f16b566f1 100644
--- a/tests/custom_cluster/test_stats_extrapolation.py
+++ b/tests/custom_cluster/test_stats_extrapolation.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import pytest
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
from tests.common.test_dimensions import (
diff --git a/tests/custom_cluster/test_thrift_debug_string_exception.py b/tests/custom_cluster/test_thrift_debug_string_exception.py
index af03abd49..eec216679 100644
--- a/tests/custom_cluster/test_thrift_debug_string_exception.py
+++ b/tests/custom_cluster/test_thrift_debug_string_exception.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
diff --git a/tests/custom_cluster/test_thrift_socket.py b/tests/custom_cluster/test_thrift_socket.py
index c276384fd..4451ccf8d 100644
--- a/tests/custom_cluster/test_thrift_socket.py
+++ b/tests/custom_cluster/test_thrift_socket.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import os
import pytest
import ssl
diff --git a/tests/custom_cluster/test_topic_update_frequency.py b/tests/custom_cluster/test_topic_update_frequency.py
index 13402714e..f5e6c92f5 100644
--- a/tests/custom_cluster/test_topic_update_frequency.py
+++ b/tests/custom_cluster/test_topic_update_frequency.py
@@ -9,7 +9,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-from __future__ import print_function
+from __future__ import absolute_import, division, print_function
from multiprocessing.pool import ThreadPool
import pytest
diff --git a/tests/custom_cluster/test_udf_concurrency.py b/tests/custom_cluster/test_udf_concurrency.py
index 8e002655d..e9f9b8575 100644
--- a/tests/custom_cluster/test_udf_concurrency.py
+++ b/tests/custom_cluster/test_udf_concurrency.py
@@ -15,7 +15,7 @@
# specific language governing permissions and limitations
# under the License.
-from __future__ import print_function
+from __future__ import absolute_import, division, print_function
import os
import pytest
import random
diff --git a/tests/custom_cluster/test_web_pages.py b/tests/custom_cluster/test_web_pages.py
index fb983cb9d..69b873ba9 100644
--- a/tests/custom_cluster/test_web_pages.py
+++ b/tests/custom_cluster/test_web_pages.py
@@ -15,7 +15,7 @@
# specific language governing permissions and limitations
# under the License.
-from __future__ import print_function
+from __future__ import absolute_import, division, print_function
import json
import random
import re
diff --git a/tests/custom_cluster/test_wide_table_operations.py b/tests/custom_cluster/test_wide_table_operations.py
index e1081aff1..26daf46aa 100644
--- a/tests/custom_cluster/test_wide_table_operations.py
+++ b/tests/custom_cluster/test_wide_table_operations.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import os
import pytest
from subprocess import call
diff --git a/tests/data_errors/test_data_errors.py b/tests/data_errors/test_data_errors.py
index d098dfed0..b0139e7ec 100644
--- a/tests/data_errors/test_data_errors.py
+++ b/tests/data_errors/test_data_errors.py
@@ -19,6 +19,7 @@
#
# Tests Impala properly handles errors when reading and writing data.
+from __future__ import absolute_import, division, print_function
import pytest
import subprocess
diff --git a/tests/experiments/test_targeted_perf.py b/tests/experiments/test_targeted_perf.py
index ceef28020..23c44e164 100644
--- a/tests/experiments/test_targeted_perf.py
+++ b/tests/experiments/test_targeted_perf.py
@@ -20,6 +20,7 @@
# Additionally, we don't get any 'extra' coverage from them, so they're
# not an essential part of functional verification.
+from __future__ import absolute_import, division, print_function
from tests.common.impala_test_suite import ImpalaTestSuite
class TestTargetedPerf(ImpalaTestSuite):
diff --git a/tests/failure/test_failpoints.py b/tests/failure/test_failpoints.py
index 9283d4919..e95aa1e86 100644
--- a/tests/failure/test_failpoints.py
+++ b/tests/failure/test_failpoints.py
@@ -18,6 +18,7 @@
# Injects failures at specific locations in each of the plan nodes. Currently supports
# two types of failures - cancellation of the query and a failure test hook.
#
+from __future__ import absolute_import, division, print_function
import pytest
import re
from time import sleep
diff --git a/tests/hs2/hs2_test_suite.py b/tests/hs2/hs2_test_suite.py
index c92b555f5..b8acbc502 100644
--- a/tests/hs2/hs2_test_suite.py
+++ b/tests/hs2/hs2_test_suite.py
@@ -17,6 +17,7 @@
#
# Superclass of all HS2 tests containing commonly used functions.
+from __future__ import absolute_import, division, print_function
from getpass import getuser
from TCLIService import TCLIService
from ImpalaService import ImpalaHiveServer2Service
@@ -286,7 +287,7 @@ class HS2TestSuite(ImpalaTestSuite):
for col_type in HS2TestSuite.HS2_V6_COLUMN_TYPES:
typed_col = getattr(c, col_type)
if typed_col != None:
- indicator = ord(typed_col.nulls[i / 8])
+ indicator = ord(typed_col.nulls[i // 8])
if indicator & (1 << (i % 8)):
row.append("NULL")
else:
diff --git a/tests/hs2/test_fetch.py b/tests/hs2/test_fetch.py
index 7b4fe396c..1d64d49b3 100644
--- a/tests/hs2/test_fetch.py
+++ b/tests/hs2/test_fetch.py
@@ -16,6 +16,7 @@
# under the License.
#
+from __future__ import absolute_import, division, print_function
import pytest
import re
diff --git a/tests/hs2/test_fetch_first.py b/tests/hs2/test_fetch_first.py
index 75d846873..d40b1d0bb 100644
--- a/tests/hs2/test_fetch_first.py
+++ b/tests/hs2/test_fetch_first.py
@@ -20,6 +20,7 @@
# via the 'impala.resultset.cache.size' confOverlay option. FETCH_FIRST will
# succeed as long all previously fetched rows fit into the bounded result cache.
+from __future__ import absolute_import, division, print_function
import pytest
from ImpalaService import ImpalaHiveServer2Service
diff --git a/tests/hs2/test_fetch_timeout.py b/tests/hs2/test_fetch_timeout.py
index 01faac7c2..ea0ec5929 100644
--- a/tests/hs2/test_fetch_timeout.py
+++ b/tests/hs2/test_fetch_timeout.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
from time import sleep
from time import time
from tests.common.errors import Timeout
diff --git a/tests/hs2/test_hs2.py b/tests/hs2/test_hs2.py
index b7f71a19c..e87877b54 100644
--- a/tests/hs2/test_hs2.py
+++ b/tests/hs2/test_hs2.py
@@ -17,6 +17,7 @@
#
# Client tests for Impala's HiveServer2 interface
+from __future__ import absolute_import, division, print_function
from getpass import getuser
from contextlib import contextmanager
import json
diff --git a/tests/hs2/test_json_endpoints.py b/tests/hs2/test_json_endpoints.py
index d661bf757..64bce618c 100644
--- a/tests/hs2/test_json_endpoints.py
+++ b/tests/hs2/test_json_endpoints.py
@@ -17,6 +17,7 @@
#
# Tests for query expiration.
+from __future__ import absolute_import, division, print_function
import json
import pytest
from time import time
diff --git a/tests/infra/test_perf_infra.py b/tests/infra/test_perf_infra.py
index 3ec461706..951cf38d9 100644
--- a/tests/infra/test_perf_infra.py
+++ b/tests/infra/test_perf_infra.py
@@ -19,6 +19,7 @@
# product or other constraints. We want to stop these assumptions from breaking at
# pre-merge time, not later.
+from __future__ import absolute_import, division, print_function
import pytest
from tests.common.impala_test_suite import ImpalaTestSuite
diff --git a/tests/infra/test_stress_infra.py b/tests/infra/test_stress_infra.py
index 20dac52e2..8dd90457d 100644
--- a/tests/infra/test_stress_infra.py
+++ b/tests/infra/test_stress_infra.py
@@ -19,6 +19,7 @@
# product or other constraints. We want to stop these assumptions from breaking at
# pre-merge time, not later.
+from __future__ import absolute_import, division, print_function
import pytest
from decimal import Decimal
diff --git a/tests/infra/test_utils.py b/tests/infra/test_utils.py
index 1c2e28271..897cd90c7 100644
--- a/tests/infra/test_utils.py
+++ b/tests/infra/test_utils.py
@@ -17,6 +17,7 @@
# This module contains tests for some of the tests/util code.
+from __future__ import absolute_import, division, print_function
from tests.util.filesystem_utils import prepend_with_fs
from tests.util.parse_util import get_bytes_summary_stats_counter
diff --git a/tests/metadata/test_catalogd_debug_actions.py b/tests/metadata/test_catalogd_debug_actions.py
index 77cfb58ef..23faf68eb 100644
--- a/tests/metadata/test_catalogd_debug_actions.py
+++ b/tests/metadata/test_catalogd_debug_actions.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import pytest
from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.skip import SkipIfFS
diff --git a/tests/metadata/test_compute_stats.py b/tests/metadata/test_compute_stats.py
index e4c919e42..7f4bdff9a 100644
--- a/tests/metadata/test_compute_stats.py
+++ b/tests/metadata/test_compute_stats.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import pytest
from subprocess import check_call
diff --git a/tests/metadata/test_ddl.py b/tests/metadata/test_ddl.py
index f633ad48a..ff9357315 100644
--- a/tests/metadata/test_ddl.py
+++ b/tests/metadata/test_ddl.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import getpass
import itertools
import pytest
@@ -622,7 +623,7 @@ class TestDdlStatements(TestDdlBase):
"location '{1}/{0}'".format(fq_tbl_name, WAREHOUSE))
# Add some partitions (first batch of two)
- for i in xrange(num_parts / 5):
+ for i in xrange(num_parts // 5):
start = time.time()
self.client.execute(
"alter table {0} add partition(j={1}, s='{1}')".format(fq_tbl_name, i))
@@ -644,7 +645,7 @@ class TestDdlStatements(TestDdlBase):
.format(fq_tbl_name, WAREHOUSE))
# Add some more partitions
- for i in xrange(num_parts / 5, num_parts):
+ for i in xrange(num_parts // 5, num_parts):
start = time.time()
self.client.execute(
"alter table {0} add partition(j={1},s='{1}')".format(fq_tbl_name, i))
diff --git a/tests/metadata/test_ddl_base.py b/tests/metadata/test_ddl_base.py
index 068d34bce..494d99315 100644
--- a/tests/metadata/test_ddl_base.py
+++ b/tests/metadata/test_ddl_base.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
#
+from __future__ import absolute_import, division, print_function
from tests.common.test_dimensions import (
ALL_NODES_ONLY,
create_exec_option_dimension,
diff --git a/tests/metadata/test_event_processing.py b/tests/metadata/test_event_processing.py
index b8d5553ca..f0358b7aa 100644
--- a/tests/metadata/test_event_processing.py
+++ b/tests/metadata/test_event_processing.py
@@ -14,6 +14,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
from subprocess import check_call
import pytest
diff --git a/tests/metadata/test_explain.py b/tests/metadata/test_explain.py
index 9e1e35697..c42f485dc 100644
--- a/tests/metadata/test_explain.py
+++ b/tests/metadata/test_explain.py
@@ -17,6 +17,7 @@
# Functional tests running EXPLAIN statements.
#
+from __future__ import absolute_import, division, print_function
import re
from tests.common.impala_test_suite import ImpalaTestSuite
diff --git a/tests/metadata/test_hdfs_encryption.py b/tests/metadata/test_hdfs_encryption.py
index 897180fd5..ff3706d0f 100644
--- a/tests/metadata/test_hdfs_encryption.py
+++ b/tests/metadata/test_hdfs_encryption.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import getpass
import pytest
diff --git a/tests/metadata/test_hdfs_permissions.py b/tests/metadata/test_hdfs_permissions.py
index 766fdfcfc..b840154b1 100644
--- a/tests/metadata/test_hdfs_permissions.py
+++ b/tests/metadata/test_hdfs_permissions.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.skip import SkipIfFS, SkipIfLocal, SkipIfCatalogV2
from tests.common.test_dimensions import (
diff --git a/tests/metadata/test_hidden_files.py b/tests/metadata/test_hidden_files.py
index 8454b83b8..d6775d27f 100644
--- a/tests/metadata/test_hidden_files.py
+++ b/tests/metadata/test_hidden_files.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
from subprocess import check_call
from tests.common.impala_test_suite import ImpalaTestSuite
diff --git a/tests/metadata/test_hms_integration.py b/tests/metadata/test_hms_integration.py
index 828033a9f..343f67877 100644
--- a/tests/metadata/test_hms_integration.py
+++ b/tests/metadata/test_hms_integration.py
@@ -23,7 +23,7 @@
# TODO: For each test, verify all the metadata available via Hive and
# Impala, in all the possible ways of validating that metadata.
-from __future__ import print_function
+from __future__ import absolute_import, division, print_function
import pytest
import random
import string
diff --git a/tests/metadata/test_last_ddl_time_update.py b/tests/metadata/test_last_ddl_time_update.py
index 36a4db358..7e44b4bd5 100644
--- a/tests/metadata/test_last_ddl_time_update.py
+++ b/tests/metadata/test_last_ddl_time_update.py
@@ -16,6 +16,7 @@
# under the License.
# Impala tests for DDL statements
+from __future__ import absolute_import, division, print_function
import time
from tests.common.impala_test_suite import ImpalaTestSuite
diff --git a/tests/metadata/test_load.py b/tests/metadata/test_load.py
index 149664d66..9dc3500e8 100644
--- a/tests/metadata/test_load.py
+++ b/tests/metadata/test_load.py
@@ -17,6 +17,7 @@
# Functional tests for LOAD DATA statements.
+from __future__ import absolute_import, division, print_function
import time
from beeswaxd.BeeswaxService import QueryState
from copy import deepcopy
diff --git a/tests/metadata/test_metadata_query_statements.py b/tests/metadata/test_metadata_query_statements.py
index 357cc8bc8..e2ecc985d 100644
--- a/tests/metadata/test_metadata_query_statements.py
+++ b/tests/metadata/test_metadata_query_statements.py
@@ -17,6 +17,7 @@
# Impala tests for queries that query metadata and set session settings
+from __future__ import absolute_import, division, print_function
import pytest
import re
diff --git a/tests/metadata/test_partition_metadata.py b/tests/metadata/test_partition_metadata.py
index 0b922e38b..0f571fc1a 100644
--- a/tests/metadata/test_partition_metadata.py
+++ b/tests/metadata/test_partition_metadata.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import pytest
from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.skip import SkipIfFS, SkipIfLocal
diff --git a/tests/metadata/test_recover_partitions.py b/tests/metadata/test_recover_partitions.py
index 40b6343d0..131c31155 100644
--- a/tests/metadata/test_recover_partitions.py
+++ b/tests/metadata/test_recover_partitions.py
@@ -17,6 +17,7 @@
#
# Impala tests for ALTER TABLE RECOVER PARTITIONS statement
+from __future__ import absolute_import, division, print_function
import os
from six.moves import urllib
from tests.common.impala_test_suite import ImpalaTestSuite
diff --git a/tests/metadata/test_recursive_listing.py b/tests/metadata/test_recursive_listing.py
index eb4d0e86f..9273387e4 100644
--- a/tests/metadata/test_recursive_listing.py
+++ b/tests/metadata/test_recursive_listing.py
@@ -10,6 +10,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+from __future__ import absolute_import, division, print_function
import pytest
import requests
import time
diff --git a/tests/metadata/test_refresh_partition.py b/tests/metadata/test_refresh_partition.py
index 984f96ebb..b3abbbe5f 100644
--- a/tests/metadata/test_refresh_partition.py
+++ b/tests/metadata/test_refresh_partition.py
@@ -11,6 +11,7 @@
# limitations under the License.
+from __future__ import absolute_import, division, print_function
from subprocess import check_call
from tests.beeswax.impala_beeswax import ImpalaBeeswaxException
diff --git a/tests/metadata/test_reset_metadata.py b/tests/metadata/test_reset_metadata.py
index 53e4ad538..6a02f003b 100644
--- a/tests/metadata/test_reset_metadata.py
+++ b/tests/metadata/test_reset_metadata.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
from test_ddl_base import TestDdlBase
diff --git a/tests/metadata/test_reuse_partitions.py b/tests/metadata/test_reuse_partitions.py
index 72bd0d8b6..24a3e36a4 100644
--- a/tests/metadata/test_reuse_partitions.py
+++ b/tests/metadata/test_reuse_partitions.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import json
import requests
from tests.common.impala_test_suite import ImpalaTestSuite
diff --git a/tests/metadata/test_set.py b/tests/metadata/test_set.py
index a1b3276e5..bbfa71f3a 100644
--- a/tests/metadata/test_set.py
+++ b/tests/metadata/test_set.py
@@ -17,6 +17,7 @@
#
# Tests for SET <query option>
+from __future__ import absolute_import, division, print_function
from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.test_dimensions import (
create_single_exec_option_dimension,
diff --git a/tests/metadata/test_show_create_table.py b/tests/metadata/test_show_create_table.py
index f1d44627e..d0f264739 100644
--- a/tests/metadata/test_show_create_table.py
+++ b/tests/metadata/test_show_create_table.py
@@ -14,6 +14,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import pprint
import pytest
import re
diff --git a/tests/metadata/test_stale_metadata.py b/tests/metadata/test_stale_metadata.py
index 9611939fd..d58a5fc22 100644
--- a/tests/metadata/test_stale_metadata.py
+++ b/tests/metadata/test_stale_metadata.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
from subprocess import check_call
from tests.beeswax.impala_beeswax import ImpalaBeeswaxException
diff --git a/tests/metadata/test_stats_extrapolation.py b/tests/metadata/test_stats_extrapolation.py
index 605bf663c..00aa721ad 100644
--- a/tests/metadata/test_stats_extrapolation.py
+++ b/tests/metadata/test_stats_extrapolation.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
from os import path
from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.skip import SkipIfEC
diff --git a/tests/metadata/test_testcase_builder.py b/tests/metadata/test_testcase_builder.py
index 30a271404..a3d5d38a6 100644
--- a/tests/metadata/test_testcase_builder.py
+++ b/tests/metadata/test_testcase_builder.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.test_dimensions import (
create_single_exec_option_dimension,
diff --git a/tests/metadata/test_views_compatibility.py b/tests/metadata/test_views_compatibility.py
index 7daedc711..8d19fceb1 100644
--- a/tests/metadata/test_views_compatibility.py
+++ b/tests/metadata/test_views_compatibility.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import pprint
import pytest
import shlex
diff --git a/tests/observability/test_jvm_metrics.py b/tests/observability/test_jvm_metrics.py
index 80deb8134..9977d1130 100644
--- a/tests/observability/test_jvm_metrics.py
+++ b/tests/observability/test_jvm_metrics.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
from tests.common.impala_test_suite import ImpalaTestSuite
diff --git a/tests/observability/test_log_fragments.py b/tests/observability/test_log_fragments.py
index f81d79edd..f1172efad 100644
--- a/tests/observability/test_log_fragments.py
+++ b/tests/observability/test_log_fragments.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.skip import SkipIfDockerizedCluster
diff --git a/tests/observability/test_profile_tool.py b/tests/observability/test_profile_tool.py
index 42ad87fb6..c370e26c0 100644
--- a/tests/observability/test_profile_tool.py
+++ b/tests/observability/test_profile_tool.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import os.path
import tempfile
from subprocess import check_call
diff --git a/tests/performance/query.py b/tests/performance/query.py
index 5884a015b..5832e5dfa 100644
--- a/tests/performance/query.py
+++ b/tests/performance/query.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
from tests.util.test_file_parser import QueryTestSectionReader
# TODO: This interface needs to be more robust; At the moment, it has two users with
diff --git a/tests/performance/query_exec_functions.py b/tests/performance/query_exec_functions.py
index 2bd2953f1..471a1ea61 100644
--- a/tests/performance/query_exec_functions.py
+++ b/tests/performance/query_exec_functions.py
@@ -16,6 +16,7 @@
# under the License.
#
+from __future__ import absolute_import, division, print_function
import logging
import re
from datetime import datetime
diff --git a/tests/performance/query_executor.py b/tests/performance/query_executor.py
index 344ebbfc9..bb47a40cb 100644
--- a/tests/performance/query_executor.py
+++ b/tests/performance/query_executor.py
@@ -29,6 +29,7 @@
# executor.run()
# result = executor.result
+from __future__ import absolute_import, division, print_function
import logging
import os
import re
diff --git a/tests/performance/scheduler.py b/tests/performance/scheduler.py
index e61245474..95d2a7a13 100644
--- a/tests/performance/scheduler.py
+++ b/tests/performance/scheduler.py
@@ -19,6 +19,7 @@
# defined as a set of queries for a given data set, scale factor and a specific test
# vector. It treats a workload an the unit of parallelism.
+from __future__ import absolute_import, division, print_function
import logging
from collections import defaultdict
diff --git a/tests/performance/workload.py b/tests/performance/workload.py
index e6d0a6e88..1fc47b2e1 100644
--- a/tests/performance/workload.py
+++ b/tests/performance/workload.py
@@ -16,6 +16,7 @@
# under the License.
#
+from __future__ import absolute_import, division, print_function
import os
import fnmatch
import re
diff --git a/tests/performance/workload_runner.py b/tests/performance/workload_runner.py
index 24b353f72..f77f730f5 100644
--- a/tests/performance/workload_runner.py
+++ b/tests/performance/workload_runner.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import logging
from tests.common.test_dimensions import (
diff --git a/tests/query_test/test_acid.py b/tests/query_test/test_acid.py
index 2b09d830a..53d52270a 100644
--- a/tests/query_test/test_acid.py
+++ b/tests/query_test/test_acid.py
@@ -17,6 +17,7 @@
# Functional tests for ACID integration with Hive.
+from __future__ import absolute_import, division, print_function
import os
import pytest
import time
diff --git a/tests/query_test/test_acid_row_validation.py b/tests/query_test/test_acid_row_validation.py
index 041e75450..21880391f 100644
--- a/tests/query_test/test_acid_row_validation.py
+++ b/tests/query_test/test_acid_row_validation.py
@@ -17,6 +17,7 @@
# Functional tests for ACID integration with Hive.
+from __future__ import absolute_import, division, print_function
import os
import pytest
diff --git a/tests/query_test/test_aggregation.py b/tests/query_test/test_aggregation.py
index 165e595ce..cf399e20d 100644
--- a/tests/query_test/test_aggregation.py
+++ b/tests/query_test/test_aggregation.py
@@ -17,7 +17,7 @@
# Validates all aggregate functions across all datatypes
#
-from __future__ import print_function
+from __future__ import absolute_import, division, print_function
import pytest
from testdata.common import widetable
diff --git a/tests/query_test/test_analytic_tpcds.py b/tests/query_test/test_analytic_tpcds.py
index 57042e815..1cdc7f861 100644
--- a/tests/query_test/test_analytic_tpcds.py
+++ b/tests/query_test/test_analytic_tpcds.py
@@ -17,6 +17,7 @@
#
# Targeted tests to validate analytic functions use TPCDS dataset.
+from __future__ import absolute_import, division, print_function
from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.test_dimensions import create_parquet_dimension, ImpalaTestDimension
diff --git a/tests/query_test/test_async_codegen.py b/tests/query_test/test_async_codegen.py
index ae7e43fb4..acc073f24 100644
--- a/tests/query_test/test_async_codegen.py
+++ b/tests/query_test/test_async_codegen.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
#
+from __future__ import absolute_import, division, print_function
import pytest
from tests.common.impala_test_suite import ImpalaTestSuite
diff --git a/tests/query_test/test_avro_schema_resolution.py b/tests/query_test/test_avro_schema_resolution.py
index 52feb8a55..b6ed92bff 100644
--- a/tests/query_test/test_avro_schema_resolution.py
+++ b/tests/query_test/test_avro_schema_resolution.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.skip import SkipIfCatalogV2
diff --git a/tests/query_test/test_beeswax.py b/tests/query_test/test_beeswax.py
index 2887d1c23..59a9ad5bc 100644
--- a/tests/query_test/test_beeswax.py
+++ b/tests/query_test/test_beeswax.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
from tests.beeswax.impala_beeswax import ImpalaBeeswaxException
from tests.common.impala_test_suite import ImpalaTestSuite
diff --git a/tests/query_test/test_cancellation.py b/tests/query_test/test_cancellation.py
index b12725c17..784c8ef24 100644
--- a/tests/query_test/test_cancellation.py
+++ b/tests/query_test/test_cancellation.py
@@ -18,6 +18,7 @@
# Tests query cancellation using the ImpalaService.Cancel API
#
+from __future__ import absolute_import, division, print_function
import pytest
import threading
from time import sleep
diff --git a/tests/query_test/test_cast_with_format.py b/tests/query_test/test_cast_with_format.py
index 2e5127661..e73a6f9cf 100644
--- a/tests/query_test/test_cast_with_format.py
+++ b/tests/query_test/test_cast_with_format.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.test_dimensions import create_client_protocol_dimension
diff --git a/tests/query_test/test_chars.py b/tests/query_test/test_chars.py
index de9ab2cf6..e4cf8ea9f 100644
--- a/tests/query_test/test_chars.py
+++ b/tests/query_test/test_chars.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import pytest
from tests.common.impala_test_suite import ImpalaTestSuite
diff --git a/tests/query_test/test_codegen.py b/tests/query_test/test_codegen.py
index 18597aa64..81a56b3b0 100644
--- a/tests/query_test/test_codegen.py
+++ b/tests/query_test/test_codegen.py
@@ -17,6 +17,7 @@
# Tests end-to-end codegen behaviour.
+from __future__ import absolute_import, division, print_function
from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.skip import SkipIf
from tests.common.test_dimensions import create_exec_option_dimension_from_dict
diff --git a/tests/query_test/test_compressed_formats.py b/tests/query_test/test_compressed_formats.py
index e9187445d..894f237f0 100644
--- a/tests/query_test/test_compressed_formats.py
+++ b/tests/query_test/test_compressed_formats.py
@@ -15,7 +15,7 @@
# specific language governing permissions and limitations
# under the License.
-from __future__ import print_function
+from __future__ import absolute_import, division, print_function
import math
import os
import pytest
@@ -259,7 +259,7 @@ class TestLargeCompressedFile(ImpalaTestSuite):
# Number of nested structures described above.
num_chunks = int(math.ceil(file_size / self.CHUNK_SIZE))
# Number of compressed snappy blocks per chunk.
- num_blocks_per_chunk = self.CHUNK_SIZE / (compressed_size + 4)
+ num_blocks_per_chunk = self.CHUNK_SIZE // (compressed_size + 4)
# Total uncompressed size of a nested structure.
total_chunk_size = num_blocks_per_chunk * payload_size
diff --git a/tests/query_test/test_datasketches.py b/tests/query_test/test_datasketches.py
index 89e2da2c5..b8925133d 100644
--- a/tests/query_test/test_datasketches.py
+++ b/tests/query_test/test_datasketches.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
from tests.common.file_utils import create_table_from_parquet
from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.test_dimensions import create_single_exec_option_dimension
diff --git a/tests/query_test/test_datastream_sender.py b/tests/query_test/test_datastream_sender.py
index c48798884..b5c266a35 100644
--- a/tests/query_test/test_datastream_sender.py
+++ b/tests/query_test/test_datastream_sender.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
from tests.common.impala_test_suite import ImpalaTestSuite
diff --git a/tests/query_test/test_date_queries.py b/tests/query_test/test_date_queries.py
index 56e37aeb2..ad51e4b03 100644
--- a/tests/query_test/test_date_queries.py
+++ b/tests/query_test/test_date_queries.py
@@ -17,6 +17,7 @@
# Targeted tests for date type.
+from __future__ import absolute_import, division, print_function
import pytest
from tests.common.file_utils import create_table_and_copy_files
from tests.common.impala_test_suite import ImpalaTestSuite
diff --git a/tests/query_test/test_decimal_casting.py b/tests/query_test/test_decimal_casting.py
index f487e8bc3..8c4ea5912 100644
--- a/tests/query_test/test_decimal_casting.py
+++ b/tests/query_test/test_decimal_casting.py
@@ -17,6 +17,7 @@
#
# Validates that casting to Decimal works.
#
+from __future__ import absolute_import, division, print_function
import pytest
from decimal import Decimal, getcontext, ROUND_DOWN, ROUND_HALF_UP
from allpairspy import AllPairs as all_pairs
diff --git a/tests/query_test/test_decimal_fuzz.py b/tests/query_test/test_decimal_fuzz.py
index b0fb57253..1f468960c 100644
--- a/tests/query_test/test_decimal_fuzz.py
+++ b/tests/query_test/test_decimal_fuzz.py
@@ -18,6 +18,7 @@
# Generates random decimal numbers and verifies that mathematical
# operations return correct results under decimal_v2.
+from __future__ import absolute_import, division, print_function
import decimal
import math
import pytest
@@ -263,7 +264,7 @@ class TestDecimalFuzz(ImpalaTestSuite):
range_size = max_range_int - min_range_int
dist_from_min = val_int - min_range_int
- return (num_buckets * dist_from_min) / range_size + 1
+ return (num_buckets * dist_from_min) // range_size + 1
def execute_one_width_bucket(self):
val, val_prec, val_scale = self.get_decimal()
diff --git a/tests/query_test/test_decimal_queries.py b/tests/query_test/test_decimal_queries.py
index a361eef59..7ed102275 100644
--- a/tests/query_test/test_decimal_queries.py
+++ b/tests/query_test/test_decimal_queries.py
@@ -17,6 +17,7 @@
# Targeted tests for decimal type.
+from __future__ import absolute_import, division, print_function
from copy import copy
import pytest
diff --git a/tests/query_test/test_delimited_text.py b/tests/query_test/test_delimited_text.py
index 779397c7b..34bd93e04 100644
--- a/tests/query_test/test_delimited_text.py
+++ b/tests/query_test/test_delimited_text.py
@@ -19,6 +19,7 @@
# and escape characters.
#
+from __future__ import absolute_import, division, print_function
from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.test_dimensions import (
create_single_exec_option_dimension,
diff --git a/tests/query_test/test_errorlog.py b/tests/query_test/test_errorlog.py
index 258dba020..0cdaa2f06 100644
--- a/tests/query_test/test_errorlog.py
+++ b/tests/query_test/test_errorlog.py
@@ -18,6 +18,7 @@
# Injects failures at specific locations in each of the plan nodes. Currently supports
# two types of failures - cancellation of the query and a failure test hook.
#
+from __future__ import absolute_import, division, print_function
from tests.beeswax.impala_beeswax import ImpalaBeeswaxException
from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.test_dimensions import (
diff --git a/tests/query_test/test_exprs.py b/tests/query_test/test_exprs.py
index 03e6a3b15..f396841d0 100644
--- a/tests/query_test/test_exprs.py
+++ b/tests/query_test/test_exprs.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import pytest
import re
from random import randint
@@ -110,7 +111,7 @@ class TestExprLimits(ImpalaTestSuite):
# CASE expr
case_query = "select case "
- for i in xrange(0, self.EXPR_CHILDREN_LIMIT/2):
+ for i in xrange(0, self.EXPR_CHILDREN_LIMIT // 2):
case_query += " when true then 1"
case_query += " end"
self.__exec_query(case_query)
diff --git a/tests/query_test/test_fetch.py b/tests/query_test/test_fetch.py
index 361d8ea9f..2b86a77fc 100644
--- a/tests/query_test/test_fetch.py
+++ b/tests/query_test/test_fetch.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import re
from time import sleep
diff --git a/tests/query_test/test_geospatial_functions.py b/tests/query_test/test_geospatial_functions.py
index 2d8108d9a..2b34e8f1e 100644
--- a/tests/query_test/test_geospatial_functions.py
+++ b/tests/query_test/test_geospatial_functions.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.skip import SkipIfApacheHive
diff --git a/tests/query_test/test_hash_join_timer.py b/tests/query_test/test_hash_join_timer.py
index a6e408e54..5a340d0a9 100644
--- a/tests/query_test/test_hash_join_timer.py
+++ b/tests/query_test/test_hash_join_timer.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import pytest
import re
@@ -118,9 +119,9 @@ class TestHashJoinTimer(ImpalaTestSuite):
join_node_name = "03:%s" % (join_type)
for line in exec_summary:
if line['operator'] == join_node_name:
- avg_time_ms = line['avg_time'] / self.NANOS_PER_MILLI
+ avg_time_ms = line['avg_time'] // self.NANOS_PER_MILLI
self.__verify_join_time(avg_time_ms, "ExecSummary Avg")
- max_time_ms = line['max_time'] / self.NANOS_PER_MILLI
+ max_time_ms = line['max_time'] // self.NANOS_PER_MILLI
self.__verify_join_time(max_time_ms, "ExecSummary Max")
check_execsummary_count += 1
assert (check_execsummary_count == 1), \
diff --git a/tests/query_test/test_hbase_queries.py b/tests/query_test/test_hbase_queries.py
index a1acbf33b..15e715dae 100644
--- a/tests/query_test/test_hbase_queries.py
+++ b/tests/query_test/test_hbase_queries.py
@@ -17,6 +17,7 @@
# Targeted Impala HBase Tests
+from __future__ import absolute_import, division, print_function
import pytest
from tests.common.impala_test_suite import ImpalaTestSuite
diff --git a/tests/query_test/test_hdfs_caching.py b/tests/query_test/test_hdfs_caching.py
index 32f91702a..105895af2 100644
--- a/tests/query_test/test_hdfs_caching.py
+++ b/tests/query_test/test_hdfs_caching.py
@@ -17,7 +17,7 @@
# Validates limit on scan nodes
-from __future__ import print_function
+from __future__ import absolute_import, division, print_function
import pytest
import re
import time
diff --git a/tests/query_test/test_hdfs_file_mods.py b/tests/query_test/test_hdfs_file_mods.py
index 48e899521..38e9a5ca2 100644
--- a/tests/query_test/test_hdfs_file_mods.py
+++ b/tests/query_test/test_hdfs_file_mods.py
@@ -16,6 +16,7 @@
# under the License.
+from __future__ import absolute_import, division, print_function
import pytest
from tests.beeswax.impala_beeswax import ImpalaBeeswaxException
diff --git a/tests/query_test/test_iceberg.py b/tests/query_test/test_iceberg.py
index 6f1938c0d..76491ce65 100644
--- a/tests/query_test/test_iceberg.py
+++ b/tests/query_test/test_iceberg.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import datetime
import logging
import os
diff --git a/tests/query_test/test_insert.py b/tests/query_test/test_insert.py
index 05a40401f..05e3aa998 100644
--- a/tests/query_test/test_insert.py
+++ b/tests/query_test/test_insert.py
@@ -17,6 +17,7 @@
# Targeted Impala insert tests
+from __future__ import absolute_import, division, print_function
import os
import pytest
import re
diff --git a/tests/query_test/test_insert_behaviour.py b/tests/query_test/test_insert_behaviour.py
index 5b20759c0..f4b269c32 100644
--- a/tests/query_test/test_insert_behaviour.py
+++ b/tests/query_test/test_insert_behaviour.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import getpass
import grp
import os
diff --git a/tests/query_test/test_insert_parquet.py b/tests/query_test/test_insert_parquet.py
index 6113664f0..33fc83af5 100644
--- a/tests/query_test/test_insert_parquet.py
+++ b/tests/query_test/test_insert_parquet.py
@@ -17,6 +17,7 @@
# Targeted Impala insert tests
+from __future__ import absolute_import, division, print_function
import os
from collections import namedtuple
diff --git a/tests/query_test/test_insert_permutation.py b/tests/query_test/test_insert_permutation.py
index 46d10903a..a3973a40e 100644
--- a/tests/query_test/test_insert_permutation.py
+++ b/tests/query_test/test_insert_permutation.py
@@ -17,6 +17,7 @@
# Targeted Impala insert tests
+from __future__ import absolute_import, division, print_function
from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.test_dimensions import (
create_exec_option_dimension,
diff --git a/tests/query_test/test_invalid_test_header.py b/tests/query_test/test_invalid_test_header.py
index 6b7b3bd49..886972e04 100644
--- a/tests/query_test/test_invalid_test_header.py
+++ b/tests/query_test/test_invalid_test_header.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import pytest
from tests.common.impala_test_suite import ImpalaTestSuite
diff --git a/tests/query_test/test_io_metrics.py b/tests/query_test/test_io_metrics.py
index 5388f85b6..621d6f418 100644
--- a/tests/query_test/test_io_metrics.py
+++ b/tests/query_test/test_io_metrics.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import pytest
from tests.common.environ import IS_DOCKERIZED_TEST_CLUSTER
diff --git a/tests/query_test/test_join_queries.py b/tests/query_test/test_join_queries.py
index 8961b6391..48abae1ad 100644
--- a/tests/query_test/test_join_queries.py
+++ b/tests/query_test/test_join_queries.py
@@ -17,6 +17,7 @@
# Targeted tests for Impala joins
#
+from __future__ import absolute_import, division, print_function
import pytest
from copy import deepcopy
diff --git a/tests/query_test/test_kudu.py b/tests/query_test/test_kudu.py
index 0f1c59a17..d637b3c35 100644
--- a/tests/query_test/test_kudu.py
+++ b/tests/query_test/test_kudu.py
@@ -15,7 +15,7 @@
# specific language governing permissions and limitations
# under the License.
-from __future__ import print_function
+from __future__ import absolute_import, division, print_function
from kudu.schema import (
BOOL,
DOUBLE,
diff --git a/tests/query_test/test_lifecycle.py b/tests/query_test/test_lifecycle.py
index 30b12c067..a14cf1e79 100644
--- a/tests/query_test/test_lifecycle.py
+++ b/tests/query_test/test_lifecycle.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import pytest
import time
from tests.beeswax.impala_beeswax import ImpalaBeeswaxException
diff --git a/tests/query_test/test_limit.py b/tests/query_test/test_limit.py
index 5233b4682..8ff09e462 100644
--- a/tests/query_test/test_limit.py
+++ b/tests/query_test/test_limit.py
@@ -17,7 +17,7 @@
# Validates limit on scan nodes
-from __future__ import print_function
+from __future__ import absolute_import, division, print_function
from tests.beeswax.impala_beeswax import ImpalaBeeswaxException
from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.test_vector import ImpalaTestDimension
diff --git a/tests/query_test/test_limit_pushdown_analytic.py b/tests/query_test/test_limit_pushdown_analytic.py
index caac68b5a..d1295eea8 100644
--- a/tests/query_test/test_limit_pushdown_analytic.py
+++ b/tests/query_test/test_limit_pushdown_analytic.py
@@ -18,6 +18,7 @@
# Test the limit pushdown to analytic sort in the presence
# of ranking functions
+from __future__ import absolute_import, division, print_function
from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.test_dimensions import (create_single_exec_option_dimension,
extend_exec_option_dimension)
diff --git a/tests/query_test/test_local_fs.py b/tests/query_test/test_local_fs.py
index dd891dbbb..a6468fbbd 100644
--- a/tests/query_test/test_local_fs.py
+++ b/tests/query_test/test_local_fs.py
@@ -17,6 +17,7 @@
# Validates table stored on the LocalFileSystem.
#
+from __future__ import absolute_import, division, print_function
from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.skip import SkipIfDockerizedCluster
from tests.common.test_dimensions import create_single_exec_option_dimension
diff --git a/tests/query_test/test_mem_usage_scaling.py b/tests/query_test/test_mem_usage_scaling.py
index 8910aa336..8660f9433 100644
--- a/tests/query_test/test_mem_usage_scaling.py
+++ b/tests/query_test/test_mem_usage_scaling.py
@@ -15,6 +15,8 @@
# specific language governing permissions and limitations
# under the License.
#
+
+from __future__ import absolute_import, division, print_function
import pytest
from copy import copy
diff --git a/tests/query_test/test_mt_dop.py b/tests/query_test/test_mt_dop.py
index 58483c9ec..94d54c90a 100644
--- a/tests/query_test/test_mt_dop.py
+++ b/tests/query_test/test_mt_dop.py
@@ -17,6 +17,7 @@
# Tests queries with the MT_DOP query option.
+from __future__ import absolute_import, division, print_function
import pytest
import logging
diff --git a/tests/query_test/test_multiple_filesystems.py b/tests/query_test/test_multiple_filesystems.py
index 91abe53a2..628baa79a 100644
--- a/tests/query_test/test_multiple_filesystems.py
+++ b/tests/query_test/test_multiple_filesystems.py
@@ -17,6 +17,7 @@
# Validates table stored on the LocalFileSystem.
#
+from __future__ import absolute_import, division, print_function
import pytest
from subprocess import check_call, call
diff --git a/tests/query_test/test_nested_types.py b/tests/query_test/test_nested_types.py
index 99d44e537..e92cf49bd 100644
--- a/tests/query_test/test_nested_types.py
+++ b/tests/query_test/test_nested_types.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import os
from copy import deepcopy
import pytest
diff --git a/tests/query_test/test_observability.py b/tests/query_test/test_observability.py
index 51124bb61..a7f42058f 100644
--- a/tests/query_test/test_observability.py
+++ b/tests/query_test/test_observability.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
from collections import defaultdict
from datetime import datetime
from tests.beeswax.impala_beeswax import ImpalaBeeswaxException
diff --git a/tests/query_test/test_orc_stats.py b/tests/query_test/test_orc_stats.py
index 25cb10616..3237ac8d0 100644
--- a/tests/query_test/test_orc_stats.py
+++ b/tests/query_test/test_orc_stats.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
from tests.common.impala_test_suite import ImpalaTestSuite
diff --git a/tests/query_test/test_parquet_bloom_filter.py b/tests/query_test/test_parquet_bloom_filter.py
index 11af6d28d..902d7e769 100644
--- a/tests/query_test/test_parquet_bloom_filter.py
+++ b/tests/query_test/test_parquet_bloom_filter.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import math
import os
diff --git a/tests/query_test/test_parquet_late_materialization.py b/tests/query_test/test_parquet_late_materialization.py
index 670daa4d6..aa1fb3b13 100644
--- a/tests/query_test/test_parquet_late_materialization.py
+++ b/tests/query_test/test_parquet_late_materialization.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
from tests.common.impala_test_suite import ImpalaTestSuite
diff --git a/tests/query_test/test_parquet_page_index.py b/tests/query_test/test_parquet_page_index.py
index cd8127767..8bf26a074 100644
--- a/tests/query_test/test_parquet_page_index.py
+++ b/tests/query_test/test_parquet_page_index.py
@@ -17,6 +17,7 @@
# Targeted Impala insert tests
+from __future__ import absolute_import, division, print_function
import os
from collections import namedtuple
diff --git a/tests/query_test/test_parquet_stats.py b/tests/query_test/test_parquet_stats.py
index dd3ce678f..41b2a8349 100644
--- a/tests/query_test/test_parquet_stats.py
+++ b/tests/query_test/test_parquet_stats.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import os
import pytest
import shlex
diff --git a/tests/query_test/test_partitioning.py b/tests/query_test/test_partitioning.py
index 649d785fc..1ff0d23cf 100644
--- a/tests/query_test/test_partitioning.py
+++ b/tests/query_test/test_partitioning.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import pytest
from tests.beeswax.impala_beeswax import ImpalaBeeswaxException
diff --git a/tests/query_test/test_queries.py b/tests/query_test/test_queries.py
index dbde99a2f..2b9ee01bf 100644
--- a/tests/query_test/test_queries.py
+++ b/tests/query_test/test_queries.py
@@ -17,6 +17,7 @@
# General Impala query tests
+from __future__ import absolute_import, division, print_function
import pytest
import re
from copy import deepcopy
diff --git a/tests/query_test/test_query_compilation.py b/tests/query_test/test_query_compilation.py
index 1d4c9caff..f880d37c1 100644
--- a/tests/query_test/test_query_compilation.py
+++ b/tests/query_test/test_query_compilation.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import pytest
from tests.common.impala_test_suite import ImpalaTestSuite
diff --git a/tests/query_test/test_query_mem_limit.py b/tests/query_test/test_query_mem_limit.py
index 209fca84f..bd3615c35 100644
--- a/tests/query_test/test_query_mem_limit.py
+++ b/tests/query_test/test_query_mem_limit.py
@@ -17,6 +17,7 @@
#
# Targeted tests to validate per-query memory limit.
+from __future__ import absolute_import, division, print_function
import pytest
import re
import sys
@@ -46,8 +47,8 @@ class TestQueryMemLimit(ImpalaTestSuite):
# A mem_limit is expressed in bytes, with values <= 0 signifying no cap.
# These values are either really small, unlimited, or have a really large cap.
MAXINT_BYTES = str(sys.maxint)
- MAXINT_MB = str(sys.maxint/(1024*1024))
- MAXINT_GB = str(sys.maxint/(1024*1024*1024))
+ MAXINT_MB = str(sys.maxint // (1024 * 1024))
+ MAXINT_GB = str(sys.maxint // (1024 * 1024 * 1024))
# We expect the tests with MAXINT_* using valid units [bmg] to succeed.
PASS_REGEX = re.compile("(%s|%s|%s)[bmg]?$" % (MAXINT_BYTES, MAXINT_MB, MAXINT_GB),
re.I)
diff --git a/tests/query_test/test_query_opts.py b/tests/query_test/test_query_opts.py
index f9ef1d573..93f925d40 100644
--- a/tests/query_test/test_query_opts.py
+++ b/tests/query_test/test_query_opts.py
@@ -20,6 +20,7 @@
# to make it easier to handle startup failures (right now it waits 60sec to
# timeout).
+from __future__ import absolute_import, division, print_function
from TCLIService import TCLIService
from tests.common.impala_test_suite import ImpalaTestSuite
diff --git a/tests/query_test/test_resource_limits.py b/tests/query_test/test_resource_limits.py
index 7644193c3..9f95b4eb8 100644
--- a/tests/query_test/test_resource_limits.py
+++ b/tests/query_test/test_resource_limits.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.skip import SkipIfEC, SkipIfLocal, SkipIfFS
from tests.common.test_dimensions import create_parquet_dimension
diff --git a/tests/query_test/test_result_spooling.py b/tests/query_test/test_result_spooling.py
index a2db1bca8..9883cd94f 100644
--- a/tests/query_test/test_result_spooling.py
+++ b/tests/query_test/test_result_spooling.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import pytest
import re
import time
diff --git a/tests/query_test/test_rows_availability.py b/tests/query_test/test_rows_availability.py
index 0bd65df8b..357929894 100644
--- a/tests/query_test/test_rows_availability.py
+++ b/tests/query_test/test_rows_availability.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import pytest
import re
from tests.common.impala_test_suite import ImpalaTestSuite
diff --git a/tests/query_test/test_runtime_filters.py b/tests/query_test/test_runtime_filters.py
index 2e91c1cf9..4be4305af 100644
--- a/tests/query_test/test_runtime_filters.py
+++ b/tests/query_test/test_runtime_filters.py
@@ -16,6 +16,7 @@
# under the License.
#
+from __future__ import absolute_import, division, print_function
from copy import deepcopy
import os
import pytest
diff --git a/tests/query_test/test_scanners.py b/tests/query_test/test_scanners.py
index 566442542..be372018b 100644
--- a/tests/query_test/test_scanners.py
+++ b/tests/query_test/test_scanners.py
@@ -21,7 +21,7 @@
# tests can run with the normal exploration strategy and the overall test runtime doesn't
# explode.
-from __future__ import print_function
+from __future__ import absolute_import, division, print_function
import os
import pytest
import random
diff --git a/tests/query_test/test_scanners_fuzz.py b/tests/query_test/test_scanners_fuzz.py
index 0576132c2..ec411abc1 100644
--- a/tests/query_test/test_scanners_fuzz.py
+++ b/tests/query_test/test_scanners_fuzz.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
from copy import copy
import itertools
import logging
diff --git a/tests/query_test/test_scratch_limit.py b/tests/query_test/test_scratch_limit.py
index 8779cd31b..2620dc0ea 100644
--- a/tests/query_test/test_scratch_limit.py
+++ b/tests/query_test/test_scratch_limit.py
@@ -10,6 +10,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+from __future__ import absolute_import, division, print_function
from tests.beeswax.impala_beeswax import ImpalaBeeswaxException
from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.test_dimensions import create_single_exec_option_dimension
diff --git a/tests/query_test/test_sfs.py b/tests/query_test/test_sfs.py
index 5db318e21..4a1ee1caf 100644
--- a/tests/query_test/test_sfs.py
+++ b/tests/query_test/test_sfs.py
@@ -18,6 +18,7 @@
# This test suite validates the functionality that allows users to create an external
# table associated with a single file.
+from __future__ import absolute_import, division, print_function
from tests.common.file_utils import copy_files_to_hdfs_dir
from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.skip import SkipIf
diff --git a/tests/query_test/test_sort.py b/tests/query_test/test_sort.py
index 2644c4c76..bcb76a766 100644
--- a/tests/query_test/test_sort.py
+++ b/tests/query_test/test_sort.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import re
from copy import copy, deepcopy
diff --git a/tests/query_test/test_spilling.py b/tests/query_test/test_spilling.py
index 3c0c04d3d..ea49fb47f 100644
--- a/tests/query_test/test_spilling.py
+++ b/tests/query_test/test_spilling.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import pytest
from copy import deepcopy
diff --git a/tests/query_test/test_tablesample.py b/tests/query_test/test_tablesample.py
index 865238205..08e73ccb7 100644
--- a/tests/query_test/test_tablesample.py
+++ b/tests/query_test/test_tablesample.py
@@ -17,6 +17,7 @@
# Tests the TABLESAMPLE clause.
+from __future__ import absolute_import, division, print_function
import pytest
import subprocess
diff --git a/tests/query_test/test_tpcds_queries.py b/tests/query_test/test_tpcds_queries.py
index a17276da4..bd497c9bd 100644
--- a/tests/query_test/test_tpcds_queries.py
+++ b/tests/query_test/test_tpcds_queries.py
@@ -17,6 +17,7 @@
# Functional tests running the TPC-DS workload
#
+from __future__ import absolute_import, division, print_function
import pytest
from tests.common.impala_test_suite import ImpalaTestSuite
diff --git a/tests/query_test/test_tpch_nested_queries.py b/tests/query_test/test_tpch_nested_queries.py
index 84ee2b58d..714852d94 100644
--- a/tests/query_test/test_tpch_nested_queries.py
+++ b/tests/query_test/test_tpch_nested_queries.py
@@ -17,6 +17,7 @@
# Functional tests running the TPCH workload.
+from __future__ import absolute_import, division, print_function
from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.test_dimensions import create_single_exec_option_dimension
diff --git a/tests/query_test/test_tpch_queries.py b/tests/query_test/test_tpch_queries.py
index 68a298400..593037d5b 100644
--- a/tests/query_test/test_tpch_queries.py
+++ b/tests/query_test/test_tpch_queries.py
@@ -16,6 +16,7 @@
# under the License.
# Functional tests running the TPCH workload.
+from __future__ import absolute_import, division, print_function
import pytest
from tests.common.impala_test_suite import ImpalaTestSuite
diff --git a/tests/query_test/test_udfs.py b/tests/query_test/test_udfs.py
index 5ac87c038..a7e93fcbe 100644
--- a/tests/query_test/test_udfs.py
+++ b/tests/query_test/test_udfs.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
from copy import copy
import os
import re
diff --git a/tests/query_test/test_utf8_strings.py b/tests/query_test/test_utf8_strings.py
index 3221eb8bc..8d583d884 100644
--- a/tests/query_test/test_utf8_strings.py
+++ b/tests/query_test/test_utf8_strings.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.test_dimensions import (create_exec_option_dimension,
create_client_protocol_dimension, hs2_parquet_constraint)
diff --git a/tests/run-tests.py b/tests/run-tests.py
index 7c1ae8559..1ec0f9722 100755
--- a/tests/run-tests.py
+++ b/tests/run-tests.py
@@ -22,7 +22,7 @@
# executing the remaining tests in parallel. To run only some of
# these, use --skip-serial, --skip-stress, or --skip-parallel.
# All additional command line options are passed to py.test.
-from __future__ import print_function
+from __future__ import absolute_import, division, print_function
from tests.common.impala_cluster import ImpalaCluster
from tests.common.impala_service import ImpaladService
from tests.conftest import configure_logging
diff --git a/tests/shell/test_cookie_util.py b/tests/shell/test_cookie_util.py
index 7e0cc54f3..5441ee334 100644
--- a/tests/shell/test_cookie_util.py
+++ b/tests/shell/test_cookie_util.py
@@ -18,6 +18,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import unittest
from datetime import datetime, timedelta
diff --git a/tests/shell/test_shell_client.py b/tests/shell/test_shell_client.py
index 1388b527f..949608613 100644
--- a/tests/shell/test_shell_client.py
+++ b/tests/shell/test_shell_client.py
@@ -18,12 +18,13 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
from shell.impala_client import ImpalaBeeswaxClient, ImpalaHS2Client
from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.test_dimensions import (
create_client_protocol_dimension, create_client_protocol_no_strict_dimension,
create_uncompressed_text_dimension, create_single_exec_option_dimension)
-from util import get_impalad_host_port
+from tests.shell.util import get_impalad_host_port
class TestShellClient(ImpalaTestSuite):
@@ -76,7 +77,7 @@ class TestShellClient(ImpalaTestSuite):
client.connect()
handle = client.execute_query(
"select * from functional.alltypes limit {0}".format(num_rows), query_options)
- self.__fetch_rows(client.fetch(handle), num_rows / fetch_size, num_rows)
+ self.__fetch_rows(client.fetch(handle), num_rows // fetch_size, num_rows)
finally:
if handle is not None: client.close_query(handle)
client.close_connection()
@@ -85,7 +86,7 @@ class TestShellClient(ImpalaTestSuite):
"""Fetches all rows using the given fetch_batches generator. Asserts that num_batches
batches are produced by the generator and that num_rows are returned."""
num_batches_count = 0
- rows_per_batch = num_rows / num_batches
+ rows_per_batch = num_rows // num_batches
for fetch_batch in fetch_batches:
assert len(fetch_batch) == rows_per_batch
num_batches_count += 1
diff --git a/tests/shell/test_shell_commandline.py b/tests/shell/test_shell_commandline.py
index 484f7b97a..f78fa332c 100644
--- a/tests/shell/test_shell_commandline.py
+++ b/tests/shell/test_shell_commandline.py
@@ -18,6 +18,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import errno
import getpass
import os
@@ -38,9 +39,9 @@ from tests.common.test_dimensions import (
create_client_protocol_dimension, create_client_protocol_strict_dimension,
create_uncompressed_text_dimension, create_single_exec_option_dimension)
from time import sleep, time
-from util import (get_impalad_host_port, assert_var_substitution, run_impala_shell_cmd,
- ImpalaShell, build_shell_env, wait_for_query_state,
- create_impala_shell_executable_dimension, get_impala_shell_executable)
+from tests.shell.util import (get_impalad_host_port, assert_var_substitution,
+ run_impala_shell_cmd, ImpalaShell, build_shell_env, wait_for_query_state,
+ create_impala_shell_executable_dimension, get_impala_shell_executable)
from contextlib import closing
diff --git a/tests/shell/test_shell_interactive.py b/tests/shell/test_shell_interactive.py
index 30ae99f16..f3469e907 100755
--- a/tests/shell/test_shell_interactive.py
+++ b/tests/shell/test_shell_interactive.py
@@ -18,7 +18,7 @@
# specific language governing permissions and limitations
# under the License.
-from __future__ import print_function
+from __future__ import absolute_import, division, print_function
import httplib
import logging
import os
@@ -44,9 +44,9 @@ from tests.common.skip import SkipIfLocal
from tests.common.test_dimensions import (
create_client_protocol_dimension, create_client_protocol_strict_dimension,
create_uncompressed_text_dimension, create_single_exec_option_dimension)
-from util import (assert_var_substitution, ImpalaShell, get_impalad_port, get_shell_cmd,
- get_open_sessions_metric, spawn_shell, get_unused_port,
- create_impala_shell_executable_dimension, get_impala_shell_executable)
+from tests.shell.util import (assert_var_substitution, ImpalaShell, get_impalad_port,
+ get_shell_cmd, get_open_sessions_metric, spawn_shell, get_unused_port,
+ create_impala_shell_executable_dimension, get_impala_shell_executable)
import SimpleHTTPServer
import SocketServer
diff --git a/tests/shell/util.py b/tests/shell/util.py
index 2097eabd3..fe30b83a7 100755
--- a/tests/shell/util.py
+++ b/tests/shell/util.py
@@ -18,7 +18,7 @@
# specific language governing permissions and limitations
# under the License.
-from __future__ import print_function
+from __future__ import absolute_import, division, print_function
import logging
import os
import socket
diff --git a/tests/statestore/test_statestore.py b/tests/statestore/test_statestore.py
index 5dfead822..55cbf9b93 100644
--- a/tests/statestore/test_statestore.py
+++ b/tests/statestore/test_statestore.py
@@ -15,7 +15,7 @@
# specific language governing permissions and limitations
# under the License.
-from __future__ import print_function
+from __future__ import absolute_import, division, print_function
from collections import defaultdict
import json
import logging
diff --git a/tests/stress/concurrent_select.py b/tests/stress/concurrent_select.py
index bb2d96829..7fa6f6d28 100755
--- a/tests/stress/concurrent_select.py
+++ b/tests/stress/concurrent_select.py
@@ -55,7 +55,7 @@
# 8) Verify the result set hash of successful queries if there are no DML queries in the
# current run.
-from __future__ import print_function
+from __future__ import absolute_import, division, print_function
import logging
import os
@@ -968,7 +968,7 @@ def populate_runtime_info(query, impala, converted_args, timeout_secs=maxint):
return
reports = reports_by_outcome[leading_outcome]
reports.sort(key=lambda r: r.runtime_secs)
- return reports[len(reports) / 2]
+ return reports[len(reports) // 2]
if not any((old_required_mem_mb_with_spilling, old_required_mem_mb_without_spilling)):
mem_estimate = estimate_query_mem_mb_usage(query, runner.impalad_conn)
@@ -1001,7 +1001,7 @@ def populate_runtime_info(query, impala, converted_args, timeout_secs=maxint):
mem_limit = old_required_mem_mb_without_spilling
old_required_mem_mb_without_spilling = None
else:
- mem_limit = (lower_bound + upper_bound) / 2
+ mem_limit = (lower_bound + upper_bound) // 2
LOG.info("Next mem_limit: {0}".format(mem_limit))
should_break = mem_limit / float(upper_bound) > 1 - mem_limit_eq_threshold_percent \
or upper_bound - mem_limit < mem_limit_eq_threshold_mb
@@ -1038,7 +1038,7 @@ def populate_runtime_info(query, impala, converted_args, timeout_secs=maxint):
mem_limit = old_required_mem_mb_with_spilling
old_required_mem_mb_with_spilling = None
else:
- mem_limit = (lower_bound + upper_bound) / 2
+ mem_limit = (lower_bound + upper_bound) // 2
LOG.info("Next mem_limit: {0}".format(mem_limit))
should_break = mem_limit / float(upper_bound) > 1 - mem_limit_eq_threshold_percent \
or upper_bound - mem_limit < mem_limit_eq_threshold_mb
diff --git a/tests/stress/extract_min_mem.py b/tests/stress/extract_min_mem.py
index 5749d164f..22301441c 100755
--- a/tests/stress/extract_min_mem.py
+++ b/tests/stress/extract_min_mem.py
@@ -33,7 +33,7 @@
#
# ./tests/stress/extract_min_mem.py mem_usage_scaling_runtime_info.json
#
-from __future__ import print_function
+from __future__ import absolute_import, division, print_function
import json
import sys
diff --git a/tests/stress/mem_broker.py b/tests/stress/mem_broker.py
index cd2c83fff..e844cbe05 100644
--- a/tests/stress/mem_broker.py
+++ b/tests/stress/mem_broker.py
@@ -17,6 +17,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import logging
import os
from contextlib import contextmanager
diff --git a/tests/stress/queries.py b/tests/stress/queries.py
index 62b703886..c4c01a783 100644
--- a/tests/stress/queries.py
+++ b/tests/stress/queries.py
@@ -20,6 +20,7 @@
# This module implements helpers for representing queries to be executed by the
# stress test, loading them and generating them.
+from __future__ import absolute_import, division, print_function
import logging
import os
from textwrap import dedent
diff --git a/tests/stress/query_retries_stress_runner.py b/tests/stress/query_retries_stress_runner.py
index f0876e1bc..d21496a83 100755
--- a/tests/stress/query_retries_stress_runner.py
+++ b/tests/stress/query_retries_stress_runner.py
@@ -24,6 +24,7 @@
# TODO: Make the script cancellable; more of a nice to have, but Ctrl+C does not kill
# the script, it has to be killed manually (e.g. kill [pid]).
+from __future__ import absolute_import, division, print_function
import logging
import pipes
import os
diff --git a/tests/stress/query_runner.py b/tests/stress/query_runner.py
index 524cf07ff..26a99fbca 100644
--- a/tests/stress/query_runner.py
+++ b/tests/stress/query_runner.py
@@ -17,6 +17,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import logging
from multiprocessing import Value
import os
diff --git a/tests/stress/runtime_info.py b/tests/stress/runtime_info.py
index d9ac7c757..a10692a22 100644
--- a/tests/stress/runtime_info.py
+++ b/tests/stress/runtime_info.py
@@ -20,7 +20,7 @@
# Utility functions used by the stress test to save and load runtime info
# about queries to and from JSON files.
-from __future__ import print_function
+from __future__ import absolute_import, division, print_function
from collections import defaultdict
import json
import logging
diff --git a/tests/stress/stress_util.py b/tests/stress/stress_util.py
index 7163515da..6bd8faa1c 100644
--- a/tests/stress/stress_util.py
+++ b/tests/stress/stress_util.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import time
import traceback
diff --git a/tests/stress/test_acid_stress.py b/tests/stress/test_acid_stress.py
index 6ba33d65f..49b94b2db 100644
--- a/tests/stress/test_acid_stress.py
+++ b/tests/stress/test_acid_stress.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import pytest
import random
import time
diff --git a/tests/stress/test_ddl_stress.py b/tests/stress/test_ddl_stress.py
index 16d71a6fe..09480b315 100644
--- a/tests/stress/test_ddl_stress.py
+++ b/tests/stress/test_ddl_stress.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import pytest
from tests.common.impala_test_suite import ImpalaTestSuite
diff --git a/tests/stress/test_insert_stress.py b/tests/stress/test_insert_stress.py
index 3b5af90d5..93eaf91d6 100644
--- a/tests/stress/test_insert_stress.py
+++ b/tests/stress/test_insert_stress.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import pytest
import random
import time
diff --git a/tests/stress/util.py b/tests/stress/util.py
index 00b581503..1b3d9fe42 100644
--- a/tests/stress/util.py
+++ b/tests/stress/util.py
@@ -19,7 +19,7 @@
# Utility functions used by the stress test that don't fit in the other modules.
-from __future__ import print_function
+from __future__ import absolute_import, division, print_function
import sys
import threading
diff --git a/tests/unittests/test_command.py b/tests/unittests/test_command.py
index a2a9e4c74..2ab58f92d 100644
--- a/tests/unittests/test_command.py
+++ b/tests/unittests/test_command.py
@@ -17,6 +17,7 @@
#
# Unit tests for collect_diagnostics.Command
+from __future__ import absolute_import, division, print_function
import os
import pytest
import sys
diff --git a/tests/unittests/test_file_parser.py b/tests/unittests/test_file_parser.py
index 5e79eae1e..e6f7a935e 100644
--- a/tests/unittests/test_file_parser.py
+++ b/tests/unittests/test_file_parser.py
@@ -17,7 +17,7 @@
# Unit tests for the test file parser
-from __future__ import print_function
+from __future__ import absolute_import, division, print_function
from tests.common.base_test_suite import BaseTestSuite
from tests.util.test_file_parser import parse_test_file_text
diff --git a/tests/unittests/test_result_verifier.py b/tests/unittests/test_result_verifier.py
index 94d2fd917..9142ca1a8 100644
--- a/tests/unittests/test_result_verifier.py
+++ b/tests/unittests/test_result_verifier.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.test_dimensions import create_uncompressed_text_dimension
from tests.common.test_result_verifier import create_query_result
diff --git a/tests/util/acid_txn.py b/tests/util/acid_txn.py
index 0bc82125b..f1830f2db 100644
--- a/tests/util/acid_txn.py
+++ b/tests/util/acid_txn.py
@@ -15,7 +15,7 @@
# specific language governing permissions and limitations
# under the License.
-from __future__ import print_function
+from __future__ import absolute_import, division, print_function
from tests.util.thrift_util import create_transport
from hive_metastore import ThriftHiveMetastore
from hive_metastore.ttypes import (AbortTxnRequest, AllocateTableWriteIdsRequest,
diff --git a/tests/util/adls_util.py b/tests/util/adls_util.py
index 3b9ea0f8e..b236a488f 100644
--- a/tests/util/adls_util.py
+++ b/tests/util/adls_util.py
@@ -20,6 +20,7 @@
# This file uses the azure-data-lake-store-python client and provides simple
# functions to the Impala test suite to access Azure Data Lake Store.
+from __future__ import absolute_import, division, print_function
from azure.datalake.store import core, lib, multithread, exceptions
from tests.util.filesystem_base import BaseFilesystem
from tests.util.filesystem_utils import ADLS_CLIENT_ID, ADLS_TENANT_ID, ADLS_CLIENT_SECRET
diff --git a/tests/util/auto_scaler.py b/tests/util/auto_scaler.py
index 1236aaa84..79b87294b 100755
--- a/tests/util/auto_scaler.py
+++ b/tests/util/auto_scaler.py
@@ -17,6 +17,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import argparse
import time
import logging
@@ -56,7 +57,7 @@ class AutoScaler(object):
self.start_batch_size = start_batch_size
# Maximum number of executor groups. We only have 10 TCP ports free on our
# miniclusters and we need one for the dedicated coordinator.
- self.max_groups = 9 / self.group_size
+ self.max_groups = 9 // self.group_size
# max_groups can further bound the maximum number of groups we are going to start,
# but we won't start more than possible.
if max_groups > 0 and max_groups < self.max_groups:
diff --git a/tests/util/calculation_util.py b/tests/util/calculation_util.py
index 263d7e3e1..657c40a92 100644
--- a/tests/util/calculation_util.py
+++ b/tests/util/calculation_util.py
@@ -20,6 +20,7 @@
# are simple enough that it is better to implement them ourselves to avoid extra
# dependencies.
+from __future__ import absolute_import, division, print_function
import math
import random
import string
@@ -38,9 +39,9 @@ def calculate_median(values):
sorted_values = sorted(values)
length = len(sorted_values)
if length % 2 == 0:
- return (sorted_values[length / 2] + sorted_values[length / 2 - 1]) / 2
+ return (sorted_values[length // 2] + sorted_values[length // 2 - 1]) / 2
else:
- return sorted_values[length / 2]
+ return sorted_values[length // 2]
def calculate_geomean(values):
""" Calculates the geometric mean of the given collection of numerics """
diff --git a/tests/util/cancel_util.py b/tests/util/cancel_util.py
index 7e7bf15ec..60dabc4c5 100644
--- a/tests/util/cancel_util.py
+++ b/tests/util/cancel_util.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import threading
from time import sleep
from tests.beeswax.impala_beeswax import ImpalaBeeswaxException
diff --git a/tests/util/cluster_controller.py b/tests/util/cluster_controller.py
index 388fa4936..048507b21 100644
--- a/tests/util/cluster_controller.py
+++ b/tests/util/cluster_controller.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
try:
import fabric.decorators
from fabric.context_managers import hide, settings
diff --git a/tests/util/compute_table_stats.py b/tests/util/compute_table_stats.py
index f982802bb..bb25e6c67 100755
--- a/tests/util/compute_table_stats.py
+++ b/tests/util/compute_table_stats.py
@@ -19,6 +19,7 @@
#
# Utility for computing table statistics of tables in the Hive Metastore
+from __future__ import absolute_import, division, print_function
from contextlib import contextmanager
from optparse import OptionParser
import logging
diff --git a/tests/util/concurrent_workload.py b/tests/util/concurrent_workload.py
index b9828066c..025f67da7 100755
--- a/tests/util/concurrent_workload.py
+++ b/tests/util/concurrent_workload.py
@@ -19,6 +19,7 @@
# This class can be used to drive a concurrent workload against a local minicluster
+from __future__ import absolute_import, division, print_function
import argparse
import logging
# Needed to work around datetime threading bug:
diff --git a/tests/util/event_processor_utils.py b/tests/util/event_processor_utils.py
index 261440cdc..4d2703cb1 100644
--- a/tests/util/event_processor_utils.py
+++ b/tests/util/event_processor_utils.py
@@ -19,6 +19,7 @@
# modifies the metadata via Hive and checks that the modification
# succeeded by querying Impala, or vice versa.
+from __future__ import absolute_import, division, print_function
import logging
import requests
import time
diff --git a/tests/util/failpoints_util.py b/tests/util/failpoints_util.py
index 2970567c6..de3f43666 100644
--- a/tests/util/failpoints_util.py
+++ b/tests/util/failpoints_util.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
from tests.common.impala_test_suite import ImpalaTestSuite, LOG
diff --git a/tests/util/filesystem_base.py b/tests/util/filesystem_base.py
index 037eefccf..8f479bcb6 100644
--- a/tests/util/filesystem_base.py
+++ b/tests/util/filesystem_base.py
@@ -17,6 +17,7 @@
#
# Filsystem access abstraction
+from __future__ import absolute_import, division, print_function
from abc import ABCMeta, abstractmethod
class BaseFilesystem(object):
diff --git a/tests/util/filesystem_utils.py b/tests/util/filesystem_utils.py
index fe13a807e..a3114e18b 100644
--- a/tests/util/filesystem_utils.py
+++ b/tests/util/filesystem_utils.py
@@ -16,6 +16,7 @@
# under the License.
#
# Utilities for supporting different filesystems.
+from __future__ import absolute_import, division, print_function
import os
# FILESYSTEM_PREFIX is the path prefix that should be used in queries. When running
diff --git a/tests/util/get_parquet_metadata.py b/tests/util/get_parquet_metadata.py
index db4ef3f41..21099acb9 100644
--- a/tests/util/get_parquet_metadata.py
+++ b/tests/util/get_parquet_metadata.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import os
import struct
diff --git a/tests/util/hdfs_util.py b/tests/util/hdfs_util.py
index 9afdf640c..ef5a0c38c 100644
--- a/tests/util/hdfs_util.py
+++ b/tests/util/hdfs_util.py
@@ -17,6 +17,7 @@
#
# Hdfs access utilities
+from __future__ import absolute_import, division, print_function
import getpass
import httplib
import os.path
diff --git a/tests/util/iceberg_util.py b/tests/util/iceberg_util.py
index 9418638e3..924b5e249 100644
--- a/tests/util/iceberg_util.py
+++ b/tests/util/iceberg_util.py
@@ -14,6 +14,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import datetime
diff --git a/tests/util/parse_util.py b/tests/util/parse_util.py
index 97c1c5c5b..46a995a96 100644
--- a/tests/util/parse_util.py
+++ b/tests/util/parse_util.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import re
from datetime import datetime
diff --git a/tests/util/plugin_runner.py b/tests/util/plugin_runner.py
index 2f2849f15..e16a28ac6 100644
--- a/tests/util/plugin_runner.py
+++ b/tests/util/plugin_runner.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import logging
import os
import pkgutil
diff --git a/tests/util/run_impyla_http_query.py b/tests/util/run_impyla_http_query.py
index df81f8793..98f95d899 100755
--- a/tests/util/run_impyla_http_query.py
+++ b/tests/util/run_impyla_http_query.py
@@ -20,7 +20,7 @@
# It can be used by other tests (e.g. LdapImpylaHttpTest.java) that start a cluster with
# an LDAP server to validate Impyla's functionality.
-from __future__ import print_function
+from __future__ import absolute_import, division, print_function
import argparse
import logging
diff --git a/tests/util/shell_util.py b/tests/util/shell_util.py
index ddbd24b1c..17e19886c 100644
--- a/tests/util/shell_util.py
+++ b/tests/util/shell_util.py
@@ -17,6 +17,7 @@
#
# Utility functions related to executing shell commands.
+from __future__ import absolute_import, division, print_function
import logging
import os
import shlex
diff --git a/tests/util/ssh_util.py b/tests/util/ssh_util.py
index a26f1bcf9..67b33889e 100644
--- a/tests/util/ssh_util.py
+++ b/tests/util/ssh_util.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import atexit
import logging
import os
diff --git a/tests/util/test_file_parser.py b/tests/util/test_file_parser.py
index 26516600c..91973c457 100644
--- a/tests/util/test_file_parser.py
+++ b/tests/util/test_file_parser.py
@@ -17,7 +17,7 @@
# This module is used for common utilities related to parsing test files
-from __future__ import print_function
+from __future__ import absolute_import, division, print_function
import codecs
import collections
import logging
diff --git a/tests/util/thrift_util.py b/tests/util/thrift_util.py
index e5f9360c7..8baff3334 100644
--- a/tests/util/thrift_util.py
+++ b/tests/util/thrift_util.py
@@ -16,6 +16,7 @@
# under the License.
#
# Thrift utility functions
+from __future__ import absolute_import, division, print_function
import getpass
import sasl
import struct
diff --git a/tests/util/web_pages_util.py b/tests/util/web_pages_util.py
index 317289e7b..a8b6b37a5 100644
--- a/tests/util/web_pages_util.py
+++ b/tests/util/web_pages_util.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
import json
diff --git a/tests/verifiers/mem_usage_verifier.py b/tests/verifiers/mem_usage_verifier.py
index 644e5fa69..0bcd40b55 100644
--- a/tests/verifiers/mem_usage_verifier.py
+++ b/tests/verifiers/mem_usage_verifier.py
@@ -17,6 +17,7 @@
#
# Verifier for memtracker usage values (Total, Peak, etc).
+from __future__ import absolute_import, division, print_function
import re
SIZE_FACTORS = {"b": 1, "kb": 1 << 10, "mb": 1 << 20, "gb": 1 << 30}
diff --git a/tests/verifiers/metric_verifier.py b/tests/verifiers/metric_verifier.py
index 59848b303..c8ee696d7 100644
--- a/tests/verifiers/metric_verifier.py
+++ b/tests/verifiers/metric_verifier.py
@@ -17,6 +17,7 @@
#
# Verifier for common impalad metrics
+from __future__ import absolute_import, division, print_function
import logging
from time import time, sleep
diff --git a/tests/verifiers/test_verify_metrics.py b/tests/verifiers/test_verify_metrics.py
index 8db6811a7..984beb05c 100644
--- a/tests/verifiers/test_verify_metrics.py
+++ b/tests/verifiers/test_verify_metrics.py
@@ -17,6 +17,7 @@
#
# Verification of impalad metrics after a test run.
+from __future__ import absolute_import, division, print_function
from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.impala_cluster import ImpalaCluster
from tests.verifiers.metric_verifier import MetricVerifier
diff --git a/tests/webserver/test_web_pages.py b/tests/webserver/test_web_pages.py
index 544e8c3d3..e1383a9aa 100644
--- a/tests/webserver/test_web_pages.py
+++ b/tests/webserver/test_web_pages.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from __future__ import absolute_import, division, print_function
from tests.common.environ import ImpalaTestClusterFlagsDetector
from tests.common.file_utils import grep_dir
from tests.common.skip import SkipIfBuildType, SkipIfDockerizedCluster
[impala] 04/06: IMPALA-11975: Fix Dictionary methods to work with Python 3
Posted by jo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
joemcdonnell pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit c233634d747e4106128627f258d1bee23855fd1e
Author: Joe McDonnell <jo...@cloudera.com>
AuthorDate: Sat Mar 4 09:52:49 2023 -0800
IMPALA-11975: Fix Dictionary methods to work with Python 3
Python 3 made the main dictionary methods lazy (items(),
keys(), values()). This means that code that uses those
methods may need to wrap the call in list() to get a
list immediately. Python 3 also removed the old iter*
lazy variants.
This changes all locations to use Python 3 dictionary
methods and wraps calls with list() appropriately.
This also changes all itemitems(), itervalues(), iterkeys()
locations to items(), values(), keys(), etc. Python 2
will not use the lazy implementation of these, so there
is a theoretical performance impact. Our python code is
mostly for tests and the performance impact is minimal.
Python 2 will be deprecated when Python 3 is functional.
This addresses these pylint warnings:
dict-iter-method
dict-keys-not-iterating
dict-values-not-iterating
Testing:
- Ran core tests
Change-Id: Ie873ece54a633a8a95ed4600b1df4be7542348da
Reviewed-on: http://gerrit.cloudera.org:8080/19590
Reviewed-by: Joe McDonnell <jo...@cloudera.com>
Tested-by: Joe McDonnell <jo...@cloudera.com>
---
bin/banned_py3k_warnings.txt | 3 +++
bin/load-data.py | 2 +-
bin/start-impala-cluster.py | 4 ++--
testdata/bin/generate-schema-statements.py | 2 +-
tests/beeswax/impala_beeswax.py | 6 +++---
tests/common/impala_cluster.py | 4 ++--
tests/common/impala_connection.py | 2 +-
tests/common/impala_test_suite.py | 6 +++---
tests/comparison/common.py | 6 +++---
tests/comparison/leopard/impala_docker_env.py | 2 +-
tests/comparison/query_generator.py | 10 +++++-----
tests/comparison/query_profile.py | 16 ++++++++--------
tests/custom_cluster/test_admission_controller.py | 5 +++--
tests/custom_cluster/test_breakpad.py | 4 ++--
tests/performance/workload.py | 2 +-
tests/query_test/test_cancellation.py | 2 +-
tests/query_test/test_decimal_fuzz.py | 4 ++--
tests/query_test/test_insert_parquet.py | 2 +-
tests/query_test/test_kudu.py | 4 ++--
tests/run-tests.py | 2 +-
tests/stress/concurrent_select.py | 7 ++++---
tests/stress/extract_min_mem.py | 2 +-
tests/stress/queries.py | 2 +-
tests/stress/query_runner.py | 6 +++---
tests/stress/runtime_info.py | 10 +++++-----
25 files changed, 60 insertions(+), 55 deletions(-)
diff --git a/bin/banned_py3k_warnings.txt b/bin/banned_py3k_warnings.txt
index 01d54fe73..a970906a0 100644
--- a/bin/banned_py3k_warnings.txt
+++ b/bin/banned_py3k_warnings.txt
@@ -7,3 +7,6 @@ zip-builtin-not-iterating
filter-builtin-not-iterating
reduce-builtin
deprecated-itertools-function
+dict-iter-method
+dict-keys-not-iterating
+dict-values-not-iterating
diff --git a/bin/load-data.py b/bin/load-data.py
index 535a4f8bf..a4cfd5a97 100755
--- a/bin/load-data.py
+++ b/bin/load-data.py
@@ -483,7 +483,7 @@ def main():
total_time = 0.0
thread_pool.close()
thread_pool.join()
- for workload, load_time in loading_time_map.iteritems():
+ for workload, load_time in loading_time_map.items():
total_time += load_time
LOG.info('Data loading for workload \'%s\' completed in: %.2fs'\
% (workload, load_time))
diff --git a/bin/start-impala-cluster.py b/bin/start-impala-cluster.py
index 5fac2c2d6..40ccd3d36 100755
--- a/bin/start-impala-cluster.py
+++ b/bin/start-impala-cluster.py
@@ -593,7 +593,7 @@ class DockerMiniClusterOperations(object):
# List all running containers on the network and kill those with the impalad name
# prefix to make sure that no running container are left over from previous clusters.
container_name_prefix = self.__gen_container_name__("impalad")
- for container_id, info in self.__get_network_info__()["Containers"].iteritems():
+ for container_id, info in self.__get_network_info__()["Containers"].items():
container_name = info["Name"]
if container_name.startswith(container_name_prefix):
LOG.info("Stopping container {0}".format(container_name))
@@ -665,7 +665,7 @@ class DockerMiniClusterOperations(object):
port_args = ["-P"]
else:
port_args = ["-p{dst}:{src}".format(src=src, dst=dst)
- for src, dst in port_map.iteritems()]
+ for src, dst in port_map.items()]
# Impersonate the current user for operations against the minicluster. This is
# necessary because the user name inside the container is "root".
# TODO: pass in the actual options
diff --git a/testdata/bin/generate-schema-statements.py b/testdata/bin/generate-schema-statements.py
index 69683a2f4..232db516b 100755
--- a/testdata/bin/generate-schema-statements.py
+++ b/testdata/bin/generate-schema-statements.py
@@ -336,7 +336,7 @@ def build_table_template(file_format, columns, partition_columns, row_format, tb
external = ""
all_tblproperties = []
- for key, value in tblproperties.iteritems():
+ for key, value in tblproperties.items():
all_tblproperties.append("'{0}' = '{1}'".format(key, value))
# If there are no properties to set avoid the TBLPROPERTIES clause altogether.
diff --git a/tests/beeswax/impala_beeswax.py b/tests/beeswax/impala_beeswax.py
index c58bb7109..dbe2e81fc 100644
--- a/tests/beeswax/impala_beeswax.py
+++ b/tests/beeswax/impala_beeswax.py
@@ -128,7 +128,7 @@ class ImpalaBeeswaxClient(object):
self.query_states = QueryState._NAMES_TO_VALUES
def __options_to_string_list(self):
- return ["%s=%s" % (k,v) for (k,v) in self.__query_options.iteritems()]
+ return ["%s=%s" % (k, v) for (k, v) in self.__query_options.items()]
def get_query_options(self):
return self.__query_options
@@ -140,7 +140,7 @@ class ImpalaBeeswaxClient(object):
if query_option_dict is None:
raise ValueError('Cannot pass None value for query options')
self.clear_query_options()
- for name, value in query_option_dict.iteritems():
+ for name, value in query_option_dict.items():
self.set_query_option(name, value)
def get_query_option(self, name):
@@ -483,7 +483,7 @@ class ImpalaBeeswaxClient(object):
result = self.close_dml(handle)
# The insert was successful
num_rows = sum(map(int, result.rows_modified.values()))
- data = ["%s: %s" % row for row in result.rows_modified.iteritems()]
+ data = ["%s: %s" % row for row in result.rows_modified.items()]
exec_result = ImpalaBeeswaxResult(query_id=handle.id, success=True, data=data)
exec_result.summary = "Inserted %d rows" % (num_rows,)
return exec_result
diff --git a/tests/common/impala_cluster.py b/tests/common/impala_cluster.py
index 8a02e952f..e8a9aff7f 100644
--- a/tests/common/impala_cluster.py
+++ b/tests/common/impala_cluster.py
@@ -282,7 +282,7 @@ class ImpalaCluster(object):
args = container_info["Args"]
executable = os.path.basename(args[0])
port_map = {}
- for k, v in container_info["NetworkSettings"]["Ports"].iteritems():
+ for k, v in container_info["NetworkSettings"]["Ports"].items():
# Key looks like "25000/tcp"..
port = int(k.split("/")[0])
# Value looks like { "HostPort": "25002", "HostIp": "" }.
@@ -626,7 +626,7 @@ def run_daemon(daemon_binary, args, build_type="latest", env_vars={}, output_fil
# achieve the same thing but it doesn't work on some platforms for some reasons.
sys_cmd = ("{set_cmds} {cmd} {redirect} &".format(
set_cmds=''.join(["export {0}={1};".format(k, pipes.quote(v))
- for k, v in env_vars.iteritems()]),
+ for k, v in env_vars.items()]),
cmd=' '.join([pipes.quote(tok) for tok in cmd]),
redirect=redirect))
os.system(sys_cmd)
diff --git a/tests/common/impala_connection.py b/tests/common/impala_connection.py
index 21128db5d..6c949ea23 100644
--- a/tests/common/impala_connection.py
+++ b/tests/common/impala_connection.py
@@ -94,7 +94,7 @@ class ImpalaConnection(object):
"""Replaces existing configuration with the given dictionary"""
assert config_option_dict is not None, "config_option_dict cannot be None"
self.clear_configuration()
- for name, value in config_option_dict.iteritems():
+ for name, value in config_option_dict.items():
self.set_configuration_option(name, value)
@abc.abstractmethod
diff --git a/tests/common/impala_test_suite.py b/tests/common/impala_test_suite.py
index 92099170e..c8c780a3b 100644
--- a/tests/common/impala_test_suite.py
+++ b/tests/common/impala_test_suite.py
@@ -408,7 +408,7 @@ class ImpalaTestSuite(BaseTestSuite):
# Populate the default query option if it's empty.
if not self.default_query_options:
query_options = impalad_client.get_default_configuration()
- for key, value in query_options.iteritems():
+ for key, value in query_options.items():
self.default_query_options[key.upper()] = value
# Restore all the changed query options.
for query_option in query_options_changed:
@@ -512,12 +512,12 @@ class ImpalaTestSuite(BaseTestSuite):
raise AssertionError("Query contains $DATABASE but no use_db specified")
if extra:
- for k, v in extra.iteritems():
+ for k, v in extra.items():
if k in repl:
raise RuntimeError("Key {0} is reserved".format(k))
repl[k] = v
- for k, v in repl.iteritems():
+ for k, v in repl.items():
s = s.replace(k, v)
return s
diff --git a/tests/comparison/common.py b/tests/comparison/common.py
index 1025e7c79..5b2e71dcd 100644
--- a/tests/comparison/common.py
+++ b/tests/comparison/common.py
@@ -162,7 +162,7 @@ class ValExpr(object):
if self.is_func:
for arg in self.args:
if isinstance(arg, ValExpr):
- for col, count in arg.count_col_refs().iteritems():
+ for col, count in arg.count_col_refs().items():
col_ref_counts[col] += count
elif self.is_col:
col_ref_counts[self] += 1
@@ -631,7 +631,7 @@ class TableExprList(list):
def joinable_cols_by_type(self):
cols_by_type = defaultdict(ValExprList)
for table_expr in self:
- for type_, cols in table_expr.joinable_cols_by_type.iteritems():
+ for type_, cols in table_expr.joinable_cols_by_type.items():
cols_by_type[type_].extend(cols)
return cols_by_type
@@ -639,7 +639,7 @@ class TableExprList(list):
def cols_by_type(self):
cols_by_type = defaultdict(ValExprList)
for table_expr in self:
- for type_, cols in table_expr.cols_by_type.iteritems():
+ for type_, cols in table_expr.cols_by_type.items():
cols_by_type[type_].extend(cols)
return cols_by_type
diff --git a/tests/comparison/leopard/impala_docker_env.py b/tests/comparison/leopard/impala_docker_env.py
index af3a09d48..1814764b6 100755
--- a/tests/comparison/leopard/impala_docker_env.py
+++ b/tests/comparison/leopard/impala_docker_env.py
@@ -141,7 +141,7 @@ class ImpalaDockerEnv(object):
volume_ops = ' '.join(
['-v {host_path}:{container_path}'.format(host_path=host_path,
container_path=container_path)
- for host_path, container_path in volume_map.iteritems()])
+ for host_path, container_path in volume_map.items()])
start_command += (
'docker run -d -t {volume_ops} -p {postgres_port}:5432 -p {ssh_port}:22 '
'-p {impala_port}:21050 {docker_image_name} /bin/docker-boot-daemon').format(
diff --git a/tests/comparison/query_generator.py b/tests/comparison/query_generator.py
index 3da76a1ab..00df0c4f2 100644
--- a/tests/comparison/query_generator.py
+++ b/tests/comparison/query_generator.py
@@ -752,7 +752,7 @@ class QueryGenerator(object):
# A root_func was chosen and it's children are in one or more of the
# null_args_by_func_allowed pools. A pool will be chosen, then a child function.
null_arg_counts_by_pool = dict((pool_category, len(pool)) for pool_category, pool
- in null_args_by_func_allowed.iteritems())
+ in null_args_by_func_allowed.items())
# There is a special case that would lead to a dead end. If there is only one
# distinct place holder across all the pools and an analytic is still needed,
# then that place holder cannot be replaced by an aggregate since aggregates
@@ -788,7 +788,7 @@ class QueryGenerator(object):
if parent_func:
# Remove the place holder from all of the other pools.
- for pool_category, pool in null_args_by_func_allowed.iteritems():
+ for pool_category, pool in null_args_by_func_allowed.items():
for null_arg_idx, (func, arg_idx) in enumerate(pool):
if func is parent_func and arg_idx == parent_arg_idx:
del pool[null_arg_idx]
@@ -849,7 +849,7 @@ class QueryGenerator(object):
continue
null_args.append((chosen_func, idx))
- if not any(null_args_by_func_allowed.itervalues()):
+ if not any(null_args_by_func_allowed.values()):
# Some analytic functions take no arguments. Ex: ROW_NUM()
break
@@ -1253,7 +1253,7 @@ class QueryGenerator(object):
root_predicate, relational_predicates = self._create_boolean_func_tree(
require_relational_func=True,
- relational_col_types=table_exprs_by_col_types.keys(),
+ relational_col_types=list(table_exprs_by_col_types.keys()),
allowed_signatures=join_signatures)
for predicate in relational_predicates:
@@ -1440,7 +1440,7 @@ class QueryGenerator(object):
# Prefer to replace Boolean leaves to get a more realistic expression.
return_type = Boolean
else:
- return_type = choice(null_args_by_type.keys())
+ return_type = choice(list(null_args_by_type.keys()))
# Rather than track if this is a child of a relational function, in which case
# the arg type needs to be preserved, just always assume that this is a child of
# a relational function.
diff --git a/tests/comparison/query_profile.py b/tests/comparison/query_profile.py
index e47f7a6c6..fae66d71e 100644
--- a/tests/comparison/query_profile.py
+++ b/tests/comparison/query_profile.py
@@ -296,9 +296,9 @@ class DefaultProfile(object):
weights = self.weights(*weight_args)
else:
weights = weight_args[0]
- total_weight = sum(weights.itervalues())
+ total_weight = sum(weights.values())
numeric_choice = randint(1, total_weight)
- for choice_, weight in weights.iteritems():
+ for choice_, weight in weights.items():
if weight <= 0:
continue
if numeric_choice <= weight:
@@ -312,7 +312,7 @@ class DefaultProfile(object):
else:
weights = weights[0]
return self._choose_from_weights(dict((choice_, weight) for choice_, weight
- in weights.iteritems() if filter_fn(choice_)))
+ in weights.items() if filter_fn(choice_)))
def _decide_from_probability(self, *keys):
return random() < self.probability(*keys)
@@ -341,7 +341,7 @@ class DefaultProfile(object):
return self._choose_from_bounds('MAX_NESTED_EXPR_COUNT')
def allowed_analytic_designs(self):
- return [design for design, is_enabled in self._flags['ANALYTIC_DESIGNS'].iteritems()
+ return [design for design, is_enabled in self._flags['ANALYTIC_DESIGNS'].items()
if is_enabled]
def use_partition_by_clause_in_analytic(self):
@@ -386,14 +386,14 @@ class DefaultProfile(object):
def choose_subquery_predicate_category(self, func_name, allow_correlated):
weights = self.weights('SUBQUERY_PREDICATE')
- func_names = set(name for name, _, _ in weights.iterkeys())
+ func_names = set(name for name, _, _ in weights.keys())
if func_name not in func_names:
func_name = 'Scalar'
allow_agg = self.weights('SELECT_ITEM_CATEGORY').get('AGG', 0)
if allow_correlated and self.bounds('TABLE_COUNT')[1] == 0:
allow_correlated = False
weights = dict(((name, use_agg, use_correlated), weight)
- for (name, use_agg, use_correlated), weight in weights.iteritems()
+ for (name, use_agg, use_correlated), weight in weights.items()
if name == func_name and
(allow_agg or use_agg == 'NON_AGG') and
weight)
@@ -563,7 +563,7 @@ class DefaultProfile(object):
if not func_weights:
raise Exception('All functions disallowed based on signature types')
distinct_signature_lengths = set(signature_length_by_func.values())
- for func, weight in func_weights.iteritems():
+ for func, weight in func_weights.items():
signature_length = signature_length_by_func[func]
func_weights[func] = reduce(
lambda x, y: x * y,
@@ -591,7 +591,7 @@ class DefaultProfile(object):
signature_weights[idx] = signature_weight
signature_lengths[idx] = signature_length
distinct_signature_lengths = set(signature_lengths.values())
- for idx, weight in signature_weights.iteritems():
+ for idx, weight in signature_weights.items():
signature_length = signature_lengths[idx]
signature_weights[idx] = reduce(
lambda x, y: x * y,
diff --git a/tests/custom_cluster/test_admission_controller.py b/tests/custom_cluster/test_admission_controller.py
index 79e4cc3a7..1fc7a14af 100644
--- a/tests/custom_cluster/test_admission_controller.py
+++ b/tests/custom_cluster/test_admission_controller.py
@@ -1852,8 +1852,9 @@ class TestAdmissionControllerStress(TestAdmissionControllerBase):
curr[impalad] = init[impalad]
while True:
- LOG.debug("wait_for_statestore_updates: curr=%s, init=%s, d=%s", curr.values(),
- init.values(), [curr[i] - init[i] for i in self.impalads])
+ LOG.debug("wait_for_statestore_updates: curr=%s, init=%s, d=%s",
+ list(curr.values()), list(init.values()),
+ [curr[i] - init[i] for i in self.impalads])
if all([curr[i] - init[i] >= heartbeats for i in self.impalads]): break
for impalad in self.impalads:
curr[impalad] = impalad.service.get_metric_value(
diff --git a/tests/custom_cluster/test_breakpad.py b/tests/custom_cluster/test_breakpad.py
index 6e43c42b5..38c827588 100644
--- a/tests/custom_cluster/test_breakpad.py
+++ b/tests/custom_cluster/test_breakpad.py
@@ -66,7 +66,7 @@ class TestBreakpadBase(CustomClusterTestSuite):
def start_cluster_with_args(self, **kwargs):
cluster_options = []
for daemon_arg in DAEMON_ARGS:
- daemon_options = " ".join("-{0}={1}".format(k, v) for k, v in kwargs.iteritems())
+ daemon_options = " ".join("-{0}={1}".format(k, v) for k, v in kwargs.items())
cluster_options.append("--{0}={1}".format(daemon_arg, daemon_options))
self._start_impala_cluster(cluster_options)
@@ -401,7 +401,7 @@ class TestLogging(TestBreakpadBase):
def start_cluster_with_args(self, cluster_size, log_dir, **kwargs):
cluster_options = []
for daemon_arg in DAEMON_ARGS:
- daemon_options = " ".join("-{0}={1}".format(k, v) for k, v in kwargs.iteritems())
+ daemon_options = " ".join("-{0}={1}".format(k, v) for k, v in kwargs.items())
cluster_options.append("--{0}={1}".format(daemon_arg, daemon_options))
self._start_impala_cluster(cluster_options, cluster_size=cluster_size,
expected_num_impalads=cluster_size, impala_log_dir=log_dir)
diff --git a/tests/performance/workload.py b/tests/performance/workload.py
index 1fc47b2e1..507c18ef4 100644
--- a/tests/performance/workload.py
+++ b/tests/performance/workload.py
@@ -74,7 +74,7 @@ class Workload(object):
"""
queries = list()
- for query_name, query_str in self._query_map.iteritems():
+ for query_name, query_str in self._query_map.items():
queries.append(Query(name=query_name,
query_str=query_str,
workload=self._name,
diff --git a/tests/query_test/test_cancellation.py b/tests/query_test/test_cancellation.py
index c6cfb50ad..63c43e383 100644
--- a/tests/query_test/test_cancellation.py
+++ b/tests/query_test/test_cancellation.py
@@ -198,7 +198,7 @@ class TestCancellation(ImpalaTestSuite):
for _ in range(5)), 'Query failed to cancel'
# Get profile and check for formatting errors
profile = client.get_runtime_profile(handle, TRuntimeProfileFormat.THRIFT)
- for (k, v) in profile.nodes[1].info_strings.iteritems():
+ for (k, v) in profile.nodes[1].info_strings.items():
# Ensure that whitespace gets removed from values.
assert v == v.rstrip(), \
"Profile value contains surrounding whitespace: %s %s" % (k, v)
diff --git a/tests/query_test/test_decimal_fuzz.py b/tests/query_test/test_decimal_fuzz.py
index d55251f00..cdb28d9a9 100644
--- a/tests/query_test/test_decimal_fuzz.py
+++ b/tests/query_test/test_decimal_fuzz.py
@@ -46,10 +46,10 @@ class TestDecimalFuzz(ImpalaTestSuite):
cls.iterations = 10000
def weighted_choice(self, options):
- total_weight = sum(options.itervalues())
+ total_weight = sum(options.values())
numeric_choice = random.uniform(0, total_weight)
last_choice = None
- for choice, weight in options.iteritems():
+ for choice, weight in options.items():
if numeric_choice <= weight:
return choice
numeric_choice -= weight
diff --git a/tests/query_test/test_insert_parquet.py b/tests/query_test/test_insert_parquet.py
index dc72c63b6..b4f9307da 100644
--- a/tests/query_test/test_insert_parquet.py
+++ b/tests/query_test/test_insert_parquet.py
@@ -425,7 +425,7 @@ class TestHdfsParquetTableWriter(ImpalaTestSuite):
def _check_only_one_member_var_is_set(obj, var_name):
"""Checks that 'var_name' is the only member of 'obj' that is not None. Useful to
check Thrift unions."""
- keys = [k for k, v in vars(obj).iteritems() if v is not None]
+ keys = [k for k, v in vars(obj).items() if v is not None]
assert keys == [var_name]
def _check_no_logical_type(self, schemas, column_name):
diff --git a/tests/query_test/test_kudu.py b/tests/query_test/test_kudu.py
index f985599e2..be3de8a58 100644
--- a/tests/query_test/test_kudu.py
+++ b/tests/query_test/test_kudu.py
@@ -604,7 +604,7 @@ class TestKuduPartitioning(KuduTestSuite):
query = "INSERT INTO %s SELECT id FROM functional.alltypes" % table_full_name
exec_options = dict((k, str(v)) for k, v
- in vector.get_value('exec_option').iteritems())
+ in vector.get_value('exec_option').items())
cursor.execute(query, configuration=exec_options)
profile = cursor.get_profile()
@@ -1357,7 +1357,7 @@ class TestKuduMemLimits(KuduTestSuite):
"""Tests that the queries specified in this test suite run under the given
memory limits."""
exec_options = dict((k, str(v)) for k, v
- in vector.get_value('exec_option').iteritems())
+ in vector.get_value('exec_option').items())
exec_options['mem_limit'] = "{0}m".format(mem_limit)
# IMPALA-9856: We disable query result spooling so that this test can run queries
# with low mem_limit.
diff --git a/tests/run-tests.py b/tests/run-tests.py
index 1ec0f9722..166adc11b 100755
--- a/tests/run-tests.py
+++ b/tests/run-tests.py
@@ -166,7 +166,7 @@ def build_test_args(base_name, valid_dirs=VALID_TEST_DIRS):
ignored_dirs = build_ignore_dir_arg_list(valid_dirs=valid_dirs)
logging_args = []
- for arg, log in LOGGING_ARGS.iteritems():
+ for arg, log in LOGGING_ARGS.items():
logging_args.extend([arg, os.path.join(RESULT_DIR, log.format(base_name))])
if valid_dirs != ['verifiers']:
diff --git a/tests/stress/concurrent_select.py b/tests/stress/concurrent_select.py
index 8c15e49ee..36ee36363 100755
--- a/tests/stress/concurrent_select.py
+++ b/tests/stress/concurrent_select.py
@@ -200,7 +200,7 @@ def print_crash_info_if_exists(impala, start_time):
LOG.error(
"Aborting after %s failed attempts to check if impalads crashed", max_attempts)
raise e
- for message in crashed_impalads.itervalues():
+ for message in crashed_impalads.values():
print(message, file=sys.stderr)
return crashed_impalads
@@ -393,8 +393,9 @@ class StressRunner(object):
# First randomly determine a query type, then choose a random query of that
# type.
if (
- QueryType.SELECT in queries_by_type and
- (len(queries_by_type.keys()) == 1 or random() < self._select_probability)
+ QueryType.SELECT in queries_by_type
+ and (len(list(queries_by_type.keys())) == 1
+ or random() < self._select_probability)
):
result = choice(queries_by_type[QueryType.SELECT])
else:
diff --git a/tests/stress/extract_min_mem.py b/tests/stress/extract_min_mem.py
index 22301441c..33db7b3f5 100755
--- a/tests/stress/extract_min_mem.py
+++ b/tests/stress/extract_min_mem.py
@@ -40,7 +40,7 @@ import sys
results = []
with open(sys.argv[1]) as f:
data = json.load(f)
- for query_data in data['db_names']['tpch_parquet'].itervalues():
+ for query_data in data['db_names']['tpch_parquet'].values():
runtime_info = query_data['[]']
# Build up list of query numbers and minimum memory.
results.append((int(runtime_info['name'][1:]),
diff --git a/tests/stress/queries.py b/tests/stress/queries.py
index 36b3a287c..4ea878f09 100644
--- a/tests/stress/queries.py
+++ b/tests/stress/queries.py
@@ -117,7 +117,7 @@ def load_tpc_queries(workload):
LOG.info("Loading %s queries", workload)
queries = []
for query_name, query_sql in test_file_parser.load_tpc_queries(workload,
- include_stress_queries=True).iteritems():
+ include_stress_queries=True).items():
query = Query()
query.name = query_name
query.sql = query_sql
diff --git a/tests/stress/query_runner.py b/tests/stress/query_runner.py
index 26a99fbca..fa6b5f5d2 100644
--- a/tests/stress/query_runner.py
+++ b/tests/stress/query_runner.py
@@ -188,10 +188,10 @@ class QueryRunner(object):
if run_set_up and query.set_up_sql:
LOG.debug("Running set up query:\n%s", query.set_up_sql)
cursor.execute(query.set_up_sql)
- for query_option, value in self.common_query_options.iteritems():
+ for query_option, value in self.common_query_options.items():
cursor.execute(
"SET {query_option}={value}".format(query_option=query_option, value=value))
- for query_option, value in query.options.iteritems():
+ for query_option, value in query.options.items():
cursor.execute(
"SET {query_option}={value}".format(query_option=query_option, value=value))
# Set a time limit if it is the expected method of cancellation, or as an additional
@@ -383,7 +383,7 @@ class QueryRunner(object):
def get_metric_vals(self):
"""Get the current values of the all metrics as a list of (k, v) pairs."""
- return [(k, v.value) for k, v in self._metrics.iteritems()]
+ return [(k, v.value) for k, v in self._metrics.items()]
def increment_metric(self, name):
"""Increment the current value of the metric called 'name'."""
diff --git a/tests/stress/runtime_info.py b/tests/stress/runtime_info.py
index a10692a22..5523aa66a 100644
--- a/tests/stress/runtime_info.py
+++ b/tests/stress/runtime_info.py
@@ -83,9 +83,9 @@ def load_runtime_info(path, impala=None):
store.get("host_names") != sorted([i.host_name for i in impala.impalads])
):
return queries_by_db_and_sql
- for db_name, queries_by_sql in store["db_names"].iteritems():
- for sql, queries_by_options in queries_by_sql.iteritems():
- for options, json_query in queries_by_options.iteritems():
+ for db_name, queries_by_sql in store["db_names"].items():
+ for sql, queries_by_options in queries_by_sql.items():
+ for options, json_query in queries_by_options.items():
query = Query()
query.__dict__.update(json_query)
query.sql = sql
@@ -123,11 +123,11 @@ def print_runtime_info_comparison(old_runtime_info, new_runtime_info):
"Old Runtime wout/Spilling",
"New Runtime wout/Spilling",
"Diff %"]))
- for db_name, old_queries in old_runtime_info.iteritems():
+ for db_name, old_queries in old_runtime_info.items():
new_queries = new_runtime_info.get(db_name)
if not new_queries:
continue
- for sql, old_query in old_queries.iteritems():
+ for sql, old_query in old_queries.items():
new_query = new_queries.get(sql)
if not new_query:
continue
[impala] 05/06: IMPALA-11976: Fix use of deprecated functions/fields removed in Python 3
Posted by jo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
joemcdonnell pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit aa4050b4d9fe3cfc20dde1386c67cea55e9ce3e5
Author: Joe McDonnell <jo...@cloudera.com>
AuthorDate: Sat Mar 4 10:21:59 2023 -0800
IMPALA-11976: Fix use of deprecated functions/fields removed in Python 3
Python 3 moved several things around or removed deprecated
functions / fields:
- sys.maxint was removed, but sys.maxsize provides similar functionality
- long was removed, but int provides the same range
- file() was removed, but open() already provided the same functionality
- Exception.message was removed, but str(exception) is equivalent
- Some encodings (like hex) were moved to codecs.encode()
- string.letters -> string.ascii_letters
- string.lowercase -> string.ascii_lowercase
- string.strip was removed
This fixes all of those locations. Python 3 also has slightly different
rounding behavior from round(), so this changes round() to use future's
builtins.round() to get the Python 3 behavior.
This fixes the following pylint warnings:
- file-builtin
- long-builtin
- invalid-str-codec
- round-builtin
- deprecated-string-function
- sys-max-int
- exception-message-attribute
Testing:
- Ran cores tests
Change-Id: I094cd7fd06b0d417fc875add401d18c90d7a792f
Reviewed-on: http://gerrit.cloudera.org:8080/19591
Reviewed-by: Joe McDonnell <jo...@cloudera.com>
Tested-by: Joe McDonnell <jo...@cloudera.com>
---
bin/banned_py3k_warnings.txt | 7 +++++++
bin/parse-thrift-profile.py | 2 +-
docker/setup_build_context.py | 4 ++--
lib/python/impala_py_lib/gdb/impala-gdb.py | 4 ++--
testdata/bin/wait-for-hiveserver2.py | 4 ++--
testdata/bin/wait-for-metastore.py | 4 ++--
tests/common/impala_connection.py | 5 +++--
tests/common/impala_test_suite.py | 4 ++--
tests/common/kudu_test_suite.py | 2 +-
tests/comparison/cluster.py | 8 ++++----
tests/comparison/discrepancy_searcher.py | 2 +-
tests/custom_cluster/test_admission_controller.py | 6 +++---
tests/custom_cluster/test_frontend_connection_limit.py | 2 +-
tests/custom_cluster/test_hs2_fault_injection.py | 1 +
tests/custom_cluster/test_parquet_max_page_header.py | 4 ++--
tests/custom_cluster/test_restart_services.py | 4 ++--
tests/metadata/test_hms_integration.py | 4 ++--
tests/metadata/test_last_ddl_time_update.py | 9 +++++----
tests/performance/query_exec_functions.py | 4 ++--
tests/query_test/test_insert_parquet.py | 2 +-
tests/query_test/test_query_mem_limit.py | 6 +++---
tests/query_test/test_scanners_fuzz.py | 4 ++--
tests/shell/test_shell_interactive.py | 2 +-
tests/stress/concurrent_select.py | 10 +++++-----
tests/stress/query_runner.py | 7 ++++---
tests/unittests/test_file_parser.py | 2 +-
tests/unittests/test_result_verifier.py | 4 ++--
27 files changed, 64 insertions(+), 53 deletions(-)
diff --git a/bin/banned_py3k_warnings.txt b/bin/banned_py3k_warnings.txt
index a970906a0..652d91075 100644
--- a/bin/banned_py3k_warnings.txt
+++ b/bin/banned_py3k_warnings.txt
@@ -10,3 +10,10 @@ deprecated-itertools-function
dict-iter-method
dict-keys-not-iterating
dict-values-not-iterating
+file-builtin
+long-builtin
+invalid-str-codec
+round-builtin
+deprecated-string-function
+sys-max-int
+exception-message-attribute
diff --git a/bin/parse-thrift-profile.py b/bin/parse-thrift-profile.py
index 8e16aa079..6062d6536 100755
--- a/bin/parse-thrift-profile.py
+++ b/bin/parse-thrift-profile.py
@@ -46,7 +46,7 @@ import sys
if len(sys.argv) == 1 or sys.argv[1] == "-":
input_data = sys.stdin
elif len(sys.argv) == 2:
- input_data = file(sys.argv[1])
+ input_data = open(sys.argv[1])
else:
print("Usage: %s [file]" % (sys.argv[0],), file=sys.stderr)
sys.exit(1)
diff --git a/docker/setup_build_context.py b/docker/setup_build_context.py
index 9d32ab1d6..82f13e0bf 100755
--- a/docker/setup_build_context.py
+++ b/docker/setup_build_context.py
@@ -194,7 +194,7 @@ if args.utility_context:
else:
# Impala Coordinator dependencies.
num_jars_on_classpath = 0
- dep_classpath = file(os.path.join(IMPALA_HOME, "fe/target/build-classpath.txt")).read()
+ dep_classpath = open(os.path.join(IMPALA_HOME, "fe/target/build-classpath.txt")).read()
for jar in dep_classpath.split(":"):
num_jars_on_classpath += 1
assert os.path.exists(jar), "missing jar from classpath: {0}".format(jar)
@@ -216,7 +216,7 @@ else:
assert num_frontend_jars == 1
# Impala Executor dependencies.
- dep_classpath = file(os.path.join(IMPALA_HOME,
+ dep_classpath = open(os.path.join(IMPALA_HOME,
"java/executor-deps/target/build-executor-deps-classpath.txt")).read()
for jar in dep_classpath.split(":"):
assert os.path.exists(jar), "missing jar from classpath: {0}".format(jar)
diff --git a/lib/python/impala_py_lib/gdb/impala-gdb.py b/lib/python/impala_py_lib/gdb/impala-gdb.py
index c5aa93d21..432ce14a8 100644
--- a/lib/python/impala_py_lib/gdb/impala-gdb.py
+++ b/lib/python/impala_py_lib/gdb/impala-gdb.py
@@ -49,8 +49,8 @@ def get_fragment_instances():
# No valid thread_debug_info
if not tdi:
break
- hi = long(tdi['instance_id_']['hi'])
- lo = long(tdi['instance_id_']['lo'])
+ hi = int(tdi['instance_id_']['hi'])
+ lo = int(tdi['instance_id_']['lo'])
fi = "%lx:%lx" % (hi, lo)
if fi != "0:0":
fragment_instances[fi.strip('"')].append(thread.num)
diff --git a/testdata/bin/wait-for-hiveserver2.py b/testdata/bin/wait-for-hiveserver2.py
index 6ba00f012..37a2898f2 100755
--- a/testdata/bin/wait-for-hiveserver2.py
+++ b/testdata/bin/wait-for-hiveserver2.py
@@ -75,10 +75,10 @@ while time.time() - now < TIMEOUT_SECONDS:
print("HiveServer2 service is up at %s." % options.hs2_hostport)
exit(0)
except Exception as e:
- if "SASL" in e.message: # Bail out on SASL failures
+ if "SASL" in str(e): # Bail out on SASL failures
print("SASL failure when attempting connection:")
raise
- if "GSS" in e.message: # Other GSSAPI failures
+ if "GSS" in str(e): # Other GSSAPI failures
print("GSS failure when attempting connection:")
raise
print("Waiting for HiveServer2 at %s..." % options.hs2_hostport)
diff --git a/testdata/bin/wait-for-metastore.py b/testdata/bin/wait-for-metastore.py
index 6b62487c0..ab898fa27 100755
--- a/testdata/bin/wait-for-metastore.py
+++ b/testdata/bin/wait-for-metastore.py
@@ -60,10 +60,10 @@ while time.time() - now < TIMEOUT_SECONDS:
print("Metastore service is up at %s." % options.metastore_hostport)
exit(0)
except Exception as e:
- if "SASL" in e.message: # Bail out on SASL failures
+ if "SASL" in str(e): # Bail out on SASL failures
print("SASL failure when attempting connection:")
raise
- if "GSS" in e.message: # Other GSSAPI failures
+ if "GSS" in str(e): # Other GSSAPI failures
print("GSS failure when attempting connection:")
raise
print("Waiting for the Metastore at %s..." % options.metastore_hostport)
diff --git a/tests/common/impala_connection.py b/tests/common/impala_connection.py
index 6c949ea23..54c706d2b 100644
--- a/tests/common/impala_connection.py
+++ b/tests/common/impala_connection.py
@@ -21,6 +21,7 @@
from __future__ import absolute_import, division, print_function
import abc
+import codecs
import logging
import re
@@ -386,8 +387,8 @@ class ImpylaHS2Connection(ImpalaConnection):
"""Return the string representation of the query id."""
guid_bytes = \
operation_handle.get_handle()._last_operation.handle.operationId.guid
- return "{0}:{1}".format(guid_bytes[7::-1].encode('hex_codec'),
- guid_bytes[16:7:-1].encode('hex_codec'))
+ return "{0}:{1}".format(codecs.encode(guid_bytes[7::-1], 'hex_codec'),
+ codecs.encode(guid_bytes[16:7:-1], 'hex_codec'))
def get_state(self, operation_handle):
LOG.info("-- getting state for operation: {0}".format(operation_handle))
diff --git a/tests/common/impala_test_suite.py b/tests/common/impala_test_suite.py
index c8c780a3b..bc83ca609 100644
--- a/tests/common/impala_test_suite.py
+++ b/tests/common/impala_test_suite.py
@@ -18,7 +18,7 @@
# The base class that should be used for almost all Impala tests
from __future__ import absolute_import, division, print_function
-from builtins import range
+from builtins import range, round
import glob
import grp
import json
@@ -1041,7 +1041,7 @@ class ImpalaTestSuite(BaseTestSuite):
# is specified; explicitly make sure there's nothing to
# read to avoid hanging, especially when running interactively
# with py.test.
- stdin=file("/dev/null"),
+ stdin=open("/dev/null"),
env=env)
(stdout, stderr) = call.communicate()
call.wait()
diff --git a/tests/common/kudu_test_suite.py b/tests/common/kudu_test_suite.py
index 0d8eca371..74791b0a9 100644
--- a/tests/common/kudu_test_suite.py
+++ b/tests/common/kudu_test_suite.py
@@ -111,7 +111,7 @@ class KuduTestSuite(ImpalaTestSuite):
@classmethod
def random_table_name(cls):
- return "".join(choice(string.lowercase) for _ in range(10))
+ return "".join(choice(string.ascii_lowercase) for _ in range(10))
@classmethod
def to_kudu_table_name(cls, db_name, tbl_name):
diff --git a/tests/comparison/cluster.py b/tests/comparison/cluster.py
index f95de563d..517df3f94 100644
--- a/tests/comparison/cluster.py
+++ b/tests/comparison/cluster.py
@@ -21,7 +21,7 @@
# module depends on db_connection which use some query generator classes.
from __future__ import absolute_import, division, print_function
-from builtins import range, zip
+from builtins import int, range, zip
import hdfs
import logging
import os
@@ -37,7 +37,7 @@ from getpass import getuser
from multiprocessing.pool import ThreadPool
from random import choice
from StringIO import StringIO
-from sys import maxint
+from sys import maxsize
from tempfile import mkdtemp
from threading import Lock
from time import mktime, strptime
@@ -629,7 +629,7 @@ class Impala(Service):
impalads = self.impalads
promise = self._thread_pool.map_async(func, impalads)
# Python doesn't handle ctrl-c well unless a timeout is provided.
- results = promise.get(maxint)
+ results = promise.get(maxsize)
if as_dict:
results = dict(zip(impalads, results))
return results
@@ -874,7 +874,7 @@ class MiniClusterImpalad(Impalad):
return int(pid)
def find_process_mem_mb_limit(self):
- return long(self.get_metric("mem-tracker.process.limit")["value"]) // 1024 ** 2
+ return int(self.get_metric("mem-tracker.process.limit")["value"]) // 1024 ** 2
def find_core_dump_dir(self):
raise NotImplementedError()
diff --git a/tests/comparison/discrepancy_searcher.py b/tests/comparison/discrepancy_searcher.py
index 2e1fdb143..079850fd4 100755
--- a/tests/comparison/discrepancy_searcher.py
+++ b/tests/comparison/discrepancy_searcher.py
@@ -25,7 +25,7 @@
# TODO: IMPALA-4600: refactor this module
from __future__ import absolute_import, division, print_function
-from builtins import range, zip
+from builtins import range, round, zip
from copy import deepcopy
from decimal import Decimal
from logging import getLogger
diff --git a/tests/custom_cluster/test_admission_controller.py b/tests/custom_cluster/test_admission_controller.py
index 1fc7a14af..c25719248 100644
--- a/tests/custom_cluster/test_admission_controller.py
+++ b/tests/custom_cluster/test_admission_controller.py
@@ -18,7 +18,7 @@
# Tests admission control
from __future__ import absolute_import, division, print_function
-from builtins import range
+from builtins import int, range, round
import itertools
import logging
import os
@@ -504,7 +504,7 @@ class TestAdmissionController(TestAdmissionControllerBase, HS2TestSuite):
self.execute_query_expect_success(self.client, query, exec_options)
# A bit too much memory to run on coordinator.
- exec_options['mem_limit'] = long(self.PROC_MEM_TEST_LIMIT * 1.1)
+ exec_options['mem_limit'] = int(self.PROC_MEM_TEST_LIMIT * 1.1)
ex = self.execute_query_expect_failure(self.client, query, exec_options)
assert ("Rejected query from pool default-pool: request memory needed "
"1.10 GB is greater than memory available for admission 1.00 GB" in
@@ -2218,7 +2218,7 @@ class TestAdmissionControllerStress(TestAdmissionControllerBase):
# should be fine. This exercises the code that does the per-pool memory
# accounting (see MemTracker::GetPoolMemReserved()) without actually being throttled.
self.run_admission_test(vector, {'request_pool': self.pool_name,
- 'mem_limit': sys.maxint})
+ 'mem_limit': sys.maxsize})
@pytest.mark.execute_serially
@SkipIfOS.redhat6
diff --git a/tests/custom_cluster/test_frontend_connection_limit.py b/tests/custom_cluster/test_frontend_connection_limit.py
index 911b808c9..409b2b9a1 100644
--- a/tests/custom_cluster/test_frontend_connection_limit.py
+++ b/tests/custom_cluster/test_frontend_connection_limit.py
@@ -47,7 +47,7 @@ class TestFrontendConnectionLimit(CustomClusterTestSuite):
client.execute(query)
except Exception as e:
client.close()
- raise ImpalaBeeswaxException(e.message)
+ raise ImpalaBeeswaxException(str(e))
client.close()
@pytest.mark.execute_serially
diff --git a/tests/custom_cluster/test_hs2_fault_injection.py b/tests/custom_cluster/test_hs2_fault_injection.py
index cb9866044..67d52c2b7 100644
--- a/tests/custom_cluster/test_hs2_fault_injection.py
+++ b/tests/custom_cluster/test_hs2_fault_injection.py
@@ -16,6 +16,7 @@
# under the License.
from __future__ import absolute_import, division, print_function
+from builtins import round
import pytest
import requests
diff --git a/tests/custom_cluster/test_parquet_max_page_header.py b/tests/custom_cluster/test_parquet_max_page_header.py
index 52e13c690..81fa03ef3 100644
--- a/tests/custom_cluster/test_parquet_max_page_header.py
+++ b/tests/custom_cluster/test_parquet_max_page_header.py
@@ -91,9 +91,9 @@ class TestParquetMaxPageHeader(CustomClusterTestSuite):
"""Creates a file in HDFS containing two MAX_STRING_LENGTH lines."""
file_name = os.path.join(dir, file)
# Create two 10MB long strings.
- random_text1 = "".join([random.choice(string.letters)
+ random_text1 = "".join([random.choice(string.ascii_letters)
for i in range(self.MAX_STRING_LENGTH)])
- random_text2 = "".join([random.choice(string.letters)
+ random_text2 = "".join([random.choice(string.ascii_letters)
for i in range(self.MAX_STRING_LENGTH)])
put = subprocess.Popen(["hdfs", "dfs", "-put", "-d", "-f", "-", file_name],
stdin=subprocess.PIPE, bufsize=-1)
diff --git a/tests/custom_cluster/test_restart_services.py b/tests/custom_cluster/test_restart_services.py
index 1492089f5..f8afe450c 100644
--- a/tests/custom_cluster/test_restart_services.py
+++ b/tests/custom_cluster/test_restart_services.py
@@ -65,8 +65,8 @@ class TestRestart(CustomClusterTestSuite):
cursor.execute("describe database functional")
return
except HiveServer2Error as e:
- assert "AnalysisException: Database does not exist: functional" in e.message,\
- "Unexpected exception: " + e.message
+ assert "AnalysisException: Database does not exist: functional" in str(e),\
+ "Unexpected exception: " + str(e)
sleep(1)
assert False, "Coordinator never received non-empty metadata from the restarted " \
"statestore after {0} seconds".format(wait_time_s)
diff --git a/tests/metadata/test_hms_integration.py b/tests/metadata/test_hms_integration.py
index 9caa5dda0..b67d0cac9 100644
--- a/tests/metadata/test_hms_integration.py
+++ b/tests/metadata/test_hms_integration.py
@@ -236,7 +236,7 @@ class TestHmsIntegration(ImpalaTestSuite):
dictionary that holds the parsed attributes."""
result = {}
output_lines = output.split('\n')
- stat_names = list(map(string.strip, output_lines[0].split(',')))
+ stat_names = [s.strip() for s in output_lines[0].split(',')]
stat_values = output_lines[3].split(',')
assert len(stat_names) == len(stat_values)
for i in range(0, len(stat_names)):
@@ -248,7 +248,7 @@ class TestHmsIntegration(ImpalaTestSuite):
dictionary that holds the parsed attributes."""
result = {}
for line in output.split('\n'):
- line_elements = list(map(string.strip, line.split(',')))
+ line_elements = [s.strip() for s in line.split(',')]
if len(line_elements) >= 2:
result[line_elements[0]] = line_elements[1]
return result
diff --git a/tests/metadata/test_last_ddl_time_update.py b/tests/metadata/test_last_ddl_time_update.py
index 7e44b4bd5..babad7391 100644
--- a/tests/metadata/test_last_ddl_time_update.py
+++ b/tests/metadata/test_last_ddl_time_update.py
@@ -17,6 +17,7 @@
# Impala tests for DDL statements
from __future__ import absolute_import, division, print_function
+from builtins import int
import time
from tests.common.impala_test_suite import ImpalaTestSuite
@@ -110,15 +111,15 @@ class TestLastDdlTimeUpdate(ImpalaTestSuite):
if expect_changed_ddl_time:
# check that the new ddlTime is strictly greater than the old one.
- assert long(afterDdlTime) > long(beforeDdlTime)
+ assert int(afterDdlTime) > int(beforeDdlTime)
else:
- assert long(afterDdlTime) == long(beforeDdlTime)
+ assert int(afterDdlTime) == int(beforeDdlTime)
if expect_changed_stats_time:
# check that the new statsTime is strictly greater than the old one.
- assert long(afterStatsTime) > long(beforeStatsTime)
+ assert int(afterStatsTime) > int(beforeStatsTime)
else:
- assert long(afterStatsTime) == long(beforeStatsTime)
+ assert int(afterStatsTime) == int(beforeStatsTime)
def _update_name(self, new_tbl_name):
""""
diff --git a/tests/performance/query_exec_functions.py b/tests/performance/query_exec_functions.py
index 471a1ea61..0da147146 100644
--- a/tests/performance/query_exec_functions.py
+++ b/tests/performance/query_exec_functions.py
@@ -22,7 +22,7 @@ import re
from datetime import datetime
from impala.dbapi import connect
from tests.beeswax.impala_beeswax import ImpalaBeeswaxClient, ImpalaBeeswaxResult
-from sys import maxint
+from sys import maxsize
from tests.performance.query import HiveQueryResult, ImpalaQueryResult
from tests.util.shell_util import exec_process
from time import time
@@ -44,7 +44,7 @@ def get_hs2_hive_cursor(hiveserver, user=None, use_kerberos=False,
user=user,
database=database,
auth_mechanism="GSSAPI" if use_kerberos else "PLAIN",
- timeout=maxint)
+ timeout=maxsize)
cursor = conn.cursor(configuration=execOptions)
LOG.info("Connected to {0}:{1}".format(host, port))
diff --git a/tests/query_test/test_insert_parquet.py b/tests/query_test/test_insert_parquet.py
index b4f9307da..9264643ab 100644
--- a/tests/query_test/test_insert_parquet.py
+++ b/tests/query_test/test_insert_parquet.py
@@ -18,7 +18,7 @@
# Targeted Impala insert tests
from __future__ import absolute_import, division, print_function
-from builtins import map, range
+from builtins import map, range, round
import os
from collections import namedtuple
diff --git a/tests/query_test/test_query_mem_limit.py b/tests/query_test/test_query_mem_limit.py
index bd3615c35..e12c18f0f 100644
--- a/tests/query_test/test_query_mem_limit.py
+++ b/tests/query_test/test_query_mem_limit.py
@@ -46,9 +46,9 @@ class TestQueryMemLimit(ImpalaTestSuite):
# dynamically, even if it is a rough approximation.
# A mem_limit is expressed in bytes, with values <= 0 signifying no cap.
# These values are either really small, unlimited, or have a really large cap.
- MAXINT_BYTES = str(sys.maxint)
- MAXINT_MB = str(sys.maxint // (1024 * 1024))
- MAXINT_GB = str(sys.maxint // (1024 * 1024 * 1024))
+ MAXINT_BYTES = str(sys.maxsize)
+ MAXINT_MB = str(sys.maxsize // (1024 * 1024))
+ MAXINT_GB = str(sys.maxsize // (1024 * 1024 * 1024))
# We expect the tests with MAXINT_* using valid units [bmg] to succeed.
PASS_REGEX = re.compile("(%s|%s|%s)[bmg]?$" % (MAXINT_BYTES, MAXINT_MB, MAXINT_GB),
re.I)
diff --git a/tests/query_test/test_scanners_fuzz.py b/tests/query_test/test_scanners_fuzz.py
index ed7ce144c..bda50f982 100644
--- a/tests/query_test/test_scanners_fuzz.py
+++ b/tests/query_test/test_scanners_fuzz.py
@@ -16,7 +16,7 @@
# under the License.
from __future__ import absolute_import, division, print_function
-from builtins import range
+from builtins import range, int
from copy import copy
import itertools
import logging
@@ -175,7 +175,7 @@ class TestScannersFuzzing(ImpalaTestSuite):
rng = random.Random()
random_seed = os.environ.get("SCANNER_FUZZ_SEED") or time.time()
LOG.info("Using random seed %d", random_seed)
- rng.seed(long(random_seed))
+ rng.seed(int(random_seed))
tmp_table_dir = tempfile.mkdtemp(prefix="tmp-scanner-fuzz-%s" % fuzz_table,
dir=os.path.join(os.environ['IMPALA_HOME'], "testdata"))
diff --git a/tests/shell/test_shell_interactive.py b/tests/shell/test_shell_interactive.py
index f3469e907..101793613 100755
--- a/tests/shell/test_shell_interactive.py
+++ b/tests/shell/test_shell_interactive.py
@@ -626,7 +626,7 @@ class TestImpalaShellInteractive(ImpalaTestSuite):
self._expect_with_cmd(child_proc, "select 'hi'", vector, ('hi'))
child_proc.sendline('exit;')
child_proc.expect(pexpect.EOF)
- history_contents = file(new_hist.name).read()
+ history_contents = open(new_hist.name).read()
assert "select 'hi'" in history_contents
def test_rerun(self, vector, tmp_history_file):
diff --git a/tests/stress/concurrent_select.py b/tests/stress/concurrent_select.py
index 36ee36363..96b8978f7 100755
--- a/tests/stress/concurrent_select.py
+++ b/tests/stress/concurrent_select.py
@@ -71,7 +71,7 @@ from copy import copy
from datetime import datetime
from multiprocessing import Lock, Process, Queue, Value
from random import choice, random, randrange, shuffle
-from sys import exit, maxint
+from sys import exit, maxsize
from tempfile import gettempdir
from textwrap import dedent
from threading import current_thread
@@ -596,7 +596,7 @@ class StressRunner(object):
else:
# Let the query run as long as necessary - it is nearly impossible to pick a
# good value that won't have false positives under load - see IMPALA-8222.
- timeout = maxint
+ timeout = maxsize
report = query_runner.run_query(query, mem_limit, timeout_secs=timeout,
cancel_mech=cancel_mech)
LOG.debug("Got execution report for query")
@@ -858,7 +858,7 @@ def populate_runtime_info_for_random_queries(impala, candidate_queries, converte
return queries
-def populate_runtime_info(query, impala, converted_args, timeout_secs=maxint):
+def populate_runtime_info(query, impala, converted_args, timeout_secs=maxsize):
"""Runs the given query by itself repeatedly until the minimum memory is determined
with and without spilling. Potentially all fields in the Query class (except
'sql') will be populated by this method. 'required_mem_mb_without_spilling' and
@@ -997,7 +997,7 @@ def populate_runtime_info(query, impala, converted_args, timeout_secs=maxint):
LOG.info("Finding minimum memory required to avoid spilling")
lower_bound = max(limit_exceeded_mem, spill_mem)
- upper_bound = min(non_spill_mem or maxint, impala.min_impalad_mem_mb)
+ upper_bound = min(non_spill_mem or maxsize, impala.min_impalad_mem_mb)
while True:
if old_required_mem_mb_without_spilling:
mem_limit = old_required_mem_mb_without_spilling
@@ -1034,7 +1034,7 @@ def populate_runtime_info(query, impala, converted_args, timeout_secs=maxint):
LOG.info("Finding absolute minimum memory required")
lower_bound = limit_exceeded_mem
upper_bound = min(
- spill_mem or maxint, non_spill_mem or maxint, impala.min_impalad_mem_mb)
+ spill_mem or maxsize, non_spill_mem or maxsize, impala.min_impalad_mem_mb)
while True:
if old_required_mem_mb_with_spilling:
mem_limit = old_required_mem_mb_with_spilling
diff --git a/tests/stress/query_runner.py b/tests/stress/query_runner.py
index fa6b5f5d2..bdf591f0a 100644
--- a/tests/stress/query_runner.py
+++ b/tests/stress/query_runner.py
@@ -18,13 +18,14 @@
# under the License.
from __future__ import absolute_import, division, print_function
+from builtins import round
import logging
from multiprocessing import Value
import os
import re
from textwrap import dedent
from time import sleep, time
-from sys import maxint
+from sys import maxsize
from tests.stress.queries import QueryType
from tests.stress.util import create_and_start_daemon_thread, increment
@@ -103,7 +104,7 @@ class QueryRunner(object):
self.impalad_conn = self.impalad.impala.connect(impalad=self.impalad)
def run_query(self, query, mem_limit_mb, run_set_up=False,
- timeout_secs=maxint, cancel_mech=None, retain_profile=False):
+ timeout_secs=maxsize, cancel_mech=None, retain_profile=False):
"""Run a query and return an execution report. If 'run_set_up' is True, set up sql
will be executed before the main query. This should be the case during the binary
search phase of the stress test. 'cancel_mech' is optionally a CancelMechanism
@@ -472,7 +473,7 @@ def _add_row_to_hash(row, curr_hash):
curr_hash += _hash_val(idx, val)
# Modulo the result to keep it "small" otherwise the math ops can be slow
# since python does infinite precision math.
- curr_hash %= maxint
+ curr_hash %= maxsize
return curr_hash
diff --git a/tests/unittests/test_file_parser.py b/tests/unittests/test_file_parser.py
index e6f7a935e..fbb5d1789 100644
--- a/tests/unittests/test_file_parser.py
+++ b/tests/unittests/test_file_parser.py
@@ -88,7 +88,7 @@ class TestTestFileParser(BaseTestSuite):
skip_unknown_sections=False)
assert 0, 'Expected error due to invalid section'
except RuntimeError as re:
- assert re.message == "Unknown subsection: TYPES"
+ assert str(re) == "Unknown subsection: TYPES"
def test_parse_query_name(self):
results = parse_test_file_text(test_text, VALID_SECTIONS, False)
diff --git a/tests/unittests/test_result_verifier.py b/tests/unittests/test_result_verifier.py
index 9142ca1a8..0663fc164 100644
--- a/tests/unittests/test_result_verifier.py
+++ b/tests/unittests/test_result_verifier.py
@@ -50,13 +50,13 @@ class TestResultVerifier(ImpalaTestSuite):
res.rows[0]['does_not_exist']
assert False, 'Expected error due to column alias not existing'
except IndexError as e:
- assert "No column with label: does_not_exist" in e.message
+ assert "No column with label: does_not_exist" in str(e)
try:
res.rows[0][2]
assert False, 'Expected error due to column position not existing'
except IndexError as e:
- assert 'list index out of range' in e.message
+ assert 'list index out of range' in str(e)
def test_compute_aggregation(self, vector):
profile = '''