You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by mi...@apache.org on 2018/02/01 16:46:09 UTC
[3/4] impala git commit: IMPALA-4319: remove some deprecated query
options
IMPALA-4319: remove some deprecated query options
Adds a concept of a "removed" query option that has no effect but does
not return an error when a user attempts to set it. These options are
not returned by "set" or "set all" commands that are executed in
impala-shell or server-side.
These query options have been deprecated for several releases:
DEFAULT_ORDER_BY_LIMIT, ABORT_ON_DEFAULT_LIMIT_EXCEEDED,
V_CPU_CORES, RESERVATION_REQUEST_TIMEOUT, RM_INITIAL_MEM,
SCAN_NODE_CODEGEN_THRESHOLD, MAX_IO_BUFFERS
RM_INITIAL_MEM did still have an effect, but it was undocumented and
MEM_LIMIT should be used in preference.
DISABLE_CACHED_READS also had an effect but it was documented as
deprecated.
Otherwise the options had no effect at all.
Testing:
Ran exhaustive build.
Updated query option tests to reflect the new behaviour.
Cherry-picks: not for 2.x.
Change-Id: I9e742e9b0eca0e5c81fd71db3122fef31522fcad
Reviewed-on: http://gerrit.cloudera.org:8080/9118
Reviewed-by: Tim Armstrong <ta...@cloudera.com>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/acfd169c
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/acfd169c
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/acfd169c
Branch: refs/heads/master
Commit: acfd169c8e814adb61a1f1d35327b38ee980f217
Parents: 1b1087e
Author: Tim Armstrong <ta...@cloudera.com>
Authored: Tue Jan 23 14:36:11 2018 -0800
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Thu Feb 1 08:26:26 2018 +0000
----------------------------------------------------------------------
be/src/exec/hdfs-scan-node-base.cc | 3 -
be/src/runtime/runtime-state.h | 3 -
be/src/scheduling/query-schedule.cc | 6 +-
be/src/scheduling/scheduler-test-util.h | 1 -
be/src/scheduling/scheduler-test.cc | 27 --------
be/src/scheduling/scheduler.cc | 7 ---
be/src/scheduling/scheduler.h | 7 ---
be/src/service/child-query.cc | 2 +
be/src/service/client-request-state.cc | 1 +
be/src/service/query-options-test.cc | 8 +--
be/src/service/query-options.cc | 65 +++++++++-----------
be/src/service/query-options.h | 20 +++---
common/thrift/ImpalaInternalService.thrift | 29 ---------
common/thrift/ImpalaService.thrift | 28 +++------
common/thrift/beeswax.thrift | 6 +-
shell/impala_shell.py | 17 +++--
.../functional-query/queries/QueryTest/set.test | 32 ++++------
tests/comparison/discrepancy_searcher.py | 2 -
.../custom_cluster/test_admission_controller.py | 5 +-
tests/hs2/test_hs2.py | 12 +++-
tests/query_test/test_observability.py | 2 +-
tests/shell/test_shell_commandline.py | 14 +++--
tests/shell/test_shell_interactive.py | 8 ++-
23 files changed, 105 insertions(+), 200 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/acfd169c/be/src/exec/hdfs-scan-node-base.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/hdfs-scan-node-base.cc b/be/src/exec/hdfs-scan-node-base.cc
index 2f3d0c5..8b065fa 100644
--- a/be/src/exec/hdfs-scan-node-base.cc
+++ b/be/src/exec/hdfs-scan-node-base.cc
@@ -232,9 +232,6 @@ Status HdfsScanNodeBase::Prepare(RuntimeState* state) {
if (expected_local && params.volume_id == -1) ++num_ranges_missing_volume_id;
bool try_cache = params.is_cached;
- if (runtime_state_->query_options().disable_cached_reads) {
- DCHECK(!try_cache) << "Params should not have had this set.";
- }
file_desc->splits.push_back(
AllocateScanRange(file_desc->fs, file_desc->filename.c_str(), split.length,
split.offset, split.partition_id, params.volume_id, expected_local,
http://git-wip-us.apache.org/repos/asf/impala/blob/acfd169c/be/src/runtime/runtime-state.h
----------------------------------------------------------------------
diff --git a/be/src/runtime/runtime-state.h b/be/src/runtime/runtime-state.h
index 4eb3e10..66b2099 100644
--- a/be/src/runtime/runtime-state.h
+++ b/be/src/runtime/runtime-state.h
@@ -102,9 +102,6 @@ class RuntimeState {
int batch_size() const { return query_options().batch_size; }
bool abort_on_error() const { return query_options().abort_on_error; }
bool strict_mode() const { return query_options().strict_mode; }
- bool abort_on_default_limit_exceeded() const {
- return query_options().abort_on_default_limit_exceeded;
- }
bool decimal_v2() const { return query_options().decimal_v2; }
const TQueryCtx& query_ctx() const;
const TPlanFragmentInstanceCtx& instance_ctx() const { return *instance_ctx_; }
http://git-wip-us.apache.org/repos/asf/impala/blob/acfd169c/be/src/scheduling/query-schedule.cc
----------------------------------------------------------------------
diff --git a/be/src/scheduling/query-schedule.cc b/be/src/scheduling/query-schedule.cc
index e14af78..f833273 100644
--- a/be/src/scheduling/query-schedule.cc
+++ b/be/src/scheduling/query-schedule.cc
@@ -193,11 +193,7 @@ int64_t QuerySchedule::GetPerHostMemoryEstimate() const {
}
int64_t per_host_mem = 0L;
- // TODO: Remove rm_initial_mem and associated logic when we're sure that clients won't
- // be affected.
- if (query_options_.__isset.rm_initial_mem && query_options_.rm_initial_mem > 0) {
- per_host_mem = query_options_.rm_initial_mem;
- } else if (has_query_option) {
+ if (has_query_option) {
per_host_mem = query_option_memory_limit;
} else {
DCHECK(request_.__isset.per_host_mem_estimate);
http://git-wip-us.apache.org/repos/asf/impala/blob/acfd169c/be/src/scheduling/scheduler-test-util.h
----------------------------------------------------------------------
diff --git a/be/src/scheduling/scheduler-test-util.h b/be/src/scheduling/scheduler-test-util.h
index c3d6b1c..254acb0 100644
--- a/be/src/scheduling/scheduler-test-util.h
+++ b/be/src/scheduling/scheduler-test-util.h
@@ -236,7 +236,6 @@ class Plan {
void SetReplicaPreference(TReplicaPreference::type p);
void SetRandomReplica(bool b) { query_options_.schedule_random_replica = b; }
- void SetDisableCachedReads(bool b) { query_options_.disable_cached_reads = b; }
const Cluster& cluster() const { return schema_.cluster(); }
const std::vector<TNetworkAddress>& referenced_datanodes() const;
http://git-wip-us.apache.org/repos/asf/impala/blob/acfd169c/be/src/scheduling/scheduler-test.cc
----------------------------------------------------------------------
diff --git a/be/src/scheduling/scheduler-test.cc b/be/src/scheduling/scheduler-test.cc
index c70e54f..ca0dbbd 100644
--- a/be/src/scheduling/scheduler-test.cc
+++ b/be/src/scheduling/scheduler-test.cc
@@ -281,33 +281,6 @@ TEST_F(SchedulerTest, TestCachedReadPreferred) {
EXPECT_EQ(0, result.NumRemoteAssignedBytes());
}
-/// Verify that disable_cached_reads is effective.
-TEST_F(SchedulerTest, TestDisableCachedReads) {
- Cluster cluster;
- cluster.AddHosts(3, true, true);
-
- Schema schema(cluster);
- schema.AddSingleBlockTable("T1", {0, 2}, {1});
-
- Plan plan(schema);
- // 1 of the 3 replicas is cached.
- plan.AddTableScan("T1");
- plan.SetDisableCachedReads(true);
-
- Result result(plan);
- SchedulerWrapper scheduler(plan);
- ASSERT_OK(scheduler.Compute(&result));
- EXPECT_EQ(0, result.NumCachedAssignedBytes());
- EXPECT_EQ(1 * Block::DEFAULT_BLOCK_SIZE, result.NumDiskAssignedBytes());
- EXPECT_EQ(0, result.NumRemoteAssignedBytes());
-
- // Compute additional assignments.
- for (int i = 0; i < 8; ++i) ASSERT_OK(scheduler.Compute(&result));
- EXPECT_EQ(0, result.NumCachedAssignedBytes());
- EXPECT_EQ(9 * Block::DEFAULT_BLOCK_SIZE, result.NumDiskAssignedBytes());
- EXPECT_EQ(0, result.NumRemoteAssignedBytes());
-}
-
/// IMPALA-3019: Test for round robin reset problem. We schedule the same plan twice but
/// send an empty statestored message in between.
/// TODO: This problem cannot occur anymore and the test is merely green for random
http://git-wip-us.apache.org/repos/asf/impala/blob/acfd169c/be/src/scheduling/scheduler.cc
----------------------------------------------------------------------
diff --git a/be/src/scheduling/scheduler.cc b/be/src/scheduling/scheduler.cc
index e924f50..527b5cf 100644
--- a/be/src/scheduling/scheduler.cc
+++ b/be/src/scheduling/scheduler.cc
@@ -510,13 +510,6 @@ Status Scheduler::ComputeScanRangeAssignment(const BackendConfig& executor_confi
// of memory distance classes see TReplicaPreference in PlanNodes.thrift.
TReplicaPreference::type base_distance = query_options.replica_preference;
- // The query option to disable cached reads adjusts the memory base distance to view
- // all replicas as having a distance disk_local or worse.
- if (query_options.disable_cached_reads
- && base_distance == TReplicaPreference::CACHE_LOCAL) {
- base_distance = TReplicaPreference::DISK_LOCAL;
- }
-
// A preference attached to the plan node takes precedence.
if (node_replica_preference) base_distance = *node_replica_preference;
http://git-wip-us.apache.org/repos/asf/impala/blob/acfd169c/be/src/scheduling/scheduler.h
----------------------------------------------------------------------
diff --git a/be/src/scheduling/scheduler.h b/be/src/scheduling/scheduler.h
index 2fe90b8..4be2996 100644
--- a/be/src/scheduling/scheduler.h
+++ b/be/src/scheduling/scheduler.h
@@ -58,8 +58,6 @@ class SchedulerWrapper;
/// TODO: Notice when there are duplicate statestore registrations (IMPALA-23)
/// TODO: Track assignments (assignment_ctx in ComputeScanRangeAssignment) per query
/// instead of per plan node?
-/// TODO: Remove disable_cached_reads query option in the next compatibility-breaking
-/// release (IMPALA-2963)
/// TODO: Replace the usage of shared_ptr with atomic_shared_ptr once compilers support
/// it. Alternatively consider using Kudu's rw locks.
/// TODO: Inject global dependencies into the class (for example ExecEnv::GetInstance(),
@@ -376,11 +374,6 @@ class Scheduler {
/// the assignments over more replicas. Allowed values are CACHE_LOCAL (default),
/// DISK_LOCAL and REMOTE.
///
- /// disable_cached_reads:
- /// Setting this value to true is equivalent to setting replica_preference to
- /// DISK_LOCAL and takes precedence over replica_preference. The default setting is
- /// false.
- ///
/// schedule_random_replica:
/// When equivalent executors with a memory distance of DISK_LOCAL are found for a
/// scan range (same memory distance, same amount of assigned work), then the first
http://git-wip-us.apache.org/repos/asf/impala/blob/acfd169c/be/src/service/child-query.cc
----------------------------------------------------------------------
diff --git a/be/src/service/child-query.cc b/be/src/service/child-query.cc
index 520a834..ac4c6be 100644
--- a/be/src/service/child-query.cc
+++ b/be/src/service/child-query.cc
@@ -112,8 +112,10 @@ void ChildQuery::SetQueryOptions(const TQueryOptions& parent_options,
val << parent_options.NAME;\
conf[#ENUM] = val.str();\
}
+#define REMOVED_QUERY_OPT_FN(NAME, ENUM)
QUERY_OPTS_TABLE
#undef QUERY_OPT_FN
+#undef REMOVED_QUERY_OPT_FN
// Ignore debug actions on child queries because they may cause deadlock.
map<string, string>::iterator it = conf.find("DEBUG_ACTION");
if (it != conf.end()) conf.erase(it);
http://git-wip-us.apache.org/repos/asf/impala/blob/acfd169c/be/src/service/client-request-state.cc
----------------------------------------------------------------------
diff --git a/be/src/service/client-request-state.cc b/be/src/service/client-request-state.cc
index ee35afa..64ab950 100644
--- a/be/src/service/client-request-state.cc
+++ b/be/src/service/client-request-state.cc
@@ -241,6 +241,7 @@ void ClientRequestState::PopulateResultForSet(bool is_set_all) {
for (; itr != config.end(); ++itr) {
const auto opt_level_id =
parent_server_->query_option_levels_[itr->first];
+ if (opt_level_id == TQueryOptionLevel::REMOVED) continue;
if (!is_set_all && (opt_level_id == TQueryOptionLevel::DEVELOPMENT ||
opt_level_id == TQueryOptionLevel::DEPRECATED)) {
continue;
http://git-wip-us.apache.org/repos/asf/impala/blob/acfd169c/be/src/service/query-options-test.cc
----------------------------------------------------------------------
diff --git a/be/src/service/query-options-test.cc b/be/src/service/query-options-test.cc
index 552c218..42681d9 100644
--- a/be/src/service/query-options-test.cc
+++ b/be/src/service/query-options-test.cc
@@ -138,7 +138,6 @@ TEST(QueryOptions, SetByteOptions) {
vector<pair<OptionDef<int64_t>, Range<int64_t>>> case_set_i64 {
{MAKE_OPTIONDEF(mem_limit), {-1, I64_MAX}},
{MAKE_OPTIONDEF(max_scan_range_length), {-1, I64_MAX}},
- {MAKE_OPTIONDEF(rm_initial_mem), {-1, I64_MAX}},
{MAKE_OPTIONDEF(buffer_pool_limit), {-1, I64_MAX}},
{MAKE_OPTIONDEF(max_row_size), {1, ROW_SIZE_LIMIT}},
{MAKE_OPTIONDEF(parquet_file_size), {-1, I32_MAX}},
@@ -244,19 +243,14 @@ TEST(QueryOptions, SetIntOptions) {
// Test options with non regular validation rule
TEST(QueryOptions, SetSpecialOptions) {
- // REPLICA_PREFERENCE cannot be set to 0 if DISABLE_CACHED_READS is true
- // It also has unsettable enum values: cache_rack(1) & disk_rack(3)
+ // REPLICA_PREFERENCE has unsettable enum values: cache_rack(1) & disk_rack(3)
TQueryOptions options;
{
OptionDef<TReplicaPreference::type> key_def = MAKE_OPTIONDEF(replica_preference);
auto TestOk = MakeTestOkFn(options, key_def);
auto TestError = MakeTestErrFn(options, key_def);
- EXPECT_OK(SetQueryOption("DISABLE_CACHED_READS", "false", &options, nullptr));
TestOk("cache_local", TReplicaPreference::CACHE_LOCAL);
TestOk("0", TReplicaPreference::CACHE_LOCAL);
- EXPECT_OK(SetQueryOption("DISABLE_CACHED_READS", "true", &options, nullptr));
- TestError("cache_local");
- TestError("0");
TestError("cache_rack");
TestError("1");
TestOk("disk_local", TReplicaPreference::DISK_LOCAL);
http://git-wip-us.apache.org/repos/asf/impala/blob/acfd169c/be/src/service/query-options.cc
----------------------------------------------------------------------
diff --git a/be/src/service/query-options.cc b/be/src/service/query-options.cc
index ff2fd4e..3f7bb7c 100644
--- a/be/src/service/query-options.cc
+++ b/be/src/service/query-options.cc
@@ -62,8 +62,10 @@ void impala::OverlayQueryOptions(const TQueryOptions& src, const QueryOptionsMas
"Size of QueryOptionsMask must be increased.";
#define QUERY_OPT_FN(NAME, ENUM, LEVEL)\
if (src.__isset.NAME && mask[TImpalaQueryOptions::ENUM]) dst->__set_##NAME(src.NAME);
+#define REMOVED_QUERY_OPT_FN(NAME, ENUM)
QUERY_OPTS_TABLE
#undef QUERY_OPT_FN
+#undef REMOVED_QUERY_OPT_FN
}
void impala::TQueryOptionsToMap(const TQueryOptions& query_options,
@@ -78,8 +80,10 @@ void impala::TQueryOptionsToMap(const TQueryOptions& query_options,
(*configuration)[#ENUM] = ""; \
}\
}
+#define REMOVED_QUERY_OPT_FN(NAME, ENUM) (*configuration)[#ENUM] = "";
QUERY_OPTS_TABLE
#undef QUERY_OPT_FN
+#undef REMOVED_QUERY_OPT_FN
}
// Resets query_options->option to its default value.
@@ -91,8 +95,10 @@ static void ResetQueryOption(const int option, TQueryOptions* query_options) {
query_options->__isset.NAME = defaults.__isset.NAME;\
query_options->NAME = defaults.NAME;\
break;
+#define REMOVED_QUERY_OPT_FN(NAME, ENUM)
QUERY_OPTS_TABLE
#undef QUERY_OPT_FN
+#undef REMOVED_QUERY_OPT_FN
}
}
@@ -113,8 +119,10 @@ string impala::DebugQueryOptions(const TQueryOptions& query_options) {
if (i++ > 0) ss << ",";\
ss << #ENUM << "=" << query_options.NAME;\
}
+#define REMOVED_QUERY_OPT_FN(NAME, ENUM)
QUERY_OPTS_TABLE
#undef QUERY_OPT_FN
+#undef REMOVED_QUERY_OPT_FN
return ss.str();
}
@@ -131,6 +139,19 @@ static int GetQueryOptionForKey(const string& key) {
return -1;
}
+// Return true if we can ignore a reference to this removed query option.
+static bool IsRemovedQueryOption(const string& key) {
+#define QUERY_OPT_FN(NAME, ENUM, LEVEL)
+#define REMOVED_QUERY_OPT_FN(NAME, ENUM) \
+ if (iequals(key, #NAME)) { \
+ return true; \
+ }
+ QUERY_OPTS_TABLE
+#undef QUERY_OPT_FN
+#undef REMOVED_QUERY_OPT_FN
+ return false;
+}
+
// Note that we allow numerical values for boolean and enum options. This is because
// TQueryOptionsToMap() will output the numerical values, and we need to parse its output
// configuration.
@@ -185,9 +206,6 @@ Status impala::SetQueryOption(const string& key, const string& value,
query_options->__set_max_scan_range_length(scan_length);
break;
}
- case TImpalaQueryOptions::MAX_IO_BUFFERS:
- query_options->__set_max_io_buffers(atoi(value.c_str()));
- break;
case TImpalaQueryOptions::NUM_SCANNER_THREADS:
query_options->__set_num_scanner_threads(atoi(value.c_str()));
break;
@@ -195,9 +213,6 @@ Status impala::SetQueryOption(const string& key, const string& value,
query_options->__set_allow_unsupported_formats(
iequals(value, "true") || iequals(value, "1"));
break;
- case TImpalaQueryOptions::DEFAULT_ORDER_BY_LIMIT:
- query_options->__set_default_order_by_limit(atoi(value.c_str()));
- break;
case TImpalaQueryOptions::DEBUG_ACTION:
query_options->__set_debug_action(value.c_str());
break;
@@ -233,10 +248,6 @@ Status impala::SetQueryOption(const string& key, const string& value,
}
break;
}
- case TImpalaQueryOptions::ABORT_ON_DEFAULT_LIMIT_EXCEEDED:
- query_options->__set_abort_on_default_limit_exceeded(
- iequals(value, "true") || iequals(value, "1"));
- break;
case TImpalaQueryOptions::HBASE_CACHING:
query_options->__set_hbase_caching(atoi(value.c_str()));
break;
@@ -278,27 +289,10 @@ Status impala::SetQueryOption(const string& key, const string& value,
case TImpalaQueryOptions::REQUEST_POOL:
query_options->__set_request_pool(value);
break;
- case TImpalaQueryOptions::V_CPU_CORES:
- query_options->__set_v_cpu_cores(atoi(value.c_str()));
- break;
- case TImpalaQueryOptions::RESERVATION_REQUEST_TIMEOUT:
- query_options->__set_reservation_request_timeout(atoi(value.c_str()));
- break;
- case TImpalaQueryOptions::DISABLE_CACHED_READS:
- if (iequals(value, "true") || iequals(value, "1")) {
- query_options->__set_disable_cached_reads(true);
- }
- break;
case TImpalaQueryOptions::DISABLE_OUTERMOST_TOPN:
query_options->__set_disable_outermost_topn(
iequals(value, "true") || iequals(value, "1"));
break;
- case TImpalaQueryOptions::RM_INITIAL_MEM: {
- int64_t reservation_size;
- RETURN_IF_ERROR(ParseMemValue(value, "RM memory limit", &reservation_size));
- query_options->__set_rm_initial_mem(reservation_size);
- break;
- }
case TImpalaQueryOptions::QUERY_TIMEOUT_S:
query_options->__set_query_timeout_s(atoi(value.c_str()));
break;
@@ -327,10 +321,6 @@ Status impala::SetQueryOption(const string& key, const string& value,
break;
case TImpalaQueryOptions::REPLICA_PREFERENCE:
if (iequals(value, "cache_local") || iequals(value, "0")) {
- if (query_options->disable_cached_reads) {
- return Status("Conflicting settings: DISABLE_CACHED_READS = true and"
- " REPLICA_PREFERENCE = CACHE_LOCAL");
- }
query_options->__set_replica_preference(TReplicaPreference::CACHE_LOCAL);
} else if (iequals(value, "disk_local") || iequals(value, "2")) {
query_options->__set_replica_preference(TReplicaPreference::DISK_LOCAL);
@@ -345,10 +335,6 @@ Status impala::SetQueryOption(const string& key, const string& value,
query_options->__set_schedule_random_replica(
iequals(value, "true") || iequals(value, "1"));
break;
- // TODO: remove this query option (IMPALA-4319).
- case TImpalaQueryOptions::SCAN_NODE_CODEGEN_THRESHOLD:
- query_options->__set_scan_node_codegen_threshold(atol(value.c_str()));
- break;
case TImpalaQueryOptions::DISABLE_STREAMING_PREAGGREGATIONS:
query_options->__set_disable_streaming_preaggregations(
iequals(value, "true") || iequals(value, "1"));
@@ -614,6 +600,10 @@ Status impala::SetQueryOption(const string& key, const string& value,
break;
}
default:
+ if (IsRemovedQueryOption(key)) {
+ LOG(WARNING) << "Ignoring attempt to set removed query option '" << key << "'";
+ return Status::OK();
+ }
// We hit this DCHECK(false) if we forgot to add the corresponding entry here
// when we add a new query option.
LOG(ERROR) << "Missing exec option implementation: " << key;
@@ -659,7 +649,12 @@ void impala::PopulateQueryOptionLevels(QueryOptionLevels* query_option_levels)
{\
(*query_option_levels)[#ENUM] = LEVEL;\
}
+#define REMOVED_QUERY_OPT_FN(NAME, ENUM)\
+ {\
+ (*query_option_levels)[#ENUM] = TQueryOptionLevel::REMOVED;\
+ }
QUERY_OPTS_TABLE
QUERY_OPT_FN(support_start_over, SUPPORT_START_OVER, TQueryOptionLevel::ADVANCED)
#undef QUERY_OPT_FN
+#undef REMOVED_QUERY_OPT_FN
}
http://git-wip-us.apache.org/repos/asf/impala/blob/acfd169c/be/src/service/query-options.h
----------------------------------------------------------------------
diff --git a/be/src/service/query-options.h b/be/src/service/query-options.h
index 9cdc935..bb07552 100644
--- a/be/src/service/query-options.h
+++ b/be/src/service/query-options.h
@@ -42,16 +42,14 @@ typedef std::unordered_map<string, beeswax::TQueryOptionLevel::type>
#define QUERY_OPTS_TABLE\
DCHECK_EQ(_TImpalaQueryOptions_VALUES_TO_NAMES.size(),\
TImpalaQueryOptions::COMPUTE_STATS_MIN_SAMPLE_SIZE + 1);\
- QUERY_OPT_FN(abort_on_default_limit_exceeded, ABORT_ON_DEFAULT_LIMIT_EXCEEDED,\
- TQueryOptionLevel::DEPRECATED)\
+ REMOVED_QUERY_OPT_FN(abort_on_default_limit_exceeded, ABORT_ON_DEFAULT_LIMIT_EXCEEDED)\
QUERY_OPT_FN(abort_on_error, ABORT_ON_ERROR, TQueryOptionLevel::REGULAR)\
QUERY_OPT_FN(allow_unsupported_formats, ALLOW_UNSUPPORTED_FORMATS,\
TQueryOptionLevel::DEPRECATED)\
QUERY_OPT_FN(batch_size, BATCH_SIZE, TQueryOptionLevel::DEVELOPMENT)\
QUERY_OPT_FN(debug_action, DEBUG_ACTION, TQueryOptionLevel::DEVELOPMENT)\
- QUERY_OPT_FN(default_order_by_limit, DEFAULT_ORDER_BY_LIMIT,\
- TQueryOptionLevel::DEPRECATED)\
- QUERY_OPT_FN(disable_cached_reads, DISABLE_CACHED_READS, TQueryOptionLevel::DEPRECATED)\
+ REMOVED_QUERY_OPT_FN(default_order_by_limit, DEFAULT_ORDER_BY_LIMIT)\
+ REMOVED_QUERY_OPT_FN(disable_cached_reads, DISABLE_CACHED_READS)\
QUERY_OPT_FN(disable_outermost_topn, DISABLE_OUTERMOST_TOPN,\
TQueryOptionLevel::DEVELOPMENT)\
QUERY_OPT_FN(disable_codegen, DISABLE_CODEGEN, TQueryOptionLevel::REGULAR)\
@@ -59,7 +57,7 @@ typedef std::unordered_map<string, beeswax::TQueryOptionLevel::type>
QUERY_OPT_FN(hbase_cache_blocks, HBASE_CACHE_BLOCKS, TQueryOptionLevel::ADVANCED)\
QUERY_OPT_FN(hbase_caching, HBASE_CACHING, TQueryOptionLevel::ADVANCED)\
QUERY_OPT_FN(max_errors, MAX_ERRORS, TQueryOptionLevel::ADVANCED)\
- QUERY_OPT_FN(max_io_buffers, MAX_IO_BUFFERS, TQueryOptionLevel::DEPRECATED)\
+ REMOVED_QUERY_OPT_FN(max_io_buffers, MAX_IO_BUFFERS)\
QUERY_OPT_FN(max_scan_range_length, MAX_SCAN_RANGE_LENGTH,\
TQueryOptionLevel::DEVELOPMENT)\
QUERY_OPT_FN(mem_limit, MEM_LIMIT, TQueryOptionLevel::REGULAR)\
@@ -68,11 +66,10 @@ typedef std::unordered_map<string, beeswax::TQueryOptionLevel::type>
QUERY_OPT_FN(compression_codec, COMPRESSION_CODEC, TQueryOptionLevel::REGULAR)\
QUERY_OPT_FN(parquet_file_size, PARQUET_FILE_SIZE, TQueryOptionLevel::ADVANCED)\
QUERY_OPT_FN(request_pool, REQUEST_POOL, TQueryOptionLevel::REGULAR)\
- QUERY_OPT_FN(reservation_request_timeout, RESERVATION_REQUEST_TIMEOUT,\
- TQueryOptionLevel::DEPRECATED)\
+ REMOVED_QUERY_OPT_FN(reservation_request_timeout, RESERVATION_REQUEST_TIMEOUT)\
QUERY_OPT_FN(sync_ddl, SYNC_DDL, TQueryOptionLevel::REGULAR)\
- QUERY_OPT_FN(v_cpu_cores, V_CPU_CORES, TQueryOptionLevel::DEPRECATED)\
- QUERY_OPT_FN(rm_initial_mem, RM_INITIAL_MEM, TQueryOptionLevel::DEPRECATED)\
+ REMOVED_QUERY_OPT_FN(v_cpu_cores, V_CPU_CORES)\
+ REMOVED_QUERY_OPT_FN(rm_initial_mem, RM_INITIAL_MEM)\
QUERY_OPT_FN(query_timeout_s, QUERY_TIMEOUT_S, TQueryOptionLevel::REGULAR)\
QUERY_OPT_FN(buffer_pool_limit, BUFFER_POOL_LIMIT, TQueryOptionLevel::ADVANCED)\
QUERY_OPT_FN(appx_count_distinct, APPX_COUNT_DISTINCT, TQueryOptionLevel::ADVANCED)\
@@ -85,8 +82,7 @@ typedef std::unordered_map<string, beeswax::TQueryOptionLevel::type>
QUERY_OPT_FN(replica_preference, REPLICA_PREFERENCE, TQueryOptionLevel::ADVANCED)\
QUERY_OPT_FN(schedule_random_replica, SCHEDULE_RANDOM_REPLICA,\
TQueryOptionLevel::ADVANCED)\
- QUERY_OPT_FN(scan_node_codegen_threshold, SCAN_NODE_CODEGEN_THRESHOLD,\
- TQueryOptionLevel::DEPRECATED)\
+ REMOVED_QUERY_OPT_FN(scan_node_codegen_threshold, SCAN_NODE_CODEGEN_THRESHOLD)\
QUERY_OPT_FN(disable_streaming_preaggregations, DISABLE_STREAMING_PREAGGREGATIONS,\
TQueryOptionLevel::REGULAR)\
QUERY_OPT_FN(runtime_filter_mode, RUNTIME_FILTER_MODE, TQueryOptionLevel::REGULAR)\
http://git-wip-us.apache.org/repos/asf/impala/blob/acfd169c/common/thrift/ImpalaInternalService.thrift
----------------------------------------------------------------------
diff --git a/common/thrift/ImpalaInternalService.thrift b/common/thrift/ImpalaInternalService.thrift
index c44189a..37bf638 100644
--- a/common/thrift/ImpalaInternalService.thrift
+++ b/common/thrift/ImpalaInternalService.thrift
@@ -97,16 +97,9 @@ struct TQueryOptions {
5: optional i32 num_nodes = NUM_NODES_ALL
6: optional i64 max_scan_range_length = 0
7: optional i32 num_scanner_threads = 0
-
- // TODO: IMPALA-4306: retire at compatibility-breaking version
- 8: optional i32 max_io_buffers = 0 // Deprecated in 1.1
9: optional bool allow_unsupported_formats = 0
- // TODO: IMPALA-4306: retire at compatibility-breaking version
- 10: optional i64 default_order_by_limit = -1 // Deprecated in 1.4
11: optional string debug_action = ""
12: optional i64 mem_limit = 0
- // TODO: IMPALA-4306: retire at compatibility-breaking version
- 13: optional bool abort_on_default_limit_exceeded = 0 // Deprecated in 1.4
14: optional CatalogObjects.THdfsCompression compression_codec
15: optional i32 hbase_caching = 0
16: optional bool hbase_cache_blocks = 0
@@ -118,27 +111,9 @@ struct TQueryOptions {
// the pool is determined based on the user.
20: optional string request_pool
- // Per-host virtual CPU cores required for query (only relevant with RM).
- // TODO: IMPALA-3271: retire at compatibility-breaking version
- 21: optional i16 v_cpu_cores
-
- // Max time in milliseconds the resource broker should wait for
- // a resource request to be granted by Llama/Yarn (only relevant with RM).
- // TODO: IMPALA-3271: retire at compatibility-breaking version
- 22: optional i64 reservation_request_timeout
-
- // Disables taking advantage of HDFS caching. This has two parts:
- // 1. disable preferring to schedule to cached replicas
- // 2. disable the cached read path.
- 23: optional bool disable_cached_reads = 0
-
// test hook to disable topn on the outermost select block.
24: optional bool disable_outermost_topn = 0
- // Override for initial memory reservation size if RM is enabled.
- // TODO: IMPALA-3271: retire at compatibility-breaking version
- 25: optional i64 rm_initial_mem = 0
-
// Time, in s, before a query will be timed out if it is inactive. May not exceed
// --idle_query_timeout if that flag > 0.
26: optional i32 query_timeout_s = 0
@@ -178,10 +153,6 @@ struct TQueryOptions {
// subsequent queries. The default is to start with the same replica for every query.
34: optional bool schedule_random_replica = 0
- // For scan nodes with any conjuncts, use codegen to evaluate the conjuncts if
- // the number of rows * number of operators in the conjuncts exceeds this threshold.
- 35: optional i64 scan_node_codegen_threshold = 1800000
-
// If true, the planner will not generate plans with streaming preaggregations.
36: optional bool disable_streaming_preaggregations = 0
http://git-wip-us.apache.org/repos/asf/impala/blob/acfd169c/common/thrift/ImpalaService.thrift
----------------------------------------------------------------------
diff --git a/common/thrift/ImpalaService.thrift b/common/thrift/ImpalaService.thrift
index 0f2d3d0..0360f6c 100644
--- a/common/thrift/ImpalaService.thrift
+++ b/common/thrift/ImpalaService.thrift
@@ -67,8 +67,7 @@ enum TImpalaQueryOptions {
// a length of 0 indicates backend default;
MAX_SCAN_RANGE_LENGTH,
- // Maximum number of io buffers (per disk)
- MAX_IO_BUFFERS,
+ MAX_IO_BUFFERS, // Removed
// Number of scanner threads.
NUM_SCANNER_THREADS,
@@ -76,10 +75,7 @@ enum TImpalaQueryOptions {
// If true, Impala will try to execute on file formats that are not fully supported yet
ALLOW_UNSUPPORTED_FORMATS,
- // if set and > -1, specifies the default limit applied to a top-level SELECT statement
- // with an ORDER BY but without a LIMIT clause (ie, if the SELECT statement also has
- // a LIMIT clause, this default is ignored)
- DEFAULT_ORDER_BY_LIMIT,
+ DEFAULT_ORDER_BY_LIMIT, // Removed
// DEBUG ONLY:
// If set to
@@ -92,8 +88,7 @@ enum TImpalaQueryOptions {
// invalid, the option is ignored.
DEBUG_ACTION,
- // If true, raise an error when the DEFAULT_ORDER_BY_LIMIT has been reached.
- ABORT_ON_DEFAULT_LIMIT_EXCEEDED,
+ ABORT_ON_DEFAULT_LIMIT_EXCEEDED, // Removed
// Compression codec when inserting into tables.
// Valid values are "snappy", "gzip", "bzip2" and "none"
@@ -133,14 +128,9 @@ enum TImpalaQueryOptions {
// the pool is determined based on the user.
REQUEST_POOL,
- // Per-host virtual CPU cores required for query (only relevant with RM).
- // TODO: IMPALA-3271: retire at compatibility-breaking version
- V_CPU_CORES,
+ V_CPU_CORES, // Removed
- // Max time in milliseconds the resource broker should wait for
- // a resource request to be granted by Llama/Yarn (only relevant with RM).
- // TODO: IMPALA-3271: retire at compatibility-breaking version
- RESERVATION_REQUEST_TIMEOUT,
+ RESERVATION_REQUEST_TIMEOUT, // Removed
// if true, disables cached reads. This option has no effect if REPLICA_PREFERENCE is
// configured.
@@ -150,9 +140,7 @@ enum TImpalaQueryOptions {
// Temporary testing flag
DISABLE_OUTERMOST_TOPN,
- // Size of initial memory reservation when RM is enabled
- // TODO: IMPALA-3271: retire at compatibility-breaking version
- RM_INITIAL_MEM,
+ RM_INITIAL_MEM, // Removed
// Time, in s, before a query will be timed out if it is inactive. May not exceed
// --idle_query_timeout if that flag > 0.
@@ -186,9 +174,7 @@ enum TImpalaQueryOptions {
// Enables random backend selection during scheduling.
SCHEDULE_RANDOM_REPLICA,
- // For scan nodes with any conjuncts, use codegen to evaluate the conjuncts if
- // the number of rows * number of operators in the conjuncts exceeds this threshold.
- SCAN_NODE_CODEGEN_THRESHOLD,
+ SCAN_NODE_CODEGEN_THRESHOLD, // Removed
// If true, the planner will not generate plans with streaming preaggregations.
DISABLE_STREAMING_PREAGGREGATIONS,
http://git-wip-us.apache.org/repos/asf/impala/blob/acfd169c/common/thrift/beeswax.thrift
----------------------------------------------------------------------
diff --git a/common/thrift/beeswax.thrift b/common/thrift/beeswax.thrift
index 5d133dd..481aa2f 100644
--- a/common/thrift/beeswax.thrift
+++ b/common/thrift/beeswax.thrift
@@ -96,12 +96,14 @@ exception QueryNotFoundException {
}
// Impala extension:
-// Levels to use when displaying query options from Impala shell
+// Levels to use when displaying query options from Impala shell. REMOVED options should
+// not be displayed in the shell, but setting them is a warning rather than an error.
enum TQueryOptionLevel {
REGULAR,
ADVANCED,
DEVELOPMENT,
- DEPRECATED
+ DEPRECATED,
+ REMOVED
}
/** Represents a Hadoop-style configuration variable. */
http://git-wip-us.apache.org/repos/asf/impala/blob/acfd169c/shell/impala_shell.py
----------------------------------------------------------------------
diff --git a/shell/impala_shell.py b/shell/impala_shell.py
index a671f2a..4a77d53 100755
--- a/shell/impala_shell.py
+++ b/shell/impala_shell.py
@@ -86,6 +86,7 @@ class QueryOptionLevels:
ADVANCED = 1
DEVELOPMENT = 2
DEPRECATED = 3
+ REMOVED = 4
class QueryOptionDisplayModes:
REGULAR_OPTIONS_ONLY = 1
@@ -237,8 +238,8 @@ class ImpalaShell(object, cmd.Cmd):
if not self.imp_client.default_query_options and not self.set_query_options:
print '\tNo options available.'
else:
- (regular_options, advanced_options, development_options, deprecated_options) = \
- self._get_query_option_grouping()
+ (regular_options, advanced_options, development_options, deprecated_options,
+ removed_options) = self._get_query_option_grouping()
self._print_option_group(regular_options)
# If the shell is connected to an Impala that predates IMPALA-2181 then
# the advanced_options would be empty and only the regular options would
@@ -260,8 +261,8 @@ class ImpalaShell(object, cmd.Cmd):
query option level for display purposes using the received query_option_levels
parameters.
If the option level can't be determined then it defaults to 'REGULAR'"""
- regular_options, advanced_options, development_options, deprecated_options = \
- {}, {}, {}, {}
+ (regular_options, advanced_options, development_options, deprecated_options,
+ removed_options) = {}, {}, {}, {}, {}
for option_name, option_value in self.imp_client.default_query_options.iteritems():
level = self.imp_client.query_option_levels.get(option_name,
QueryOptionLevels.REGULAR)
@@ -271,9 +272,12 @@ class ImpalaShell(object, cmd.Cmd):
development_options[option_name] = option_value
elif level == QueryOptionLevels.DEPRECATED:
deprecated_options[option_name] = option_value
+ elif level == QueryOptionLevels.REMOVED:
+ removed_options[option_name] = option_value
else:
advanced_options[option_name] = option_value
- return (regular_options, advanced_options, development_options, deprecated_options)
+ return (regular_options, advanced_options, development_options, deprecated_options,
+ removed_options)
def _print_option_group(self, query_options):
"""Gets query options and prints them. Value is inside [] for the ones having
@@ -683,6 +687,9 @@ class ImpalaShell(object, cmd.Cmd):
print "Available query options, with their values (defaults shown in []):"
self._print_options(QueryOptionDisplayModes.REGULAR_OPTIONS_ONLY)
return CmdStatus.ERROR
+ if self.imp_client.query_option_levels[option_upper] == QueryOptionLevels.REMOVED:
+ self._print_if_verbose("Ignoring removed query option: '{0}'".format(tokens[0]))
+ return CmdStatus.SUCCESS
self.set_query_options[option_upper] = tokens[1]
self._print_if_verbose('%s set to %s' % (option_upper, tokens[1]))
http://git-wip-us.apache.org/repos/asf/impala/blob/acfd169c/testdata/workloads/functional-query/queries/QueryTest/set.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/set.test b/testdata/workloads/functional-query/queries/QueryTest/set.test
index 32ad938..57c5131 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/set.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/set.test
@@ -7,21 +7,17 @@ set buffer_pool_limit=7;
---- QUERY
set all;
---- RESULTS: VERIFY_IS_SUBSET
-'ABORT_ON_DEFAULT_LIMIT_EXCEEDED','0','DEPRECATED'
'ABORT_ON_ERROR','0','REGULAR'
'ALLOW_UNSUPPORTED_FORMATS','0','DEPRECATED'
'BATCH_SIZE','0','DEVELOPMENT'
'BUFFER_POOL_LIMIT','','ADVANCED'
'DEBUG_ACTION','','DEVELOPMENT'
-'DEFAULT_ORDER_BY_LIMIT','-1','DEPRECATED'
-'DISABLE_CACHED_READS','0','DEPRECATED'
'DISABLE_CODEGEN','0','REGULAR'
'DISABLE_OUTERMOST_TOPN','0','DEVELOPMENT'
'EXPLAIN_LEVEL','1','REGULAR'
'HBASE_CACHE_BLOCKS','0','ADVANCED'
'HBASE_CACHING','0','ADVANCED'
'MAX_ERRORS','100','ADVANCED'
-'MAX_IO_BUFFERS','0','DEPRECATED'
'MAX_SCAN_RANGE_LENGTH','0','DEVELOPMENT'
'MEM_LIMIT','0','REGULAR'
'NUM_NODES','0','DEVELOPMENT'
@@ -29,10 +25,7 @@ set all;
'COMPRESSION_CODEC','','REGULAR'
'PARQUET_FILE_SIZE','0','ADVANCED'
'REQUEST_POOL','','REGULAR'
-'RESERVATION_REQUEST_TIMEOUT','','DEPRECATED'
-'RM_INITIAL_MEM','0','DEPRECATED'
'SYNC_DDL','0','REGULAR'
-'V_CPU_CORES','','DEPRECATED'
---- TYPES
STRING, STRING, STRING
====
@@ -40,21 +33,17 @@ STRING, STRING, STRING
set explain_level=3;
set all;
---- RESULTS: VERIFY_IS_SUBSET
-'ABORT_ON_DEFAULT_LIMIT_EXCEEDED','0','DEPRECATED'
'ABORT_ON_ERROR','0','REGULAR'
'ALLOW_UNSUPPORTED_FORMATS','0','DEPRECATED'
'BATCH_SIZE','0','DEVELOPMENT'
'BUFFER_POOL_LIMIT','','ADVANCED'
'DEBUG_ACTION','','DEVELOPMENT'
-'DEFAULT_ORDER_BY_LIMIT','-1','DEPRECATED'
-'DISABLE_CACHED_READS','0','DEPRECATED'
'DISABLE_CODEGEN','0','REGULAR'
'DISABLE_OUTERMOST_TOPN','0','DEVELOPMENT'
'EXPLAIN_LEVEL','3','REGULAR'
'HBASE_CACHE_BLOCKS','0','ADVANCED'
'HBASE_CACHING','0','ADVANCED'
'MAX_ERRORS','100','ADVANCED'
-'MAX_IO_BUFFERS','0','DEPRECATED'
'MAX_SCAN_RANGE_LENGTH','0','DEVELOPMENT'
'MEM_LIMIT','0','REGULAR'
'NUM_NODES','0','DEVELOPMENT'
@@ -62,10 +51,7 @@ set all;
'COMPRESSION_CODEC','','REGULAR'
'PARQUET_FILE_SIZE','0','ADVANCED'
'REQUEST_POOL','','REGULAR'
-'RESERVATION_REQUEST_TIMEOUT','','DEPRECATED'
-'RM_INITIAL_MEM','0','DEPRECATED'
'SYNC_DDL','0','REGULAR'
-'V_CPU_CORES','','DEPRECATED'
---- TYPES
STRING, STRING, STRING
====
@@ -73,21 +59,17 @@ STRING, STRING, STRING
set explain_level='0';
set all;
---- RESULTS: VERIFY_IS_SUBSET
-'ABORT_ON_DEFAULT_LIMIT_EXCEEDED','0','DEPRECATED'
'ABORT_ON_ERROR','0','REGULAR'
'ALLOW_UNSUPPORTED_FORMATS','0','DEPRECATED'
'BATCH_SIZE','0','DEVELOPMENT'
'BUFFER_POOL_LIMIT','','ADVANCED'
'DEBUG_ACTION','','DEVELOPMENT'
-'DEFAULT_ORDER_BY_LIMIT','-1','DEPRECATED'
-'DISABLE_CACHED_READS','0','DEPRECATED'
'DISABLE_CODEGEN','0','REGULAR'
'DISABLE_OUTERMOST_TOPN','0','DEVELOPMENT'
'EXPLAIN_LEVEL','0','REGULAR'
'HBASE_CACHE_BLOCKS','0','ADVANCED'
'HBASE_CACHING','0','ADVANCED'
'MAX_ERRORS','100','ADVANCED'
-'MAX_IO_BUFFERS','0','DEPRECATED'
'MAX_SCAN_RANGE_LENGTH','0','DEVELOPMENT'
'MEM_LIMIT','0','REGULAR'
'NUM_NODES','0','DEVELOPMENT'
@@ -95,10 +77,7 @@ set all;
'COMPRESSION_CODEC','','REGULAR'
'PARQUET_FILE_SIZE','0','ADVANCED'
'REQUEST_POOL','','REGULAR'
-'RESERVATION_REQUEST_TIMEOUT','','DEPRECATED'
-'RM_INITIAL_MEM','0','DEPRECATED'
'SYNC_DDL','0','REGULAR'
-'V_CPU_CORES','','DEPRECATED'
---- TYPES
STRING, STRING, STRING
====
@@ -241,3 +220,14 @@ set max_row_size=0;
---- CATCH
Invalid max row size of 0. Valid sizes are in [1, 1099511627776]
====
+---- QUERY
+# Setting some removed query options should be a no-op.
+set DEFAULT_ORDER_BY_LIMIT="foo";
+set ABORT_ON_DEFAULT_LIMIT_EXCEEDED = "foo";
+set V_CPU_CORES = "foo";
+set RESERVATION_REQUEST_TIMEOUT = "foo";
+set RM_INITIAL_MEM = "foo";
+set SCAN_NODE_CODEGEN_THRESHOLD = "foo";
+set max_io_buffers="foo";
+---- RESULTS
+====
http://git-wip-us.apache.org/repos/asf/impala/blob/acfd169c/tests/comparison/discrepancy_searcher.py
----------------------------------------------------------------------
diff --git a/tests/comparison/discrepancy_searcher.py b/tests/comparison/discrepancy_searcher.py
index e0e1725..df9c562 100755
--- a/tests/comparison/discrepancy_searcher.py
+++ b/tests/comparison/discrepancy_searcher.py
@@ -316,7 +316,6 @@ class QueryExecutor(object):
SET DISABLE_UNSAFE_SPILLS={disable_unsafe_spills};
SET EXEC_SINGLE_NODE_ROWS_THRESHOLD={exec_single_node_rows_threshold};
SET BUFFER_POOL_LIMIT={buffer_pool_limit};
- SET MAX_IO_BUFFERS={max_io_buffers};
SET MAX_SCAN_RANGE_LENGTH={max_scan_range_length};
SET NUM_NODES={num_nodes};
SET NUM_SCANNER_THREADS={num_scanner_threads};
@@ -334,7 +333,6 @@ class QueryExecutor(object):
disable_unsafe_spills=choice((0, 1)),
exec_single_node_rows_threshold=randint(1, 100000000),
buffer_pool_limit=randint(1, 100000000),
- max_io_buffers=randint(1, 100000000),
max_scan_range_length=randint(1, 100000000),
num_nodes=randint(3, 3),
num_scanner_threads=randint(1, 100),
http://git-wip-us.apache.org/repos/asf/impala/blob/acfd169c/tests/custom_cluster/test_admission_controller.py
----------------------------------------------------------------------
diff --git a/tests/custom_cluster/test_admission_controller.py b/tests/custom_cluster/test_admission_controller.py
index ccbbd32..69cabd8 100644
--- a/tests/custom_cluster/test_admission_controller.py
+++ b/tests/custom_cluster/test_admission_controller.py
@@ -298,12 +298,11 @@ class TestAdmissionController(TestAdmissionControllerBase, HS2TestSuite):
# Should be able to set query options (overriding defaults if applicable) with the
# config overlay sent with the query RPC. mem_limit is a pool-level override and
# max_io_buffers has no proc/pool default.
- client.set_configuration({'request_pool': 'root.queueA', 'mem_limit': '12345',
- 'max_io_buffers': '100'})
+ client.set_configuration({'request_pool': 'root.queueA', 'mem_limit': '12345'})
result = client.execute("select 1")
self.__check_query_options(result.runtime_profile,\
['MEM_LIMIT=12345', 'QUERY_TIMEOUT_S=5', 'REQUEST_POOL=root.queueA',\
- 'ABORT_ON_ERROR=1', 'MAX_IO_BUFFERS=100'])
+ 'ABORT_ON_ERROR=1'])
# Once options are reset to their defaults, the queue
# configuration should kick back in. We'll see the
http://git-wip-us.apache.org/repos/asf/impala/blob/acfd169c/tests/hs2/test_hs2.py
----------------------------------------------------------------------
diff --git a/tests/hs2/test_hs2.py b/tests/hs2/test_hs2.py
index c90113c..ccdcdea 100644
--- a/tests/hs2/test_hs2.py
+++ b/tests/hs2/test_hs2.py
@@ -93,7 +93,10 @@ class TestHS2(HS2TestSuite):
assert "MAX_ERRORS" in vals2
assert levels["MAX_ERRORS"] == "ADVANCED"
assert "DEBUG_ACTION" not in vals2
- assert "SCAN_NODE_CODEGEN_THRESHOLD" not in vals2
+ assert "ALLOW_UNSUPPORTED_FORMATS" not in vals2
+
+ # Removed options should not be returned.
+ assert "MAX_IO_BUFFERS" not in vals2
@needs_session()
def test_session_option_levels_via_set_all(self):
@@ -106,12 +109,15 @@ class TestHS2(HS2TestSuite):
assert "SYNC_DDL" in vals
assert "MAX_ERRORS" in vals
assert "DEBUG_ACTION" in vals
- assert "SCAN_NODE_CODEGEN_THRESHOLD" in vals
+ assert "ALLOW_UNSUPPORTED_FORMATS" in vals
assert levels["COMPRESSION_CODEC"] == "REGULAR"
assert levels["SYNC_DDL"] == "REGULAR"
assert levels["MAX_ERRORS"] == "ADVANCED"
assert levels["DEBUG_ACTION"] == "DEVELOPMENT"
- assert levels["SCAN_NODE_CODEGEN_THRESHOLD"] == "DEPRECATED"
+ assert levels["ALLOW_UNSUPPORTED_FORMATS"] == "DEPRECATED"
+
+ # Removed options should not be returned.
+ assert "MAX_IO_BUFFERS" not in vals
def test_open_session_http_addr(self):
"""Check that OpenSession returns the coordinator's http address."""
http://git-wip-us.apache.org/repos/asf/impala/blob/acfd169c/tests/query_test/test_observability.py
----------------------------------------------------------------------
diff --git a/tests/query_test/test_observability.py b/tests/query_test/test_observability.py
index 85fc4f1..e838081 100644
--- a/tests/query_test/test_observability.py
+++ b/tests/query_test/test_observability.py
@@ -104,7 +104,7 @@ class TestObservability(ImpalaTestSuite):
# Set a query option explicitly through client
self.execute_query("set MEM_LIMIT = 8589934592")
# Make sure explicitly set default values are not shown in the profile
- self.execute_query("set MAX_IO_BUFFERS = 0")
+ self.execute_query("set runtime_filter_wait_time_ms = 0")
runtime_profile = self.execute_query("select 1").runtime_profile
assert "Query Options (set by configuration): MEM_LIMIT=8589934592" in runtime_profile
# For this query, the planner sets NUM_NODES=1, NUM_SCANNER_THREADS=1,
http://git-wip-us.apache.org/repos/asf/impala/blob/acfd169c/tests/shell/test_shell_commandline.py
----------------------------------------------------------------------
diff --git a/tests/shell/test_shell_commandline.py b/tests/shell/test_shell_commandline.py
index c2e7509..ee9b4db 100644
--- a/tests/shell/test_shell_commandline.py
+++ b/tests/shell/test_shell_commandline.py
@@ -200,6 +200,12 @@ class TestImpalaShell(ImpalaTestSuite):
result = run_impala_shell_cmd(args)
assert 'WARNINGS:' not in result.stderr
+ def test_removed_query_option(self):
+ """Test that removed query options produce warning."""
+ result = run_impala_shell_cmd("-q 'set disable_cached_reads=true'",
+ expect_success=True)
+ assert "Ignoring removed query option: 'disable_cached_reads'" in result.stderr
+
def test_output_format(self):
expected_output = ['1'] * 3
args = '-q "select 1,1,1" -B --quiet'
@@ -230,10 +236,10 @@ class TestImpalaShell(ImpalaTestSuite):
run_impala_shell_cmd(args)
# set
# spaces around the = sign
- args = '-q "set default_order_by_limit = 10"'
+ args = '-q "set batch_size = 10"'
run_impala_shell_cmd(args)
# no spaces around the = sign
- args = '-q "set default_order_by_limit=10"'
+ args = '-q "set batch_size=10"'
run_impala_shell_cmd(args)
# test query options displayed
args = '-q "set"'
@@ -251,10 +257,10 @@ class TestImpalaShell(ImpalaTestSuite):
assert 'MEM_LIMIT: [0]' not in result_set.stdout
# Negative tests for set
# use : instead of =
- args = '-q "set default_order_by_limit:10"'
+ args = '-q "set batch_size:10"'
run_impala_shell_cmd(args, expect_success=False)
# use 2 = signs
- args = '-q "set default_order_by_limit=10=50"'
+ args = '-q "set batch_size=10=50"'
run_impala_shell_cmd(args, expect_success=False)
# describe and desc should return the same result.
args = '-q "describe %s" -B' % empty_table
http://git-wip-us.apache.org/repos/asf/impala/blob/acfd169c/tests/shell/test_shell_interactive.py
----------------------------------------------------------------------
diff --git a/tests/shell/test_shell_interactive.py b/tests/shell/test_shell_interactive.py
index a104809..7f7f955 100755
--- a/tests/shell/test_shell_interactive.py
+++ b/tests/shell/test_shell_interactive.py
@@ -367,10 +367,12 @@ class TestImpalaShellInteractive(object):
assert "Advanced Query Options:" in result.stdout
assert "APPX_COUNT_DISTINCT" in result.stdout
assert "SUPPORT_START_OVER" in result.stdout
+ # Development, deprecated and removed options should not be shown.
assert "Development Query Options:" not in result.stdout
assert "DEBUG_ACTION" not in result.stdout
assert "Deprecated Query Options:" not in result.stdout
- assert "ABORT_ON_DEFAULT_LIMIT_EXCEEDED" not in result.stdout
+ assert "ALLOW_UNSUPPORTED_FORMATS" not in result.stdout
+ assert "MAX_IO_BUFFERS" not in result.stdout
shell2 = ImpalaShell()
shell2.send_cmd("set all")
@@ -388,7 +390,9 @@ class TestImpalaShellInteractive(object):
assert "APPX_COUNT_DISTINCT" in advanced_part
assert "SUPPORT_START_OVER" in advanced_part
assert "DEBUG_ACTION" in development_part
- assert "ABORT_ON_DEFAULT_LIMIT_EXCEEDED" in result.stdout[deprecated_part_start_idx:]
+ assert "ALLOW_UNSUPPORTED_FORMATS" in result.stdout[deprecated_part_start_idx:]
+ # Removed options should not be shown.
+ assert "MAX_IO_BUFFERS" not in result.stdout
def check_command_case_sensitivity(self, command, expected):
shell = ImpalaShell()