You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by tm...@apache.org on 2019/06/19 16:57:36 UTC
[impala] 01/05: IMPALA-8667. Remove --pull_incremental_stats flag
This is an automated email from the ASF dual-hosted git repository.
tmarshall pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 800f6358552bfab0bb4cfc0cf518854305379d61
Author: Todd Lipcon <to...@apache.org>
AuthorDate: Tue Jun 18 11:29:05 2019 -0700
IMPALA-8667. Remove --pull_incremental_stats flag
This flag was added as a "chicken bit" -- so we could disable the new
feature if we had some problems with it. It's been out in the wild for a
number of months and we haven't seen any such problems, so at this point
let's stop maintaining the old code path.
Change-Id: I8878fcd8a2462963c7db3183a003bb9816dda8f9
Reviewed-on: http://gerrit.cloudera.org:8080/13671
Reviewed-by: Bharath Vissapragada <bh...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
be/src/common/global-flags.cc | 7 +--
be/src/util/backend-gflag-util.cc | 2 -
common/thrift/BackendGflags.thrift | 2 -
docs/topics/impala_metadata.xml | 37 -------------
.../apache/impala/analysis/ComputeStatsStmt.java | 18 +++----
.../impala/catalog/CatalogServiceCatalog.java | 3 +-
.../org/apache/impala/catalog/FeCatalogUtils.java | 6 +--
.../java/org/apache/impala/catalog/HdfsTable.java | 19 +------
.../apache/impala/catalog/local/LocalCatalog.java | 7 +--
.../apache/impala/catalog/local/LocalFsTable.java | 6 +--
.../org/apache/impala/service/BackendConfig.java | 4 --
.../org/apache/impala/catalog/CatalogTest.java | 61 +++++++++-------------
tests/common/custom_cluster_test_suite.py | 4 --
tests/conftest.py | 4 --
tests/custom_cluster/test_incremental_stats.py | 46 ----------------
15 files changed, 41 insertions(+), 185 deletions(-)
diff --git a/be/src/common/global-flags.cc b/be/src/common/global-flags.cc
index a28846f..4185b98 100644
--- a/be/src/common/global-flags.cc
+++ b/be/src/common/global-flags.cc
@@ -220,12 +220,6 @@ DEFINE_bool_hidden(disable_catalog_data_ops_debug_only, false,
// same way, is error prone. One fix for this flag is to set it only on
// catalogd, propagate the setting as a property of the Catalog object, and let
// impalad uses act on this setting.
-DEFINE_bool(pull_incremental_statistics, true,
- "When set, impalad coordinators pull incremental statistics from catalogd on-demand "
- "and catalogd does not broadcast incremental statistics via statestored to "
- "coordinators. If used, the flag must be set on both catalogd and all impalad "
- "coordinators. This feature should not be used when --use_local_catalog is true.");
-
DEFINE_int32(invalidate_tables_timeout_s, 0, "If a table has not been referenced in a "
"SQL statement for more than the configured amount of time, the catalog server will "
"automatically evict its cached metadata about this table. This has the same effect "
@@ -315,6 +309,7 @@ REMOVED_FLAG(llama_registration_timeout_secs);
REMOVED_FLAG(llama_registration_wait_secs);
REMOVED_FLAG(local_nodemanager_url);
REMOVED_FLAG(max_free_io_buffers);
+REMOVED_FLAG(pull_incremental_statistics);
REMOVED_FLAG(report_status_retry_interval_ms);
REMOVED_FLAG(resource_broker_cnxn_attempts);
REMOVED_FLAG(resource_broker_cnxn_retry_interval_ms);
diff --git a/be/src/util/backend-gflag-util.cc b/be/src/util/backend-gflag-util.cc
index 99b2926..c393e6b 100644
--- a/be/src/util/backend-gflag-util.cc
+++ b/be/src/util/backend-gflag-util.cc
@@ -57,7 +57,6 @@ DECLARE_string(sentry_config);
DECLARE_double(max_filter_error_rate);
DECLARE_int64(min_buffer_size);
DECLARE_bool(disable_catalog_data_ops_debug_only);
-DECLARE_bool(pull_incremental_statistics);
DECLARE_int32(invalidate_tables_timeout_s);
DECLARE_bool(invalidate_tables_on_memory_pressure);
DECLARE_double(invalidate_tables_gc_old_gen_full_threshold);
@@ -125,7 +124,6 @@ Status GetThriftBackendGflags(JNIEnv* jni_env, jbyteArray* cfg_bytes) {
cfg.__set_authorized_proxy_group_config(FLAGS_authorized_proxy_group_config);
cfg.__set_disable_catalog_data_ops_debug_only(
FLAGS_disable_catalog_data_ops_debug_only);
- cfg.__set_pull_incremental_statistics(FLAGS_pull_incremental_statistics);
cfg.__set_catalog_topic_mode(FLAGS_catalog_topic_mode);
cfg.__set_invalidate_tables_timeout_s(FLAGS_invalidate_tables_timeout_s);
cfg.__set_invalidate_tables_on_memory_pressure(
diff --git a/common/thrift/BackendGflags.thrift b/common/thrift/BackendGflags.thrift
index 6574faf..063329f 100644
--- a/common/thrift/BackendGflags.thrift
+++ b/common/thrift/BackendGflags.thrift
@@ -86,8 +86,6 @@ struct TBackendGflags {
30: required i32 local_catalog_cache_expiration_s
- 31: required bool pull_incremental_statistics
-
32: required string catalog_topic_mode
33: required i32 invalidate_tables_timeout_s
diff --git a/docs/topics/impala_metadata.xml b/docs/topics/impala_metadata.xml
index 1c36574..4139e80 100644
--- a/docs/topics/impala_metadata.xml
+++ b/docs/topics/impala_metadata.xml
@@ -87,43 +87,6 @@ under the License.
</concept>
- <concept id="pull_incremental_statistics">
-
- <title>Loading Incremental Statistics from Catalog Server</title>
-
- <conbody>
-
- <p>
- Starting in Impala 3.1, a new configuration setting,
- <codeph>‑‑pull_incremental_statistics</codeph>, was added and set to
- <codeph>true</codeph> by default. When you start Impala <codeph>catalogd</codeph> and
- <codeph>impalad</codeph> coordinators with this setting enabled:
- </p>
-
- <ul>
- <li>
- Newly created incremental stats will be smaller in size thus reducing memory pressure
- on the <codeph>catalogd</codeph> daemon. Your users can keep more tables and
- partitions in the same catalog and have lower chances of crashing
- <codeph>catalogd</codeph> due to out-of-memory issues.
- </li>
-
- <li>
- Incremental stats will not be replicated to <codeph>impalad</codeph> and will be
- accessed on demand from <codeph>catalogd</codeph>, resulting in a reduced memory
- footprint of <codeph>impalad</codeph>.
- </li>
- </ul>
-
- <p>
- We do not recommend you change the default setting of
- <codeph>‑‑pull_incremental_statistics</codeph>.
- </p>
-
- </conbody>
-
- </concept>
-
<concept id="auto_poll_hms_notification">
<title>Automatic Metadata Sync using Hive Metastore Notification Events</title>
diff --git a/fe/src/main/java/org/apache/impala/analysis/ComputeStatsStmt.java b/fe/src/main/java/org/apache/impala/analysis/ComputeStatsStmt.java
index a23df47..43b34b3 100644
--- a/fe/src/main/java/org/apache/impala/analysis/ComputeStatsStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/ComputeStatsStmt.java
@@ -595,8 +595,9 @@ public class ComputeStatsStmt extends StatementBase {
/**
* Get partition statistics from the list of partitions, omitting those in
* excludedPartitions and those for which incremental statistics are not present.
- * If configured to pull incremental statistics directly from the
- * catalog, partition statistics are fetched from the catalog.
+ *
+ * If incremental stats data is not present already in the local catlaog,
+ * partition statistics are fetched from the catalog daemon.
*/
private static Map<Long, TPartitionStats> getOrFetchPartitionStats(Analyzer analyzer,
FeFsTable table, Collection<? extends FeFsPartition> partitions,
@@ -606,12 +607,10 @@ public class ComputeStatsStmt extends StatementBase {
int expectedNumStats = partitions.size() - excludedPartitions.size();
Preconditions.checkArgument(expectedNumStats >= 0);
- // Incremental stats are fetched only when configured to do so except
- // when also using a local catalog or when testing. When using a local
- // catalog, it makes more sense to use the getPartitions api which is
- // designed to fetch specific fields and specific partitions.
- if (BackendConfig.INSTANCE.pullIncrementalStatistics()
- && !BackendConfig.INSTANCE.getBackendCfg().use_local_catalog
+ // Incremental stats are already present locally when using LocalCatalog
+ // or when testing. TODO(IMPALA-7535) fetch incremental stats separately
+ // for LocalCatlaog as well.
+ if (!BackendConfig.INSTANCE.getBackendCfg().use_local_catalog
&& !RuntimeEnv.INSTANCE.isTestEnv()) {
// We're configured to fetch the statistics from catalogd, so collect the relevant
// partition ids.
@@ -647,8 +646,7 @@ public class ComputeStatsStmt extends StatementBase {
private static Map<Long, TPartitionStats> fetchPartitionStats(Analyzer analyzer,
FeFsTable table, List<FeFsPartition> partitions) throws AnalysisException {
Preconditions.checkNotNull(partitions);
- Preconditions.checkState(BackendConfig.INSTANCE.pullIncrementalStatistics()
- && !RuntimeEnv.INSTANCE.isTestEnv());
+ Preconditions.checkState(!RuntimeEnv.INSTANCE.isTestEnv());
if (partitions.isEmpty()) return Collections.emptyMap();
Stopwatch sw = new Stopwatch().start();
int numCompressedBytes = 0;
diff --git a/fe/src/main/java/org/apache/impala/catalog/CatalogServiceCatalog.java b/fe/src/main/java/org/apache/impala/catalog/CatalogServiceCatalog.java
index db143fb..0c8ea59 100644
--- a/fe/src/main/java/org/apache/impala/catalog/CatalogServiceCatalog.java
+++ b/fe/src/main/java/org/apache/impala/catalog/CatalogServiceCatalog.java
@@ -506,8 +506,7 @@ public class CatalogServiceCatalog extends Catalog {
*/
public Map<String, ByteBuffer> getPartitionStats(TGetPartitionStatsRequest request)
throws CatalogException {
- Preconditions.checkState(BackendConfig.INSTANCE.pullIncrementalStatistics()
- && !RuntimeEnv.INSTANCE.isTestEnv());
+ Preconditions.checkState(!RuntimeEnv.INSTANCE.isTestEnv());
TTableName tableName = request.table_name;
LOG.info("Fetching partition statistics for: " + tableName.getDb_name() + "."
+ tableName.getTable_name());
diff --git a/fe/src/main/java/org/apache/impala/catalog/FeCatalogUtils.java b/fe/src/main/java/org/apache/impala/catalog/FeCatalogUtils.java
index 8ab8671..8898fa1 100644
--- a/fe/src/main/java/org/apache/impala/catalog/FeCatalogUtils.java
+++ b/fe/src/main/java/org/apache/impala/catalog/FeCatalogUtils.java
@@ -317,7 +317,7 @@ public abstract class FeCatalogUtils {
}
public static THdfsPartition fsPartitionToThrift(FeFsPartition part,
- ThriftObjectType type, boolean includePartitionStats) {
+ ThriftObjectType type) {
HdfsStorageDescriptor sd = part.getInputFormatDescriptor();
THdfsPartition thriftHdfsPart = new THdfsPartition(
sd.getLineDelim(),
@@ -341,10 +341,6 @@ public abstract class FeCatalogUtils {
// and another thread doing DDL may modify the map.
thriftHdfsPart.setHms_parameters(Maps.newHashMap(part.getParameters()));
thriftHdfsPart.setHas_incremental_stats(part.hasIncrementalStats());
- if (includePartitionStats && part.getPartitionStatsCompressed() != null) {
- thriftHdfsPart.setPartition_stats(
- Preconditions.checkNotNull(part.getPartitionStatsCompressed()));
- }
// Add block location information
long numBlocks = 0;
diff --git a/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java b/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java
index 980cf47..779a96e 100644
--- a/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java
+++ b/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java
@@ -57,7 +57,6 @@ import org.apache.impala.common.ImpalaException;
import org.apache.impala.common.Pair;
import org.apache.impala.compat.MetastoreShim;
import org.apache.impala.fb.FbFileBlock;
-import org.apache.impala.service.BackendConfig;
import org.apache.impala.thrift.CatalogLookupStatus;
import org.apache.impala.thrift.CatalogObjectsConstants;
import org.apache.impala.thrift.TAccessLevel;
@@ -1448,19 +1447,6 @@ public class HdfsTable extends Table implements FeFsTable {
}
/**
- * Determines whether incremental stats should be sent from the catalogd to impalad.
- * Incremental stats will be sent if their size is less than the configured limit
- * (function of numPartitions) and they are sent via statestore (and not a direct
- * fetch from catalogd).
- */
- private boolean shouldSendIncrementalStats(int numPartitions) {
- long statsSizeEstimate =
- numPartitions * getColumns().size() * STATS_SIZE_PER_COLUMN_BYTES;
- return statsSizeEstimate < BackendConfig.INSTANCE.getIncStatsMaxSize()
- && !BackendConfig.INSTANCE.pullIncrementalStatistics();
- }
-
- /**
* Create a THdfsTable corresponding to this HdfsTable. If serializing the "FULL"
* information, then then all partitions and THdfsFileDescs of each partition should be
* included. Otherwise, don't include any THdfsFileDescs, and include only those
@@ -1484,14 +1470,13 @@ public class HdfsTable extends Table implements FeFsTable {
int numPartitions =
(refPartitions == null) ? partitionMap_.values().size() : refPartitions.size();
memUsageEstimate += numPartitions * PER_PARTITION_MEM_USAGE_BYTES;
- boolean includeIncrementalStats = shouldSendIncrementalStats(numPartitions);
FileMetadataStats stats = new FileMetadataStats();
Map<Long, THdfsPartition> idToPartition = new HashMap<>();
for (HdfsPartition partition: partitionMap_.values()) {
long id = partition.getId();
if (refPartitions == null || refPartitions.contains(id)) {
THdfsPartition tHdfsPartition = FeCatalogUtils.fsPartitionToThrift(
- partition, type, includeIncrementalStats);
+ partition, type);
if (partition.hasIncrementalStats()) {
memUsageEstimate += getColumns().size() * STATS_SIZE_PER_COLUMN_BYTES;
hasIncrementalStats_ = true;
@@ -1510,7 +1495,7 @@ public class HdfsTable extends Table implements FeFsTable {
if (type == ThriftObjectType.FULL) fileMetadataStats_.set(stats);
THdfsPartition prototypePartition = FeCatalogUtils.fsPartitionToThrift(
- prototypePartition_, ThriftObjectType.DESCRIPTOR_ONLY, false);
+ prototypePartition_, ThriftObjectType.DESCRIPTOR_ONLY);
memUsageEstimate += fileMetadataStats_.numFiles * PER_FD_MEM_USAGE_BYTES +
fileMetadataStats_.numBlocks * PER_BLOCK_MEM_USAGE_BYTES;
diff --git a/fe/src/main/java/org/apache/impala/catalog/local/LocalCatalog.java b/fe/src/main/java/org/apache/impala/catalog/local/LocalCatalog.java
index 1868073..2a6f438 100644
--- a/fe/src/main/java/org/apache/impala/catalog/local/LocalCatalog.java
+++ b/fe/src/main/java/org/apache/impala/catalog/local/LocalCatalog.java
@@ -208,11 +208,8 @@ public class LocalCatalog implements FeCatalog {
@Override
public TGetPartitionStatsResponse getPartitionStats(
TableName table) throws InternalException {
- // TODO(vercegovac): add validation to ensure that both pulling incremental
- // statistics and a local catalog are not specified.
- throw new UnsupportedOperationException("--pull_incremental_statistics and "
- + "--use_local_catalog cannot both be enabled."
- );
+ // TODO(IMPALA-7535) lazy-fetch incremental stats for LocalCatalog
+ throw new UnsupportedOperationException("Stats are eagerly fetched in LocalCatalog");
}
@Override
diff --git a/fe/src/main/java/org/apache/impala/catalog/local/LocalFsTable.java b/fe/src/main/java/org/apache/impala/catalog/local/LocalFsTable.java
index d90609f..19db140 100644
--- a/fe/src/main/java/org/apache/impala/catalog/local/LocalFsTable.java
+++ b/fe/src/main/java/org/apache/impala/catalog/local/LocalFsTable.java
@@ -294,13 +294,11 @@ public class LocalFsTable extends LocalTable implements FeFsTable {
for (FeFsPartition partition : partitions) {
idToPartition.put(partition.getId(),
FeCatalogUtils.fsPartitionToThrift(partition,
- ThriftObjectType.DESCRIPTOR_ONLY,
- /*includeIncrementalStats=*/false));
+ ThriftObjectType.DESCRIPTOR_ONLY));
}
THdfsPartition tPrototypePartition = FeCatalogUtils.fsPartitionToThrift(
- createPrototypePartition(), ThriftObjectType.DESCRIPTOR_ONLY,
- /*includeIncrementalStats=*/false);
+ createPrototypePartition(), ThriftObjectType.DESCRIPTOR_ONLY);
THdfsTable hdfsTable = new THdfsTable(getHdfsBaseDir(), getColumnNames(),
getNullPartitionKeyValue(), nullColumnValue_, idToPartition,
diff --git a/fe/src/main/java/org/apache/impala/service/BackendConfig.java b/fe/src/main/java/org/apache/impala/service/BackendConfig.java
index 44bf0a9..aef3f41 100644
--- a/fe/src/main/java/org/apache/impala/service/BackendConfig.java
+++ b/fe/src/main/java/org/apache/impala/service/BackendConfig.java
@@ -90,10 +90,6 @@ public class BackendConfig {
return backendCfg_.disable_catalog_data_ops_debug_only;
}
- public boolean pullIncrementalStatistics() {
- return backendCfg_.pull_incremental_statistics;
- }
-
public int getInvalidateTablesTimeoutS() {
return backendCfg_.invalidate_tables_timeout_s;
}
diff --git a/fe/src/test/java/org/apache/impala/catalog/CatalogTest.java b/fe/src/test/java/org/apache/impala/catalog/CatalogTest.java
index 8fd46f2..73c47f7 100644
--- a/fe/src/test/java/org/apache/impala/catalog/CatalogTest.java
+++ b/fe/src/test/java/org/apache/impala/catalog/CatalogTest.java
@@ -54,10 +54,8 @@ import org.apache.impala.authorization.AuthorizationPolicy;
import org.apache.impala.catalog.MetaStoreClientPool.MetaStoreClient;
import org.apache.impala.common.ImpalaException;
import org.apache.impala.common.Reference;
-import org.apache.impala.service.BackendConfig;
import org.apache.impala.testutil.CatalogServiceTestCatalog;
import org.apache.impala.testutil.TestUtils;
-import org.apache.impala.thrift.TBackendGflags;
import org.apache.impala.thrift.TFunctionBinaryType;
import org.apache.impala.thrift.TGetPartitionStatsRequest;
import org.apache.impala.thrift.TPartitionKeyValue;
@@ -639,41 +637,30 @@ public class CatalogTest {
@Test
public void testPullIncrementalStats() throws CatalogException {
- // Save the current setting for pull_incremental_statistics.
- TBackendGflags gflags = BackendConfig.INSTANCE.getBackendCfg();
- boolean pullStats = gflags.isSetPull_incremental_statistics();
-
- try {
- // Restored in the finally clause.
- gflags.setPull_incremental_statistics(true);
-
- // Partitioned table with stats. Load the table prior to fetching.
- catalog_.getOrLoadTable("functional", "alltypesagg", "test");
- expectStatistics("functional", "alltypesagg", 11);
-
- // Partitioned table with stats. Invalidate the table prior to fetching.
- Reference<Boolean> tblWasRemoved = new Reference<Boolean>();
- Reference<Boolean> dbWasAdded = new Reference<Boolean>();
- catalog_.invalidateTable(
- new TTableName("functional", "alltypesagg"), tblWasRemoved, dbWasAdded);
- expectStatistics("functional", "alltypesagg", 11);
-
- // Unpartitioned table with no stats.
- expectStatistics("functional", "table_no_newline", 0);
-
- // Unpartitioned table with stats.
- expectStatistics("functional", "dimtbl", 0);
-
- // Bogus table.
- expectStatisticsException("functional", "doesnotexist",
- "Requested partition statistics for table that does not exist");
-
- // Case of IncompleteTable due to loading error.
- expectStatisticsException("functional", "bad_serde",
- "No statistics available for incompletely loaded table");
- } finally {
- gflags.setPull_incremental_statistics(pullStats);
- }
+ // Partitioned table with stats. Load the table prior to fetching.
+ catalog_.getOrLoadTable("functional", "alltypesagg", "test");
+ expectStatistics("functional", "alltypesagg", 11);
+
+ // Partitioned table with stats. Invalidate the table prior to fetching.
+ Reference<Boolean> tblWasRemoved = new Reference<Boolean>();
+ Reference<Boolean> dbWasAdded = new Reference<Boolean>();
+ catalog_.invalidateTable(
+ new TTableName("functional", "alltypesagg"), tblWasRemoved, dbWasAdded);
+ expectStatistics("functional", "alltypesagg", 11);
+
+ // Unpartitioned table with no stats.
+ expectStatistics("functional", "table_no_newline", 0);
+
+ // Unpartitioned table with stats.
+ expectStatistics("functional", "dimtbl", 0);
+
+ // Bogus table.
+ expectStatisticsException("functional", "doesnotexist",
+ "Requested partition statistics for table that does not exist");
+
+ // Case of IncompleteTable due to loading error.
+ expectStatisticsException("functional", "bad_serde",
+ "No statistics available for incompletely loaded table");
}
@Test
diff --git a/tests/common/custom_cluster_test_suite.py b/tests/common/custom_cluster_test_suite.py
index d651b84..dee5848 100644
--- a/tests/common/custom_cluster_test_suite.py
+++ b/tests/common/custom_cluster_test_suite.py
@@ -238,10 +238,6 @@ class CustomClusterTestSuite(ImpalaTestSuite):
cmd.append("--impalad_args=--use_local_catalog=1")
cmd.append("--catalogd_args=--catalog_topic_mode=minimal")
- if pytest.config.option.pull_incremental_statistics:
- cmd.append("--impalad_args=%s --catalogd_args=%s" %
- ("--pull_incremental_statistics", "--pull_incremental_statistics"))
-
default_query_option_kvs = []
# Put any defaults first, then any arguments after that so they can override defaults.
if os.environ.get("ERASURE_CODING") == "true":
diff --git a/tests/conftest.py b/tests/conftest.py
index e5904ba..411f13b 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -127,10 +127,6 @@ def pytest_addoption(parser):
default=False, help="Run all tests against Impala configured with "
"LocalCatalog.")
- parser.addoption("--pull_incremental_statistics", dest="pull_incremental_statistics",
- action="store_true", default=False, help="Run all tests against Impala"
- " where impalads pull incremental statistics directly from catalogd.")
-
parser.addoption("--sanity", action="store_true", default=False,
help="Runs a single test vector from each test to provide a quick "
"sanity check at the cost of lower test coverage.")
diff --git a/tests/custom_cluster/test_incremental_stats.py b/tests/custom_cluster/test_incremental_stats.py
deleted file mode 100644
index e58d5fd..0000000
--- a/tests/custom_cluster/test_incremental_stats.py
+++ /dev/null
@@ -1,46 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-import pytest
-from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
-
-class TestIncrementalStatistics(CustomClusterTestSuite):
- """Tests flag that controls pulling directly from catalogd using the
- --pull_incremental_statistics flag."""
-
- @classmethod
- def get_workload(self):
- return 'functional-query'
-
- @pytest.mark.execute_serially
- @CustomClusterTestSuite.with_args(impalad_args="--pull_incremental_statistics=false",
- catalogd_args="--pull_incremental_statistics=false")
- def test_push_stats(self, vector, unique_database):
- """
- Tests compute incremental stats when incremental stats are pushed via statestore.
- """
- self.run_test_case('QueryTest/compute-stats-incremental', vector, unique_database)
-
- @pytest.mark.execute_serially
- @CustomClusterTestSuite.with_args(
- impalad_args="--pull_incremental_statistics=true --use_local_catalog=true",
- catalogd_args="--pull_incremental_statistics=true --catalog_topic_mode=minimal")
- def test_with_local_catalog(self, vector, unique_database):
- """
- Tests that when local catalog is used, the pull incremental stats flag has no effect.
- """
- self.run_test_case('QueryTest/compute-stats-incremental', vector, unique_database)