You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by mi...@apache.org on 2026/07/21 16:32:17 UTC
(impala) 01/02: IMPALA-15121: Fix query failure when catalogd is in standby mode during HA failover
This is an automated email from the ASF dual-hosted git repository.
asf-gitbox-commits pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 450aad373856167c18136adce92749550a979a71
Author: Yida Wu <yi...@cloudera.com>
AuthorDate: Fri Jun 26 22:36:58 2026 -0700
IMPALA-15121: Fix query failure when catalogd is in standby mode during HA failover
During a catalogd HA failover, an impalad coordinator might try
to fetch metadata from a catalogd that was previously in active
mode but has just restarted and in standby mode now. In this
case, the metadata request is rejected with a standby mode
error, causing the query to fail.
This change recognizes the standby mode rejection error and
converts it into an InconsistentMetadataFetchException, allowing
the existing Frontend retry mechanism to retry the metadata
fetch. As a result, the query retries against the new active
catalogd at the end instead of failing.
Added thrift error code CATALOG_IN_STANDBY_MODE for rejection
on standby mode.
Tests:
Added a new ee test test_standby_retry_during_failover.
Passed core tests.
Change-Id: Ia511e22a596733e6a2dd752894d6da43e91faca6
Reviewed-on: http://gerrit.cloudera.org:8080/24600
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
be/src/catalog/catalog-server.cc | 4 +--
common/thrift/generate_error_codes.py | 5 ++-
.../impala/catalog/local/CatalogdMetaProvider.java | 9 +++++-
tests/custom_cluster/test_catalogd_ha.py | 37 ++++++++++++++++++++++
4 files changed, 50 insertions(+), 5 deletions(-)
diff --git a/be/src/catalog/catalog-server.cc b/be/src/catalog/catalog-server.cc
index 76c85d9c2..96d5d922f 100644
--- a/be/src/catalog/catalog-server.cc
+++ b/be/src/catalog/catalog-server.cc
@@ -835,9 +835,7 @@ Status CatalogServer::WaitPendingMetadataResetStarts(const string& server_addres
while (true) {
// Return error if this catalogd is not active.
if (!is_active_.Load()) {
- return Status(Substitute("Request for Catalog service is rejected since "
- "catalogd $0 is in standby mode",
- server_address));
+ return Status::Expected(TErrorCode::CATALOG_IN_STANDBY_MODE, server_address);
}
// Early return if first reset has completed.
diff --git a/common/thrift/generate_error_codes.py b/common/thrift/generate_error_codes.py
index 2d7a4318d..c1b139789 100755
--- a/common/thrift/generate_error_codes.py
+++ b/common/thrift/generate_error_codes.py
@@ -501,7 +501,10 @@ error_codes = (
("TUPLE_CACHE_OUTSTANDING_WRITE_LIMIT_EXCEEDED", 163, "Outstanding tuple cache writes "
"exceeded the limit ($0 bytes)"),
- ("CHARSET_CONVERSION_ERROR", 164, "Error during buffer conversion: $0")
+ ("CHARSET_CONVERSION_ERROR", 164, "Error during buffer conversion: $0"),
+
+ ("CATALOG_IN_STANDBY_MODE", 165, "Request for Catalog service is rejected since "
+ "catalogd $0 is in standby mode")
)
import sys
diff --git a/fe/src/main/java/org/apache/impala/catalog/local/CatalogdMetaProvider.java b/fe/src/main/java/org/apache/impala/catalog/local/CatalogdMetaProvider.java
index 44f23d733..2da685f7f 100644
--- a/fe/src/main/java/org/apache/impala/catalog/local/CatalogdMetaProvider.java
+++ b/fe/src/main/java/org/apache/impala/catalog/local/CatalogdMetaProvider.java
@@ -493,7 +493,14 @@ public class CatalogdMetaProvider implements MetaProvider {
resp = new TGetPartialCatalogObjectResponse();
new TDeserializer().deserialize(resp, ret);
if (resp.isSetStatus() && resp.status.status_code != TErrorCode.OK) {
- throw new TException(String.join("\n", resp.status.error_msgs));
+ String errorMsg = String.join("\n", resp.status.error_msgs);
+ if (resp.status.status_code == TErrorCode.CATALOG_IN_STANDBY_MODE) {
+ LOG.warn("Catalogd is in standby mode. Triggering metadata fetch retry.");
+ throw new InconsistentMetadataFetchException(
+ CatalogLookupStatus.CATALOG_SERVICE_CHANGED,
+ "Catalogd is in standby mode: " + errorMsg);
+ }
+ throw new TException(errorMsg);
}
if (resp.isSetCatalog_service_id()
&& witnessCatalogServiceId(resp.getCatalog_service_id())) {
diff --git a/tests/custom_cluster/test_catalogd_ha.py b/tests/custom_cluster/test_catalogd_ha.py
index 2be15b7e6..aa7a68559 100644
--- a/tests/custom_cluster/test_catalogd_ha.py
+++ b/tests/custom_cluster/test_catalogd_ha.py
@@ -19,6 +19,7 @@ import json
import logging
import re
import requests
+import threading
import time
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite, IMPALA_HOME
@@ -823,3 +824,39 @@ class TestCatalogdHA(CustomClusterTestSuite):
else:
page = requests.get(self.CATALOG_HA_INFO_URL.format(port))
assert page.status_code == requests.codes.not_found
+
+ @CustomClusterTestSuite.with_args(
+ statestored_args=SS_AUTO_FAILOVER_ARGS,
+ catalogd_args="--catalogd_ha_reset_metadata_on_failover=false "
+ "--enable_reload_events=true",
+ impalad_args="--use_local_catalog=true --local_catalog_max_fetch_retries=10",
+ start_args="--enable_catalogd_ha",
+ disable_log_buffering=True)
+ def test_standby_retry_during_failover(self):
+ """Tests that a query retries when it encounters a standby mode rejection
+ during an HA failover, instead of failing."""
+ active_catalogd, standby_catalogd = self.__get_catalogds()
+ query_error = []
+
+ def run_query():
+ with self.create_impala_client() as client:
+ try:
+ self.execute_query_unchecked(client, "refresh functional.alltypes")
+ except Exception as e:
+ query_error.append(str(e))
+
+ active_catalogd.kill()
+ t = threading.Thread(target=run_query)
+ t.start()
+ active_catalogd.wait_for_exit()
+ # Wait until the standby catalog becomes active.
+ standby_catalogd.service.wait_for_metric_value('catalog-server.active-status',
+ expected_value=True,
+ timeout=30)
+ # Restart the killed catalogd.
+ active_catalogd.start(wait_until_ready=True)
+ # Wait for the query to finish.
+ t.join(timeout=30)
+ # Verify the behavior.
+ if query_error:
+ assert False, "Query failed unexpectedly: " + query_error[0]