You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by ka...@apache.org on 2025/08/11 17:32:18 UTC
(phoenix) branch PHOENIX-7562-feature updated: PHOENIX-7566 [1/n] HAGroupStoreClient/Manager changes for Consistent HA (#2244)
This is an automated email from the ASF dual-hosted git repository.
kadir pushed a commit to branch PHOENIX-7562-feature
in repository https://gitbox.apache.org/repos/asf/phoenix.git
The following commit(s) were added to refs/heads/PHOENIX-7562-feature by this push:
new 6195138285 PHOENIX-7566 [1/n] HAGroupStoreClient/Manager changes for Consistent HA (#2244)
6195138285 is described below
commit 619513828509cf7ddf286891acdb06387ce21e46
Author: ritegarg <58...@users.noreply.github.com>
AuthorDate: Mon Aug 11 10:32:13 2025 -0700
PHOENIX-7566 [1/n] HAGroupStoreClient/Manager changes for Consistent HA (#2244)
Co-authored-by: Ritesh Garg <ri...@riteshg-ltmd34g.internal.salesforce.com>
---
.../BaseScannerRegionObserverConstants.java | 5 +
.../InvalidClusterRoleTransitionException.java | 41 +
.../exception/MutationBlockedIOException.java | 36 +
.../StaleHAGroupStoreRecordVersionException.java | 42 +
.../org/apache/phoenix/execute/MutationState.java | 8 +-
.../org/apache/phoenix/jdbc/ClusterRoleRecord.java | 31 +-
.../apache/phoenix/jdbc/HAGroupStoreClient.java | 666 ++++++++++++---
.../apache/phoenix/jdbc/HAGroupStoreManager.java | 167 +++-
.../apache/phoenix/jdbc/HAGroupStoreRecord.java | 264 ++++++
.../apache/phoenix/jdbc/HighAvailabilityGroup.java | 57 +-
.../org/apache/phoenix/jdbc/PhoenixConnection.java | 13 +
.../phoenix/jdbc/PhoenixDatabaseMetaData.java | 16 +
.../org/apache/phoenix/jdbc/PhoenixHAAdmin.java | 127 ++-
.../phoenix/query/ConnectionQueryServicesImpl.java | 90 +-
.../query/ConnectionlessQueryServicesImpl.java | 63 +-
.../org/apache/phoenix/query/QueryConstants.java | 30 +
.../org/apache/phoenix/query/QueryServices.java | 1 -
.../org/apache/phoenix/util/WALAnnotationUtil.java | 6 +-
.../protobuf/RegionServerEndpointService.proto | 20 +
.../coprocessor/PhoenixRegionServerEndpoint.java | 58 +-
.../UngroupedAggregateRegionObserver.java | 18 -
.../UngroupedAggregateRegionScanner.java | 5 +-
.../phoenix/hbase/index/IndexRegionObserver.java | 37 +-
.../MigrateSystemTablesToSystemNamespaceIT.java | 4 +-
.../end2end/PhoenixRegionServerEndpointIT.java | 12 -
...gionServerEndpointITWithConsistentFailover.java | 158 ++++
.../phoenix/end2end/QueryDatabaseMetaDataIT.java | 6 +-
.../SystemTablesCreationOnConnectionIT.java | 4 +-
.../phoenix/end2end/TenantSpecificTablesDDLIT.java | 4 +-
.../IndexRegionObserverMutationBlockingIT.java | 217 +++++
.../apache/phoenix/jdbc/HAGroupStoreClientIT.java | 916 ++++++++++++++++-----
.../apache/phoenix/jdbc/HAGroupStoreManagerIT.java | 323 +++++++-
.../phoenix/jdbc/HighAvailabilityGroupTestIT.java | 5 +-
.../phoenix/jdbc/ParallelPhoenixConnectionIT.java | 85 +-
.../org/apache/phoenix/jdbc/PhoenixHAAdminIT.java | 481 +++++++++++
.../apache/phoenix/jdbc/ClusterRoleRecordTest.java | 11 +-
.../phoenix/jdbc/HAGroupStoreRecordTest.java | 359 ++++++++
.../phoenix/jdbc/PhoenixHAAdminToolTest.java | 2 +-
.../apache/phoenix/util/HAGroupStoreTestUtil.java | 113 +++
39 files changed, 3994 insertions(+), 507 deletions(-)
diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/coprocessorclient/BaseScannerRegionObserverConstants.java b/phoenix-core-client/src/main/java/org/apache/phoenix/coprocessorclient/BaseScannerRegionObserverConstants.java
index 74f6c67026..41becf9374 100644
--- a/phoenix-core-client/src/main/java/org/apache/phoenix/coprocessorclient/BaseScannerRegionObserverConstants.java
+++ b/phoenix-core-client/src/main/java/org/apache/phoenix/coprocessorclient/BaseScannerRegionObserverConstants.java
@@ -190,4 +190,9 @@ public class BaseScannerRegionObserverConstants {
/** Exposed for testing */
public static final String SCANNER_OPENED_TRACE_INFO = "Scanner opened on server";
+
+ /**
+ * Attribute name used to pass HAGroupName in Mutations.
+ */
+ public static final String HA_GROUP_NAME_ATTRIB = "_HAGroupName";
}
diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/exception/InvalidClusterRoleTransitionException.java b/phoenix-core-client/src/main/java/org/apache/phoenix/exception/InvalidClusterRoleTransitionException.java
new file mode 100644
index 0000000000..e5fa01e958
--- /dev/null
+++ b/phoenix-core-client/src/main/java/org/apache/phoenix/exception/InvalidClusterRoleTransitionException.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.exception;
+
+/**
+ * Exception thrown when attempting an invalid cluster role transition.
+ */
+public class InvalidClusterRoleTransitionException extends Exception {
+ private static final long serialVersionUID = 1L;
+
+ /**
+ * @param msg reason for the exception
+ */
+ public InvalidClusterRoleTransitionException(String msg) {
+ super(msg);
+ }
+
+ /**
+ * @param msg reason for the exception
+ * @param cause the underlying cause
+ */
+ public InvalidClusterRoleTransitionException(String msg, Throwable cause) {
+ super(msg, cause);
+ }
+
+}
diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/exception/MutationBlockedIOException.java b/phoenix-core-client/src/main/java/org/apache/phoenix/exception/MutationBlockedIOException.java
new file mode 100644
index 0000000000..590c3615d6
--- /dev/null
+++ b/phoenix-core-client/src/main/java/org/apache/phoenix/exception/MutationBlockedIOException.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.exception;
+
+import java.io.IOException;
+
+/**
+ * Exception thrown when CLUSTER_ROLE_BASED_MUTATION_BLOCK_ENABLED is set
+ * and the current cluster role is ACTIVE_TO_STANDBY.
+ */
+public class MutationBlockedIOException extends IOException {
+ private static final long serialVersionUID = 1L;
+
+ /**
+ * @param msg reason for the exception
+ */
+ public MutationBlockedIOException(String msg) {
+ super(msg);
+ }
+
+}
diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/exception/StaleHAGroupStoreRecordVersionException.java b/phoenix-core-client/src/main/java/org/apache/phoenix/exception/StaleHAGroupStoreRecordVersionException.java
new file mode 100644
index 0000000000..ca55305917
--- /dev/null
+++ b/phoenix-core-client/src/main/java/org/apache/phoenix/exception/StaleHAGroupStoreRecordVersionException.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.exception;
+
+/**
+ * Exception thrown when attempting to update HAGroupStoreRecord with a stale ZK stat version,
+ * indicating that the record has been modified by another process.
+ */
+public class StaleHAGroupStoreRecordVersionException extends Exception {
+ private static final long serialVersionUID = 1L;
+
+ /**
+ * @param msg reason for the exception
+ */
+ public StaleHAGroupStoreRecordVersionException(String msg) {
+ super(msg);
+ }
+
+ /**
+ * @param msg reason for the exception
+ * @param cause the underlying cause
+ */
+ public StaleHAGroupStoreRecordVersionException(String msg, Throwable cause) {
+ super(msg, cause);
+ }
+
+}
diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/execute/MutationState.java b/phoenix-core-client/src/main/java/org/apache/phoenix/execute/MutationState.java
index b92d699867..2a52426ffd 100644
--- a/phoenix-core-client/src/main/java/org/apache/phoenix/execute/MutationState.java
+++ b/phoenix-core-client/src/main/java/org/apache/phoenix/execute/MutationState.java
@@ -55,6 +55,7 @@ import java.util.Set;
import javax.annotation.Nonnull;
import javax.annotation.concurrent.Immutable;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.TableName;
@@ -640,8 +641,8 @@ public class MutationState implements SQLCloseable {
final MultiRowMutationState values, final long mutationTimestamp, final long serverTimestamp,
boolean includeAllIndexes, final boolean sendAll) {
final PTable table = tableRef.getTable();
- final List<PTable> indexList = includeAllIndexes ?
- Lists.newArrayList(IndexMaintainer.maintainedIndexes(table.getIndexes().iterator())) :
+ final List<PTable> indexList = includeAllIndexes ?
+ Lists.newArrayList(IndexMaintainer.maintainedIndexes(table.getIndexes().iterator())) :
IndexUtil.getClientMaintainedIndexes(table);
final Iterator<PTable> indexes = indexList.iterator();
final List<Mutation> mutationList = Lists.newArrayListWithExpectedSize(values.size());
@@ -864,7 +865,8 @@ public class MutationState implements SQLCloseable {
Bytes.toBytes(table.getExternalSchemaId()) : null;
byte[] lastDDLTimestamp =
table.getLastDDLTimestamp() != null ? Bytes.toBytes(table.getLastDDLTimestamp()) : null;
- WALAnnotationUtil.annotateMutation(mutation, tenantId, schemaName, tableName, tableType, lastDDLTimestamp);
+ byte[] haGroupName = StringUtils.isNotBlank(connection.getHAGroupName()) ? Bytes.toBytes(connection.getHAGroupName()) : null;
+ WALAnnotationUtil.annotateMutation(mutation, tenantId, schemaName, tableName, tableType, lastDDLTimestamp, haGroupName);
}
/**
diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/ClusterRoleRecord.java b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/ClusterRoleRecord.java
index cc463b7972..3b71f8f9cd 100644
--- a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/ClusterRoleRecord.java
+++ b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/ClusterRoleRecord.java
@@ -60,13 +60,14 @@ public class ClusterRoleRecord {
* take traffic, standby and offline do not, and unknown is used if the state cannot be determined.
*/
public enum ClusterRole {
- ACTIVE, STANDBY, OFFLINE, UNKNOWN, ACTIVE_TO_STANDBY;
+ ACTIVE, STANDBY, OFFLINE, UNKNOWN, ACTIVE_TO_STANDBY, STANDBY_TO_ACTIVE;
/**
* @return true if a cluster with this role can be connected, otherwise false
*/
public boolean canConnect() {
- return this == ACTIVE || this == STANDBY || this == ACTIVE_TO_STANDBY;
+ return this == ACTIVE || this == STANDBY
+ || this == ACTIVE_TO_STANDBY || this == STANDBY_TO_ACTIVE;
}
public static ClusterRole from(byte[] bytes) {
@@ -76,6 +77,32 @@ public class ClusterRoleRecord {
.findFirst()
.orElse(UNKNOWN);
}
+
+ public boolean isMutationBlocked() {
+ return this == ACTIVE_TO_STANDBY;
+ }
+
+ /**
+ * @return the default HAGroupState for this ClusterRole
+ */
+ @JsonIgnore
+ public HAGroupStoreRecord.HAGroupState getDefaultHAGroupState() {
+ switch (this) {
+ case ACTIVE:
+ return HAGroupStoreRecord.HAGroupState.ACTIVE_NOT_IN_SYNC;
+ case STANDBY:
+ return HAGroupStoreRecord.HAGroupState.DEGRADED_STANDBY;
+ case OFFLINE:
+ return HAGroupStoreRecord.HAGroupState.OFFLINE;
+ case ACTIVE_TO_STANDBY:
+ return HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY;
+ case STANDBY_TO_ACTIVE:
+ return HAGroupStoreRecord.HAGroupState.STANDBY_TO_ACTIVE;
+ case UNKNOWN:
+ default:
+ return HAGroupStoreRecord.HAGroupState.UNKNOWN;
+ }
+ }
}
/**
diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HAGroupStoreClient.java b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HAGroupStoreClient.java
index c3606ddad2..123dc28d5c 100644
--- a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HAGroupStoreClient.java
+++ b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HAGroupStoreClient.java
@@ -17,191 +17,625 @@
*/
package org.apache.phoenix.jdbc;
+import java.io.Closeable;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.tuple.Pair;
import org.apache.curator.framework.recipes.cache.ChildData;
import org.apache.curator.framework.recipes.cache.PathChildrenCache;
import org.apache.curator.framework.recipes.cache.PathChildrenCacheListener;
import org.apache.curator.utils.ZKPaths;
import org.apache.hadoop.conf.Configuration;
+import org.apache.phoenix.exception.InvalidClusterRoleTransitionException;
+import org.apache.phoenix.exception.StaleHAGroupStoreRecordVersionException;
+import org.apache.phoenix.jdbc.ClusterRoleRecord.ClusterRole;
+import org.apache.phoenix.jdbc.ClusterRoleRecord.RegistryType;
+import org.apache.phoenix.jdbc.HAGroupStoreRecord.HAGroupState;
import org.apache.phoenix.thirdparty.com.google.common.annotations.VisibleForTesting;
-import org.apache.phoenix.thirdparty.com.google.common.collect.ImmutableList;
+import org.apache.phoenix.thirdparty.com.google.common.base.Preconditions;
+import org.apache.phoenix.util.JDBCUtil;
+import org.apache.zookeeper.data.Stat;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.List;
-import java.util.Objects;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.TimeUnit;
-import java.util.stream.Collectors;
+import static org.apache.hadoop.hbase.HConstants.DEFAULT_ZK_SESSION_TIMEOUT;
+import static org.apache.hadoop.hbase.HConstants.ZK_SESSION_TIMEOUT;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.CLUSTER_ROLE_1;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.CLUSTER_ROLE_2;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.CLUSTER_URL_1;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.CLUSTER_URL_2;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.HA_GROUP_NAME;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.POLICY;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_HA_GROUP_NAME;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VERSION;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.ZK_URL_1;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.ZK_URL_2;
+import static org.apache.phoenix.jdbc.PhoenixHAAdmin.getLocalZkUrl;
+import static org.apache.phoenix.jdbc.PhoenixHAAdmin.toPath;
+import static org.apache.phoenix.util.PhoenixRuntime.JDBC_PROTOCOL_SEPARATOR;
+import static org.apache.phoenix.util.PhoenixRuntime.JDBC_PROTOCOL_ZK;
/**
- * Write-through cache for HAGroupStore.
+ * Main implementation of HAGroupStoreClient with peer support.
+ * Write-through cache for HAGroupStore based on {@link HAGroupStoreRecord}.
* Uses {@link PathChildrenCache} from {@link org.apache.curator.framework.CuratorFramework}.
*/
public class HAGroupStoreClient implements Closeable {
+ public static final String ZK_CONSISTENT_HA_NAMESPACE =
+ "phoenix" + ZKPaths.PATH_SEPARATOR + "consistentHA";
private static final long HA_GROUP_STORE_CLIENT_INITIALIZATION_TIMEOUT_MS = 30000L;
- private static volatile HAGroupStoreClient haGroupStoreClientInstance;
+ // Multiplier for ZK session timeout to account for time it will take for HMaster to abort
+ // the region server in case ZK connection is lost from the region server.
+ private static final double ZK_SESSION_TIMEOUT_MULTIPLIER = 1.1;
+ private static final String CACHE_TYPE_LOCAL = "LOCAL";
+ private static final String CACHE_TYPE_PEER = "PEER";
private PhoenixHAAdmin phoenixHaAdmin;
+ private PhoenixHAAdmin peerPhoenixHaAdmin;
private static final Logger LOGGER = LoggerFactory.getLogger(HAGroupStoreClient.class);
- // Map contains <ClusterRole, Map<HAGroupName(String), ClusterRoleRecord>>
- private final ConcurrentHashMap<ClusterRoleRecord.ClusterRole, ConcurrentHashMap<String, ClusterRoleRecord>> clusterRoleToCRRMap
- = new ConcurrentHashMap<>();
- private PathChildrenCache pathChildrenCache;
- private volatile boolean isHealthy;
+ // Map of <ZKUrl, <HAGroupName, HAGroupStoreClientInstance>>
+ private static final Map<String, ConcurrentHashMap<String, HAGroupStoreClient>> instances =
+ new ConcurrentHashMap<>();
+ // HAGroupName for this instance
+ private final String haGroupName;
+ // PathChildrenCache for current cluster and HAGroupName
+ private PathChildrenCache pathChildrenCache = null;
+ // PathChildrenCache for peer cluster and HAGroupName
+ private PathChildrenCache peerPathChildrenCache = null;
+ // Whether the client is healthy
+ private volatile boolean isHealthy = false;
+ // Configuration
+ private final Configuration conf;
+ // ZK URL for the current cluster and HAGroupName
+ private String zkUrl;
+ // Peer ZK URL for peer cluster and HAGroupName
+ private String peerZKUrl = null;
+ // Peer Custom Event Listener
+ private final PathChildrenCacheListener peerCustomPathChildrenCacheListener;
+ // Wait time for sync mode
+ private final long waitTimeForSyncModeInMs;
+ // Policy for the HA group
+ private HighAvailabilityPolicy policy;
+ private ClusterRole clusterRole;
+ private ClusterRole peerClusterRole;
+ private String clusterUrl;
+ private String peerClusterUrl;
+ private long clusterRoleRecordVersion;
+
+ public static HAGroupStoreClient getInstance(Configuration conf, String haGroupName) throws SQLException {
+ return getInstanceForZkUrl(conf, haGroupName, null);
+ }
/**
* Creates/gets an instance of HAGroupStoreClient.
* Can return null instance if unable to initialize.
*
* @param conf configuration
+ * @param haGroupName name of the HA group. Only specified HA group is tracked.
+ * @param zkUrl zkUrl to use for the client. Prefer providing this parameter to avoid
+ * the overhead of getting the local zkUrl from the configuration.
* @return HAGroupStoreClient instance
*/
- public static HAGroupStoreClient getInstance(Configuration conf) {
- if (haGroupStoreClientInstance == null || !haGroupStoreClientInstance.isHealthy) {
+ public static HAGroupStoreClient getInstanceForZkUrl(Configuration conf, String haGroupName,
+ String zkUrl) throws SQLException {
+ Preconditions.checkNotNull(haGroupName, "haGroupName cannot be null");
+ String localZkUrl = Objects.toString(zkUrl, getLocalZkUrl(conf));
+ Preconditions.checkNotNull(localZkUrl, "zkUrl cannot be null");
+ HAGroupStoreClient result = instances.getOrDefault(localZkUrl, new ConcurrentHashMap<>())
+ .getOrDefault(haGroupName, null);
+ if (result == null || !result.isHealthy) {
synchronized (HAGroupStoreClient.class) {
- if (haGroupStoreClientInstance == null || !haGroupStoreClientInstance.isHealthy) {
- haGroupStoreClientInstance = new HAGroupStoreClient(conf, null);
- if (!haGroupStoreClientInstance.isHealthy) {
- haGroupStoreClientInstance.close();
- haGroupStoreClientInstance = null;
+ result = instances.getOrDefault(localZkUrl, new ConcurrentHashMap<>())
+ .getOrDefault(haGroupName, null);
+ if (result == null || !result.isHealthy) {
+ result = new HAGroupStoreClient(conf, null, null, haGroupName, zkUrl);
+ if (!result.isHealthy) {
+ result.close();
+ result = null;
+ } else {
+ instances.putIfAbsent(localZkUrl, new ConcurrentHashMap<>());
+ instances.get(localZkUrl).put(haGroupName, result);
}
}
}
}
- return haGroupStoreClientInstance;
+ return result;
+ }
+
+ /**
+ * Get the list of HAGroupNames from system table.
+ * We can also get the list of HAGroupNames from the system table by providing the zkUrl in
+ * where clause but we need to match the formatted zkUrl with the zkUrl in the system table so
+ * that matching is done correctly.
+ *
+ * @param zkUrl for connecting to Table
+ * @return the list of HAGroupNames
+ * @throws SQLException in case of unexpected error
+ */
+ public static List<String> getHAGroupNames(String zkUrl) throws SQLException {
+ List<String> result = new ArrayList<>();
+ String queryString = String.format("SELECT %s,%s,%s FROM %s", HA_GROUP_NAME, ZK_URL_1,
+ ZK_URL_2, SYSTEM_HA_GROUP_NAME);
+ try (PhoenixConnection conn = (PhoenixConnection) DriverManager.getConnection(
+ JDBC_PROTOCOL_ZK + JDBC_PROTOCOL_SEPARATOR + zkUrl);
+ Statement stmt = conn.createStatement();
+ ResultSet rs = stmt.executeQuery(queryString)) {
+ while (rs.next()) {
+ String zkUrl1 = rs.getString(ZK_URL_1);
+ String zkUrl2 = rs.getString(ZK_URL_2);
+ String formattedZkUrl1 = JDBCUtil.formatUrl(zkUrl1, RegistryType.ZK);
+ String formattedZkUrl2 = JDBCUtil.formatUrl(zkUrl2, RegistryType.ZK);
+ String formattedZkUrl = JDBCUtil.formatUrl(zkUrl, RegistryType.ZK);
+ if (StringUtils.equals(formattedZkUrl1, formattedZkUrl) ||
+ StringUtils.equals(formattedZkUrl2, formattedZkUrl)) {
+ result.add(rs.getString(HA_GROUP_NAME));
+ }
+ }
+ }
+ return result;
}
@VisibleForTesting
- HAGroupStoreClient(final Configuration conf, final PathChildrenCacheListener pathChildrenCacheListener) {
+ HAGroupStoreClient(final Configuration conf,
+ final PathChildrenCacheListener pathChildrenCacheListener,
+ final PathChildrenCacheListener peerPathChildrenCacheListener,
+ final String haGroupName,
+ final String zkUrl) {
+ this.conf = conf;
+ this.haGroupName = haGroupName;
+ this.zkUrl = zkUrl;
+ this.waitTimeForSyncModeInMs = (long) Math.ceil(conf.getLong(ZK_SESSION_TIMEOUT, DEFAULT_ZK_SESSION_TIMEOUT)
+ * ZK_SESSION_TIMEOUT_MULTIPLIER);
+ // Custom Event Listener
+ this.peerCustomPathChildrenCacheListener = peerPathChildrenCacheListener;
try {
- this.phoenixHaAdmin = new PhoenixHAAdmin(conf);
- final PathChildrenCache pathChildrenCache;
- pathChildrenCache = new PathChildrenCache(phoenixHaAdmin.getCurator(), ZKPaths.PATH_SEPARATOR, true);
- final CountDownLatch latch = new CountDownLatch(1);
- if (pathChildrenCacheListener != null) {
- pathChildrenCache.getListenable().addListener(pathChildrenCacheListener);
- } else {
- pathChildrenCache.getListenable().addListener((client, event) -> {
- LOGGER.info("HAGroupStoreClient PathChildrenCache Received event for type {}", event.getType());
- final ChildData childData = event.getData();
- ClusterRoleRecord eventCRR = extractCRROrNull(childData);
- switch (event.getType()) {
- case CHILD_ADDED:
- case CHILD_UPDATED:
- if (eventCRR != null && eventCRR.getHaGroupName() != null) {
- updateClusterRoleRecordMap(eventCRR);
- }
- break;
- case CHILD_REMOVED:
- // In case of CHILD_REMOVED, we get the old version of data that was just deleted in event.
- if (eventCRR != null && eventCRR.getHaGroupName() != null
- && !eventCRR.getHaGroupName().isEmpty()
- && eventCRR.getRole(phoenixHaAdmin.getZkUrl()) != null) {
- LOGGER.info("Received CHILD_REMOVED event, Removing CRR {} from existing CRR Map {}", eventCRR, clusterRoleToCRRMap);
- final ClusterRoleRecord.ClusterRole role = eventCRR.getRole(phoenixHaAdmin.getZkUrl());
- clusterRoleToCRRMap.putIfAbsent(role, new ConcurrentHashMap<>());
- clusterRoleToCRRMap.get(role).remove(eventCRR.getHaGroupName());
- }
- break;
- case INITIALIZED:
- latch.countDown();
- break;
- case CONNECTION_LOST:
- case CONNECTION_SUSPENDED:
- isHealthy = false;
- break;
- case CONNECTION_RECONNECTED:
- isHealthy = true;
- break;
- default:
- LOGGER.warn("Unexpected event type {}, complete event {}", event.getType(), event);
- }
- });
+ // Initialize HAGroupStoreClient attributes
+ initializeHAGroupStoreClientAttributes(haGroupName);
+ // Initialize Phoenix HA Admin
+ this.phoenixHaAdmin = new PhoenixHAAdmin(this.zkUrl, conf, ZK_CONSISTENT_HA_NAMESPACE);
+ // Initialize local cache
+ this.pathChildrenCache = initializePathChildrenCache(phoenixHaAdmin,
+ pathChildrenCacheListener, CACHE_TYPE_LOCAL);
+ // Initialize ZNode if not present in ZK
+ initializeZNodeIfNeeded();
+ if (this.pathChildrenCache != null) {
+ this.isHealthy = true;
+ // Initialize peer cache
+ maybeInitializePeerPathChildrenCache();
}
- pathChildrenCache.start(PathChildrenCache.StartMode.POST_INITIALIZED_EVENT);
- this.pathChildrenCache = pathChildrenCache;
- isHealthy = latch.await(HA_GROUP_STORE_CLIENT_INITIALIZATION_TIMEOUT_MS, TimeUnit.MILLISECONDS);
- buildClusterRoleToCRRMap();
+
} catch (Exception e) {
- isHealthy = false;
- LOGGER.error("Unexpected error occurred while initializing HAGroupStoreClient, marking cache as unhealthy", e);
+ this.isHealthy = false;
+ close();
+ LOGGER.error("Unexpected error occurred while initializing HAGroupStoreClient, "
+ + "marking cache as unhealthy", e);
}
}
- private ClusterRoleRecord extractCRROrNull(final ChildData childData) {
- if (childData != null) {
- byte[] data = childData.getData();
- return ClusterRoleRecord.fromJson(data).orElse(null);
+ /**
+ * Rebuilds the internal cache by querying for all needed data.
+ * This is a blocking operation that does not generate events to listeners.
+ *
+ * @throws Exception if rebuild fails or client is not healthy
+ */
+ public void rebuild() throws Exception {
+ if (!isHealthy) {
+ throw new IOException("HAGroupStoreClient is not healthy");
}
- return null;
+ LOGGER.info("Rebuilding HAGroupStoreClient for HA group {}", haGroupName);
+ initializeHAGroupStoreClientAttributes(haGroupName);
+ initializeZNodeIfNeeded();
+ maybeInitializePeerPathChildrenCache();
+
+ // NOTE: this is a BLOCKING method.
+ // Completely rebuild the internal cache by querying for all needed data
+ // WITHOUT generating any events to send to listeners.
+ if (pathChildrenCache != null) {
+ pathChildrenCache.rebuild();
+ }
+ if (peerPathChildrenCache != null) {
+ peerPathChildrenCache.rebuild();
+ }
+ LOGGER.info("Rebuild Complete for HAGroupStoreClient for HA group {}", haGroupName);
}
- private void updateClusterRoleRecordMap(final ClusterRoleRecord crr) {
- if (crr != null && crr.getHaGroupName() != null && crr.getRole(phoenixHaAdmin.getZkUrl()) != null) {
- ClusterRoleRecord.ClusterRole role = crr.getRole(phoenixHaAdmin.getZkUrl());
- LOGGER.info("Updating Existing CRR Map {} with new CRR {}", clusterRoleToCRRMap, crr);
- clusterRoleToCRRMap.putIfAbsent(role, new ConcurrentHashMap<>());
- clusterRoleToCRRMap.get(role).put(crr.getHaGroupName(), crr);
- LOGGER.info("Added new CRR {} to CRR Map", crr);
- // Remove any pre-existing mapping with any other role for this HAGroupName
- for (ClusterRoleRecord.ClusterRole mapRole : clusterRoleToCRRMap.keySet()) {
- if (mapRole != role) {
- ConcurrentHashMap<String, ClusterRoleRecord> roleWiseMap = clusterRoleToCRRMap.get(mapRole);
- if (roleWiseMap.containsKey(crr.getHaGroupName())) {
- LOGGER.info("Removing any pre-existing mapping with role {} for HAGroupName {}", mapRole, crr.getHaGroupName());
- roleWiseMap.remove(crr.getHaGroupName());
- }
- }
- }
- LOGGER.info("Final Updated CRR Map {}", clusterRoleToCRRMap);
+ /**
+ * Get HAGroupStoreRecord from local quorum.
+ *
+ * @return HAGroupStoreRecord for the specified HA group name, or null if not found
+ * @throws IOException if the client is not healthy
+ */
+ public HAGroupStoreRecord getHAGroupStoreRecord() throws IOException {
+ if (!isHealthy) {
+ throw new IOException("HAGroupStoreClient is not healthy");
}
+ return fetchCacheRecord(this.pathChildrenCache, CACHE_TYPE_LOCAL).getLeft();
}
- private void buildClusterRoleToCRRMap() {
- List<ClusterRoleRecord> clusterRoleRecords = pathChildrenCache.getCurrentData().stream().map(this::extractCRROrNull)
- .filter(Objects::nonNull).collect(Collectors.toList());
- for (ClusterRoleRecord crr : clusterRoleRecords) {
- updateClusterRoleRecordMap(crr);
+ /**
+ * Set the HA group status for the specified HA group name.
+ * Checks if the status is needed to be updated based on logic in isUpdateNeeded function.
+ *
+ * @param haGroupState the HA group state to set
+ * @throws IOException if the client is not healthy or the operation fails
+ * @throws StaleHAGroupStoreRecordVersionException if the version is stale
+ * @throws InvalidClusterRoleTransitionException when transition is not valid
+ */
+ public void setHAGroupStatusIfNeeded(HAGroupStoreRecord.HAGroupState haGroupState)
+ throws IOException, StaleHAGroupStoreRecordVersionException,
+ InvalidClusterRoleTransitionException {
+ Preconditions.checkNotNull(haGroupState, "haGroupState cannot be null");
+ if (!isHealthy) {
+ throw new IOException("HAGroupStoreClient is not healthy");
+ }
+ Pair<HAGroupStoreRecord, Stat> cacheRecord = fetchCacheRecord(
+ this.pathChildrenCache, CACHE_TYPE_LOCAL);
+ HAGroupStoreRecord currentHAGroupStoreRecord = cacheRecord.getLeft();
+ Stat currentHAGroupStoreRecordStat = cacheRecord.getRight();
+ if (currentHAGroupStoreRecord == null) {
+ throw new IOException("Current HAGroupStoreRecordStat in cache is null, "
+ + "cannot update HAGroupStoreRecord, the record should be initialized "
+ + "in System Table first" + haGroupName);
+ }
+ if (isUpdateNeeded(currentHAGroupStoreRecord.getHAGroupState(),
+ currentHAGroupStoreRecordStat.getMtime(), haGroupState)) {
+ HAGroupStoreRecord newHAGroupStoreRecord = new HAGroupStoreRecord(
+ currentHAGroupStoreRecord.getProtocolVersion(),
+ currentHAGroupStoreRecord.getHaGroupName(),
+ haGroupState
+ );
+ // TODO: Check if cluster role is changing, if so, we need to update
+ // the system table first
+ // Lock the row in System Table and make sure update is reflected
+ // in all regionservers
+ // It should automatically update the ZK record as well.
+ phoenixHaAdmin.updateHAGroupStoreRecordInZooKeeper(haGroupName,
+ newHAGroupStoreRecord, currentHAGroupStoreRecordStat.getVersion());
+ } else {
+ LOGGER.info("Not updating HAGroupStoreRecord for HA group {} with state {}",
+ haGroupName, haGroupState);
}
}
- public void rebuild() throws Exception {
+ /**
+ * Returns the ClusterRoleRecord for the cluster pair.
+ * Information in System Table for peer cluster might be stale,
+ * so we need to get the latest information from ZK.
+ *
+ * @return ClusterRoleRecord for the cluster pair
+ * @throws IOException
+ */
+ public ClusterRoleRecord getClusterRoleRecord() throws IOException {
+ HAGroupStoreRecord peerHAGroupStoreRecord = getHAGroupStoreRecordFromPeer();
+ ClusterRoleRecord.ClusterRole peerClusterRole = peerHAGroupStoreRecord != null
+ ? peerHAGroupStoreRecord.getClusterRole()
+ : ClusterRole.UNKNOWN;
+ return new ClusterRoleRecord(this.haGroupName,
+ this.policy,
+ this.clusterUrl,
+ this.clusterRole,
+ this.peerClusterUrl,
+ peerClusterRole,
+ this.clusterRoleRecordVersion);
+ }
+
+ /**
+ * Get HAGroupStoreRecord from peer cluster.
+ *
+ * @return HAGroupStoreRecord for the specified HA group name, or null if not found
+ * @throws IOException if the client is not healthy
+ */
+ private HAGroupStoreRecord getHAGroupStoreRecordFromPeer() throws IOException {
if (!isHealthy) {
throw new IOException("HAGroupStoreClient is not healthy");
}
- LOGGER.info("Rebuilding HAGroupStoreClient for HA groups");
- // NOTE: this is a BLOCKING method.
- // Completely rebuild the internal cache by querying for all needed data
- // WITHOUT generating any events to send to listeners.
- pathChildrenCache.rebuild();
- buildClusterRoleToCRRMap();
- LOGGER.info("Rebuild Complete for HAGroupStoreClient");
+ return fetchCacheRecord(this.peerPathChildrenCache, CACHE_TYPE_PEER).getLeft();
+ }
+
+ private void initializeZNodeIfNeeded() throws IOException,
+ StaleHAGroupStoreRecordVersionException {
+ // Sometimes the ZNode might not be available in local cache yet, so better to check
+ // in ZK directly if we need to initialize
+ Pair<HAGroupStoreRecord, Stat> cacheRecordFromZK =
+ phoenixHaAdmin.getHAGroupStoreRecordInZooKeeper(this.haGroupName);
+ HAGroupStoreRecord haGroupStoreRecord = cacheRecordFromZK.getLeft();
+ HAGroupStoreRecord newHAGroupStoreRecord = new HAGroupStoreRecord(
+ HAGroupStoreRecord.DEFAULT_PROTOCOL_VERSION,
+ haGroupName,
+ this.clusterRole.getDefaultHAGroupState()
+ );
+ // Only update current ZNode if it doesn't have the same role as present in System Table.
+ // If not exists, then create ZNode
+ if (haGroupStoreRecord != null &&
+ !haGroupStoreRecord.getClusterRole().equals(this.clusterRole)) {
+ phoenixHaAdmin.updateHAGroupStoreRecordInZooKeeper(haGroupName,
+ newHAGroupStoreRecord, cacheRecordFromZK.getRight().getVersion());
+ } else if (haGroupStoreRecord == null) {
+ phoenixHaAdmin.createHAGroupStoreRecordInZooKeeper(newHAGroupStoreRecord);
+ }
+ }
+
+ private void initializeHAGroupStoreClientAttributes(String haGroupName) throws SQLException {
+ String queryString = String.format("SELECT * FROM %s WHERE %s = '%s'",
+ SYSTEM_HA_GROUP_NAME, HA_GROUP_NAME, haGroupName);
+ try (PhoenixConnection conn = (PhoenixConnection) DriverManager.getConnection(
+ JDBC_PROTOCOL_ZK + JDBC_PROTOCOL_SEPARATOR + zkUrl);
+ Statement stmt = conn.createStatement();
+ ResultSet rs = stmt.executeQuery(queryString)) {
+ if (rs.next()) {
+ this.policy = HighAvailabilityPolicy.valueOf(rs.getString(POLICY));
+ String zkUrl1 = rs.getString(ZK_URL_1);
+ String zkUrl2 = rs.getString(ZK_URL_2);
+ String clusterRole1 = rs.getString(CLUSTER_ROLE_1);
+ String clusterRole2 = rs.getString(CLUSTER_ROLE_2);
+ String clusterUrl1 = rs.getString(CLUSTER_URL_1);
+ String clusterUrl2 = rs.getString(CLUSTER_URL_2);
+ this.clusterRoleRecordVersion = rs.getLong(VERSION);
+ Preconditions.checkNotNull(zkUrl1, "ZK_URL_1 in System Table cannot be null");
+ Preconditions.checkNotNull(zkUrl2, "ZK_URL_2 in System Table cannot be null");
+ String formattedZkUrl1 = JDBCUtil.formatUrl(zkUrl1, RegistryType.ZK);
+ String formattedZkUrl2 = JDBCUtil.formatUrl(zkUrl2, RegistryType.ZK);
+ String formattedZkUrl = JDBCUtil.formatUrl(this.zkUrl, RegistryType.ZK);
+ if (StringUtils.equals(formattedZkUrl1, formattedZkUrl)) {
+ this.peerZKUrl = formattedZkUrl2;
+ this.clusterRole = ClusterRoleRecord.ClusterRole.from(
+ clusterRole1.getBytes(StandardCharsets.UTF_8));
+ this.peerClusterRole = ClusterRoleRecord.ClusterRole.from(
+ clusterRole2.getBytes(StandardCharsets.UTF_8));
+ this.clusterUrl = clusterUrl1;
+ this.peerClusterUrl = clusterUrl2;
+ } else if (StringUtils.equals(formattedZkUrl2, formattedZkUrl)) {
+ this.peerZKUrl = JDBCUtil.formatUrl(zkUrl1, RegistryType.ZK);
+ this.clusterRole = ClusterRoleRecord.ClusterRole.from(
+ clusterRole2.getBytes(StandardCharsets.UTF_8));
+ this.peerClusterRole = ClusterRoleRecord.ClusterRole.from(
+ clusterRole1.getBytes(StandardCharsets.UTF_8));
+ this.clusterUrl = clusterUrl2;
+ this.peerClusterUrl = clusterUrl1;
+ }
+ } else {
+ throw new SQLException("HAGroupStoreRecord not found for HA group name: " +
+ haGroupName + " in System Table " + SYSTEM_HA_GROUP_NAME);
+ }
+ }
+ Preconditions.checkNotNull(this.clusterRole,
+ "Cluster role in System Table cannot be null");
+ Preconditions.checkNotNull(this.peerClusterRole,
+ "Peer cluster role in System Table cannot be null");
+ Preconditions.checkNotNull(this.clusterUrl,
+ "Cluster URL in System Table cannot be null");
+ Preconditions.checkNotNull(this.peerZKUrl,
+ "Peer ZK URL in System Table cannot be null");
+ Preconditions.checkNotNull(this.peerClusterUrl,
+ "Peer Cluster URL in System Table cannot be null");
+ Preconditions.checkNotNull(this.clusterRoleRecordVersion,
+ "Cluster role record version in System Table cannot be null");
+ }
+
+ private void maybeInitializePeerPathChildrenCache() {
+ if (StringUtils.isNotBlank(this.peerZKUrl)) {
+ try {
+ // Setup peer connection if needed (first time or ZK Url changed)
+ if (peerPathChildrenCache == null
+ || (peerPhoenixHaAdmin != null &&
+ !StringUtils.equals(this.peerZKUrl, peerPhoenixHaAdmin.getZkUrl()))) {
+ // Clean up existing peer connection if it exists
+ closePeerConnection();
+ // Setup new peer connection
+ this.peerPhoenixHaAdmin
+ = new PhoenixHAAdmin(this.peerZKUrl, conf, ZK_CONSISTENT_HA_NAMESPACE);
+ // Create new PeerPathChildrenCache
+ this.peerPathChildrenCache = initializePathChildrenCache(peerPhoenixHaAdmin,
+ this.peerCustomPathChildrenCacheListener, CACHE_TYPE_PEER);
+ }
+ } catch (Exception e) {
+ closePeerConnection();
+ LOGGER.error("Unable to initialize PeerPathChildrenCache for HAGroupStoreClient",
+ e);
+ // Don't think we should mark HAGroupStoreClient as unhealthy if
+ // peerCache is unhealthy, if needed we can introduce a config to control behavior.
+ }
+ } else {
+ // Close Peer Cache for this HAGroupName if currentClusterRecord is null
+ // or peerZKUrl is blank
+ closePeerConnection();
+ LOGGER.error("Not initializing PeerPathChildrenCache for HAGroupStoreClient "
+ + "with HAGroupName {} as peerZKUrl is blank", haGroupName);
+ }
}
+ private PathChildrenCache initializePathChildrenCache(PhoenixHAAdmin admin,
+ PathChildrenCacheListener customListener, String cacheType) {
+ LOGGER.info("Initializing {} PathChildrenCache with URL {}", cacheType, admin.getZkUrl());
+ PathChildrenCache newPathChildrenCache = null;
+ try {
+ newPathChildrenCache = new PathChildrenCache(admin.getCurator(),
+ ZKPaths.PATH_SEPARATOR, true);
+ final CountDownLatch latch = new CountDownLatch(1);
+ newPathChildrenCache.getListenable().addListener(customListener != null
+ ? customListener
+ : createCacheListener(latch, cacheType));
+ newPathChildrenCache.start(PathChildrenCache.StartMode.POST_INITIALIZED_EVENT);
+ boolean initialized = latch.await(HA_GROUP_STORE_CLIENT_INITIALIZATION_TIMEOUT_MS,
+ TimeUnit.MILLISECONDS);
+ return initialized ? newPathChildrenCache : null;
+ } catch (Exception e) {
+ if (newPathChildrenCache != null) {
+ try {
+ newPathChildrenCache.close();
+ } catch (IOException ioe) {
+ LOGGER.error("Failed to close {} PathChildrenCache with ZKUrl",
+ cacheType, ioe);
+ }
+ }
+ LOGGER.error("Failed to initialize {} PathChildrenCache", cacheType, e);
+ return null;
+ }
+ }
+
+ private PathChildrenCacheListener createCacheListener(CountDownLatch latch, String cacheType) {
+ return (client, event) -> {
+ final ChildData childData = event.getData();
+ HAGroupStoreRecord eventRecord = extractHAGroupStoreRecordOrNull(childData);
+ LOGGER.info("HAGroupStoreClient Cache {} received event {} type {} at {}",
+ cacheType, eventRecord, event.getType(), System.currentTimeMillis());
+ switch (event.getType()) {
+ // TODO: Add support for event watcher for HAGroupStoreRecord
+ // case CHILD_ADDED:
+ // case CHILD_UPDATED:
+ // case CHILD_REMOVED:
+ case INITIALIZED:
+ latch.countDown();
+ break;
+ case CONNECTION_LOST:
+ case CONNECTION_SUSPENDED:
+ if (CACHE_TYPE_LOCAL.equals(cacheType)) {
+ isHealthy = false;
+ }
+ LOGGER.warn("{} HAGroupStoreClient cache connection lost/suspended",
+ cacheType);
+ break;
+ case CONNECTION_RECONNECTED:
+ if (CACHE_TYPE_LOCAL.equals(cacheType)) {
+ isHealthy = true;
+ }
+ LOGGER.info("{} HAGroupStoreClient cache connection reconnected", cacheType);
+ break;
+ default:
+ LOGGER.warn("Unexpected {} event type {}, complete event {}",
+ cacheType, event.getType(), event);
+ }
+ };
+ }
+
+
+ private Pair<HAGroupStoreRecord, Stat> fetchCacheRecord(PathChildrenCache cache,
+ String cacheType) {
+ if (cache == null) {
+ LOGGER.warn("{} HAGroupStoreClient cache is null, returning null", cacheType);
+ return Pair.of(null, null);
+ }
+
+ String targetPath = toPath(this.haGroupName);
+ // Try to get record from current cache data
+ Pair<HAGroupStoreRecord, Stat> result = extractRecordAndStat(cache, targetPath, cacheType);
+ if (result.getLeft() != null) {
+ return result;
+ }
+
+ if (cacheType.equals(CACHE_TYPE_PEER)) {
+ return Pair.of(null, null);
+ }
+ // If no record found, try to rebuild and fetch again
+ LOGGER.info("No record found at path {} for {} cluster, trying to initialize ZNode "
+ + "from System Table in case it might have been deleted",
+ targetPath, cacheType);
+ try {
+ rebuild();
+ return extractRecordAndStat(cache, targetPath, cacheType);
+ } catch (Exception e) {
+ LOGGER.error("Failed to initialize ZNode from System Table, giving up "
+ + "and returning null", e);
+ return Pair.of(null, null);
+ }
+ }
+
+ private Pair<HAGroupStoreRecord, Stat> extractRecordAndStat(PathChildrenCache cache,
+ String targetPath, String cacheType) {
+ ChildData childData = cache.getCurrentData(targetPath);
+ if (childData != null) {
+ HAGroupStoreRecord record = extractHAGroupStoreRecordOrNull(childData);
+ Stat currentStat = childData.getStat();
+ LOGGER.info("Built {} cluster record: {}", cacheType, record);
+ return Pair.of(record, currentStat);
+ }
+ return Pair.of(null, null);
+ }
+
+ private HAGroupStoreRecord extractHAGroupStoreRecordOrNull(final ChildData childData) {
+ if (childData != null) {
+ byte[] data = childData.getData();
+ return HAGroupStoreRecord.fromJson(data).orElse(null);
+ }
+ return null;
+ }
+
+
+ /**
+ * Closes the peer connection and cleans up peer-related resources.
+ */
+ private void closePeerConnection() {
+ try {
+ if (peerPathChildrenCache != null) {
+ peerPathChildrenCache.close();
+ peerPathChildrenCache = null;
+ }
+ if (peerPhoenixHaAdmin != null) {
+ peerPhoenixHaAdmin.close();
+ peerPhoenixHaAdmin = null;
+ }
+ } catch (Exception e) {
+ LOGGER.warn("Failed to close peer connection", e);
+ }
+ }
@Override
public void close() {
try {
LOGGER.info("Closing HAGroupStoreClient");
- clusterRoleToCRRMap.clear();
if (pathChildrenCache != null) {
pathChildrenCache.close();
+ pathChildrenCache = null;
}
+ closePeerConnection();
LOGGER.info("Closed HAGroupStoreClient");
} catch (IOException e) {
LOGGER.error("Exception closing HAGroupStoreClient", e);
}
}
- public List<ClusterRoleRecord> getCRRsByClusterRole(ClusterRoleRecord.ClusterRole clusterRole) throws IOException {
- if (!isHealthy) {
- throw new IOException("HAGroupStoreClient is not healthy");
+ /**
+ * Checks if the HAGroupStoreRecord needs to be updated.
+ * If the cluster role is allowed to transition to the new state and the status refresh
+ * interval has expired, the HAGroupStoreRecord needs to be updated.
+ * If the transition is not allowed, an exception is thrown.
+ *
+ * @param currentHAGroupState the current HAGroupState of the HAGroupStoreRecord
+ * @param currentHAGroupStoreRecordMtime the last modified time of the current
+ * HAGroupStoreRecord
+ * @param newHAGroupState the cluster state to check
+ * @return true if the HAGroupStoreRecord needs to be updated, false otherwise
+ * @throws InvalidClusterRoleTransitionException if the cluster role transition is invalid
+ */
+ private boolean isUpdateNeeded(HAGroupStoreRecord.HAGroupState currentHAGroupState,
+ long currentHAGroupStoreRecordMtime,
+ HAGroupStoreRecord.HAGroupState newHAGroupState)
+ throws InvalidClusterRoleTransitionException {
+ long waitTime = 0L;
+ if (currentHAGroupState.isTransitionAllowed(newHAGroupState)) {
+ if (currentHAGroupState == HAGroupState.ACTIVE_NOT_IN_SYNC
+ && newHAGroupState == HAGroupState.ACTIVE_IN_SYNC) {
+ waitTime = waitTimeForSyncModeInMs;
+ }
+ } else {
+ throw new InvalidClusterRoleTransitionException("Cannot transition from "
+ + currentHAGroupState + " to " + newHAGroupState);
}
- return ImmutableList.copyOf(clusterRoleToCRRMap.getOrDefault(clusterRole, new ConcurrentHashMap<>()).values());
+ return ((System.currentTimeMillis() - currentHAGroupStoreRecordMtime) > waitTime);
}
-}
+
+}
\ No newline at end of file
diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HAGroupStoreManager.java b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HAGroupStoreManager.java
index 690953a7b0..e04d1a2dac 100644
--- a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HAGroupStoreManager.java
+++ b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HAGroupStoreManager.java
@@ -18,13 +18,31 @@
package org.apache.phoenix.jdbc;
import org.apache.hadoop.conf.Configuration;
+import org.apache.phoenix.exception.InvalidClusterRoleTransitionException;
+import org.apache.phoenix.exception.StaleHAGroupStoreRecordVersionException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import java.io.IOException;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+
+import static org.apache.phoenix.jdbc.PhoenixHAAdmin.getLocalZkUrl;
import static org.apache.phoenix.query.QueryServices.CLUSTER_ROLE_BASED_MUTATION_BLOCK_ENABLED;
-import static org.apache.phoenix.query.QueryServicesOptions.DEFAULT_CLUSTER_ROLE_BASED_MUTATION_BLOCK_ENABLED;
+import static org.apache.phoenix.query.QueryServicesOptions
+ .DEFAULT_CLUSTER_ROLE_BASED_MUTATION_BLOCK_ENABLED;
+/**
+ * Implementation of HAGroupStoreManager that uses HAGroupStoreClient.
+ * Manages all HAGroupStoreClient instances.
+ */
public class HAGroupStoreManager {
private static volatile HAGroupStoreManager haGroupStoreManagerInstance;
+ private static final Logger LOGGER = LoggerFactory.getLogger(HAGroupStoreManager.class);
private final boolean mutationBlockEnabled;
+ private final String zkUrl;
private final Configuration conf;
/**
@@ -33,7 +51,7 @@ public class HAGroupStoreManager {
* @param conf configuration
* @return HAGroupStoreManager instance
*/
- public static HAGroupStoreManager getInstance(Configuration conf) {
+ public static HAGroupStoreManager getInstance(final Configuration conf) {
if (haGroupStoreManagerInstance == null) {
synchronized (HAGroupStoreManager.class) {
if (haGroupStoreManagerInstance == null) {
@@ -47,18 +65,37 @@ public class HAGroupStoreManager {
private HAGroupStoreManager(final Configuration conf) {
this.mutationBlockEnabled = conf.getBoolean(CLUSTER_ROLE_BASED_MUTATION_BLOCK_ENABLED,
DEFAULT_CLUSTER_ROLE_BASED_MUTATION_BLOCK_ENABLED);
+ this.zkUrl = getLocalZkUrl(conf);
this.conf = conf;
}
/**
- * Checks whether mutation is blocked or not.
+ * Returns the list of all HA group names.
+ * @return list of all HA group names.
+ * @throws SQLException if there is an error with querying the table.
+ */
+ public List<String> getHAGroupNames() throws SQLException {
+ return HAGroupStoreClient.getHAGroupNames(this.zkUrl);
+ }
+
+
+ /**
+ * Checks whether mutation is blocked or not for a specific HA group.
+ *
+ * @param haGroupName name of the HA group, null for default HA group which tracks
+ * all HA groups.
+ * @return true if mutation is blocked, false otherwise.
* @throws IOException when HAGroupStoreClient is not healthy.
*/
- public boolean isMutationBlocked() throws IOException {
+ public boolean isMutationBlocked(String haGroupName) throws IOException, SQLException {
if (mutationBlockEnabled) {
- HAGroupStoreClient haGroupStoreClient = HAGroupStoreClient.getInstance(conf);
+ HAGroupStoreClient haGroupStoreClient = HAGroupStoreClient.getInstanceForZkUrl(conf,
+ haGroupName, zkUrl);
if (haGroupStoreClient != null) {
- return !haGroupStoreClient.getCRRsByClusterRole(ClusterRoleRecord.ClusterRole.ACTIVE_TO_STANDBY).isEmpty();
+ return haGroupStoreClient.getHAGroupStoreRecord() != null
+ && haGroupStoreClient.getHAGroupStoreRecord().getClusterRole() != null
+ && haGroupStoreClient.getHAGroupStoreRecord().getClusterRole()
+ .isMutationBlocked();
}
throw new IOException("HAGroupStoreClient is not initialized");
}
@@ -66,15 +103,123 @@ public class HAGroupStoreManager {
}
/**
- * Force rebuilds the HAGroupStoreClient
- * @throws Exception
+ * Force rebuilds the HAGroupStoreClient instance for all HA groups.
+ * If any HAGroupStoreClient instance is not created, it will be created.
+ * @param broadcastUpdate if true, the update will be broadcasted to all
+ * regionserver endpoints.
+ * @throws Exception in case of an error with dependencies or table.
*/
- public void invalidateHAGroupStoreClient() throws Exception {
- HAGroupStoreClient haGroupStoreClient = HAGroupStoreClient.getInstance(conf);
+ public void invalidateHAGroupStoreClient(boolean broadcastUpdate) throws Exception {
+ List<String> haGroupNames = HAGroupStoreClient.getHAGroupNames(this.zkUrl);
+ List<String> failedHAGroupNames = new ArrayList<>();
+ for (String haGroupName : haGroupNames) {
+ try {
+ invalidateHAGroupStoreClient(haGroupName, broadcastUpdate);
+ } catch (Exception e) {
+ failedHAGroupNames.add(haGroupName);
+ LOGGER.error("Failed to invalidate HAGroupStoreClient for " + haGroupName, e);
+ }
+ }
+ if (!failedHAGroupNames.isEmpty()) {
+ throw new IOException("Failed to invalidate HAGroupStoreClient for " +
+ failedHAGroupNames
+ + "successfully invalidated HAGroupStoreClient instance for "
+ + haGroupNames.removeAll(failedHAGroupNames) + " HA groups");
+ }
+ }
+
+ /**
+ * Force rebuilds the HAGroupStoreClient for a specific HA group.
+ *
+ * @param haGroupName name of the HA group, null for default HA group and tracks all HA groups.
+ * @param broadcastUpdate if true, the update will be broadcasted to all
+ * regionserver endpoints.
+ * @throws Exception in case of an error with dependencies or table.
+ */
+ public void invalidateHAGroupStoreClient(final String haGroupName,
+ boolean broadcastUpdate) throws Exception {
+ HAGroupStoreClient haGroupStoreClient = HAGroupStoreClient.getInstanceForZkUrl(conf,
+ haGroupName, zkUrl);
if (haGroupStoreClient != null) {
haGroupStoreClient.rebuild();
} else {
throw new IOException("HAGroupStoreClient is not initialized");
}
}
-}
+
+ /**
+ * Returns the HAGroupStoreRecord for a specific HA group.
+ *
+ * @param haGroupName name of the HA group
+ * @return Optional HAGroupStoreRecord for the HA group, can be empty if the HA group
+ * is not found.
+ * @throws IOException when HAGroupStoreClient is not healthy.
+ */
+ public Optional<HAGroupStoreRecord> getHAGroupStoreRecord(final String haGroupName)
+ throws IOException, SQLException {
+ HAGroupStoreClient haGroupStoreClient = HAGroupStoreClient.getInstanceForZkUrl(conf,
+ haGroupName, zkUrl);
+ if (haGroupStoreClient != null) {
+ return Optional.ofNullable(haGroupStoreClient.getHAGroupStoreRecord());
+ }
+ throw new IOException("HAGroupStoreClient is not initialized");
+ }
+
+ /**
+ * Sets the HAGroupStoreRecord to StoreAndForward mode in local cluster.
+ *
+ * @param haGroupName name of the HA group
+ * @throws IOException when HAGroupStoreClient is not healthy.
+ */
+ public void setHAGroupStatusToStoreAndForward(final String haGroupName)
+ throws IOException, StaleHAGroupStoreRecordVersionException,
+ InvalidClusterRoleTransitionException, SQLException {
+ HAGroupStoreClient haGroupStoreClient = HAGroupStoreClient.getInstanceForZkUrl(conf,
+ haGroupName, zkUrl);
+ if (haGroupStoreClient != null) {
+ haGroupStoreClient.setHAGroupStatusIfNeeded(
+ HAGroupStoreRecord.HAGroupState.ACTIVE_NOT_IN_SYNC);
+ } else {
+ throw new IOException("HAGroupStoreClient is not initialized");
+ }
+ }
+
+ /**
+ * Sets the HAGroupStoreRecord to Sync mode in local cluster.
+ *
+ * @param haGroupName name of the HA group
+ * @throws IOException when HAGroupStoreClient is not healthy.
+ */
+ public void setHAGroupStatusRecordToSync(final String haGroupName)
+ throws IOException, StaleHAGroupStoreRecordVersionException,
+ InvalidClusterRoleTransitionException, SQLException {
+ HAGroupStoreClient haGroupStoreClient = HAGroupStoreClient.getInstanceForZkUrl(conf,
+ haGroupName, zkUrl);
+ if (haGroupStoreClient != null) {
+ haGroupStoreClient.setHAGroupStatusIfNeeded(
+ HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+ } else {
+ throw new IOException("HAGroupStoreClient is not initialized");
+ }
+ }
+
+
+ /**
+ * Returns the ClusterRoleRecord for the cluster pair.
+ * If the peer cluster is not connected or peer cluster is not configured, it will
+ * return UNKNOWN for peer cluster.
+ * Only implemented by HAGroupStoreManagerImpl.
+ *
+ * @return ClusterRoleRecord for the cluster pair
+ * @throws IOException when HAGroupStoreClient is not healthy.
+ */
+ public ClusterRoleRecord getClusterRoleRecord(String haGroupName)
+ throws IOException, SQLException {
+ HAGroupStoreClient haGroupStoreClient = HAGroupStoreClient.getInstanceForZkUrl(conf,
+ haGroupName, zkUrl);
+ if (haGroupStoreClient != null) {
+ return haGroupStoreClient.getClusterRoleRecord();
+ }
+ throw new IOException("HAGroupStoreClient is not initialized");
+ }
+}
\ No newline at end of file
diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HAGroupStoreRecord.java b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HAGroupStoreRecord.java
new file mode 100644
index 0000000000..aa49a22eb2
--- /dev/null
+++ b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HAGroupStoreRecord.java
@@ -0,0 +1,264 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.phoenix.jdbc;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import org.apache.commons.lang3.builder.EqualsBuilder;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
+import org.apache.phoenix.thirdparty.com.google.common.base.Preconditions;
+import org.apache.phoenix.thirdparty.com.google.common.collect.ImmutableSet;
+import org.apache.phoenix.util.JacksonUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+
+/**
+ * Immutable class representing an HA group store record with simplified fields.
+ * This is a simplified version of ClusterRoleRecord that contains essential
+ * information about an HA group only for a single cluster.
+ */
+public class HAGroupStoreRecord {
+
+ private static final Logger LOG = LoggerFactory.getLogger(HAGroupStoreRecord.class);
+ public static final String DEFAULT_PROTOCOL_VERSION = "1.0";
+
+ /**
+ * Enum representing the HA group state with each state having a corresponding ClusterRole.
+ */
+ public enum HAGroupState {
+ ABORT_TO_ACTIVE_IN_SYNC,
+ ABORT_TO_ACTIVE_NOT_IN_SYNC,
+ ABORT_TO_STANDBY,
+ ACTIVE_IN_SYNC,
+ ACTIVE_NOT_IN_SYNC,
+ ACTIVE_NOT_IN_SYNC_TO_STANDBY,
+ ACTIVE_NOT_IN_SYNC_WITH_OFFLINE_PEER,
+ ACTIVE_IN_SYNC_TO_STANDBY,
+ ACTIVE_WITH_OFFLINE_PEER,
+ DEGRADED_STANDBY,
+ DEGRADED_STANDBY_FOR_READER,
+ DEGRADED_STANDBY_FOR_WRITER,
+ OFFLINE,
+ STANDBY,
+ STANDBY_TO_ACTIVE,
+ UNKNOWN;
+
+ private Set<HAGroupState> allowedTransitions;
+
+ /**
+ * Gets the corresponding ClusterRole for this HAGroupState.
+ * @return the corresponding ClusterRole
+ */
+ public ClusterRoleRecord.ClusterRole getClusterRole() {
+ switch (this) {
+ case ABORT_TO_ACTIVE_IN_SYNC:
+ case ABORT_TO_ACTIVE_NOT_IN_SYNC:
+ case ACTIVE_IN_SYNC:
+ case ACTIVE_NOT_IN_SYNC:
+ case ACTIVE_NOT_IN_SYNC_WITH_OFFLINE_PEER:
+ case ACTIVE_WITH_OFFLINE_PEER:
+ return ClusterRoleRecord.ClusterRole.ACTIVE;
+ case ACTIVE_IN_SYNC_TO_STANDBY:
+ case ACTIVE_NOT_IN_SYNC_TO_STANDBY:
+ return ClusterRoleRecord.ClusterRole.ACTIVE_TO_STANDBY;
+ case ABORT_TO_STANDBY:
+ case DEGRADED_STANDBY:
+ case DEGRADED_STANDBY_FOR_READER:
+ case DEGRADED_STANDBY_FOR_WRITER:
+ case STANDBY:
+ return ClusterRoleRecord.ClusterRole.STANDBY;
+ case STANDBY_TO_ACTIVE:
+ return ClusterRoleRecord.ClusterRole.STANDBY_TO_ACTIVE;
+ case OFFLINE:
+ return ClusterRoleRecord.ClusterRole.OFFLINE;
+ case UNKNOWN:
+ default:
+ return ClusterRoleRecord.ClusterRole.UNKNOWN;
+ }
+ }
+
+ static {
+ // Initialize allowed transitions
+ ACTIVE_NOT_IN_SYNC.allowedTransitions = ImmutableSet.of(
+ ACTIVE_NOT_IN_SYNC, ACTIVE_IN_SYNC,
+ ACTIVE_NOT_IN_SYNC_TO_STANDBY, ACTIVE_NOT_IN_SYNC_WITH_OFFLINE_PEER
+ );
+
+ ACTIVE_IN_SYNC.allowedTransitions = ImmutableSet.of(
+ ACTIVE_NOT_IN_SYNC, ACTIVE_WITH_OFFLINE_PEER, ACTIVE_IN_SYNC_TO_STANDBY
+ );
+
+ STANDBY.allowedTransitions = ImmutableSet.of(STANDBY_TO_ACTIVE,
+ DEGRADED_STANDBY_FOR_READER, DEGRADED_STANDBY_FOR_WRITER);
+ // This needs to be manually recovered by operator
+ OFFLINE.allowedTransitions = ImmutableSet.of();
+ // This needs to be manually recovered by operator
+ UNKNOWN.allowedTransitions = ImmutableSet.of();
+ ACTIVE_NOT_IN_SYNC_TO_STANDBY.allowedTransitions = ImmutableSet.of(ABORT_TO_ACTIVE_NOT_IN_SYNC,
+ ACTIVE_IN_SYNC_TO_STANDBY);
+ ACTIVE_IN_SYNC_TO_STANDBY.allowedTransitions = ImmutableSet.of(ABORT_TO_ACTIVE_IN_SYNC, STANDBY);
+ STANDBY_TO_ACTIVE.allowedTransitions = ImmutableSet.of(ABORT_TO_STANDBY,
+ ACTIVE_IN_SYNC);
+ DEGRADED_STANDBY.allowedTransitions
+ = ImmutableSet.of(DEGRADED_STANDBY_FOR_READER, DEGRADED_STANDBY_FOR_WRITER);
+ DEGRADED_STANDBY_FOR_WRITER.allowedTransitions = ImmutableSet.of(STANDBY,
+ DEGRADED_STANDBY);
+ DEGRADED_STANDBY_FOR_READER.allowedTransitions = ImmutableSet.of(STANDBY,
+ DEGRADED_STANDBY);
+ ACTIVE_WITH_OFFLINE_PEER.allowedTransitions = ImmutableSet.of(ACTIVE_NOT_IN_SYNC);
+ ABORT_TO_ACTIVE_IN_SYNC.allowedTransitions = ImmutableSet.of(ACTIVE_IN_SYNC);
+ ABORT_TO_ACTIVE_NOT_IN_SYNC.allowedTransitions = ImmutableSet.of(ACTIVE_NOT_IN_SYNC);
+ ABORT_TO_STANDBY.allowedTransitions = ImmutableSet.of(STANDBY);
+ ACTIVE_NOT_IN_SYNC_WITH_OFFLINE_PEER.allowedTransitions =
+ ImmutableSet.of(ACTIVE_NOT_IN_SYNC);
+ }
+
+ /**
+ * Checks if the transition from this state to the target state is allowed.
+ * @param targetState the state to transition to
+ * @return true if the transition is allowed, false otherwise
+ */
+ public boolean isTransitionAllowed(HAGroupState targetState) {
+ return allowedTransitions.contains(targetState);
+ }
+
+ public static HAGroupState from(byte[] bytes) {
+ if (bytes == null) {
+ return UNKNOWN;
+ }
+ String value = new String(bytes, StandardCharsets.UTF_8);
+ return Arrays.stream(HAGroupState.values())
+ .filter(r -> r.name().equalsIgnoreCase(value))
+ .findFirst()
+ .orElse(UNKNOWN);
+ }
+ }
+
+ private final String protocolVersion;
+ private final String haGroupName;
+ private final HAGroupState haGroupState;
+
+ @JsonCreator
+ public HAGroupStoreRecord(@JsonProperty("protocolVersion") String protocolVersion,
+ @JsonProperty("haGroupName") String haGroupName,
+ @JsonProperty("haGroupState") HAGroupState haGroupState) {
+ Preconditions.checkNotNull(haGroupName, "HA group name cannot be null!");
+ Preconditions.checkNotNull(haGroupState, "HA group state cannot be null!");
+
+ this.protocolVersion = Objects.toString(protocolVersion, DEFAULT_PROTOCOL_VERSION);
+ this.haGroupName = haGroupName;
+ this.haGroupState = haGroupState;
+ }
+
+ public static Optional<HAGroupStoreRecord> fromJson(byte[] bytes) {
+ if (bytes == null) {
+ return Optional.empty();
+ }
+ try {
+ return Optional.of(JacksonUtil.getObjectReader(HAGroupStoreRecord.class)
+ .readValue(bytes));
+ } catch (Exception e) {
+ LOG.error("Fail to deserialize data to an HA group store record", e);
+ return Optional.empty();
+ }
+ }
+
+ public static byte[] toJson(HAGroupStoreRecord record) throws IOException {
+ return JacksonUtil.getObjectWriter().withoutAttribute("clusterRole")
+ .writeValueAsBytes(record);
+ }
+
+ public boolean hasSameInfo(HAGroupStoreRecord other) {
+ return haGroupName.equals(other.haGroupName) &&
+ haGroupState.equals(other.haGroupState) &&
+ protocolVersion.equals(other.protocolVersion);
+ }
+
+ public String getProtocolVersion() {
+ return protocolVersion;
+ }
+
+ public String getHaGroupName() {
+ return haGroupName;
+ }
+
+ @JsonProperty("haGroupState")
+ public HAGroupState getHAGroupState() {
+ return haGroupState;
+ }
+
+ @JsonIgnore
+ public ClusterRoleRecord.ClusterRole getClusterRole() {
+ return haGroupState.getClusterRole();
+ }
+
+ @Override
+ public int hashCode() {
+ return new HashCodeBuilder()
+ .append(protocolVersion)
+ .append(haGroupName)
+ .append(haGroupState)
+ .hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == this) {
+ return true;
+ } else if (other == null) {
+ return false;
+ } else if (!(other instanceof HAGroupStoreRecord)) {
+ return false;
+ } else {
+ HAGroupStoreRecord otherRecord = (HAGroupStoreRecord) other;
+ return new EqualsBuilder()
+ .append(protocolVersion, otherRecord.protocolVersion)
+ .append(haGroupName, otherRecord.haGroupName)
+ .append(haGroupState, otherRecord.haGroupState)
+ .isEquals();
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "HAGroupStoreRecord{"
+ + "protocolVersion='" + protocolVersion + '\''
+ + ", haGroupName='" + haGroupName + '\''
+ + ", haGroupState=" + haGroupState
+ + '}';
+ }
+
+ public String toPrettyString() {
+ try {
+ return JacksonUtil.getObjectWriterPretty().writeValueAsString(this);
+ } catch (Exception e) {
+ LOG.error("Fail to wrap this object as JSON, returning the oneliner "
+ + "using toString", e);
+ return toString();
+ }
+ }
+}
\ No newline at end of file
diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HighAvailabilityGroup.java b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HighAvailabilityGroup.java
index 9b704973c4..3d743315e8 100644
--- a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HighAvailabilityGroup.java
+++ b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HighAvailabilityGroup.java
@@ -148,7 +148,7 @@ public class HighAvailabilityGroup {
.expireAfterWrite(PHOENIX_HA_TRANSITION_TIMEOUT_MS_DEFAULT, TimeUnit.MILLISECONDS)
.build();
/**
- * The Curator client cache, one client instance per cluster.
+ * The Curator client cache, one client instance per cluster and namespace combination.
*/
@VisibleForTesting
static final Cache<String, CuratorFramework> CURATOR_CACHE = CacheBuilder.newBuilder()
@@ -354,8 +354,8 @@ public class HighAvailabilityGroup {
GROUPS.remove(info);
haGroup.close();
try {
- CuratorFramework curator1 = CURATOR_CACHE.getIfPresent(info.getUrl1());
- CuratorFramework curator2 = CURATOR_CACHE.getIfPresent(info.getUrl2());
+ CuratorFramework curator1 = CURATOR_CACHE.getIfPresent(generateCacheKey(info.getUrl1(), null));
+ CuratorFramework curator2 = CURATOR_CACHE.getIfPresent(generateCacheKey(info.getUrl2(), null));
if (curator1 != null && curator2 != null) {
Stat node1 = curator1.checkExists().forPath(info.getZkPath());
Stat node2 = curator2.checkExists().forPath(info.getZkPath());
@@ -423,6 +423,19 @@ public class HighAvailabilityGroup {
return Optional.of(fallbackCluster);
}
+ /**
+ * Generate cache key for curator cache based on URL and namespace.
+ *
+ * @param jdbcUrl the ZK endpoint host:port or the JDBC connection String host:port:/hbase
+ * @param namespace the ZooKeeper namespace, uses default if null
+ * @return cache key string
+ */
+ @VisibleForTesting
+ static String generateCacheKey(String jdbcUrl, String namespace) {
+ String effectiveNamespace = namespace != null ? namespace : PHOENIX_HA_ZOOKEEPER_ZNODE_NAMESPACE;
+ return jdbcUrl + ":" + effectiveNamespace;
+ }
+
/**
* Get an active curator ZK client for the given properties and ZK endpoint.
* <p>
@@ -435,9 +448,28 @@ public class HighAvailabilityGroup {
@SuppressWarnings("UnstableApiUsage")
public static CuratorFramework getCurator(String jdbcUrl, Properties properties)
throws IOException {
+ return getCurator(jdbcUrl, properties, PHOENIX_HA_ZOOKEEPER_ZNODE_NAMESPACE);
+ }
+
+ /**
+ * Get an active curator ZK client for the given properties, ZK endpoint and namespace.
+ * <p>
+ * This can be from cached object since Curator should be shared per cluster.
+ *
+ * @param jdbcUrl the ZK endpoint host:port or the JDBC connection String host:port:/hbase
+ * @param properties the properties defining time out values and retry count
+ * @param namespace the ZooKeeper namespace to use, defaults to PHOENIX_HA_ZOOKEEPER_ZNODE_NAMESPACE if null
+ * @return a new Curator framework client
+ */
+ @SuppressWarnings("UnstableApiUsage")
+ public static CuratorFramework getCurator(String jdbcUrl, Properties properties, String namespace)
+ throws IOException {
+ // Use namespace as part of cache key to avoid conflicts between different namespaces
+ String effectiveNamespace = namespace != null ? namespace : PHOENIX_HA_ZOOKEEPER_ZNODE_NAMESPACE;
+ String cacheKey = generateCacheKey(jdbcUrl, namespace);
try {
- return CURATOR_CACHE.get(jdbcUrl, () -> {
- CuratorFramework curator = createCurator(jdbcUrl, properties);
+ return CURATOR_CACHE.get(cacheKey, () -> {
+ CuratorFramework curator = createCurator(jdbcUrl, properties, effectiveNamespace);
try {
if (!curator.blockUntilConnected(PHOENIX_HA_ZK_CONNECTION_TIMEOUT_MS_DEFAULT,
TimeUnit.MILLISECONDS)) {
@@ -458,17 +490,22 @@ public class HighAvailabilityGroup {
} catch (Exception e) {
LOG.error("Fail to get an active curator for url {}", jdbcUrl, e);
// invalidate the cache when getting/creating throws exception
- CURATOR_CACHE.invalidate(jdbcUrl);
+ CURATOR_CACHE.invalidate(cacheKey);
throw new IOException(e);
}
}
/**
- * Create a curator ZK client for the given properties and ZK endpoint.
+ * Create a curator ZK client for the given properties, ZK endpoint and namespace.
* <p>
* Unless caller needs a new curator, it should use {@link #getCurator(String, Properties)}.
+ *
+ * @param jdbcUrl the ZK endpoint host:port or the JDBC connection String host:port:/hbase
+ * @param properties the properties defining time out values and retry count
+ * @param namespace the ZooKeeper namespace to use, defaults to PHOENIX_HA_ZOOKEEPER_ZNODE_NAMESPACE if null
+ * @return a new Curator framework client
*/
- private static CuratorFramework createCurator(String jdbcUrl, Properties properties) {
+ private static CuratorFramework createCurator(String jdbcUrl, Properties properties, String namespace) {
// Get the ZK endpoint in host:port format by removing JDBC protocol and HBase root node
final String zkUrl;
if (jdbcUrl.startsWith(PhoenixRuntime.JDBC_PROTOCOL)) {
@@ -498,7 +535,7 @@ public class HighAvailabilityGroup {
CuratorFramework curator = CuratorFrameworkFactory
.builder()
.connectString(zkUrl)
- .namespace(PHOENIX_HA_ZOOKEEPER_ZNODE_NAMESPACE)
+ .namespace(namespace != null ? namespace : PHOENIX_HA_ZOOKEEPER_ZNODE_NAMESPACE)
.connectionTimeoutMs(connectionTimeoutMs)
.sessionTimeoutMs(sessionTimeoutMs)
.retryPolicy(retryPolicy)
@@ -1073,4 +1110,4 @@ public class HighAvailabilityGroup {
}
}
-}
+}
\ No newline at end of file
diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/PhoenixConnection.java b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/PhoenixConnection.java
index e2830f19a8..53da422dba 100644
--- a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/PhoenixConnection.java
+++ b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/PhoenixConnection.java
@@ -189,6 +189,7 @@ public class PhoenixConnection implements MetaDataMutated, SQLCloseable, Phoenix
private LogLevel auditLogLevel;
private Double logSamplingRate;
private String sourceOfOperation;
+ private String haGroupName;
private volatile SQLException reasonForClose;
private static final String[] CONNECTION_PROPERTIES;
@@ -1252,6 +1253,18 @@ public class PhoenixConnection implements MetaDataMutated, SQLCloseable, Phoenix
this.consistency = val;
}
+ /**
+ * This is temporary method to set HAGroupName for the connection.
+ * This will be removed once we have a proper way to set HAGroupName in the connection.
+ */
+ public void setHAGroupName(String haGroupName) {
+ this.haGroupName = haGroupName;
+ }
+
+ public String getHAGroupName() {
+ return this.haGroupName;
+ }
+
@Override
public void setCatalog(String catalog) throws SQLException {
checkOpen();
diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
index 3625dd006e..c40f4521e1 100644
--- a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
+++ b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
@@ -458,6 +458,13 @@ public class PhoenixDatabaseMetaData implements DatabaseMetaData {
public static final String SYSTEM_CDC_STREAM_TABLE = "CDC_STREAM";
public static final String SYSTEM_CDC_STREAM_NAME =
SchemaUtil.getTableName(SYSTEM_CATALOG_SCHEMA, SYSTEM_CDC_STREAM_TABLE);
+
+ public static final String SYSTEM_HA_GROUP_TABLE = "HA_GROUP";
+ public static final String SYSTEM_HA_GROUP_NAME =
+ SchemaUtil.getTableName(SYSTEM_CATALOG_SCHEMA, SYSTEM_HA_GROUP_TABLE);
+ public static final byte[] SYSTEM_HA_GROUP_NAME_BYTES = Bytes.toBytes(SYSTEM_HA_GROUP_NAME);
+ public static final TableName SYSTEM_HA_GROUP_HBASE_TABLE_NAME = TableName.valueOf(SYSTEM_HA_GROUP_NAME);
+
public static final String STREAM_NAME = "STREAM_NAME";
public static final String STREAM_STATUS = "STREAM_STATUS";
public static final String STREAM_TYPE = "STREAM_TYPE";
@@ -468,6 +475,15 @@ public class PhoenixDatabaseMetaData implements DatabaseMetaData {
public static final String PARTITION_START_KEY = "PARTITION_START_KEY";
public static final String PARTITION_END_KEY = "PARTITION_END_KEY";
public static final String PARENT_PARTITION_START_TIME = "PARENT_PARTITION_START_TIME";
+ public static final String HA_GROUP_NAME = "HA_GROUP_NAME";
+ public static final String POLICY = "POLICY";
+ public static final String VERSION = "VERSION";
+ public static final String ZK_URL_1 = "ZK_URL_1";
+ public static final String ZK_URL_2 = "ZK_URL_2";
+ public static final String CLUSTER_URL_1 = "CLUSTER_URL_1";
+ public static final String CLUSTER_URL_2 = "CLUSTER_URL_2";
+ public static final String CLUSTER_ROLE_1 = "CLUSTER_ROLE_1";
+ public static final String CLUSTER_ROLE_2 = "CLUSTER_ROLE_2";
public static final String QUERY_ID = "QUERY_ID";
public static final String USER = "USER";
diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/PhoenixHAAdmin.java b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/PhoenixHAAdmin.java
index 0ef78372e0..6735288734 100644
--- a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/PhoenixHAAdmin.java
+++ b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/PhoenixHAAdmin.java
@@ -18,6 +18,7 @@
package org.apache.phoenix.jdbc;
import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.tuple.Pair;
import org.apache.curator.RetryPolicy;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.recipes.atomic.AtomicValue;
@@ -26,10 +27,12 @@ import org.apache.curator.utils.ZKPaths;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.util.PairOfSameType;
+import org.apache.phoenix.exception.StaleHAGroupStoreRecordVersionException;
import org.apache.phoenix.thirdparty.com.google.common.base.Preconditions;
import org.apache.phoenix.util.JDBCUtil;
import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.data.Stat;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -40,9 +43,12 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
import java.util.Optional;
import java.util.Properties;
+import static org.apache.phoenix.jdbc.HighAvailabilityGroup.PHOENIX_HA_ZOOKEEPER_ZNODE_NAMESPACE;
+
/**
* Helper class to update cluster role record for a ZK cluster.
* The ZK client this accessor has is confined to a single ZK cluster, but it can be used to operate
@@ -63,6 +69,13 @@ public class PhoenixHAAdmin implements Closeable {
public CuratorFramework getCurator(String zkUrl, Properties properties) throws IOException {
return HighAvailabilityGroup.getCurator(zkUrl, properties);
}
+
+ /**
+ * Gets curator with specific namespace blocking if necessary to create it
+ */
+ public CuratorFramework getCurator(String zkUrl, Properties properties, String namespace) throws IOException {
+ return HighAvailabilityGroup.getCurator(zkUrl, properties, namespace);
+ }
}
private static final Logger LOG = LoggerFactory.getLogger(PhoenixHAAdmin.class);
@@ -76,17 +89,29 @@ public class PhoenixHAAdmin implements Closeable {
/** Curator Provider **/
private final HighAvailibilityCuratorProvider
highAvailibilityCuratorProvider;
+ /** Namespace for Curator **/
+ private final String namespace;
public PhoenixHAAdmin(Configuration conf) {
- this(getLocalZkUrl(conf), conf, HighAvailibilityCuratorProvider.INSTANCE);
+ this(getLocalZkUrl(conf), conf, HighAvailibilityCuratorProvider.INSTANCE, null);
+ }
+
+ public PhoenixHAAdmin(Configuration conf, String namespace) {
+ this(getLocalZkUrl(conf), conf, HighAvailibilityCuratorProvider.INSTANCE, namespace);
}
- public PhoenixHAAdmin(String zkUrl, Configuration conf) {
- this(zkUrl, conf, HighAvailibilityCuratorProvider.INSTANCE);
+ public PhoenixHAAdmin(String zkUrl, Configuration conf, String namespace) {
+ this(zkUrl, conf, HighAvailibilityCuratorProvider.INSTANCE, namespace);
}
public PhoenixHAAdmin(String zkUrl, Configuration conf,
- HighAvailibilityCuratorProvider highAvailibilityCuratorProvider) {
+ HighAvailibilityCuratorProvider highAvailibilityCuratorProvider) {
+ this(zkUrl, conf, highAvailibilityCuratorProvider, null);
+ }
+
+ public PhoenixHAAdmin(String zkUrl, Configuration conf,
+ HighAvailibilityCuratorProvider highAvailibilityCuratorProvider,
+ String namespace) {
Preconditions.checkNotNull(zkUrl);
Preconditions.checkNotNull(conf);
Preconditions.checkNotNull(highAvailibilityCuratorProvider);
@@ -94,6 +119,7 @@ public class PhoenixHAAdmin implements Closeable {
this.conf = conf;
conf.iterator().forEachRemaining(k -> properties.setProperty(k.getKey(), k.getValue()));
this.highAvailibilityCuratorProvider = highAvailibilityCuratorProvider;
+ this.namespace = Objects.toString(namespace, PHOENIX_HA_ZOOKEEPER_ZNODE_NAMESPACE);
}
/**
@@ -129,7 +155,7 @@ public class PhoenixHAAdmin implements Closeable {
* Gets curator from the cache if available otherwise calls into getCurator to make it.
*/
public CuratorFramework getCurator() throws IOException {
- return highAvailibilityCuratorProvider.getCurator(zkUrl, properties);
+ return highAvailibilityCuratorProvider.getCurator(zkUrl, properties, namespace);
}
@@ -160,20 +186,22 @@ public class PhoenixHAAdmin implements Closeable {
}
}
- /**
- * This lists all cluster role records stored in the zookeeper nodes.
- * This read-only operation and hence no side effect on the ZK cluster.
- */
- public List<ClusterRoleRecord> listAllClusterRoleRecordsOnZookeeper() throws IOException {
- List<String> haGroupNames;
+ public List<String> getHAGroupNames() throws IOException {
try {
- haGroupNames = getCurator().getChildren().forPath(ZKPaths.PATH_SEPARATOR);
+ return getCurator().getChildren().forPath(ZKPaths.PATH_SEPARATOR);
} catch (Exception e) {
String msg = String.format("Got exception when listing all HA groups in %s", zkUrl);
LOG.error(msg);
throw new IOException(msg, e);
}
+ }
+ /**
+ * This lists all cluster role records stored in the zookeeper nodes.
+ * This read-only operation and hence no side effect on the ZK cluster.
+ */
+ public List<ClusterRoleRecord> listAllClusterRoleRecordsOnZookeeper() throws IOException {
+ List<String> haGroupNames = getHAGroupNames();
List<ClusterRoleRecord> records = new ArrayList<>();
List<String> failedHaGroups = new ArrayList<>();
for (String haGroupName : haGroupNames) {
@@ -327,6 +355,15 @@ public class PhoenixHAAdmin implements Closeable {
return inconsistentHaGroups;
}
+ public void deleteHAGroupStoreRecordInZooKeeper(String haGroupName) throws IOException {
+ try {
+ getCurator().delete().quietly().deletingChildrenIfNeeded().forPath(toPath(haGroupName));
+ } catch (Exception e) {
+ LOG.error("Failed to delete HAGroupStoreRecord for HA group {}", haGroupName, e);
+ throw new IOException("Failed to delete HAGroupStoreRecord for HA group " + haGroupName, e);
+ }
+ }
+
/**
* This updates the cluster role data on the zookeeper it connects to.
* To avoid conflicts, it does CAS (compare-and-set) when updating. The constraint is that the
@@ -460,6 +497,72 @@ public class PhoenixHAAdmin implements Closeable {
}
}
+ /**
+ * Creates a new HAGroupStoreRecord in ZooKeeper.
+ *
+ * @param newHAGroupStoreRecord the new record to create
+ * @throws IOException if any error occurs during the creation
+ */
+ public void createHAGroupStoreRecordInZooKeeper(HAGroupStoreRecord newHAGroupStoreRecord)
+ throws IOException {
+ try {
+ getCurator()
+ .create()
+ .creatingParentsIfNeeded()
+ .forPath(toPath(newHAGroupStoreRecord.getHaGroupName()), HAGroupStoreRecord.toJson(newHAGroupStoreRecord));
+ } catch (Exception e) {
+ throw new IOException("Failed to create HAGroupStoreRecord for HA group " + newHAGroupStoreRecord.getHaGroupName(), e);
+ }
+ }
+
+ /**
+ * Updates the HAGroupStoreRecord in ZooKeeper with version checking.
+ *
+ * @param haGroupName the HA group name
+ * @param newHAGroupStoreRecord the new record to store
+ * @param currentStatVersion the expected stat version for optimistic locking
+ * @throws StaleHAGroupStoreRecordVersionException if the version is stale
+ * @throws IOException if any other error occurs during the update
+ */
+ public void updateHAGroupStoreRecordInZooKeeper(String haGroupName, HAGroupStoreRecord newHAGroupStoreRecord, int currentStatVersion)
+ throws IOException, StaleHAGroupStoreRecordVersionException {
+ try {
+ getCurator()
+ .setData()
+ .withVersion(currentStatVersion)
+ .forPath(toPath(haGroupName), HAGroupStoreRecord.toJson(newHAGroupStoreRecord));
+ } catch (KeeperException.BadVersionException e) {
+ LOG.error("Failed to set HAGroupStoreRecord for HA group {}, stale stat version", haGroupName, e);
+ throw new StaleHAGroupStoreRecordVersionException("Failed to set HAGroupStoreRecord for HA group "
+ + haGroupName + " with cached stat version " + currentStatVersion, e);
+ } catch (Exception e) {
+ LOG.error("Failed to set HAGroupStoreRecord for HA group {}, unexpected error", haGroupName, e);
+ throw new IOException("Failed to set HAGroupStoreRecord for HA group " + haGroupName, e);
+ }
+ }
+
+ /**
+ * Gets the HAGroupStoreRecord and Stat from ZooKeeper.
+ *
+ * @param haGroupName the HA group name
+ * @return a pair of HAGroupStoreRecord and Stat
+ * @throws IOException if any error occurs during the retrieval
+ */
+ public Pair<HAGroupStoreRecord, Stat> getHAGroupStoreRecordInZooKeeper(String haGroupName) throws IOException {
+ try {
+ byte[] data = getCurator().getData().forPath(toPath(haGroupName));
+ HAGroupStoreRecord record = HAGroupStoreRecord.fromJson(data).orElse(null);
+ Stat stat = getCurator().checkExists().forPath(toPath(haGroupName));
+ return Pair.of(record, stat);
+ } catch (KeeperException.NoNodeException nne) {
+ LOG.warn("No HAGroupStoreRecord for HA group {} in ZK", haGroupName, nne);
+ return Pair.of(null, null);
+ } catch (Exception e) {
+ LOG.error("Failed to get HAGroupStoreRecord for HA group {}", haGroupName, e);
+ throw new IOException("Failed to get HAGroupStoreRecord for HA group " + haGroupName, e);
+ }
+ }
+
/**
* Helper method to get ZK path for an HA group given the HA group name.
* It assumes the ZK namespace (prefix) has been set.
diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java b/phoenix-core-client/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
index b2c7f3b4e3..d2efaf1460 100644
--- a/phoenix-core-client/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
+++ b/phoenix-core-client/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
@@ -1698,7 +1698,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
private void pollForUpdatedTableDescriptor(final Admin admin, final TableDescriptor newTableDescriptor,
final byte[] tableName) throws InterruptedException, TimeoutException {
- checkAndRetry(new RetriableOperation() {
+ checkAndRetry (new RetriableOperation() {
@Override
public String getOperationName() {
@@ -1713,7 +1713,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
});
}
- private void checkAndRetry(RetriableOperation op) throws InterruptedException, TimeoutException {
+ private void checkAndRetry (RetriableOperation op) throws InterruptedException, TimeoutException {
int maxRetries = ConnectionQueryServicesImpl.this.props.getInt(
QueryServices.NUM_RETRIES_FOR_SCHEMA_UPDATE_CHECK,
QueryServicesOptions.DEFAULT_RETRIES_FOR_SCHEMA_UPDATE_CHECK);
@@ -3875,6 +3875,10 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
return setSystemDDLProperties(QueryConstants.CREATE_CDC_STREAM_METADATA);
}
+ protected String getHAGroupDDL() {
+ return setSystemDDLProperties(QueryConstants.CREATE_HA_GROUP_METADATA);
+ }
+
private String setSystemDDLProperties(String ddl) {
return String.format(ddl,
props.getInt(DEFAULT_SYSTEM_MAX_VERSIONS_ATTRIB, QueryServicesOptions.DEFAULT_SYSTEM_MAX_VERSIONS),
@@ -4140,8 +4144,8 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
}
private void createOtherSystemTables(PhoenixConnection metaConnection) throws SQLException, IOException {
- try {
- metaConnection.createStatement().execute(getSystemSequenceTableDDL(nSequenceSaltBuckets));
+ try (Statement stmt = metaConnection.createStatement()) {
+ stmt.execute(getSystemSequenceTableDDL(nSequenceSaltBuckets));
// When creating the table above, DDL statements are
// used. However, the CFD level properties are not set
// via DDL commands, hence we are explicitly setting
@@ -4150,33 +4154,46 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
} catch (TableAlreadyExistsException e) {
nSequenceSaltBuckets = getSaltBuckets(e);
}
- try {
- metaConnection.createStatement().execute(QueryConstants.CREATE_STATS_TABLE_METADATA);
- } catch (TableAlreadyExistsException ignore) {}
- try {
- metaConnection.createStatement().execute(getFunctionTableDDL());
- } catch (TableAlreadyExistsException ignore) {}
- try {
- metaConnection.createStatement().execute(getLogTableDDL());
- } catch (TableAlreadyExistsException ignore) {}
- try {
- metaConnection.createStatement().executeUpdate(getChildLinkDDL());
- } catch (TableAlreadyExistsException ignore) {}
- try {
- metaConnection.createStatement().executeUpdate(getMutexDDL());
- } catch (TableAlreadyExistsException ignore) {}
- try {
- metaConnection.createStatement().executeUpdate(getTaskDDL());
- } catch (TableAlreadyExistsException ignore) {}
- try {
- metaConnection.createStatement().executeUpdate(getTransformDDL());
- } catch (TableAlreadyExistsException ignore) {}
- try {
- metaConnection.createStatement().executeUpdate(getCDCStreamStatusDDL());
- } catch (TableAlreadyExistsException ignore) {}
- try {
- metaConnection.createStatement().executeUpdate(getCDCStreamDDL());
- } catch (TableAlreadyExistsException ignore) {}
+ try (Statement stmt = metaConnection.createStatement()) {
+ stmt.execute(QueryConstants.CREATE_STATS_TABLE_METADATA);
+ } catch (TableAlreadyExistsException ignore) {
+ }
+ try (Statement stmt = metaConnection.createStatement()) {
+ stmt.execute(getFunctionTableDDL());
+ } catch (TableAlreadyExistsException ignore) {
+ }
+ try (Statement stmt = metaConnection.createStatement()) {
+ stmt.execute(getLogTableDDL());
+ } catch (TableAlreadyExistsException ignore) {
+ }
+ try (Statement stmt = metaConnection.createStatement()) {
+ stmt.execute(getChildLinkDDL());
+ } catch (TableAlreadyExistsException ignore) {
+ }
+ try (Statement stmt = metaConnection.createStatement()) {
+ stmt.executeUpdate(getMutexDDL());
+ } catch (TableAlreadyExistsException ignore) {
+ }
+ try (Statement stmt = metaConnection.createStatement()) {
+ stmt.executeUpdate(getTaskDDL());
+ } catch (TableAlreadyExistsException ignore) {
+ }
+ try (Statement stmt = metaConnection.createStatement()) {
+ stmt.executeUpdate(getTransformDDL());
+ } catch (TableAlreadyExistsException ignore) {
+ }
+ try (Statement stmt = metaConnection.createStatement()) {
+ stmt.executeUpdate(getCDCStreamStatusDDL());
+ } catch (TableAlreadyExistsException ignore) {
+ }
+ try (Statement stmt = metaConnection.createStatement()) {
+ stmt.executeUpdate(getCDCStreamDDL());
+ } catch (TableAlreadyExistsException ignore) {
+ }
+ try (Statement stmt = metaConnection.createStatement()) {
+ stmt.executeUpdate(getHAGroupDDL());
+ } catch (TableAlreadyExistsException ignore) {
+ }
}
/**
@@ -4811,7 +4828,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
metaConnection = upgradeSystemMutex(metaConnection);
metaConnection = upgradeSystemCDCStreamStatus(metaConnection);
metaConnection = upgradeSystemCDCStream(metaConnection);
-
+ metaConnection = upgradeSystemHAGroup(metaConnection);
// As this is where the most time will be spent during an upgrade,
// especially when there are large number of views.
// Upgrade the SYSTEM.CHILD_LINK towards the end,
@@ -5164,6 +5181,15 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
return metaConnection;
}
+ private PhoenixConnection upgradeSystemHAGroup(PhoenixConnection metaConnection)
+ throws SQLException {
+ try (Statement stmt = metaConnection.createStatement()) {
+ stmt.executeUpdate(getHAGroupDDL());
+ } catch (TableAlreadyExistsException ignored) {
+ }
+ return metaConnection;
+ }
+
// Special method for adding the column qualifier column for 4.10.
private PhoenixConnection addColumnQualifierColumn(PhoenixConnection oldMetaConnection, Long timestamp) throws SQLException {
diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java b/phoenix-core-client/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
index e2de008e73..cfce561b0a 100644
--- a/phoenix-core-client/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
+++ b/phoenix-core-client/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
@@ -21,6 +21,7 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.INDEX_STATE_BYTES;
import static org.apache.phoenix.schema.PTableImpl.getColumnsToClone;
import java.sql.SQLException;
+import java.sql.Statement;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
@@ -216,6 +217,10 @@ public class ConnectionlessQueryServicesImpl extends DelegateQueryServices imple
return setSystemDDLProperties(QueryConstants.CREATE_CDC_STREAM_METADATA);
}
+ protected String getHAGroupDDL() {
+ return setSystemDDLProperties(QueryConstants.CREATE_HA_GROUP_METADATA);
+ }
+
private String setSystemDDLProperties(String ddl) {
return String.format(ddl,
props.getInt(DEFAULT_SYSTEM_MAX_VERSIONS_ATTRIB, QueryServicesOptions.DEFAULT_SYSTEM_MAX_VERSIONS),
@@ -430,22 +435,22 @@ public class ConnectionlessQueryServicesImpl extends DelegateQueryServices imple
String globalUrl = JDBCUtil.removeProperty(url, PhoenixRuntime.TENANT_ID_ATTRIB);
metaConnection = new PhoenixConnection(this, globalUrl, scnProps);
metaConnection.setRunningUpgrade(true);
- try {
- metaConnection.createStatement().executeUpdate(getSystemCatalogTableDDL());
+ try (Statement stmt = metaConnection.createStatement()) {
+ stmt.executeUpdate(getSystemCatalogTableDDL());
} catch (TableAlreadyExistsException ignore) {
// Ignore, as this will happen if the SYSTEM.TABLE already exists at this fixed timestamp.
// A TableAlreadyExistsException is not thrown, since the table only exists *after* this fixed timestamp.
}
- try {
+ try (Statement stmt = metaConnection.createStatement()) {
int nSaltBuckets = getSequenceSaltBuckets();
String createTableStatement = getSystemSequenceTableDDL(nSaltBuckets);
- metaConnection.createStatement().executeUpdate(createTableStatement);
+ stmt.executeUpdate(createTableStatement);
} catch (NewerTableAlreadyExistsException ignore) {
// Ignore, as this will happen if the SYSTEM.SEQUENCE already exists at this fixed timestamp.
// A TableAlreadyExistsException is not thrown, since the table only exists *after* this fixed timestamp.
}
- try {
- metaConnection.createStatement().executeUpdate(QueryConstants.CREATE_STATS_TABLE_METADATA);
+ try (Statement stmt = metaConnection.createStatement()) {
+ stmt.executeUpdate(QueryConstants.CREATE_STATS_TABLE_METADATA);
} catch (NewerTableAlreadyExistsException ignore) {
// Ignore, as this will happen if the SYSTEM.SEQUENCE already exists at this fixed
// timestamp.
@@ -453,39 +458,41 @@ public class ConnectionlessQueryServicesImpl extends DelegateQueryServices imple
// fixed timestamp.
}
- try {
- metaConnection.createStatement().executeUpdate(getFunctionTableDDL());
+ try (Statement stmt = metaConnection.createStatement()) {
+ stmt.executeUpdate(getFunctionTableDDL());
} catch (NewerTableAlreadyExistsException ignore) {
}
- try {
- metaConnection.createStatement().executeUpdate(getLogTableDDL());
+ try (Statement stmt = metaConnection.createStatement()) {
+ stmt.executeUpdate(getLogTableDDL());
} catch (NewerTableAlreadyExistsException ignore) {}
- try {
- metaConnection.createStatement()
- .executeUpdate(getChildLinkDDL());
+ try (Statement stmt = metaConnection.createStatement()) {
+ stmt.executeUpdate(getChildLinkDDL());
} catch (NewerTableAlreadyExistsException ignore) {
}
- try {
- metaConnection.createStatement()
- .executeUpdate(getMutexDDL());
+ try (Statement stmt = metaConnection.createStatement()) {
+ stmt.executeUpdate(getMutexDDL());
} catch (NewerTableAlreadyExistsException ignore) {
}
- try {
- metaConnection.createStatement()
- .executeUpdate(getTaskDDL());
+ try (Statement stmt = metaConnection.createStatement()) {
+ stmt.executeUpdate(getTaskDDL());
} catch (NewerTableAlreadyExistsException ignore) {
}
- try {
- metaConnection.createStatement()
- .executeUpdate(getTransformDDL());
+ try (Statement stmt = metaConnection.createStatement()) {
+ stmt.executeUpdate(getTransformDDL());
} catch (NewerTableAlreadyExistsException ignore) {
}
- try {
- metaConnection.createStatement().executeUpdate(getCDCStreamStatusDDL());
- } catch (TableAlreadyExistsException ignore) {}
- try {
- metaConnection.createStatement().executeUpdate(getCDCStreamDDL());
- } catch (TableAlreadyExistsException ignore) {}
+ try (Statement stmt = metaConnection.createStatement()) {
+ stmt.executeUpdate(getCDCStreamStatusDDL());
+ } catch (TableAlreadyExistsException ignore) {
+ }
+ try (Statement stmt = metaConnection.createStatement()) {
+ stmt.executeUpdate(getCDCStreamDDL());
+ } catch (TableAlreadyExistsException ignore) {
+ }
+ try (Statement stmt = metaConnection.createStatement()) {
+ stmt.executeUpdate(getHAGroupDDL());
+ } catch (TableAlreadyExistsException ignore) {
+ }
} catch (SQLException e) {
sqlE = e;
} finally {
diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/query/QueryConstants.java b/phoenix-core-client/src/main/java/org/apache/phoenix/query/QueryConstants.java
index 9911f5f0a3..f7011aabc0 100644
--- a/phoenix-core-client/src/main/java/org/apache/phoenix/query/QueryConstants.java
+++ b/phoenix-core-client/src/main/java/org/apache/phoenix/query/QueryConstants.java
@@ -48,6 +48,10 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.CHANGE_DETECTION_E
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.CHAR_OCTET_LENGTH;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.CLASS_NAME;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.CLIENT_IP;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.CLUSTER_ROLE_1;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.CLUSTER_ROLE_2;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.CLUSTER_URL_1;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.CLUSTER_URL_2;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_COUNT;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_DEF;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_FAMILY;
@@ -72,6 +76,7 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.GLOBAL_SCAN_DETAIL
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.GUIDE_POSTS_ROW_COUNT;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.GUIDE_POST_KEY;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.HA_GROUP_NAME;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.IMMUTABLE_ROWS;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.IMMUTABLE_STORAGE_SCHEME;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.INCREMENT_BY;
@@ -117,6 +122,7 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.PHOENIX_TTL_HWM;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.PHYSICAL_NAME;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.PHYSICAL_TABLE_NAME;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.PK_NAME;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.POLICY;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.QUERY;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.QUERY_ID;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.QUERY_STATUS;
@@ -150,6 +156,7 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_CDC_STREAM_
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_CDC_STREAM_TABLE;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_CHILD_LINK_TABLE;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_FUNCTION_TABLE;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_HA_GROUP_TABLE;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_LOG_TABLE;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_MUTEX_TABLE_NAME;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_STATS_TABLE;
@@ -185,11 +192,14 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TYPE_SEQUENCE;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.UPDATE_CACHE_FREQUENCY;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.USER;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.USE_STATS_FOR_PARALLELIZATION;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VERSION;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_CONSTANT;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_INDEX_ID;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_INDEX_ID_DATA_TYPE;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_STATEMENT;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_TYPE;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.ZK_URL_1;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.ZK_URL_2;
/**
*
@@ -685,4 +695,24 @@ public interface QueryConstants {
ColumnFamilyDescriptorBuilder.KEEP_DELETED_CELLS + "=%s,\n" +
TRANSACTIONAL + "=" + Boolean.FALSE + ",\n" +
UPDATE_CACHE_FREQUENCY + "=" + "7200000";
+
+ String CREATE_HA_GROUP_METADATA = "CREATE TABLE " + SYSTEM_CATALOG_SCHEMA + ".\"" +
+ SYSTEM_HA_GROUP_TABLE + "\"(\n" +
+ // PK column
+ HA_GROUP_NAME + " VARCHAR NOT NULL," + // This is the unique identifier for the HA group
+ // Non-PK columns
+ ZK_URL_1 + " VARCHAR," + // This will be used to associate the ClusterRole and URL with local or peer cluster
+ ZK_URL_2 + " VARCHAR," + // This will be used to associate the ClusterRole and URL with local or peer cluster
+ CLUSTER_URL_1 + " VARCHAR," + // This will be returned to the client
+ CLUSTER_URL_2 + " VARCHAR," + // This will be returned to the client
+ CLUSTER_ROLE_1 + " VARCHAR," + // Role for peer cluster might not be accurate, we will use only local role for recovery if needed
+ CLUSTER_ROLE_2 + " VARCHAR," + // Role for peer cluster might not be accurate, we will use only local role for recovery if needed
+ POLICY + " VARCHAR," + // There should be only one policy for an HA group
+ VERSION + " BIGINT," + // Version should be incremented for Admin Updates, only for CLUSTER_URLs and REGISTRY_TYPE
+ "CONSTRAINT " + SYSTEM_TABLE_PK_NAME +
+ " PRIMARY KEY (" + HA_GROUP_NAME + "))\n" +
+ HConstants.VERSIONS + "=%s,\n" +
+ ColumnFamilyDescriptorBuilder.KEEP_DELETED_CELLS + "=%s,\n" +
+ TRANSACTIONAL + "=" + Boolean.FALSE + ",\n" +
+ UPDATE_CACHE_FREQUENCY + "=" + "7200000";
}
diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/query/QueryServices.java b/phoenix-core-client/src/main/java/org/apache/phoenix/query/QueryServices.java
index 4f6926144a..0ba347918c 100644
--- a/phoenix-core-client/src/main/java/org/apache/phoenix/query/QueryServices.java
+++ b/phoenix-core-client/src/main/java/org/apache/phoenix/query/QueryServices.java
@@ -140,7 +140,6 @@ public interface QueryServices extends SQLCloseable {
public static final String MAX_SERVER_METADATA_CACHE_TIME_TO_LIVE_MS_ATTRIB = "phoenix.coprocessor.maxMetaDataCacheTimeToLiveMs";
public static final String MAX_SERVER_METADATA_CACHE_SIZE_ATTRIB = "phoenix.coprocessor.maxMetaDataCacheSize";
public static final String MAX_CLIENT_METADATA_CACHE_SIZE_ATTRIB = "phoenix.client.maxMetaDataCacheSize";
- public static final String HA_GROUP_NAME_ATTRIB = "phoenix.ha.group";
public static final String AUTO_UPGRADE_WHITELIST_ATTRIB = "phoenix.client.autoUpgradeWhiteList";
// Mainly for testing to force spilling
public static final String MAX_MEMORY_SIZE_ATTRIB = "phoenix.query.maxGlobalMemorySize";
diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/util/WALAnnotationUtil.java b/phoenix-core-client/src/main/java/org/apache/phoenix/util/WALAnnotationUtil.java
index 69afe0eb72..3dfb9f996f 100644
--- a/phoenix-core-client/src/main/java/org/apache/phoenix/util/WALAnnotationUtil.java
+++ b/phoenix-core-client/src/main/java/org/apache/phoenix/util/WALAnnotationUtil.java
@@ -17,6 +17,7 @@ package org.apache.phoenix.util;
import org.apache.hadoop.hbase.client.Durability;
import org.apache.hadoop.hbase.client.Mutation;
+import org.apache.phoenix.coprocessorclient.BaseScannerRegionObserverConstants;
import org.apache.phoenix.execute.MutationState;
/**
@@ -42,7 +43,7 @@ public class WALAnnotationUtil {
}
public static void annotateMutation(Mutation m, byte[] tenantId, byte[] schemaName,
- byte[] logicalTableName, byte[] tableType, byte[] ddlTimestamp) {
+ byte[] logicalTableName, byte[] tableType, byte[] ddlTimestamp, byte[] haGroupName) {
if (!m.getDurability().equals(Durability.SKIP_WAL)) {
if (tenantId != null) {
m.setAttribute(MutationState.MutationMetadataType.TENANT_ID.toString(), tenantId);
@@ -52,6 +53,9 @@ public class WALAnnotationUtil {
logicalTableName);
m.setAttribute(MutationState.MutationMetadataType.TABLE_TYPE.toString(), tableType);
m.setAttribute(MutationState.MutationMetadataType.TIMESTAMP.toString(), ddlTimestamp);
+ if (haGroupName != null) {
+ m.setAttribute(BaseScannerRegionObserverConstants.HA_GROUP_NAME_ATTRIB, haGroupName);
+ }
}
}
diff --git a/phoenix-core-client/src/main/protobuf/RegionServerEndpointService.proto b/phoenix-core-client/src/main/protobuf/RegionServerEndpointService.proto
index 725f3fc763..3df27aebaf 100644
--- a/phoenix-core-client/src/main/protobuf/RegionServerEndpointService.proto
+++ b/phoenix-core-client/src/main/protobuf/RegionServerEndpointService.proto
@@ -51,11 +51,28 @@ message InvalidateServerMetadataCacheRequest {
}
message InvalidateHAGroupStoreClientRequest {
+ required bytes haGroupName = 1;
+ required bool broadcastUpdate = 2;
}
message InvalidateHAGroupStoreClientResponse {
}
+message GetClusterRoleRecordRequest {
+ required bytes haGroupName = 1;
+}
+
+message GetClusterRoleRecordResponse {
+ // In case of single cluster configuration, one of the urls and roles can be empty
+ required bytes haGroupName = 1;
+ required bytes policy = 2;
+ optional bytes url1 = 3;
+ optional bytes role1 = 4;
+ optional bytes url2 = 5;
+ optional bytes role2 = 6;
+ optional int64 version = 7;
+}
+
service RegionServerEndpointService {
rpc validateLastDDLTimestamp(ValidateLastDDLTimestampRequest)
returns (ValidateLastDDLTimestampResponse);
@@ -65,4 +82,7 @@ service RegionServerEndpointService {
rpc invalidateHAGroupStoreClient(InvalidateHAGroupStoreClientRequest)
returns (InvalidateHAGroupStoreClientResponse);
+
+ rpc getClusterRoleRecord(GetClusterRoleRecordRequest)
+ returns (GetClusterRoleRecordResponse);
}
\ No newline at end of file
diff --git a/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/PhoenixRegionServerEndpoint.java b/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/PhoenixRegionServerEndpoint.java
index 97f71d0862..ce314020eb 100644
--- a/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/PhoenixRegionServerEndpoint.java
+++ b/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/PhoenixRegionServerEndpoint.java
@@ -17,6 +17,7 @@
*/
package org.apache.phoenix.coprocessor;
+import com.google.protobuf.ByteString;
import com.google.protobuf.RpcCallback;
import com.google.protobuf.RpcController;
import com.google.protobuf.Service;
@@ -33,6 +34,7 @@ import org.apache.phoenix.cache.ServerMetadataCacheImpl;
import org.apache.phoenix.coprocessor.generated.RegionServerEndpointProtos;
import org.apache.phoenix.coprocessorclient.metrics.MetricsMetadataCachingSource;
import org.apache.phoenix.coprocessorclient.metrics.MetricsPhoenixCoprocessorSourceFactory;
+import org.apache.phoenix.jdbc.ClusterRoleRecord;
import org.apache.phoenix.jdbc.HAGroupStoreManager;
import org.apache.phoenix.protobuf.ProtobufUtil;
import org.apache.phoenix.util.ClientUtil;
@@ -41,6 +43,8 @@ import org.apache.phoenix.util.ServerUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static org.apache.phoenix.jdbc.PhoenixHAAdmin.getLocalZkUrl;
+
/**
* This is first implementation of RegionServer coprocessor introduced by Phoenix.
*/
@@ -50,12 +54,14 @@ public class PhoenixRegionServerEndpoint
private static final Logger LOGGER = LoggerFactory.getLogger(PhoenixRegionServerEndpoint.class);
private MetricsMetadataCachingSource metricsSource;
protected Configuration conf;
+ private String zkUrl;
@Override
public void start(CoprocessorEnvironment env) throws IOException {
this.conf = env.getConfiguration();
this.metricsSource = MetricsPhoenixCoprocessorSourceFactory
.getInstance().getMetadataCachingSource();
+ this.zkUrl = getLocalZkUrl(conf);
}
@Override
@@ -119,10 +125,17 @@ public class PhoenixRegionServerEndpoint
RegionServerEndpointProtos.InvalidateHAGroupStoreClientRequest request,
RpcCallback<RegionServerEndpointProtos.InvalidateHAGroupStoreClientResponse> done) {
LOGGER.info("PhoenixRegionServerEndpoint invalidating HAGroupStoreClient");
- HAGroupStoreManager haGroupStoreManager;
try {
- haGroupStoreManager = HAGroupStoreManager.getInstance(conf);
- haGroupStoreManager.invalidateHAGroupStoreClient();
+ HAGroupStoreManager haGroupStoreManager
+ = HAGroupStoreManager.getInstance(conf);
+ if (haGroupStoreManager != null) {
+ haGroupStoreManager
+ .invalidateHAGroupStoreClient(request.getHaGroupName().toStringUtf8(),
+ request.getBroadcastUpdate());
+ } else {
+ throw new IOException("HAGroupStoreManager is null for "
+ + "current cluster, check configuration");
+ }
} catch (Throwable t) {
String errorMsg = "Invalidating HAGroupStoreClient FAILED, check exception for "
+ "specific details";
@@ -132,6 +145,43 @@ public class PhoenixRegionServerEndpoint
}
}
+ @Override
+ public void getClusterRoleRecord(RpcController controller,
+ RegionServerEndpointProtos.GetClusterRoleRecordRequest request,
+ RpcCallback<RegionServerEndpointProtos.GetClusterRoleRecordResponse> done) {
+ try {
+ HAGroupStoreManager haGroupStoreManager
+ = HAGroupStoreManager.getInstance(conf);
+ if (haGroupStoreManager != null) {
+ ClusterRoleRecord clusterRoleRecord = haGroupStoreManager
+ .getClusterRoleRecord(request.getHaGroupName().toStringUtf8());
+ RegionServerEndpointProtos.GetClusterRoleRecordResponse.Builder responseBuilder
+ = RegionServerEndpointProtos.GetClusterRoleRecordResponse.newBuilder();
+ responseBuilder.setHaGroupName(request.getHaGroupName());
+ responseBuilder
+ .setPolicy(ByteString.copyFromUtf8(clusterRoleRecord.getPolicy().name()));
+ responseBuilder.setUrl1(ByteString.copyFromUtf8(clusterRoleRecord.getUrl1()));
+ responseBuilder
+ .setRole1(ByteString.copyFromUtf8(clusterRoleRecord.getRole1().name()));
+ responseBuilder
+ .setUrl2(ByteString.copyFromUtf8(clusterRoleRecord.getUrl2()));
+ responseBuilder.
+ setRole2(ByteString.copyFromUtf8(clusterRoleRecord.getRole2().name()));
+ responseBuilder.setVersion(clusterRoleRecord.getVersion());
+ done.run(responseBuilder.build());
+ } else {
+ throw new IOException("HAGroupStoreManager is null for "
+ + "current cluster, check configuration");
+ }
+ } catch (Throwable t) {
+ String errorMsg = "Getting ClusterRoleRecord FAILED, check exception for "
+ + "specific details";
+ LOGGER.error(errorMsg, t);
+ IOException ioe = ClientUtil.createIOException(errorMsg, t);
+ ProtobufUtil.setControllerException(controller, ioe);
+ }
+ }
+
@Override
public Iterable<Service> getServices() {
return Collections.singletonList(this);
@@ -141,4 +191,4 @@ public class PhoenixRegionServerEndpoint
return ServerMetadataCacheImpl.getInstance(conf);
}
-}
\ No newline at end of file
+}
diff --git a/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java b/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
index cf8797ecfe..4b0ba455a1 100644
--- a/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
+++ b/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
@@ -61,7 +61,6 @@ import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.ipc.RpcControllerFactory;
import org.apache.hadoop.hbase.ipc.controller.InterRegionServerIndexRpcControllerFactory;
import org.apache.hadoop.hbase.regionserver.InternalScanner;
-import org.apache.hadoop.hbase.regionserver.MiniBatchOperationInProgress;
import org.apache.hadoop.hbase.regionserver.Region;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
import org.apache.hadoop.hbase.regionserver.ScanType;
@@ -89,7 +88,6 @@ import org.apache.phoenix.index.PhoenixIndexCodec;
import org.apache.phoenix.index.PhoenixIndexFailurePolicyHelper;
import org.apache.phoenix.index.PhoenixIndexFailurePolicyHelper.MutateCommand;
import org.apache.phoenix.index.PhoenixIndexMetaData;
-import org.apache.phoenix.jdbc.HAGroupStoreManager;
import org.apache.phoenix.jdbc.PhoenixConnection;
import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
import org.apache.phoenix.join.HashJoinInfo;
@@ -1055,20 +1053,4 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver
protected boolean isRegionObserverFor(Scan scan) {
return scan.getAttribute(BaseScannerRegionObserverConstants.UNGROUPED_AGG) != null;
}
-
- @Override
- public void preBatchMutate(ObserverContext<RegionCoprocessorEnvironment> c,
- MiniBatchOperationInProgress<Mutation> miniBatchOp)
- throws IOException {
- final Configuration conf = c.getEnvironment().getConfiguration();
- try {
- final HAGroupStoreManager haGroupStoreManager = HAGroupStoreManager.getInstance(conf);
- if (haGroupStoreManager.isMutationBlocked()) {
- throw new IOException("Blocking Mutation as Some CRRs are in ACTIVE_TO_STANDBY "
- + "state and CLUSTER_ROLE_BASED_MUTATION_BLOCK_ENABLED is true");
- }
- } catch (Exception e) {
- throw new IOException(e);
- }
- }
}
diff --git a/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionScanner.java b/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionScanner.java
index 32a837c080..951e541893 100644
--- a/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionScanner.java
+++ b/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionScanner.java
@@ -793,9 +793,12 @@ public class UngroupedAggregateRegionScanner extends BaseRegionScanner {
scan.getAttribute(MutationState.MutationMetadataType.TABLE_TYPE.toString());
byte[] ddlTimestamp =
scan.getAttribute(MutationState.MutationMetadataType.TIMESTAMP.toString());
+ byte[] haGroupName =
+ scan.getAttribute(BaseScannerRegionObserverConstants.HA_GROUP_NAME_ATTRIB);
for (Mutation m : mutationsList) {
- annotateMutation(m, tenantId, schemaName, logicalTableName, tableType, ddlTimestamp);
+ annotateMutation(m, tenantId, schemaName, logicalTableName,
+ tableType, ddlTimestamp, haGroupName);
}
}
diff --git a/phoenix-core-server/src/main/java/org/apache/phoenix/hbase/index/IndexRegionObserver.java b/phoenix-core-server/src/main/java/org/apache/phoenix/hbase/index/IndexRegionObserver.java
index 7ade7ca48e..1aeb7787ce 100644
--- a/phoenix-core-server/src/main/java/org/apache/phoenix/hbase/index/IndexRegionObserver.java
+++ b/phoenix-core-server/src/main/java/org/apache/phoenix/hbase/index/IndexRegionObserver.java
@@ -17,6 +17,7 @@
*/
package org.apache.phoenix.hbase.index;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellComparator;
@@ -59,6 +60,7 @@ import org.apache.phoenix.coprocessor.DelegateRegionCoprocessorEnvironment;
import org.apache.phoenix.coprocessor.generated.PTableProtos;
import org.apache.phoenix.coprocessorclient.BaseScannerRegionObserverConstants;
import org.apache.phoenix.exception.DataExceedsCapacityException;
+import org.apache.phoenix.exception.MutationBlockedIOException;
import org.apache.phoenix.execute.MutationState;
import org.apache.phoenix.expression.CaseExpression;
import org.apache.phoenix.expression.Expression;
@@ -126,6 +128,7 @@ import java.io.ByteArrayInputStream;
import java.io.DataInputStream;
import java.io.EOFException;
import java.io.IOException;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
@@ -621,10 +624,22 @@ public class IndexRegionObserver implements RegionCoprocessor, RegionObserver {
}
try {
final Configuration conf = c.getEnvironment().getConfiguration();
- final HAGroupStoreManager haGroupStoreManager = HAGroupStoreManager.getInstance(conf);
- if (haGroupStoreManager.isMutationBlocked()) {
- throw new IOException("Blocking Mutation as Some CRRs are in ACTIVE_TO_STANDBY "
- + "state and CLUSTER_ROLE_BASED_MUTATION_BLOCK_ENABLED is true");
+ final HAGroupStoreManager haGroupStoreManager
+ = HAGroupStoreManager.getInstance(conf);
+ if (haGroupStoreManager == null) {
+ throw new IOException("HAGroupStoreManager is null "
+ + "for current cluster, check configuration");
+ }
+ // Extract HAGroupName from the mutations
+ final Set<String> haGroupNames = extractHAGroupNameAttribute(miniBatchOp);
+ // Check if mutation is blocked for any of the HAGroupNames
+ for (String haGroupName : haGroupNames) {
+ if (StringUtils.isNotBlank(haGroupName)
+ && haGroupStoreManager.isMutationBlocked(haGroupName)) {
+ throw new MutationBlockedIOException("Blocking Mutation as Some CRRs are in "
+ + "ACTIVE_TO_STANDBY state and "
+ + "CLUSTER_ROLE_BASED_MUTATION_BLOCK_ENABLED is true");
+ }
}
preBatchMutateWithExceptions(c, miniBatchOp);
return;
@@ -635,6 +650,20 @@ public class IndexRegionObserver implements RegionCoprocessor, RegionObserver {
"Somehow didn't return an index update but also didn't propagate the failure to the client!");
}
+ private Set<String> extractHAGroupNameAttribute(
+ MiniBatchOperationInProgress<Mutation> miniBatchOp) {
+ Set<String> haGroupNames = new HashSet<>();
+ for (int i = 0; i < miniBatchOp.size(); i++) {
+ Mutation m = miniBatchOp.getOperation(i);
+ byte[] haGroupName = m.getAttribute(
+ BaseScannerRegionObserverConstants.HA_GROUP_NAME_ATTRIB);
+ if (haGroupName != null) {
+ haGroupNames.add(new String(haGroupName, StandardCharsets.UTF_8));
+ }
+ }
+ return haGroupNames;
+ }
+
@Override
public void preWALRestore(
org.apache.hadoop.hbase.coprocessor.ObserverContext<? extends RegionCoprocessorEnvironment> ctx,
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/MigrateSystemTablesToSystemNamespaceIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/MigrateSystemTablesToSystemNamespaceIT.java
index 6aaba958ff..31fb30ee40 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/MigrateSystemTablesToSystemNamespaceIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/MigrateSystemTablesToSystemNamespaceIT.java
@@ -67,11 +67,11 @@ public class MigrateSystemTablesToSystemNamespaceIT extends BaseTest {
private static final Set<String> PHOENIX_SYSTEM_TABLES = new HashSet<>(Arrays.asList(
"SYSTEM.CATALOG", "SYSTEM.SEQUENCE", "SYSTEM.STATS", "SYSTEM.FUNCTION",
"SYSTEM.MUTEX","SYSTEM.LOG", "SYSTEM.CHILD_LINK", "SYSTEM.TASK", "SYSTEM.TRANSFORM",
- "SYSTEM.CDC_STREAM_STATUS", "SYSTEM.CDC_STREAM"));
+ "SYSTEM.CDC_STREAM_STATUS", "SYSTEM.CDC_STREAM", "SYSTEM.HA_GROUP"));
private static final Set<String> PHOENIX_NAMESPACE_MAPPED_SYSTEM_TABLES = new HashSet<>(
Arrays.asList("SYSTEM:CATALOG", "SYSTEM:SEQUENCE", "SYSTEM:STATS", "SYSTEM:FUNCTION",
"SYSTEM:MUTEX","SYSTEM:LOG", "SYSTEM:CHILD_LINK", "SYSTEM:TASK", "SYSTEM:TRANSFORM",
- "SYSTEM:CDC_STREAM_STATUS", "SYSTEM:CDC_STREAM"));
+ "SYSTEM:CDC_STREAM_STATUS", "SYSTEM:CDC_STREAM", "SYSTEM:HA_GROUP"));
private static final String SCHEMA_NAME = "MIGRATETEST";
private static final String TABLE_NAME =
SCHEMA_NAME + "." + MigrateSystemTablesToSystemNamespaceIT.class.getSimpleName().toUpperCase();
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/PhoenixRegionServerEndpointIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/PhoenixRegionServerEndpointIT.java
index 670aba7f15..7fe8dd48c7 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/PhoenixRegionServerEndpointIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/PhoenixRegionServerEndpointIT.java
@@ -157,18 +157,6 @@ public class PhoenixRegionServerEndpointIT extends BaseTest {
}
}
- @Test
- public void testInvalidateHAGroupStoreClient() {
- HRegionServer regionServer = utility.getMiniHBaseCluster().getRegionServer(0);
- PhoenixRegionServerEndpoint coprocessor = getPhoenixRegionServerEndpoint(regionServer);
- assertNotNull(coprocessor);
- ServerRpcController controller = new ServerRpcController();
- RegionServerEndpointProtos.InvalidateHAGroupStoreClientRequest request
- = RegionServerEndpointProtos.InvalidateHAGroupStoreClientRequest.newBuilder().build();
- coprocessor.invalidateHAGroupStoreClient(controller, request, null);
- assertFalse(controller.failed());
- }
-
private String getCreateTableStmt(String tableName) {
return "CREATE TABLE " + tableName +
" (a_string varchar not null, col1 integer" +
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/PhoenixRegionServerEndpointITWithConsistentFailover.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/PhoenixRegionServerEndpointITWithConsistentFailover.java
new file mode 100644
index 0000000000..9aa6173fd1
--- /dev/null
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/PhoenixRegionServerEndpointITWithConsistentFailover.java
@@ -0,0 +1,158 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.end2end;
+
+import com.google.protobuf.RpcCallback;
+import org.apache.hadoop.hbase.ipc.ServerRpcController;
+import org.apache.hadoop.hbase.regionserver.HRegionServer;
+import org.apache.hadoop.hbase.util.ByteStringer;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.phoenix.coprocessor.PhoenixRegionServerEndpoint;
+import org.apache.phoenix.coprocessor.generated.RegionServerEndpointProtos;
+import org.apache.phoenix.jdbc.ClusterRoleRecord;
+import org.apache.phoenix.jdbc.HAGroupStoreRecord;
+import org.apache.phoenix.jdbc.HighAvailabilityPolicy;
+import org.apache.phoenix.jdbc.HighAvailabilityTestingUtility;
+import org.apache.phoenix.jdbc.PhoenixHAAdmin;
+import org.apache.phoenix.jdbc.HAGroupStoreRecord.HAGroupState;
+import org.apache.phoenix.query.BaseTest;
+import org.apache.phoenix.thirdparty.com.google.common.collect.Maps;
+import org.apache.phoenix.util.HAGroupStoreTestUtil;
+import org.apache.phoenix.util.ReadOnlyProps;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.rules.TestName;
+
+import java.util.Map;
+
+import static org.apache.phoenix.jdbc.HAGroupStoreClient.ZK_CONSISTENT_HA_NAMESPACE;
+import static org.apache.phoenix.jdbc.PhoenixHAAdmin.getLocalZkUrl;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+
+@Category({NeedsOwnMiniClusterTest.class })
+public class PhoenixRegionServerEndpointITWithConsistentFailover extends BaseTest {
+
+ private static final Long ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS = 5000L;
+ private static final HighAvailabilityTestingUtility.HBaseTestingUtilityPair CLUSTERS = new HighAvailabilityTestingUtility.HBaseTestingUtilityPair();
+ private String zkUrl;
+ private String peerZkUrl;
+
+ @Rule
+ public TestName testName = new TestName();
+
+ @BeforeClass
+ public static synchronized void doSetup() throws Exception {
+ Map<String, String> props = Maps.newHashMapWithExpectedSize(1);
+ setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator()));
+ CLUSTERS.start();
+ }
+
+ @Before
+ public void setUp() throws Exception {
+ zkUrl = getLocalZkUrl(config);
+ peerZkUrl = CLUSTERS.getZkUrl2();
+ HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(testName.getMethodName(), zkUrl, peerZkUrl,
+ ClusterRoleRecord.ClusterRole.ACTIVE, ClusterRoleRecord.ClusterRole.STANDBY, null);
+ }
+
+ @Test
+ public void testGetClusterRoleRecordAndInvalidate() throws Exception {
+ String haGroupName = testName.getMethodName();
+ HRegionServer regionServer = utility.getMiniHBaseCluster().getRegionServer(0);
+ PhoenixRegionServerEndpoint coprocessor = getPhoenixRegionServerEndpoint(regionServer);
+ assertNotNull(coprocessor);
+ ServerRpcController controller = new ServerRpcController();
+
+ try (PhoenixHAAdmin peerHAAdmin = new PhoenixHAAdmin(CLUSTERS.getHBaseCluster2().getConfiguration(), ZK_CONSISTENT_HA_NAMESPACE)) {
+ HAGroupStoreRecord peerHAGroupStoreRecord = new HAGroupStoreRecord(HAGroupStoreRecord.DEFAULT_PROTOCOL_VERSION, haGroupName, HAGroupState.STANDBY);
+ peerHAAdmin.createHAGroupStoreRecordInZooKeeper(peerHAGroupStoreRecord);
+ }
+ Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
+
+ // First getClusterRoleRecord to check if HAGroupStoreClient is working as expected
+ ClusterRoleRecord expectedRecord = buildExpectedClusterRoleRecord(haGroupName, ClusterRoleRecord.ClusterRole.ACTIVE, ClusterRoleRecord.ClusterRole.STANDBY);
+ executeGetClusterRoleRecordAndVerify(coprocessor, controller, haGroupName, expectedRecord);
+
+ // Change the role of local cluster to ACTIVE_TO_STANDBY in System Table
+ HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(testName.getMethodName(), zkUrl, peerZkUrl,
+ ClusterRoleRecord.ClusterRole.ACTIVE_TO_STANDBY, ClusterRoleRecord.ClusterRole.STANDBY, null);
+
+ // Cluster Role will still be same as before as cache is not invalidated yet
+ executeGetClusterRoleRecordAndVerify(coprocessor, controller, haGroupName, expectedRecord);
+
+ // Now Invalidate the Cache
+ coprocessor.invalidateHAGroupStoreClient(controller, getInvalidateHAGroupStoreClientRequest(haGroupName), null);
+ assertFalse(controller.failed());
+
+ // Local Cluster Role will be updated to ACTIVE_TO_STANDBY as cache is invalidated
+ ClusterRoleRecord expectedRecordAfterInvalidation = buildExpectedClusterRoleRecord(haGroupName, ClusterRoleRecord.ClusterRole.ACTIVE_TO_STANDBY, ClusterRoleRecord.ClusterRole.STANDBY);
+ executeGetClusterRoleRecordAndVerify(coprocessor, controller, haGroupName, expectedRecordAfterInvalidation);
+ }
+
+ private ClusterRoleRecord buildExpectedClusterRoleRecord(String haGroupName, ClusterRoleRecord.ClusterRole localRole, ClusterRoleRecord.ClusterRole peerRole) {
+ return new ClusterRoleRecord(haGroupName, HighAvailabilityPolicy.FAILOVER, zkUrl, localRole, peerZkUrl, peerRole, 1);
+ }
+
+ private void executeGetClusterRoleRecordAndVerify(PhoenixRegionServerEndpoint coprocessor, ServerRpcController controller,
+ String haGroupName, ClusterRoleRecord expectedRecord) {
+ RpcCallback<RegionServerEndpointProtos.GetClusterRoleRecordResponse> rpcCallback = createValidationCallback(haGroupName, expectedRecord);
+ coprocessor.getClusterRoleRecord(controller, getClusterRoleRecordRequest(haGroupName), rpcCallback);
+ assertFalse(controller.failed());
+ }
+
+ private RpcCallback<RegionServerEndpointProtos.GetClusterRoleRecordResponse> createValidationCallback(String haGroupName, ClusterRoleRecord expectedRecord) {
+ return response -> {
+ assertNotNull(response);
+ ClusterRoleRecord actual = new ClusterRoleRecord(haGroupName,
+ HighAvailabilityPolicy.valueOf(response.getPolicy().toStringUtf8()),
+ response.getUrl1().toStringUtf8(),
+ ClusterRoleRecord.ClusterRole.valueOf(response.getRole1().toStringUtf8()),
+ response.getUrl2().toStringUtf8(),
+ ClusterRoleRecord.ClusterRole.valueOf(response.getRole2().toStringUtf8()),
+ response.getVersion());
+ assertEquals(actual, expectedRecord);
+ };
+ }
+
+ private PhoenixRegionServerEndpoint getPhoenixRegionServerEndpoint(HRegionServer regionServer) {
+ PhoenixRegionServerEndpoint coproc = regionServer
+ .getRegionServerCoprocessorHost()
+ .findCoprocessor(PhoenixRegionServerEndpoint.class);
+ return coproc;
+ }
+
+ private RegionServerEndpointProtos.GetClusterRoleRecordRequest getClusterRoleRecordRequest(String haGroupName) {
+ RegionServerEndpointProtos.GetClusterRoleRecordRequest.Builder requestBuilder
+ = RegionServerEndpointProtos.GetClusterRoleRecordRequest.newBuilder();
+ requestBuilder.setHaGroupName(ByteStringer.wrap(Bytes.toBytes(haGroupName)));
+ return requestBuilder.build();
+ }
+
+ private RegionServerEndpointProtos.InvalidateHAGroupStoreClientRequest getInvalidateHAGroupStoreClientRequest(String haGroupName) {
+ RegionServerEndpointProtos.InvalidateHAGroupStoreClientRequest.Builder requestBuilder
+ = RegionServerEndpointProtos.InvalidateHAGroupStoreClientRequest.newBuilder();
+ requestBuilder.setBroadcastUpdate(false);
+ requestBuilder.setHaGroupName(ByteStringer.wrap(Bytes.toBytes(haGroupName)));
+ return requestBuilder.build();
+ }
+}
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryDatabaseMetaDataIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryDatabaseMetaDataIT.java
index 2549517fd0..fa6752733b 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryDatabaseMetaDataIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryDatabaseMetaDataIT.java
@@ -191,6 +191,10 @@ public class QueryDatabaseMetaDataIT extends ParallelStatsDisabledIT {
assertEquals(PTableType.SYSTEM.toString(), rs.getString("TABLE_TYPE"));
assertTrue(rs.next());
assertEquals(SYSTEM_CATALOG_SCHEMA, rs.getString("TABLE_SCHEM"));
+ assertEquals(PhoenixDatabaseMetaData.SYSTEM_HA_GROUP_TABLE, rs.getString("TABLE_NAME"));
+ assertEquals(PTableType.SYSTEM.toString(), rs.getString("TABLE_TYPE"));
+ assertTrue(rs.next());
+ assertEquals(SYSTEM_CATALOG_SCHEMA, rs.getString("TABLE_SCHEM"));
assertEquals(PhoenixDatabaseMetaData.SYSTEM_LOG_TABLE, rs.getString("TABLE_NAME"));
assertEquals(PTableType.SYSTEM.toString(), rs.getString("TABLE_TYPE"));
assertTrue(rs.next());
@@ -430,7 +434,7 @@ public class QueryDatabaseMetaDataIT extends ParallelStatsDisabledIT {
tables.add(rs.getString("TABLE_NAME"));
assertEquals("SYSTEM", rs.getString("TABLE_SCHEM"));
}
- assertEquals(11, tables.size());
+ assertEquals(12, tables.size());
assertTrue(tables.contains("CATALOG"));
assertTrue(tables.contains("FUNCTION"));
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SystemTablesCreationOnConnectionIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SystemTablesCreationOnConnectionIT.java
index 84074c349a..b997922640 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SystemTablesCreationOnConnectionIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SystemTablesCreationOnConnectionIT.java
@@ -112,12 +112,12 @@ public class SystemTablesCreationOnConnectionIT {
private static final Set<String> PHOENIX_SYSTEM_TABLES = new HashSet<>(Arrays.asList(
"SYSTEM.CATALOG", "SYSTEM.SEQUENCE", "SYSTEM.STATS", "SYSTEM.FUNCTION",
"SYSTEM.MUTEX", "SYSTEM.LOG", "SYSTEM.CHILD_LINK", "SYSTEM.TASK","SYSTEM.TRANSFORM",
- "SYSTEM.CDC_STREAM_STATUS", "SYSTEM.CDC_STREAM"));
+ "SYSTEM.CDC_STREAM_STATUS", "SYSTEM.CDC_STREAM", "SYSTEM.HA_GROUP"));
private static final Set<String> PHOENIX_NAMESPACE_MAPPED_SYSTEM_TABLES = new HashSet<>(
Arrays.asList("SYSTEM:CATALOG", "SYSTEM:SEQUENCE", "SYSTEM:STATS", "SYSTEM:FUNCTION",
"SYSTEM:MUTEX", "SYSTEM:LOG", "SYSTEM:CHILD_LINK", "SYSTEM:TASK", "SYSTEM:TRANSFORM",
- "SYSTEM:CDC_STREAM_STATUS", "SYSTEM:CDC_STREAM"));
+ "SYSTEM:CDC_STREAM_STATUS", "SYSTEM:CDC_STREAM", "SYSTEM:HA_GROUP"));
private static class PhoenixSysCatCreationServices extends ConnectionQueryServicesImpl {
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/TenantSpecificTablesDDLIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/TenantSpecificTablesDDLIT.java
index ab6e183103..8ae6e4cd74 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/TenantSpecificTablesDDLIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/TenantSpecificTablesDDLIT.java
@@ -561,6 +561,8 @@ public class TenantSpecificTablesDDLIT extends BaseTenantSpecificTablesIT {
assertTrue(rs.next());
assertTableMetaData(rs, SYSTEM_CATALOG_SCHEMA, SYSTEM_FUNCTION_TABLE, SYSTEM);
assertTrue(rs.next());
+ assertTableMetaData(rs, PhoenixDatabaseMetaData.SYSTEM_CATALOG_SCHEMA, PhoenixDatabaseMetaData.SYSTEM_HA_GROUP_TABLE, PTableType.SYSTEM);
+ assertTrue(rs.next());
assertTableMetaData(rs, PhoenixDatabaseMetaData.SYSTEM_CATALOG_SCHEMA, PhoenixDatabaseMetaData.SYSTEM_LOG_TABLE, PTableType.SYSTEM);
assertTrue(rs.next());
assertTableMetaData(rs, PhoenixDatabaseMetaData.SYSTEM_CATALOG_SCHEMA, PhoenixDatabaseMetaData.SYSTEM_MUTEX_TABLE_NAME, PTableType.SYSTEM);
@@ -695,4 +697,4 @@ public class TenantSpecificTablesDDLIT extends BaseTenantSpecificTablesIT {
assertColumnMetaData(rs, schema, table, column);
assertEquals(ordinalPosition, rs.getInt(ORDINAL_POSITION));
}
-}
\ No newline at end of file
+}
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexRegionObserverMutationBlockingIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexRegionObserverMutationBlockingIT.java
new file mode 100644
index 0000000000..fdf64dd2df
--- /dev/null
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexRegionObserverMutationBlockingIT.java
@@ -0,0 +1,217 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.end2end.index;
+
+import static org.apache.phoenix.jdbc.HAGroupStoreClient.ZK_CONSISTENT_HA_NAMESPACE;
+import static org.apache.phoenix.jdbc.PhoenixHAAdmin.getLocalZkUrl;
+import static org.apache.phoenix.query.QueryServices.CLUSTER_ROLE_BASED_MUTATION_BLOCK_ENABLED;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.sql.DriverManager;
+import java.util.Map;
+
+import org.apache.hadoop.hbase.client.RetriesExhaustedWithDetailsException;
+import org.apache.phoenix.end2end.NeedsOwnMiniClusterTest;
+import org.apache.phoenix.exception.MutationBlockedIOException;
+import org.apache.phoenix.execute.CommitException;
+import org.apache.phoenix.jdbc.ClusterRoleRecord;
+import org.apache.phoenix.jdbc.HAGroupStoreRecord;
+import org.apache.phoenix.jdbc.HighAvailabilityTestingUtility;
+import org.apache.phoenix.jdbc.PhoenixConnection;
+import org.apache.phoenix.jdbc.PhoenixHAAdmin;
+import org.apache.phoenix.query.BaseTest;
+import org.apache.phoenix.thirdparty.com.google.common.collect.Maps;
+import org.apache.phoenix.util.HAGroupStoreTestUtil;
+import org.apache.phoenix.util.ReadOnlyProps;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.rules.TestName;
+
+/**
+ * Integration test for mutation blocking functionality in IndexRegionObserver.preBatchMutate()
+ * Tests that MutationBlockedIOException is properly thrown when cluster role-based mutation
+ * blocking is enabled and CRRs are in ACTIVE_TO_STANDBY state.
+ */
+@Category(NeedsOwnMiniClusterTest.class)
+public class IndexRegionObserverMutationBlockingIT extends BaseTest {
+
+ private static final Long ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS = 1000L;
+ private PhoenixHAAdmin haAdmin;
+ private static final HighAvailabilityTestingUtility.HBaseTestingUtilityPair CLUSTERS = new HighAvailabilityTestingUtility.HBaseTestingUtilityPair();
+
+ @Rule
+ public TestName testName = new TestName();
+
+ @BeforeClass
+ public static synchronized void doSetup() throws Exception {
+ Map<String, String> props = Maps.newHashMapWithExpectedSize(1);
+ // Enable cluster role-based mutation blocking
+ props.put(CLUSTER_ROLE_BASED_MUTATION_BLOCK_ENABLED, "true");
+ // No retries so that tests fail faster.
+ props.put("hbase.client.retries.number", "0");
+ setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator()));
+ CLUSTERS.start();
+ }
+
+ @Before
+ public void setUp() throws Exception {
+ haAdmin = new PhoenixHAAdmin(config, ZK_CONSISTENT_HA_NAMESPACE);
+ Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
+ String zkUrl = getLocalZkUrl(config);
+ String peerZkUrl = CLUSTERS.getZkUrl2();
+ HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(testName.getMethodName(), zkUrl, peerZkUrl,
+ ClusterRoleRecord.ClusterRole.ACTIVE, ClusterRoleRecord.ClusterRole.STANDBY, null);
+ }
+
+ @Test
+ public void testMutationBlockedOnDataTableWithIndex() throws Exception {
+ String dataTableName = generateUniqueName();
+ String indexName = generateUniqueName();
+ String haGroupName = testName.getMethodName();
+
+ try (PhoenixConnection conn = (PhoenixConnection) DriverManager.getConnection(getUrl())) {
+ conn.setHAGroupName(haGroupName);
+ // Create data table and index
+ conn.createStatement().execute("CREATE TABLE " + dataTableName +
+ " (id VARCHAR PRIMARY KEY, name VARCHAR, age INTEGER)");
+ conn.createStatement().execute("CREATE INDEX " + indexName +
+ " ON " + dataTableName + "(name)");
+
+ // Initially, mutations should work - verify baseline
+ conn.createStatement().execute("UPSERT INTO " + dataTableName +
+ " VALUES ('1', 'John', 25)");
+ conn.commit();
+
+ // Set up HAGroupStoreRecord that will block mutations (ACTIVE_TO_STANDBY state)
+ HAGroupStoreRecord haGroupStoreRecord
+ = new HAGroupStoreRecord(HAGroupStoreRecord.DEFAULT_PROTOCOL_VERSION,
+ haGroupName, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY);
+ haAdmin.updateHAGroupStoreRecordInZooKeeper(haGroupName, haGroupStoreRecord, -1);
+
+ // Wait for the event to propagate
+ Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
+
+ // Test that UPSERT throws MutationBlockedIOException
+ try {
+ conn.createStatement().execute("UPSERT INTO " + dataTableName +
+ " VALUES ('2', 'Jane', 30)");
+ conn.commit();
+ fail("Expected MutationBlockedIOException to be thrown");
+ } catch (CommitException e) {
+ // Verify the exception chain contains MutationBlockedIOException
+ assertTrue("Expected MutationBlockedIOException in exception chain",
+ containsMutationBlockedException(e));
+ }
+ }
+ }
+
+
+ @Test
+ public void testMutationAllowedWhenNotBlocked() throws Exception {
+ String dataTableName = generateUniqueName();
+ String indexName = generateUniqueName();
+
+ try (PhoenixConnection conn = (PhoenixConnection) DriverManager.getConnection(getUrl())) {
+ conn.setHAGroupName(testName.getMethodName());
+ // Create data table and index
+ conn.createStatement().execute("CREATE TABLE " + dataTableName +
+ " (id VARCHAR PRIMARY KEY, name VARCHAR, age INTEGER)");
+ conn.createStatement().execute("CREATE INDEX " + indexName +
+ " ON " + dataTableName + "(name)");
+
+ // Mutations should work fine
+ conn.createStatement().execute("UPSERT INTO " + dataTableName +
+ " VALUES ('1', 'Bob', 35)");
+ conn.createStatement().execute("UPSERT INTO " + dataTableName +
+ " VALUES ('2', 'Carol', 27)");
+ conn.commit();
+
+ // Verify data was inserted successfully
+ try (java.sql.ResultSet rs = conn.createStatement().executeQuery(
+ "SELECT COUNT(*) FROM " + dataTableName)) {
+ assertTrue(rs.next());
+ assertEquals(2, rs.getInt(1));
+ }
+ }
+ }
+
+ @Test
+ public void testMutationBlockingTransition() throws Exception {
+ String dataTableName = generateUniqueName();
+ String indexName = generateUniqueName();
+ String haGroupName = testName.getMethodName();
+
+ try (PhoenixConnection conn = (PhoenixConnection) DriverManager.getConnection(getUrl())) {
+ conn.setHAGroupName(testName.getMethodName());
+ // Create data table and index
+ conn.createStatement().execute("CREATE TABLE " + dataTableName +
+ " (id VARCHAR PRIMARY KEY, name VARCHAR, age INTEGER)");
+ conn.createStatement().execute("CREATE INDEX " + indexName +
+ " ON " + dataTableName + "(name)");
+
+ // Mutation should work
+ conn.createStatement().execute("UPSERT INTO " + dataTableName +
+ " VALUES ('1', 'David', 40)");
+ conn.commit();
+ // Set up HAGroupStoreRecord that will block mutations (ACTIVE_TO_STANDBY state)
+ HAGroupStoreRecord haGroupStoreRecord
+ = new HAGroupStoreRecord(HAGroupStoreRecord.DEFAULT_PROTOCOL_VERSION,
+ haGroupName, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY);
+ haAdmin.updateHAGroupStoreRecordInZooKeeper(haGroupName, haGroupStoreRecord, -1);
+ Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
+
+ // Now mutations should be blocked
+ try {
+ conn.createStatement().execute("UPSERT INTO " + dataTableName +
+ " VALUES ('2', 'Eve', 33)");
+ conn.commit();
+ fail("Expected MutationBlockedIOException after transition to ACTIVE_TO_STANDBY");
+ } catch (CommitException e) {
+ assertTrue("Expected mutation blocked exception after transition",
+ containsMutationBlockedException(e));
+ }
+
+ // Transition back to ACTIVE (non-blocking state) and peer cluster is in ATS state
+ // Set up HAGroupStoreRecord that will block mutations (ACTIVE_TO_STANDBY state)
+ haGroupStoreRecord
+ = new HAGroupStoreRecord(HAGroupStoreRecord.DEFAULT_PROTOCOL_VERSION,
+ haGroupName, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+ haAdmin.updateHAGroupStoreRecordInZooKeeper(haGroupName, haGroupStoreRecord, -1);
+ Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
+
+ // Mutations should work again
+ conn.createStatement().execute("UPSERT INTO " + dataTableName +
+ " VALUES ('3', 'Frank', 45)");
+ conn.commit();
+ }
+ }
+
+ private boolean containsMutationBlockedException(CommitException e) {
+ Throwable cause = e.getCause();
+ if (cause instanceof RetriesExhaustedWithDetailsException) {
+ RetriesExhaustedWithDetailsException re = (RetriesExhaustedWithDetailsException) cause;
+ return re.getCause(0) instanceof MutationBlockedIOException;
+ }
+ return false;
+ }
+}
\ No newline at end of file
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HAGroupStoreClientIT.java b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HAGroupStoreClientIT.java
index 5a382f77e8..5a062b3305 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HAGroupStoreClientIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HAGroupStoreClientIT.java
@@ -17,31 +17,54 @@
*/
package org.apache.phoenix.jdbc;
-import org.apache.curator.framework.recipes.cache.PathChildrenCacheListener;
-import org.apache.curator.utils.ZKPaths;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.tuple.Pair;
import org.apache.phoenix.end2end.NeedsOwnMiniClusterTest;
+import org.apache.phoenix.exception.InvalidClusterRoleTransitionException;
import org.apache.phoenix.query.BaseTest;
import org.apache.phoenix.thirdparty.com.google.common.collect.Maps;
+import org.apache.phoenix.util.HAGroupStoreTestUtil;
import org.apache.phoenix.util.ReadOnlyProps;
+import org.apache.zookeeper.data.Stat;
+import org.junit.After;
import org.junit.Before;
import org.junit.BeforeClass;
+import org.junit.Rule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
+import org.junit.rules.TestName;
import java.io.IOException;
+import java.lang.reflect.Field;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.sql.Statement;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.Executors;
import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
+import org.apache.curator.framework.recipes.cache.PathChildrenCacheListener;
+
+import static org.apache.hadoop.hbase.HConstants.DEFAULT_ZK_SESSION_TIMEOUT;
+import static org.apache.phoenix.jdbc.HAGroupStoreClient.ZK_CONSISTENT_HA_NAMESPACE;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_HA_GROUP_NAME;
+import static org.apache.phoenix.jdbc.PhoenixHAAdmin.getLocalZkUrl;
import static org.apache.phoenix.jdbc.PhoenixHAAdmin.toPath;
+import static org.apache.phoenix.util.PhoenixRuntime.JDBC_PROTOCOL_SEPARATOR;
+import static org.apache.phoenix.util.PhoenixRuntime.JDBC_PROTOCOL_ZK;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertThrows;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
/**
* Integration tests for {@link HAGroupStoreClient}
@@ -49,171 +72,299 @@ import static org.junit.Assert.assertThrows;
@Category(NeedsOwnMiniClusterTest.class)
public class HAGroupStoreClientIT extends BaseTest {
- private final PhoenixHAAdmin haAdmin = new PhoenixHAAdmin(config);
private static final Long ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS = 5000L;
+ private static final HighAvailabilityTestingUtility.HBaseTestingUtilityPair CLUSTERS = new HighAvailabilityTestingUtility.HBaseTestingUtilityPair();
+ private PhoenixHAAdmin haAdmin;
+ private PhoenixHAAdmin peerHaAdmin;
+ private String zkUrl;
+ private String peerZKUrl;
+
+ @Rule
+ public TestName testName = new TestName();
@BeforeClass
public static synchronized void doSetup() throws Exception {
Map<String, String> props = Maps.newHashMapWithExpectedSize(1);
setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator()));
+ CLUSTERS.start();
}
@Before
public void before() throws Exception {
- // Clean up all the existing CRRs
- List<ClusterRoleRecord> crrs = haAdmin.listAllClusterRoleRecordsOnZookeeper();
- for (ClusterRoleRecord crr : crrs) {
- haAdmin.getCurator().delete().forPath(toPath(crr.getHaGroupName()));
+ haAdmin = new PhoenixHAAdmin(CLUSTERS.getHBaseCluster1().getConfiguration(), ZK_CONSISTENT_HA_NAMESPACE);
+ peerHaAdmin = new PhoenixHAAdmin(CLUSTERS.getHBaseCluster2().getConfiguration(), ZK_CONSISTENT_HA_NAMESPACE);
+ haAdmin.getCurator().delete().quietly().forPath(toPath(testName.getMethodName()));
+ peerHaAdmin.getCurator().delete().quietly().forPath(toPath(testName.getMethodName()));
+ zkUrl = getLocalZkUrl(CLUSTERS.getHBaseCluster1().getConfiguration());
+ // Clean existing records in system table
+ List<String> haGroupNames = HAGroupStoreClient.getHAGroupNames(zkUrl);
+ for (String groupName : haGroupNames) {
+ HAGroupStoreTestUtil.deleteHAGroupRecordInSystemTable(groupName, zkUrl);
+ }
+ // Insert a HAGroupStoreRecord into the system table
+ String haGroupName = testName.getMethodName();
+ this.peerZKUrl = CLUSTERS.getZkUrl2();
+ HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(haGroupName, this.zkUrl, this.peerZKUrl, ClusterRoleRecord.ClusterRole.ACTIVE, ClusterRoleRecord.ClusterRole.STANDBY, null);
+ }
+
+ @After
+ public void after() throws Exception {
+ haAdmin.getCurator().delete().quietly().forPath(toPath(testName.getMethodName()));
+ peerHaAdmin.getCurator().delete().quietly().forPath(toPath(testName.getMethodName()));
+ haAdmin.close();
+ peerHaAdmin.close();
+ HAGroupStoreTestUtil.deleteHAGroupRecordInSystemTable(testName.getMethodName(), zkUrl);
+
+ }
+
+ @Test
+ public void testHAGroupStoreClientWithBothNullZKUrl() throws Exception {
+ String haGroupName = testName.getMethodName();
+ // Clean existing record
+ HAGroupStoreTestUtil.deleteHAGroupRecordInSystemTable(haGroupName, zkUrl);
+ HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(haGroupName, null, null, ClusterRoleRecord.ClusterRole.ACTIVE, ClusterRoleRecord.ClusterRole.STANDBY, this.zkUrl);
+ HAGroupStoreClient haGroupStoreClient = HAGroupStoreClient.getInstanceForZkUrl(CLUSTERS.getHBaseCluster1().getConfiguration(), haGroupName, zkUrl);
+ assertNull(haGroupStoreClient);
+ }
+
+ @Test
+ public void testHAGroupStoreClientChangingPeerZKUrlToNullUrlToValidUrlToInvalidUrl() throws Exception {
+ String haGroupName = testName.getMethodName();
+ HAGroupStoreRecord record = new HAGroupStoreRecord("v1.0", haGroupName,
+ HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+ createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, record);
+ HAGroupStoreClient haGroupStoreClient = HAGroupStoreClient.getInstanceForZkUrl(CLUSTERS.getHBaseCluster1().getConfiguration(), haGroupName, zkUrl);
+
+ HAGroupStoreRecord currentRecord = haGroupStoreClient.getHAGroupStoreRecord();
+ assert currentRecord != null && currentRecord.getHAGroupState() == HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC;
+
+ // Base case: Check that peerPathChildrenCache is not null in HAGroupStoreClient via reflection
+ Field peerPathChildrenCache = HAGroupStoreClient.class.getDeclaredField("peerPathChildrenCache");
+ peerPathChildrenCache.setAccessible(true);
+ assertNotNull(peerPathChildrenCache.get(haGroupStoreClient));
+
+ // Clean existing record
+ HAGroupStoreTestUtil.deleteHAGroupRecordInSystemTable(haGroupName, zkUrl);
+ // Now update peerZKUrl to null and rebuild
+ HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(haGroupName, this.zkUrl, null, ClusterRoleRecord.ClusterRole.ACTIVE, ClusterRoleRecord.ClusterRole.STANDBY, null);
+ try {
+ haGroupStoreClient.rebuild();
+ fail("Should have thrown NullPointerException");
+ } catch (NullPointerException npe) {
+ // This exception is expected here.
}
+
+ // Now update System table to contain valid peer ZK URL and also change local cluster role to STANDBY
+ HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(haGroupName,this.zkUrl, this.peerZKUrl, ClusterRoleRecord.ClusterRole.STANDBY, ClusterRoleRecord.ClusterRole.ACTIVE, null);
+ haGroupStoreClient.rebuild();
+ currentRecord = haGroupStoreClient.getHAGroupStoreRecord();
+ assertNotNull(currentRecord);
+ assertEquals(HAGroupStoreRecord.HAGroupState.DEGRADED_STANDBY, currentRecord.getHAGroupState());
+
+ // Check that peerPathChildrenCache is not null now in HAGroupStoreClient via reflection
+ assertNotNull(peerPathChildrenCache.get(haGroupStoreClient));
+
+ // Now update local HAGroupStoreRecord to STANDBY to verify that HAGroupStoreClient is working as normal
+ record = new HAGroupStoreRecord("v1.0", haGroupName,
+ HAGroupStoreRecord.HAGroupState.STANDBY);
+ createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, record);
+ Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
+
+ currentRecord = haGroupStoreClient.getHAGroupStoreRecord();
+ assertNotNull(currentRecord);
+ assertEquals(HAGroupStoreRecord.HAGroupState.STANDBY, currentRecord.getHAGroupState());
+
+
+ // Now update peerZKUrl to invalid but non-null url and rebuild
+ // This URL can also be considered unreachable url due to a connectivity issue.
+ String invalidUrl = "invalidURL";
+ HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(haGroupName, this.zkUrl, invalidUrl, ClusterRoleRecord.ClusterRole.STANDBY, ClusterRoleRecord.ClusterRole.ACTIVE, null);
+ haGroupStoreClient.rebuild();
+ currentRecord = haGroupStoreClient.getHAGroupStoreRecord();
+ assertNotNull(currentRecord);
+ assertEquals(HAGroupStoreRecord.HAGroupState.STANDBY, currentRecord.getHAGroupState());
+
+ // Check CRR, the role for peer cluster with invalid url should be UNKNOWN.
+ ClusterRoleRecord clusterRoleRecord = haGroupStoreClient.getClusterRoleRecord();
+ assertNotNull(clusterRoleRecord);
+ ClusterRoleRecord expected = new ClusterRoleRecord(haGroupName,
+ HighAvailabilityPolicy.FAILOVER,
+ this.zkUrl,
+ ClusterRoleRecord.ClusterRole.STANDBY,
+ invalidUrl,
+ ClusterRoleRecord.ClusterRole.UNKNOWN,
+ 1);
+ assertEquals(expected, clusterRoleRecord);
+
+ // Check that peerPathChildrenCache is null now in HAGroupStoreClient via reflection
+ assertNull(peerPathChildrenCache.get(haGroupStoreClient));
+
+
+ }
+
+ @Test
+ public void testHAGroupStoreClientWithoutPeerZK() throws Exception {
+ String haGroupName = testName.getMethodName();
+ // Clean existing record
+ HAGroupStoreTestUtil.deleteHAGroupRecordInSystemTable(haGroupName, this.zkUrl);
+ HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(haGroupName,this.zkUrl, null, ClusterRoleRecord.ClusterRole.ACTIVE, ClusterRoleRecord.ClusterRole.STANDBY, null);
+ HAGroupStoreRecord record = new HAGroupStoreRecord("v1.0", haGroupName,
+ HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+ createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, record);
+ assertNull(HAGroupStoreClient.getInstanceForZkUrl(CLUSTERS.getHBaseCluster1().getConfiguration(), haGroupName, zkUrl));
}
@Test
- public void testHAGroupStoreClientWithSingleCRR() throws Exception {
- HAGroupStoreClient haGroupStoreClient = HAGroupStoreClient.getInstance(config);
- ClusterRoleRecord crr = new ClusterRoleRecord("failover",
- HighAvailabilityPolicy.FAILOVER, haAdmin.getZkUrl(), ClusterRoleRecord.ClusterRole.ACTIVE,
- "random-zk-url", ClusterRoleRecord.ClusterRole.STANDBY, 1L);
- haAdmin.createOrUpdateDataOnZookeeper(crr);
+ public void testHAGroupStoreClientWithSingleHAGroupStoreRecord() throws Exception {
+ String haGroupName = testName.getMethodName();
+ HAGroupStoreClient haGroupStoreClient = HAGroupStoreClient.getInstanceForZkUrl(CLUSTERS.getHBaseCluster1().getConfiguration(), haGroupName, zkUrl);
+
+ // Create and store HAGroupStoreRecord with ACTIVE state
+ HAGroupStoreRecord record = new HAGroupStoreRecord("v1.0", haGroupName,
+ HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+ createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, record);
Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
- assert haGroupStoreClient.getCRRsByClusterRole(ClusterRoleRecord.ClusterRole.ACTIVE).size() == 1;
- assert haGroupStoreClient.getCRRsByClusterRole(ClusterRoleRecord.ClusterRole.ACTIVE_TO_STANDBY).isEmpty();
+ HAGroupStoreRecord currentRecord = haGroupStoreClient.getHAGroupStoreRecord();
+ assert currentRecord != null && currentRecord.getHAGroupState() == HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC;
- // Now Update CRR so that current cluster has state ACTIVE_TO_STANDBY
- crr = new ClusterRoleRecord("failover",
- HighAvailabilityPolicy.FAILOVER, haAdmin.getZkUrl(), ClusterRoleRecord.ClusterRole.ACTIVE_TO_STANDBY,
- "random-zk-url", ClusterRoleRecord.ClusterRole.STANDBY, 2L);
- haAdmin.createOrUpdateDataOnZookeeper(crr);
+ // Now Update HAGroupStoreRecord so that current cluster has state ACTIVE_TO_STANDBY
+ record = new HAGroupStoreRecord("v1.0", haGroupName,
+ HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY);
+ createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, record);
Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
// Check that now the cluster should be in ActiveToStandby
- assert haGroupStoreClient.getCRRsByClusterRole(ClusterRoleRecord.ClusterRole.ACTIVE).isEmpty();
- assert haGroupStoreClient.getCRRsByClusterRole(ClusterRoleRecord.ClusterRole.ACTIVE_TO_STANDBY).size() == 1;
-
+ currentRecord = haGroupStoreClient.getHAGroupStoreRecord();
+ assert currentRecord != null && currentRecord.getHAGroupState() == HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY;
// Change it back to ACTIVE so that cluster is not in ACTIVE_TO_STANDBY state
- crr = new ClusterRoleRecord("failover",
- HighAvailabilityPolicy.FAILOVER, haAdmin.getZkUrl(), ClusterRoleRecord.ClusterRole.ACTIVE,
- "random-zk-url", ClusterRoleRecord.ClusterRole.STANDBY, 3L);
- haAdmin.createOrUpdateDataOnZookeeper(crr);
+ record = new HAGroupStoreRecord("v1.0", haGroupName,
+ HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+ createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, record);
Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
- assert haGroupStoreClient.getCRRsByClusterRole(ClusterRoleRecord.ClusterRole.ACTIVE).size() == 1;
- assert haGroupStoreClient.getCRRsByClusterRole(ClusterRoleRecord.ClusterRole.ACTIVE_TO_STANDBY).isEmpty();
-
+ currentRecord = haGroupStoreClient.getHAGroupStoreRecord();
+ assert currentRecord != null && currentRecord.getHAGroupState() == HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC;
// Change it again to ACTIVE_TO_STANDBY so that we can validate watcher works repeatedly
- crr = new ClusterRoleRecord("failover",
- HighAvailabilityPolicy.FAILOVER, haAdmin.getZkUrl(), ClusterRoleRecord.ClusterRole.ACTIVE_TO_STANDBY,
- "random-zk-url", ClusterRoleRecord.ClusterRole.STANDBY, 4L);
- haAdmin.createOrUpdateDataOnZookeeper(crr);
+ record = new HAGroupStoreRecord("v1.0", haGroupName,
+ HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY);
+ createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, record);
Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
- assert haGroupStoreClient.getCRRsByClusterRole(ClusterRoleRecord.ClusterRole.ACTIVE).isEmpty();
- assert haGroupStoreClient.getCRRsByClusterRole(ClusterRoleRecord.ClusterRole.ACTIVE_TO_STANDBY).size() == 1;
-
+ currentRecord = haGroupStoreClient.getHAGroupStoreRecord();
+ assert currentRecord != null && currentRecord.getHAGroupState() == HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY;
- // Change peer cluster to ACTIVE_TO_STANDBY so that we can still process mutation on this cluster
- crr = new ClusterRoleRecord("failover",
- HighAvailabilityPolicy.FAILOVER, haAdmin.getZkUrl(), ClusterRoleRecord.ClusterRole.ACTIVE,
- "random-zk-url", ClusterRoleRecord.ClusterRole.ACTIVE_TO_STANDBY, 5L);
- haAdmin.createOrUpdateDataOnZookeeper(crr);
+ // Change it back to ACTIVE to verify transition works
+ record = new HAGroupStoreRecord("v1.0", haGroupName,
+ HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+ createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, record);
Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
- assert haGroupStoreClient.getCRRsByClusterRole(ClusterRoleRecord.ClusterRole.ACTIVE).size() == 1;
- assert haGroupStoreClient.getCRRsByClusterRole(ClusterRoleRecord.ClusterRole.ACTIVE_TO_STANDBY).isEmpty();
+ currentRecord = haGroupStoreClient.getHAGroupStoreRecord();
+ assert currentRecord != null && currentRecord.getHAGroupState() == HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC;
}
+ /**
+ * Helper method to create or update HAGroupStoreRecord on ZooKeeper
+ */
+ private void createOrUpdateHAGroupStoreRecordOnZookeeper(PhoenixHAAdmin haAdmin, String haGroupName, HAGroupStoreRecord record) throws Exception {
+ String path = toPath(haGroupName);
+ if (haAdmin.getCurator().checkExists().forPath(path) == null) {
+ haAdmin.createHAGroupStoreRecordInZooKeeper(record);
+ } else {
+ final Pair<HAGroupStoreRecord, Stat> currentRecord = haAdmin.getHAGroupStoreRecordInZooKeeper(haGroupName);
+ if (currentRecord.getRight() != null && currentRecord.getLeft() != null) {
+ haAdmin.updateHAGroupStoreRecordInZooKeeper(haGroupName, record, currentRecord.getRight().getVersion());
+ } else {
+ throw new IOException("Current HAGroupStoreRecord in ZK is null, cannot update HAGroupStoreRecord " + haGroupName);
+ }
+ }
+ }
+
@Test
- public void testHAGroupStoreClientWithMultipleCRRs() throws Exception {
- HAGroupStoreClient haGroupStoreClient = HAGroupStoreClient.getInstance(config);
- // Setup initial CRRs
- ClusterRoleRecord crr1 = new ClusterRoleRecord("failover",
- HighAvailabilityPolicy.FAILOVER, haAdmin.getZkUrl(), ClusterRoleRecord.ClusterRole.ACTIVE,
- "random-zk-url", ClusterRoleRecord.ClusterRole.STANDBY, 1L);
- ClusterRoleRecord crr2 = new ClusterRoleRecord("parallel",
- HighAvailabilityPolicy.PARALLEL, haAdmin.getZkUrl(), ClusterRoleRecord.ClusterRole.ACTIVE,
- "random-zk-url", ClusterRoleRecord.ClusterRole.STANDBY, 1L);
- haAdmin.createOrUpdateDataOnZookeeper(crr1);
- haAdmin.createOrUpdateDataOnZookeeper(crr2);
+ public void testHAGroupStoreClientWithMultipleHAGroupStoreRecords() throws Exception {
+ String haGroupName1 = testName.getMethodName() + "1";
+ String haGroupName2 = testName.getMethodName() + "2";
+ HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(haGroupName1,this.zkUrl, this.peerZKUrl, ClusterRoleRecord.ClusterRole.ACTIVE, ClusterRoleRecord.ClusterRole.STANDBY, null);
+ HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(haGroupName2,this.zkUrl, this.peerZKUrl, ClusterRoleRecord.ClusterRole.ACTIVE, ClusterRoleRecord.ClusterRole.STANDBY, null);
- Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
- assert haGroupStoreClient.getCRRsByClusterRole(ClusterRoleRecord.ClusterRole.ACTIVE).size() == 2;
- assert haGroupStoreClient.getCRRsByClusterRole(ClusterRoleRecord.ClusterRole.ACTIVE_TO_STANDBY).isEmpty();
-
- // Now Update CRR so that current cluster has state ACTIVE_TO_STANDBY for only 1 crr
- crr1 = new ClusterRoleRecord("failover",
- HighAvailabilityPolicy.FAILOVER, haAdmin.getZkUrl(), ClusterRoleRecord.ClusterRole.ACTIVE_TO_STANDBY,
- "random-zk-url", ClusterRoleRecord.ClusterRole.STANDBY, 2L);
- crr2 = new ClusterRoleRecord("parallel",
- HighAvailabilityPolicy.PARALLEL, haAdmin.getZkUrl(), ClusterRoleRecord.ClusterRole.ACTIVE,
- "random-zk-url", ClusterRoleRecord.ClusterRole.STANDBY, 2L);
- haAdmin.createOrUpdateDataOnZookeeper(crr1);
- haAdmin.createOrUpdateDataOnZookeeper(crr2);
+ // Setup initial HAGroupStoreRecords
+ HAGroupStoreRecord record1 = new HAGroupStoreRecord("v1.0", haGroupName1,
+ HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+ HAGroupStoreRecord record2 = new HAGroupStoreRecord("v1.0", haGroupName2,
+ HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+
+ createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName1, record1);
+ createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName2, record2);
+
+ HAGroupStoreClient haGroupStoreClient1 = HAGroupStoreClient.getInstanceForZkUrl(CLUSTERS.getHBaseCluster1().getConfiguration(), haGroupName1, zkUrl);
+ HAGroupStoreClient haGroupStoreClient2 = HAGroupStoreClient.getInstanceForZkUrl(CLUSTERS.getHBaseCluster1().getConfiguration(), haGroupName2, zkUrl);
Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
- // Check that now the cluster should be in ActiveToStandby
- assert haGroupStoreClient.getCRRsByClusterRole(ClusterRoleRecord.ClusterRole.ACTIVE).size() == 1;
- assert haGroupStoreClient.getCRRsByClusterRole(ClusterRoleRecord.ClusterRole.ACTIVE_TO_STANDBY).size() == 1;
+ HAGroupStoreRecord currentRecord1 = haGroupStoreClient1.getHAGroupStoreRecord();
+ HAGroupStoreRecord currentRecord2 = haGroupStoreClient2.getHAGroupStoreRecord();
+ assert currentRecord1 != null && currentRecord1.getHAGroupState() == HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC;
+ assert currentRecord2 != null && currentRecord2.getHAGroupState() == HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC;
+ // Now Update HAGroupStoreRecord so that current cluster has state ACTIVE_TO_STANDBY for only 1 record
+ record1 = new HAGroupStoreRecord("v1.0", haGroupName1,
+ HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY);
+ record2 = new HAGroupStoreRecord("v1.0", haGroupName2,
+ HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
- // Change it back to ACTIVE so that cluster is not in ACTIVE_TO_STANDBY state
- crr1 = new ClusterRoleRecord("failover",
- HighAvailabilityPolicy.FAILOVER, haAdmin.getZkUrl(), ClusterRoleRecord.ClusterRole.ACTIVE,
- "random-zk-url", ClusterRoleRecord.ClusterRole.STANDBY, 3L);
- crr2 = new ClusterRoleRecord("parallel",
- HighAvailabilityPolicy.PARALLEL, haAdmin.getZkUrl(), ClusterRoleRecord.ClusterRole.ACTIVE,
- "random-zk-url", ClusterRoleRecord.ClusterRole.STANDBY, 3L);
- haAdmin.createOrUpdateDataOnZookeeper(crr1);
- haAdmin.createOrUpdateDataOnZookeeper(crr2);
+ createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName1, record1);
+ createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName2, record2);
Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
- assert haGroupStoreClient.getCRRsByClusterRole(ClusterRoleRecord.ClusterRole.ACTIVE).size() == 2;
- assert haGroupStoreClient.getCRRsByClusterRole(ClusterRoleRecord.ClusterRole.ACTIVE_TO_STANDBY).isEmpty();
+ // Check that now the cluster should be in ActiveToStandby
+ currentRecord1 = haGroupStoreClient1.getHAGroupStoreRecord();
+ currentRecord2 = haGroupStoreClient2.getHAGroupStoreRecord();
+ assert currentRecord1 != null && currentRecord1.getHAGroupState() == HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY;
+ assert currentRecord2 != null && currentRecord2.getHAGroupState() == HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC;
+ // Change it back to ACTIVE so that cluster is not in ACTIVE_TO_STANDBY state
+ record1 = new HAGroupStoreRecord("v1.0", haGroupName1,
+ HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+ record2 = new HAGroupStoreRecord("v1.0", haGroupName2,
+ HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
- // Change other crr to ACTIVE_TO_STANDBY and one in ACTIVE state so that we can validate watcher works repeatedly
- crr1 = new ClusterRoleRecord("failover",
- HighAvailabilityPolicy.FAILOVER, haAdmin.getZkUrl(), ClusterRoleRecord.ClusterRole.ACTIVE,
- "random-zk-url", ClusterRoleRecord.ClusterRole.STANDBY, 4L);
- crr2 = new ClusterRoleRecord("parallel",
- HighAvailabilityPolicy.PARALLEL, haAdmin.getZkUrl(), ClusterRoleRecord.ClusterRole.ACTIVE_TO_STANDBY,
- "random-zk-url", ClusterRoleRecord.ClusterRole.STANDBY, 4L);
- haAdmin.createOrUpdateDataOnZookeeper(crr1);
- haAdmin.createOrUpdateDataOnZookeeper(crr2);
+ createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName1, record1);
+ createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName2, record2);
Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
- assert haGroupStoreClient.getCRRsByClusterRole(ClusterRoleRecord.ClusterRole.ACTIVE).size() == 1;
- assert haGroupStoreClient.getCRRsByClusterRole(ClusterRoleRecord.ClusterRole.ACTIVE_TO_STANDBY).size() == 1;
+ currentRecord1 = haGroupStoreClient1.getHAGroupStoreRecord();
+ currentRecord2 = haGroupStoreClient2.getHAGroupStoreRecord();
+ assert currentRecord1 != null && currentRecord1.getHAGroupState() == HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC;
+ assert currentRecord2 != null && currentRecord2.getHAGroupState() == HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC;
+ // Change other record to ACTIVE_TO_STANDBY and one in ACTIVE state so that we can validate watcher works repeatedly
+ record1 = new HAGroupStoreRecord("v1.0", haGroupName1,
+ HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+ record2 = new HAGroupStoreRecord("v1.0", haGroupName2,
+ HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY);
- // Change peer cluster to ACTIVE_TO_STANDBY so that we can still process mutation on this cluster
- crr1 = new ClusterRoleRecord("failover",
- HighAvailabilityPolicy.FAILOVER, haAdmin.getZkUrl(), ClusterRoleRecord.ClusterRole.ACTIVE,
- "random-zk-url", ClusterRoleRecord.ClusterRole.ACTIVE_TO_STANDBY, 5L);
- crr2 = new ClusterRoleRecord("parallel",
- HighAvailabilityPolicy.PARALLEL, haAdmin.getZkUrl(), ClusterRoleRecord.ClusterRole.ACTIVE,
- "random-zk-url", ClusterRoleRecord.ClusterRole.ACTIVE_TO_STANDBY, 5L);
- haAdmin.createOrUpdateDataOnZookeeper(crr1);
- haAdmin.createOrUpdateDataOnZookeeper(crr2);
+ createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName1, record1);
+ createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName2, record2);
Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
- assert haGroupStoreClient.getCRRsByClusterRole(ClusterRoleRecord.ClusterRole.ACTIVE).size() == 2;
- assert haGroupStoreClient.getCRRsByClusterRole(ClusterRoleRecord.ClusterRole.ACTIVE_TO_STANDBY).isEmpty();
+ currentRecord1 = haGroupStoreClient1.getHAGroupStoreRecord();
+ currentRecord2 = haGroupStoreClient2.getHAGroupStoreRecord();
+ assert currentRecord1 != null && currentRecord1.getHAGroupState() == HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC;
+ assert currentRecord2 != null && currentRecord2.getHAGroupState() == HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY;
}
@Test
public void testMultiThreadedAccessToHACache() throws Exception {
- HAGroupStoreClient haGroupStoreClient = HAGroupStoreClient.getInstance(config);
- // Setup initial CRRs
- ClusterRoleRecord crr1 = new ClusterRoleRecord("failover",
- HighAvailabilityPolicy.FAILOVER, haAdmin.getZkUrl(), ClusterRoleRecord.ClusterRole.ACTIVE,
- "random-zk-url", ClusterRoleRecord.ClusterRole.STANDBY, 1L);
- ClusterRoleRecord crr2 = new ClusterRoleRecord("parallel",
- HighAvailabilityPolicy.PARALLEL, haAdmin.getZkUrl(), ClusterRoleRecord.ClusterRole.ACTIVE,
- "random-zk-url", ClusterRoleRecord.ClusterRole.STANDBY, 1L);
- haAdmin.createOrUpdateDataOnZookeeper(crr1);
- haAdmin.createOrUpdateDataOnZookeeper(crr2);
+ String haGroupName = testName.getMethodName();
+
+ // Setup initial HAGroupStoreRecord
+ HAGroupStoreRecord record = new HAGroupStoreRecord("v1.0", haGroupName,
+ HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+
+ createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, record);
+ HAGroupStoreClient haGroupStoreClient = HAGroupStoreClient.getInstanceForZkUrl(CLUSTERS.getHBaseCluster1().getConfiguration(), haGroupName, zkUrl);
Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
@@ -223,8 +374,8 @@ public class HAGroupStoreClientIT extends BaseTest {
for (int i = 0; i < threadCount; i++) {
executor.submit(() -> {
try {
- assert haGroupStoreClient.getCRRsByClusterRole(ClusterRoleRecord.ClusterRole.ACTIVE).size() == 2;
- assert haGroupStoreClient.getCRRsByClusterRole(ClusterRoleRecord.ClusterRole.ACTIVE_TO_STANDBY).isEmpty();
+ HAGroupStoreRecord currentRecord = haGroupStoreClient.getHAGroupStoreRecord();
+ assert currentRecord != null && currentRecord.getHAGroupState() == HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC;
latch.countDown();
} catch (Exception e) {
throw new RuntimeException(e);
@@ -233,15 +384,11 @@ public class HAGroupStoreClientIT extends BaseTest {
}
assert latch.await(10, TimeUnit.SECONDS);
- // Update CRRs
- crr1 = new ClusterRoleRecord("failover",
- HighAvailabilityPolicy.FAILOVER, haAdmin.getZkUrl(), ClusterRoleRecord.ClusterRole.ACTIVE_TO_STANDBY,
- "random-zk-url", ClusterRoleRecord.ClusterRole.STANDBY, 2L);
- crr2 = new ClusterRoleRecord("parallel",
- HighAvailabilityPolicy.PARALLEL, haAdmin.getZkUrl(), ClusterRoleRecord.ClusterRole.ACTIVE,
- "random-zk-url", ClusterRoleRecord.ClusterRole.STANDBY, 2L);
- haAdmin.createOrUpdateDataOnZookeeper(crr1);
- haAdmin.createOrUpdateDataOnZookeeper(crr2);
+ // Update HAGroupStoreRecord
+ record = new HAGroupStoreRecord("v1.0", haGroupName,
+ HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY);
+
+ createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, record);
Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
@@ -250,8 +397,8 @@ public class HAGroupStoreClientIT extends BaseTest {
for (int i = 0; i < threadCount; i++) {
executor.submit(() -> {
try {
- assert haGroupStoreClient.getCRRsByClusterRole(ClusterRoleRecord.ClusterRole.ACTIVE).size() == 1;
- assert haGroupStoreClient.getCRRsByClusterRole(ClusterRoleRecord.ClusterRole.ACTIVE_TO_STANDBY).size() == 1;
+ HAGroupStoreRecord currentRecord = haGroupStoreClient.getHAGroupStoreRecord();
+ assert currentRecord != null && currentRecord.getHAGroupState() == HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY;
latch2.countDown();
} catch (Exception e) {
throw new RuntimeException(e);
@@ -263,100 +410,251 @@ public class HAGroupStoreClientIT extends BaseTest {
@Test
public void testHAGroupStoreClientWithRootPathDeletion() throws Exception {
- HAGroupStoreClient haGroupStoreClient = HAGroupStoreClient.getInstance(config);
- ClusterRoleRecord crr1 = new ClusterRoleRecord("failover",
- HighAvailabilityPolicy.FAILOVER, haAdmin.getZkUrl(), ClusterRoleRecord.ClusterRole.ACTIVE_TO_STANDBY,
- "random-zk-url", ClusterRoleRecord.ClusterRole.STANDBY, 1L);
- ClusterRoleRecord crr2 = new ClusterRoleRecord("parallel",
- HighAvailabilityPolicy.PARALLEL, haAdmin.getZkUrl(), ClusterRoleRecord.ClusterRole.ACTIVE,
- "random-zk-url", ClusterRoleRecord.ClusterRole.STANDBY, 1L);
- haAdmin.createOrUpdateDataOnZookeeper(crr1);
- haAdmin.createOrUpdateDataOnZookeeper(crr2);
+ String haGroupName = testName.getMethodName();
+
+ HAGroupStoreRecord record1 = new HAGroupStoreRecord("v1.0", haGroupName,
+ HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+
+ createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, record1);
+ HAGroupStoreClient haGroupStoreClient = HAGroupStoreClient.getInstanceForZkUrl(CLUSTERS.getHBaseCluster1().getConfiguration(), haGroupName, zkUrl);
Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
- assert haGroupStoreClient.getCRRsByClusterRole(ClusterRoleRecord.ClusterRole.ACTIVE).size() == 1;
- assert haGroupStoreClient.getCRRsByClusterRole(ClusterRoleRecord.ClusterRole.ACTIVE_TO_STANDBY).size() == 1;
+ HAGroupStoreRecord currentRecord = haGroupStoreClient.getHAGroupStoreRecord();
+ assertNotNull(currentRecord);
+ assertEquals(HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, currentRecord.getHAGroupState());
- haAdmin.getCurator().delete().deletingChildrenIfNeeded().forPath(ZKPaths.PATH_SEPARATOR);
+ haAdmin.getCurator().delete().deletingChildrenIfNeeded().forPath(toPath(StringUtils.EMPTY));
Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
- assert haGroupStoreClient.getCRRsByClusterRole(ClusterRoleRecord.ClusterRole.ACTIVE).isEmpty();
- assert haGroupStoreClient.getCRRsByClusterRole(ClusterRoleRecord.ClusterRole.ACTIVE_TO_STANDBY).isEmpty();
-
-
- crr1 = new ClusterRoleRecord("failover",
- HighAvailabilityPolicy.FAILOVER, haAdmin.getZkUrl(), ClusterRoleRecord.ClusterRole.ACTIVE_TO_STANDBY,
- "random-zk-url", ClusterRoleRecord.ClusterRole.STANDBY, 2L);
- crr2 = new ClusterRoleRecord("parallel",
- HighAvailabilityPolicy.PARALLEL, haAdmin.getZkUrl(), ClusterRoleRecord.ClusterRole.ACTIVE,
- "random-zk-url", ClusterRoleRecord.ClusterRole.STANDBY, 2L);
- haAdmin.createOrUpdateDataOnZookeeper(crr1);
- haAdmin.createOrUpdateDataOnZookeeper(crr2);
+ currentRecord = haGroupStoreClient.getHAGroupStoreRecord();
+
+ // The record should be automatically rebuilt from System Table as it is not in ZK
+ assertNotNull(currentRecord);
+ assertEquals(HAGroupStoreRecord.HAGroupState.ACTIVE_NOT_IN_SYNC, currentRecord.getHAGroupState());
+
+ record1 = new HAGroupStoreRecord("v1.0", haGroupName,
+ HAGroupStoreRecord.HAGroupState.ACTIVE_NOT_IN_SYNC_TO_STANDBY);
+
+ createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, record1);
Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
- assert haGroupStoreClient.getCRRsByClusterRole(ClusterRoleRecord.ClusterRole.ACTIVE).size() == 1;
- assert haGroupStoreClient.getCRRsByClusterRole(ClusterRoleRecord.ClusterRole.ACTIVE_TO_STANDBY).size() == 1;
+ currentRecord = haGroupStoreClient.getHAGroupStoreRecord();
+ assertNotNull(currentRecord);
+ assertEquals(HAGroupStoreRecord.HAGroupState.ACTIVE_NOT_IN_SYNC_TO_STANDBY, currentRecord.getHAGroupState());
}
@Test
public void testThrowsExceptionWithZKDisconnectionAndThenConnection() throws Exception {
- HAGroupStoreClient haGroupStoreClient = HAGroupStoreClient.getInstance(config);
- // Setup initial CRRs
- ClusterRoleRecord crr1 = new ClusterRoleRecord("failover",
- HighAvailabilityPolicy.FAILOVER, haAdmin.getZkUrl(), ClusterRoleRecord.ClusterRole.ACTIVE,
- "random-zk-url", ClusterRoleRecord.ClusterRole.STANDBY, 1L);
- ClusterRoleRecord crr2 = new ClusterRoleRecord("parallel",
- HighAvailabilityPolicy.PARALLEL, haAdmin.getZkUrl(), ClusterRoleRecord.ClusterRole.ACTIVE,
- "random-zk-url", ClusterRoleRecord.ClusterRole.STANDBY, 1L);
- haAdmin.createOrUpdateDataOnZookeeper(crr1);
- haAdmin.createOrUpdateDataOnZookeeper(crr2);
+ String haGroupName = testName.getMethodName();
+
+ // Setup initial HAGroupStoreRecord
+ HAGroupStoreRecord record = new HAGroupStoreRecord("v1.0", haGroupName,
+ HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+
+ createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, record);
+ HAGroupStoreClient haGroupStoreClient = HAGroupStoreClient.getInstanceForZkUrl(CLUSTERS.getHBaseCluster1().getConfiguration(), haGroupName, zkUrl);
Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
- assert haGroupStoreClient.getCRRsByClusterRole(ClusterRoleRecord.ClusterRole.ACTIVE).size() == 2;
+ HAGroupStoreRecord currentRecord = haGroupStoreClient.getHAGroupStoreRecord();
+ assert currentRecord != null && currentRecord.getHAGroupState() == HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC;
// Shutdown the ZK Cluster to simulate CONNECTION_SUSPENDED event
- utility.shutdownMiniZKCluster();
+ CLUSTERS.getHBaseCluster1().shutdownMiniZKCluster();
Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
//Check that HAGroupStoreClient instance is not healthy and throws IOException
- assertThrows(IOException.class, () -> haGroupStoreClient.getCRRsByClusterRole(ClusterRoleRecord.ClusterRole.ACTIVE));
+ assertThrows(IOException.class, () -> haGroupStoreClient.getHAGroupStoreRecord());
+ // Check that the HAGroupStoreClient instance is not healthy via reflection
+ Field isHealthyField = HAGroupStoreClient.class.getDeclaredField("isHealthy");
+ isHealthyField.setAccessible(true);
+ assertFalse((boolean)isHealthyField.get(haGroupStoreClient));
// Start ZK on the same port to simulate CONNECTION_RECONNECTED event
- utility.startMiniZKCluster(1,Integer.parseInt(getZKClientPort(config)));
+ CLUSTERS.getHBaseCluster1().startMiniZKCluster(1,
+ Integer.parseInt(CLUSTERS.getHBaseCluster1().getConfiguration().get("hbase.zookeeper.property.clientPort")));
Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
//Check that HAGroupStoreClient instance is back to healthy and provides correct response
- assert haGroupStoreClient.getCRRsByClusterRole(ClusterRoleRecord.ClusterRole.ACTIVE).size() == 2;
+ currentRecord = haGroupStoreClient.getHAGroupStoreRecord();
+ assert currentRecord != null && currentRecord.getHAGroupState() == HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC;
+ // Check that the HAGroupStoreClient instance is healthy via reflection
+ assertTrue((boolean)isHealthyField.get(haGroupStoreClient));
}
+
+ // Tests for setHAGroupStatusIfNeeded method
+ @Test
+ public void testSetHAGroupStatusIfNeededDeleteZKAndSystemTableRecord() throws Exception {
+ String haGroupName = testName.getMethodName();
+
+ // Create HAGroupStoreClient without any existing record
+ HAGroupStoreClient haGroupStoreClient = HAGroupStoreClient.getInstanceForZkUrl(CLUSTERS.getHBaseCluster1().getConfiguration(), haGroupName, zkUrl);
+
+ // Delete the record from ZK
+ haAdmin.getCurator().delete().deletingChildrenIfNeeded().forPath(toPath(haGroupName));
+ Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
+
+ // Delete the record from System Table
+ try (PhoenixConnection conn = (PhoenixConnection) DriverManager.getConnection(JDBC_PROTOCOL_ZK + JDBC_PROTOCOL_SEPARATOR + zkUrl);
+ Statement stmt = conn.createStatement()) {
+ stmt.execute("DELETE FROM " + SYSTEM_HA_GROUP_NAME + " WHERE HA_GROUP_NAME = '" + haGroupName + "'");
+ conn.commit();
+ }
+
+ // This should fail because no record exists in either ZK or System Table
+ try {
+ haGroupStoreClient.setHAGroupStatusIfNeeded(HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+ fail("Expected IOException for missing system table record");
+ } catch (IOException e) {
+ assertTrue("Exception should mention system table", e.getMessage().contains("Current HAGroupStoreRecordStat in cache is null, "
+ + "cannot update HAGroupStoreRecord, the record should be initialized in System Table first"));
+ }
+ }
+
@Test
- public void testHAGroupStoreClientWithDifferentZKURLFormats() throws Exception {
- HAGroupStoreClient haGroupStoreClient = HAGroupStoreClient.getInstance(config);
- final String zkClientPort = getZKClientPort(config);
- // Setup initial CRRs
- final String format1 = "127.0.0.1\\:"+zkClientPort+"::/hbase"; // 127.0.0.1\:53228::/hbase
- final String format2 = "127.0.0.1:"+zkClientPort+"::/hbase"; // 127.0.0.1:53228::/hbase
- final String format3 = "127.0.0.1\\:"+zkClientPort+":/hbase"; // 127.0.0.1\:53228:/hbase
-
- ClusterRoleRecord crr1 = new ClusterRoleRecord("parallel1",
- HighAvailabilityPolicy.PARALLEL, format1, ClusterRoleRecord.ClusterRole.ACTIVE,
- "random-zk-url", ClusterRoleRecord.ClusterRole.STANDBY, 1L);
- haAdmin.createOrUpdateDataOnZookeeper(crr1);
-
- ClusterRoleRecord crr2 = new ClusterRoleRecord("parallel2",
- HighAvailabilityPolicy.PARALLEL, format2, ClusterRoleRecord.ClusterRole.STANDBY,
- "random-zk-url", ClusterRoleRecord.ClusterRole.STANDBY, 1L);
- haAdmin.createOrUpdateDataOnZookeeper(crr2);
-
- ClusterRoleRecord crr3 = new ClusterRoleRecord("parallel3",
- HighAvailabilityPolicy.PARALLEL, format3, ClusterRoleRecord.ClusterRole.ACTIVE_TO_STANDBY,
- "random-zk-url", ClusterRoleRecord.ClusterRole.STANDBY, 1L);
- haAdmin.createOrUpdateDataOnZookeeper(crr3);
+ public void testSetHAGroupStatusIfNeededUpdateExistingRecord() throws Exception {
+ String haGroupName = testName.getMethodName();
+
+ // Create initial record
+ HAGroupStoreRecord initialRecord = new HAGroupStoreRecord("v1.0", haGroupName,
+ HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+ createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, initialRecord);
+
+ HAGroupStoreClient haGroupStoreClient = HAGroupStoreClient.getInstanceForZkUrl(CLUSTERS.getHBaseCluster1().getConfiguration(), haGroupName, zkUrl);
+ Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
+ // Verify initial state
+ HAGroupStoreRecord currentRecord = haGroupStoreClient.getHAGroupStoreRecord();
+ assertEquals(HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, currentRecord.getHAGroupState());
+
+ // Update to STANDBY (this should succeed as it's a valid transition)
+ haGroupStoreClient.setHAGroupStatusIfNeeded(HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY);
Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
- assert haGroupStoreClient.getCRRsByClusterRole(ClusterRoleRecord.ClusterRole.ACTIVE).size() == 1;
- assert haGroupStoreClient.getCRRsByClusterRole(ClusterRoleRecord.ClusterRole.ACTIVE_TO_STANDBY).size() == 1;
- assert haGroupStoreClient.getCRRsByClusterRole(ClusterRoleRecord.ClusterRole.STANDBY).size() == 1;
+
+ // Verify the record was updated
+ currentRecord = haGroupStoreClient.getHAGroupStoreRecord();
+ assertEquals(HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY, currentRecord.getHAGroupState());
}
+ @Test
+ public void testSetHAGroupStatusIfNeededNoUpdateWhenNotNeeded() throws Exception {
+ String haGroupName = testName.getMethodName();
+
+ // Create initial record with current timestamp
+ HAGroupStoreRecord initialRecord = new HAGroupStoreRecord("v1.0", haGroupName,
+ HAGroupStoreRecord.HAGroupState.ACTIVE_NOT_IN_SYNC);
+ createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, initialRecord);
+ Stat initialRecordInZKStat = haAdmin.getHAGroupStoreRecordInZooKeeper(haGroupName).getRight();
+ int initialRecordVersion = initialRecordInZKStat.getVersion();
+
+ HAGroupStoreClient haGroupStoreClient = HAGroupStoreClient.getInstanceForZkUrl(CLUSTERS.getHBaseCluster1().getConfiguration(), haGroupName, zkUrl);
+
+ // Get the current record
+ HAGroupStoreRecord currentRecord = haGroupStoreClient.getHAGroupStoreRecord();
+
+ // Try to set to ACTIVE_IN_SYNC immediately (should not update due to timing)
+ haGroupStoreClient.setHAGroupStatusIfNeeded(HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+
+ // Add sleep if due to any bug the update might have gone through and we can assert below this.
+ Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
+ // Verify no update occurred
+ HAGroupStoreRecord afterRecord = haGroupStoreClient.getHAGroupStoreRecord();
+ Stat afterRecordInZKStat = haAdmin.getHAGroupStoreRecordInZooKeeper(haGroupName).getRight();
+ int afterRecordVersion = afterRecordInZKStat.getVersion();
+
+ assertEquals(initialRecordVersion, afterRecordVersion);
+ assertEquals("State should not change", HAGroupStoreRecord.HAGroupState.ACTIVE_NOT_IN_SYNC, afterRecord.getHAGroupState());
+ }
+
+ @Test
+ public void testSetHAGroupStatusIfNeededWithTimingLogic() throws Exception {
+ String haGroupName = testName.getMethodName();
+ // Create initial record
+ HAGroupStoreRecord initialRecord = new HAGroupStoreRecord("v1.0", haGroupName,
+ HAGroupStoreRecord.HAGroupState.ACTIVE_NOT_IN_SYNC);
+ createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, initialRecord);
+
+ HAGroupStoreClient haGroupStoreClient = HAGroupStoreClient.getInstanceForZkUrl(CLUSTERS.getHBaseCluster1().getConfiguration(), haGroupName, zkUrl);
+ Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS + DEFAULT_ZK_SESSION_TIMEOUT);
+
+ haGroupStoreClient.setHAGroupStatusIfNeeded(HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+ Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
+
+ // Verify the record was updated
+ HAGroupStoreRecord updatedRecord = haGroupStoreClient.getHAGroupStoreRecord();
+ assertEquals(HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, updatedRecord.getHAGroupState());
+ }
+
+ @Test
+ public void testSetHAGroupStatusIfNeededWithInvalidTransition() throws Exception {
+ String haGroupName = testName.getMethodName();
+
+ // Create initial record with ACTIVE state
+ HAGroupStoreRecord initialRecord = new HAGroupStoreRecord("v1.0", haGroupName,
+ HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+ createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, initialRecord);
+
+ HAGroupStoreClient haGroupStoreClient = HAGroupStoreClient.getInstanceForZkUrl(CLUSTERS.getHBaseCluster1().getConfiguration(), haGroupName, zkUrl);
+ Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
+
+ // Try to transition to STANDBY (invalid transition from ACTIVE)
+ try {
+ haGroupStoreClient.setHAGroupStatusIfNeeded(HAGroupStoreRecord.HAGroupState.STANDBY);
+ fail("Expected InvalidClusterRoleTransitionException");
+ } catch (InvalidClusterRoleTransitionException e) {
+ // Should get InvalidClusterRoleTransitionException (might be wrapped)
+ assertTrue("Exception should be about invalid transition",
+ e.getMessage().contains("Cannot transition from ACTIVE_IN_SYNC to STANDBY") ||
+ e.getCause() != null && e.getCause().getMessage().contains("Cannot transition from ACTIVE_IN_SYNC to STANDBY"));
+ }
+ }
+
+ @Test
+ public void testSetHAGroupStatusIfNeededWithUnhealthyClient() throws Exception {
+ String haGroupName = testName.getMethodName();
+
+ HAGroupStoreClient haGroupStoreClient = HAGroupStoreClient.getInstanceForZkUrl(CLUSTERS.getHBaseCluster1().getConfiguration(), haGroupName, zkUrl);
+
+ // Make client unhealthy by accessing private field
+ Field isHealthyField = HAGroupStoreClient.class.getDeclaredField("isHealthy");
+ isHealthyField.setAccessible(true);
+ isHealthyField.set(haGroupStoreClient, false);
+
+ // Try to set status on unhealthy client
+ try {
+ haGroupStoreClient.setHAGroupStatusIfNeeded(HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+ fail("Expected IOException for unhealthy client");
+ } catch (IOException e) {
+ assertTrue("Exception should mention unhealthy client", e.getMessage().contains("not healthy"));
+ }
+ }
+
+
+ @Test
+ public void testSetHAGroupStatusIfNeededMultipleTransitions() throws Exception {
+ String haGroupName = testName.getMethodName();
+
+ // Create initial record with old timestamp
+ HAGroupStoreRecord initialRecord = new HAGroupStoreRecord("v1.0", haGroupName,
+ HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+ createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, initialRecord);
+
+ HAGroupStoreClient haGroupStoreClient = HAGroupStoreClient.getInstanceForZkUrl(CLUSTERS.getHBaseCluster1().getConfiguration(), haGroupName, zkUrl);
+ Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
+
+ // First transition: ACTIVE -> ACTIVE_TO_STANDBY
+ haGroupStoreClient.setHAGroupStatusIfNeeded(HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY);
+ Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
+
+ HAGroupStoreRecord afterFirst = haGroupStoreClient.getHAGroupStoreRecord();
+ assertEquals(HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY, afterFirst.getHAGroupState());
+
+ // Wait and make another transition: ACTIVE_TO_STANDBY -> STANDBY
+ Thread.sleep(100); // Small delay to ensure timestamp difference
+ haGroupStoreClient.setHAGroupStatusIfNeeded(HAGroupStoreRecord.HAGroupState.STANDBY);
+ Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
+
+ HAGroupStoreRecord afterSecond = haGroupStoreClient.getHAGroupStoreRecord();
+ assertEquals(HAGroupStoreRecord.HAGroupState.STANDBY, afterSecond.getHAGroupState());
+ }
/**
* This test verifies that the updates coming via PathChildrenCacheListener are in order in which updates are sent to ZK
@@ -364,22 +662,27 @@ public class HAGroupStoreClientIT extends BaseTest {
*/
@Test
public void testHAGroupStoreClientWithMultiThreadedUpdates() throws Exception {
+ String haGroupName = testName.getMethodName();
+
// Number of threads to execute
int threadCount = 5;
- // Capture versions of crr in a list(crrEventVersions) in order they are received.
- List<Integer> crrEventVersions = new ArrayList<>();
+ // Capture versions of records in a list(recordEventVersions) in order they are received.
+ List<Integer> recordEventVersions = new ArrayList<>();
CountDownLatch eventsLatch = new CountDownLatch(threadCount);
PathChildrenCacheListener pathChildrenCacheListener = (client, event) -> {
- if(event.getData() != null && event.getData().getData() != null && ClusterRoleRecord.fromJson(event.getData().getData()).isPresent()) {
- ClusterRoleRecord crr = ClusterRoleRecord.fromJson(event.getData().getData()).get();
- crrEventVersions.add((int)crr.getVersion());
+ if(event.getData() != null && event.getData().getData() != null && HAGroupStoreRecord.fromJson(event.getData().getData()).isPresent()) {
+ HAGroupStoreRecord record = HAGroupStoreRecord.fromJson(event.getData().getData()).get();
+ if (record.getHaGroupName().equals(haGroupName)) {
+ recordEventVersions.add(event.getData().getStat().getVersion());
eventsLatch.countDown();
+ }
}
};
- // Start a new HAGroupStoreClient.
- new HAGroupStoreClient(config, pathChildrenCacheListener);
+ // Start a new HAGroupStoreClient with custom listener.
+ new HAGroupStoreClient(CLUSTERS.getHBaseCluster1().getConfiguration(),
+ pathChildrenCacheListener, null, haGroupName, zkUrl);
// Create multiple threads for update to ZK.
final CountDownLatch updateLatch = new CountDownLatch(threadCount);
@@ -389,10 +692,10 @@ public class HAGroupStoreClientIT extends BaseTest {
List<Integer> updateList = new ArrayList<>();
// Create a queue which can be polled to send updates to ZK.
- ConcurrentLinkedQueue<ClusterRoleRecord> updateQueue = new ConcurrentLinkedQueue<>();
+ ConcurrentLinkedQueue<HAGroupStoreRecord> updateQueue = new ConcurrentLinkedQueue<>();
for(int i = 0; i < threadCount; i++) {
- updateQueue.add(createCRR(i+1));
- updateList.add(i+1);
+ updateQueue.add(createHAGroupStoreRecord(haGroupName));
+ updateList.add(i);
}
// Submit updates to ZK.
@@ -400,7 +703,8 @@ public class HAGroupStoreClientIT extends BaseTest {
executor.submit(() -> {
try {
synchronized (HAGroupStoreClientIT.class) {
- haAdmin.createOrUpdateDataOnZookeeper(Objects.requireNonNull(updateQueue.poll()));
+ HAGroupStoreRecord record = Objects.requireNonNull(updateQueue.poll());
+ createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, record);
}
updateLatch.countDown();
} catch (Exception e) {
@@ -414,14 +718,204 @@ public class HAGroupStoreClientIT extends BaseTest {
assert updateLatch.await(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS*threadCount, TimeUnit.MILLISECONDS);
// Assert that the order of updates is same as order of events.
- assert updateList.equals(crrEventVersions);
+ assert updateList.equals(recordEventVersions);
}
- private ClusterRoleRecord createCRR(Integer version) {
- return new ClusterRoleRecord("parallel",
- HighAvailabilityPolicy.PARALLEL, haAdmin.getZkUrl(), ClusterRoleRecord.ClusterRole.ACTIVE,
- "random-zk-url", ClusterRoleRecord.ClusterRole.STANDBY, version);
+ @Test
+ public void testGetClusterRoleRecordNormalCase() throws Exception {
+ String haGroupName = testName.getMethodName();
+ HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(haGroupName, this.zkUrl, this.peerZKUrl,
+ ClusterRoleRecord.ClusterRole.ACTIVE, ClusterRoleRecord.ClusterRole.STANDBY, null);
+
+ // Create HAGroupStoreRecord for local cluster
+ HAGroupStoreRecord localRecord = new HAGroupStoreRecord("v1.0", haGroupName,
+ HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+ createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, localRecord);
+
+ // Create HAGroupStoreRecord for peer cluster
+ HAGroupStoreRecord peerRecord = new HAGroupStoreRecord("v1.0", haGroupName,
+ HAGroupStoreRecord.HAGroupState.STANDBY);
+ createOrUpdateHAGroupStoreRecordOnZookeeper(peerHaAdmin, haGroupName, peerRecord);
+
+ Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
+
+ HAGroupStoreClient haGroupStoreClient = HAGroupStoreClient.getInstanceForZkUrl(CLUSTERS.getHBaseCluster1().getConfiguration(), haGroupName, zkUrl);
+ assertNotNull(haGroupStoreClient);
+
+ // Test getClusterRoleRecord
+ ClusterRoleRecord clusterRoleRecord = haGroupStoreClient.getClusterRoleRecord();
+ assertNotNull(clusterRoleRecord);
+ ClusterRoleRecord expectedClusterRoleRecord = new ClusterRoleRecord(haGroupName,
+ HighAvailabilityPolicy.FAILOVER, zkUrl, ClusterRoleRecord.ClusterRole.ACTIVE, peerZKUrl,
+ ClusterRoleRecord.ClusterRole.STANDBY, 1);
+ assertEquals(expectedClusterRoleRecord, clusterRoleRecord);
+ }
+
+
+ @Test
+ public void testGetClusterRoleRecordWithValidPeerZKUrlButNoPeerRecord() throws Exception {
+ String haGroupName = testName.getMethodName();
+ HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(haGroupName, this.zkUrl, this.peerZKUrl,
+ ClusterRoleRecord.ClusterRole.ACTIVE, ClusterRoleRecord.ClusterRole.STANDBY, null);
+
+ // Create HAGroupStoreRecord for local cluster only (no peer record)
+ HAGroupStoreRecord localRecord = new HAGroupStoreRecord("v1.0", haGroupName,
+ HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+ createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, localRecord);
+
+ // Explicitly ensure no peer record exists
+ String peerPath = toPath(haGroupName);
+ if (peerHaAdmin.getCurator().checkExists().forPath(peerPath) != null) {
+ peerHaAdmin.getCurator().delete().quietly().forPath(peerPath);
+ }
+
+ Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
+
+ HAGroupStoreClient haGroupStoreClient = HAGroupStoreClient.getInstanceForZkUrl(CLUSTERS.getHBaseCluster1().getConfiguration(), haGroupName, zkUrl);
+ assertNotNull(haGroupStoreClient);
+
+ // Test getClusterRoleRecord when peer ZK URL is valid but no peer HAGroupStoreRecord exists in peer ZK
+ ClusterRoleRecord clusterRoleRecord = haGroupStoreClient.getClusterRoleRecord();
+ assertNotNull(clusterRoleRecord);
+ ClusterRoleRecord expectedClusterRoleRecord = new ClusterRoleRecord(haGroupName,
+ HighAvailabilityPolicy.FAILOVER, zkUrl, ClusterRoleRecord.ClusterRole.ACTIVE, this.peerZKUrl,
+ ClusterRoleRecord.ClusterRole.UNKNOWN, 1);
+ assertEquals(expectedClusterRoleRecord, clusterRoleRecord);
+ }
+
+ private HAGroupStoreRecord createHAGroupStoreRecord(String haGroupName) {
+ return new HAGroupStoreRecord("v1.0", haGroupName,
+ HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+ }
+
+ // Tests for getHAGroupNames static method
+ @Test
+ public void testGetHAGroupNamesWithSingleGroup() throws Exception {
+ String haGroupName = testName.getMethodName();
+ HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(haGroupName, this.zkUrl, this.peerZKUrl,
+ ClusterRoleRecord.ClusterRole.ACTIVE, ClusterRoleRecord.ClusterRole.STANDBY, null);
+
+ List<String> haGroupNames = HAGroupStoreClient.getHAGroupNames(zkUrl);
+
+ assertNotNull("HA group names list should not be null", haGroupNames);
+ assertTrue("HA group names list should contain the test group", haGroupNames.contains(haGroupName));
+
+ // Clean up
+ HAGroupStoreTestUtil.deleteHAGroupRecordInSystemTable(haGroupName, zkUrl);
+ }
+
+ @Test
+ public void testGetHAGroupNamesWithMultipleGroups() throws Exception {
+ // Delete old HA entry created by Before method
+ HAGroupStoreTestUtil.deleteHAGroupRecordInSystemTable(testName.getMethodName(), this.zkUrl);
+ // Verify that there are no groups returned
+ assertEquals(0, HAGroupStoreClient.getHAGroupNames(zkUrl).size());
+
+
+ String haGroupName1 = testName.getMethodName() + "_1";
+ String haGroupName2 = testName.getMethodName() + "_2";
+ String haGroupName3 = testName.getMethodName() + "_3";
+ String haGroupName4 = testName.getMethodName() + "_4";
+
+ // Insert multiple HA group records
+ HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(haGroupName1, this.zkUrl, this.peerZKUrl,
+ ClusterRoleRecord.ClusterRole.ACTIVE, ClusterRoleRecord.ClusterRole.STANDBY, null);
+ HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(haGroupName2, this.zkUrl, this.peerZKUrl,
+ ClusterRoleRecord.ClusterRole.STANDBY, ClusterRoleRecord.ClusterRole.ACTIVE, null);
+ HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(haGroupName3, this.zkUrl, this.peerZKUrl,
+ ClusterRoleRecord.ClusterRole.ACTIVE, ClusterRoleRecord.ClusterRole.STANDBY, null);
+ HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(haGroupName4, "bad_zk_url", this.peerZKUrl,
+ ClusterRoleRecord.ClusterRole.ACTIVE, ClusterRoleRecord.ClusterRole.STANDBY, this.zkUrl);
+
+ List<String> haGroupNames = HAGroupStoreClient.getHAGroupNames(zkUrl);
+
+ assertNotNull("HA group names list should not be null", haGroupNames);
+ assertTrue("HA group names list should contain haGroupName1", haGroupNames.contains(haGroupName1));
+ assertTrue("HA group names list should contain haGroupName2", haGroupNames.contains(haGroupName2));
+ assertTrue("HA group names list should contain haGroupName3", haGroupNames.contains(haGroupName3));
+ assertTrue("HA group names list should not contain haGroupName4", !haGroupNames.contains(haGroupName4));
+ assertEquals(3, haGroupNames.size());
+
+
+ // Clean up
+ HAGroupStoreTestUtil.deleteHAGroupRecordInSystemTable(haGroupName1, zkUrl);
+ HAGroupStoreTestUtil.deleteHAGroupRecordInSystemTable(haGroupName2, zkUrl);
+ HAGroupStoreTestUtil.deleteHAGroupRecordInSystemTable(haGroupName3, zkUrl);
+ }
+
+ @Test
+ public void testGetHAGroupNamesWithEmptyTable() throws Exception {
+ // First, delete any existing records that might interfere
+ List<String> existingGroups = HAGroupStoreClient.getHAGroupNames(zkUrl);
+ for (String groupName : existingGroups) {
+ if (groupName.startsWith("test")) { // Only clean up test groups
+ HAGroupStoreTestUtil.deleteHAGroupRecordInSystemTable(groupName, zkUrl);
+ }
+ }
+
+ // Verify the table is empty of test records
+ List<String> haGroupNames = HAGroupStoreClient.getHAGroupNames(zkUrl);
+ long testGroupCount = haGroupNames.stream()
+ .filter(name -> name.startsWith("test"))
+ .count();
+ assertEquals("Should have no test groups", 0, testGroupCount);
+ }
+
+ @Test
+ public void testGetHAGroupNamesAfterDeletingGroups() throws Exception {
+ String haGroupName1 = testName.getMethodName() + "_delete_1";
+ String haGroupName2 = testName.getMethodName() + "_delete_2";
+
+ // Insert HA group records
+ HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(haGroupName1, this.zkUrl, this.peerZKUrl,
+ ClusterRoleRecord.ClusterRole.ACTIVE, ClusterRoleRecord.ClusterRole.STANDBY, null);
+ HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(haGroupName2, this.zkUrl, this.peerZKUrl,
+ ClusterRoleRecord.ClusterRole.STANDBY, ClusterRoleRecord.ClusterRole.ACTIVE, null);
+
+ // Verify both groups exist
+ List<String> haGroupNames = HAGroupStoreClient.getHAGroupNames(zkUrl);
+ assertTrue("Should contain haGroupName1", haGroupNames.contains(haGroupName1));
+ assertTrue("Should contain haGroupName2", haGroupNames.contains(haGroupName2));
+
+ // Delete one group
+ HAGroupStoreTestUtil.deleteHAGroupRecordInSystemTable(haGroupName1, zkUrl);
+
+ // Verify only one group remains
+ haGroupNames = HAGroupStoreClient.getHAGroupNames(zkUrl);
+ assertFalse("Should not contain deleted haGroupName1", haGroupNames.contains(haGroupName1));
+ assertTrue("Should still contain haGroupName2", haGroupNames.contains(haGroupName2));
+
+ // Clean up remaining group
+ HAGroupStoreTestUtil.deleteHAGroupRecordInSystemTable(haGroupName2, zkUrl);
+
+ // Verify the group is gone
+ haGroupNames = HAGroupStoreClient.getHAGroupNames(zkUrl);
+ assertFalse("Should not contain deleted haGroupName2", haGroupNames.contains(haGroupName2));
+ }
+
+ @Test
+ public void testGetHAGroupNamesWithInvalidZkUrl() throws Exception {
+ String invalidZkUrl = "invalid:2181";
+
+ try {
+ HAGroupStoreClient.getHAGroupNames(invalidZkUrl);
+ fail("Expected SQLException for invalid ZK URL");
+ } catch (SQLException e) {
+ // Expected - the connection should fail with invalid ZK URL
+ assertNotNull("SQLException should have a message", e.getMessage());
+ }
+ }
+
+ @Test
+ public void testGetHAGroupNamesWithNullZkUrl() throws Exception {
+ try {
+ HAGroupStoreClient.getHAGroupNames(null);
+ fail("Expected SQLException for null ZK URL");
+ } catch (SQLException e) {
+ // Expected - the connection should fail with null ZK URL
+ assertNotNull("SQLException should have a message", e.getMessage());
+ }
}
-}
+}
\ No newline at end of file
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HAGroupStoreManagerIT.java b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HAGroupStoreManagerIT.java
index 9dfbb6cb4f..34d143e5e6 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HAGroupStoreManagerIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HAGroupStoreManagerIT.java
@@ -17,72 +17,337 @@
*/
package org.apache.phoenix.jdbc;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HConstants;
import org.apache.phoenix.end2end.NeedsOwnMiniClusterTest;
import org.apache.phoenix.query.BaseTest;
import org.apache.phoenix.thirdparty.com.google.common.collect.Maps;
+import org.apache.phoenix.util.HAGroupStoreTestUtil;
import org.apache.phoenix.util.ReadOnlyProps;
import org.junit.Before;
import org.junit.BeforeClass;
+import org.junit.Rule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
+import org.junit.rules.TestName;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
+import java.util.stream.Collectors;
+import static org.apache.phoenix.jdbc.HAGroupStoreClient.ZK_CONSISTENT_HA_NAMESPACE;
+import static org.apache.phoenix.jdbc.PhoenixHAAdmin.getLocalZkUrl;
import static org.apache.phoenix.jdbc.PhoenixHAAdmin.toPath;
import static org.apache.phoenix.query.QueryServices.CLUSTER_ROLE_BASED_MUTATION_BLOCK_ENABLED;
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
-
+/**
+ * Integration tests for {@link HAGroupStoreManager}.
+ */
@Category(NeedsOwnMiniClusterTest.class)
public class HAGroupStoreManagerIT extends BaseTest {
- private final PhoenixHAAdmin haAdmin = new PhoenixHAAdmin(config);
- private static final Long ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS = 1000L;
+
+ @Rule
+ public TestName testName = new TestName();
+
+ private PhoenixHAAdmin haAdmin;
+ private static final Long ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS = 2000L;
+ private String zkUrl;
+ private String peerZKUrl;
+ private static final HighAvailabilityTestingUtility.HBaseTestingUtilityPair CLUSTERS = new HighAvailabilityTestingUtility.HBaseTestingUtilityPair();
@BeforeClass
public static synchronized void doSetup() throws Exception {
- Map<String, String> props = Maps.newHashMapWithExpectedSize(1);
+ Map<String, String> props = Maps.newHashMapWithExpectedSize(2);
props.put(CLUSTER_ROLE_BASED_MUTATION_BLOCK_ENABLED, "true");
setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator()));
+ CLUSTERS.start();
}
@Before
public void before() throws Exception {
- // Clean up all the existing CRRs
- List<ClusterRoleRecord> crrs = haAdmin.listAllClusterRoleRecordsOnZookeeper();
- for (ClusterRoleRecord crr : crrs) {
- haAdmin.getCurator().delete().forPath(toPath(crr.getHaGroupName()));
+ haAdmin = new PhoenixHAAdmin(config, ZK_CONSISTENT_HA_NAMESPACE);
+ zkUrl = getLocalZkUrl(config);
+ this.peerZKUrl = CLUSTERS.getZkUrl2();
+
+ // Clean up existing HAGroupStoreRecords
+ try {
+ List<String> haGroupNames = HAGroupStoreClient.getHAGroupNames(zkUrl);
+ for (String haGroupName : haGroupNames) {
+ haAdmin.getCurator().delete().quietly().forPath(toPath(haGroupName));
+ }
+ } catch (Exception e) {
+ // Ignore cleanup errors
}
+ HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(testName.getMethodName(), zkUrl, peerZKUrl,
+ ClusterRoleRecord.ClusterRole.ACTIVE, ClusterRoleRecord.ClusterRole.STANDBY, null);
+ }
+
+ @Test
+ public void testMutationBlockingWithSingleHAGroup() throws Exception {
+ String haGroupName = testName.getMethodName();
+ HAGroupStoreManager haGroupStoreManager = HAGroupStoreManager.getInstance(config);
+
+ // Initially no mutation should be blocked
+ assertFalse(haGroupStoreManager.isMutationBlocked(haGroupName));
+
+ // Update to ACTIVE_TO_STANDBY role (should block mutations)
+ HAGroupStoreRecord transitionRecord = new HAGroupStoreRecord(
+ "1.0", haGroupName, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY);
+
+ haAdmin.updateHAGroupStoreRecordInZooKeeper(haGroupName, transitionRecord, 0);
+ Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
+
+ // Now mutations should be blocked
+ assertTrue(haGroupStoreManager.isMutationBlocked(haGroupName));
}
@Test
- public void testHAGroupStoreManagerWithSingleCRR() throws Exception {
+ public void testMutationBlockingWithMultipleHAGroups() throws Exception {
+ String haGroupName1 = testName.getMethodName() + "_1";
+ String haGroupName2 = testName.getMethodName() + "_2";
HAGroupStoreManager haGroupStoreManager = HAGroupStoreManager.getInstance(config);
- // Setup initial CRRs
- ClusterRoleRecord crr1 = new ClusterRoleRecord("failover",
- HighAvailabilityPolicy.FAILOVER, haAdmin.getZkUrl(), ClusterRoleRecord.ClusterRole.ACTIVE,
- "random-zk-url", ClusterRoleRecord.ClusterRole.STANDBY, 1L);
- ClusterRoleRecord crr2 = new ClusterRoleRecord("parallel",
- HighAvailabilityPolicy.PARALLEL, haAdmin.getZkUrl(), ClusterRoleRecord.ClusterRole.ACTIVE,
- "random-zk-url", ClusterRoleRecord.ClusterRole.STANDBY, 1L);
- haAdmin.createOrUpdateDataOnZookeeper(crr1);
- haAdmin.createOrUpdateDataOnZookeeper(crr2);
+ // Create two HA groups with ACTIVE and ACTIVE_NOT_IN_SYNC roles
+ HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(haGroupName1, zkUrl,
+ this.peerZKUrl, ClusterRoleRecord.ClusterRole.ACTIVE, ClusterRoleRecord.ClusterRole.ACTIVE, null);
+ HAGroupStoreRecord activeRecord1 = new HAGroupStoreRecord(
+ "1.0", haGroupName1, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+ haAdmin.createHAGroupStoreRecordInZooKeeper(activeRecord1);
+
+ HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(haGroupName2, zkUrl,
+ this.peerZKUrl, ClusterRoleRecord.ClusterRole.ACTIVE, ClusterRoleRecord.ClusterRole.ACTIVE, null);
+
+ // No mutations should be blocked
+ assertFalse(haGroupStoreManager.isMutationBlocked(haGroupName1));
+ assertFalse(haGroupStoreManager.isMutationBlocked(haGroupName2));
+
+ // Update only second group to ACTIVE_NOT_IN_SYNC_TO_STANDBY
+ HAGroupStoreRecord transitionRecord2 = new HAGroupStoreRecord(
+ "1.0", haGroupName2, HAGroupStoreRecord.HAGroupState.ACTIVE_NOT_IN_SYNC_TO_STANDBY);
+
+ haAdmin.updateHAGroupStoreRecordInZooKeeper(haGroupName2, transitionRecord2, 0);
Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
- assertFalse(haGroupStoreManager.isMutationBlocked());
+ // Global mutations should be blocked due to second group
+ assertFalse(haGroupStoreManager.isMutationBlocked(haGroupName1));
+ assertTrue(haGroupStoreManager.isMutationBlocked(haGroupName2));
+ }
+
+ @Test
+ public void testGetHAGroupStoreRecord() throws Exception {
+ String haGroupName = testName.getMethodName();
+ HAGroupStoreManager haGroupStoreManager = HAGroupStoreManager.getInstance(config);
+
+ // Get record from HAGroupStoreManager
+ Optional<HAGroupStoreRecord> recordOpt = haGroupStoreManager.getHAGroupStoreRecord(haGroupName);
+ // Should be present
+ assertTrue(recordOpt.isPresent());
+
+
+ // Delete record from System Table
+ HAGroupStoreTestUtil.deleteHAGroupRecordInSystemTable(haGroupName, zkUrl);
+ // Delete record from ZK
+ haAdmin.deleteHAGroupStoreRecordInZooKeeper(haGroupName);
+ // Sleep for propagation time so that it is now reflected in cache.
+ Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
+ // Get record from HAGroupStoreManager
+ recordOpt = haGroupStoreManager.getHAGroupStoreRecord(haGroupName);
+ // Should not be present
+ assertFalse(recordOpt.isPresent());
+
+ // Create record again in System Table
+ HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(haGroupName, zkUrl, this.peerZKUrl,
+ ClusterRoleRecord.ClusterRole.ACTIVE, ClusterRoleRecord.ClusterRole.STANDBY, null);
+ // Now it should be present
+ Optional<HAGroupStoreRecord> retrievedOpt = haGroupStoreManager.getHAGroupStoreRecord(haGroupName);
+ assertTrue(retrievedOpt.isPresent());
+
+ // Record for comparison
+ HAGroupStoreRecord record = new HAGroupStoreRecord(
+ HAGroupStoreRecord.DEFAULT_PROTOCOL_VERSION, haGroupName, HAGroupStoreRecord.HAGroupState.ACTIVE_NOT_IN_SYNC);
+
+ // Complete object comparison instead of field-by-field
+ assertEquals(record, retrievedOpt.get());
+ }
+
+ @Test
+ public void testInvalidateHAGroupStoreClient() throws Exception {
+ String haGroupName = testName.getMethodName();
+ HAGroupStoreManager haGroupStoreManager = HAGroupStoreManager.getInstance(config);
+
+ // Create a HAGroupStoreRecord first
+ HAGroupStoreRecord record = new HAGroupStoreRecord(
+ "1.0", haGroupName, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+
+ haAdmin.createHAGroupStoreRecordInZooKeeper(record);
+ Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
+
+ // Ensure we can get the record
+ Optional<HAGroupStoreRecord> recordOpt = haGroupStoreManager.getHAGroupStoreRecord(haGroupName);
+ assertTrue(recordOpt.isPresent());
+
+ // Invalidate the specific HA group client
+ haGroupStoreManager.invalidateHAGroupStoreClient(haGroupName, false);
+
+ // Should still be able to get the record after invalidation
+ recordOpt = haGroupStoreManager.getHAGroupStoreRecord(haGroupName);
+ assertTrue(recordOpt.isPresent());
+
+ // Test global invalidation
+ haGroupStoreManager.invalidateHAGroupStoreClient(false);
+
+ // Should still be able to get the record after global invalidation
+ recordOpt = haGroupStoreManager.getHAGroupStoreRecord(haGroupName);
+ assertTrue(recordOpt.isPresent());
+ }
+
+ @Test
+ public void testMutationBlockDisabled() throws Exception {
+ String haGroupName = testName.getMethodName();
+
+ // Create configuration with mutation block disabled
+ Configuration conf = new Configuration();
+ conf.set(CLUSTER_ROLE_BASED_MUTATION_BLOCK_ENABLED, "false");
+ conf.set(HConstants.ZOOKEEPER_QUORUM, getLocalZkUrl(config));
+
+ HAGroupStoreManager haGroupStoreManager = HAGroupStoreManager.getInstance(config);
- crr1 = new ClusterRoleRecord("failover",
- HighAvailabilityPolicy.FAILOVER, haAdmin.getZkUrl(), ClusterRoleRecord.ClusterRole.ACTIVE,
- "random-zk-url", ClusterRoleRecord.ClusterRole.STANDBY, 2L);
- crr2 = new ClusterRoleRecord("parallel",
- HighAvailabilityPolicy.PARALLEL, haAdmin.getZkUrl(), ClusterRoleRecord.ClusterRole.ACTIVE_TO_STANDBY,
- "random-zk-url", ClusterRoleRecord.ClusterRole.STANDBY, 2L);
- haAdmin.createOrUpdateDataOnZookeeper(crr1);
- haAdmin.createOrUpdateDataOnZookeeper(crr2);
+ // Create HAGroupStoreRecord with ACTIVE_TO_STANDBY role
+ HAGroupStoreRecord transitionRecord = new HAGroupStoreRecord(
+ "1.0", haGroupName, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY);
+ haAdmin.createHAGroupStoreRecordInZooKeeper(transitionRecord);
Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
- assert haGroupStoreManager.isMutationBlocked();
+ // Mutations should not be blocked even with ACTIVE_TO_STANDBY role
+ assertFalse(haGroupStoreManager.isMutationBlocked(haGroupName));
}
-}
+
+ @Test
+ public void testSetHAGroupStatusToStoreAndForward() throws Exception {
+ String haGroupName = testName.getMethodName();
+ HAGroupStoreManager haGroupStoreManager = HAGroupStoreManager.getInstance(config);
+
+ // Create an initial HAGroupStoreRecord with ACTIVE status
+ HAGroupStoreRecord initialRecord = new HAGroupStoreRecord(
+ "1.0", haGroupName, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+
+ haAdmin.createHAGroupStoreRecordInZooKeeper(initialRecord);
+ Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
+
+ // Set the HA group status to store and forward (ACTIVE_NOT_IN_SYNC)
+ haGroupStoreManager.setHAGroupStatusToStoreAndForward(haGroupName);
+ Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
+
+ // Verify the status was updated to ACTIVE_NOT_IN_SYNC
+ Optional<HAGroupStoreRecord> updatedRecordOpt = haGroupStoreManager.getHAGroupStoreRecord(haGroupName);
+ assertTrue(updatedRecordOpt.isPresent());
+ HAGroupStoreRecord updatedRecord = updatedRecordOpt.get();
+ assertEquals(HAGroupStoreRecord.HAGroupState.ACTIVE_NOT_IN_SYNC, updatedRecord.getHAGroupState());
+ }
+
+ @Test
+ public void testSetHAGroupStatusRecordToSync() throws Exception {
+ String haGroupName = testName.getMethodName();
+ HAGroupStoreManager haGroupStoreManager = HAGroupStoreManager.getInstance(config);
+
+ // Create an initial HAGroupStoreRecord with ACTIVE_NOT_IN_SYNC status
+ HAGroupStoreRecord initialRecord = new HAGroupStoreRecord(
+ "1.0", haGroupName, HAGroupStoreRecord.HAGroupState.ACTIVE_NOT_IN_SYNC);
+
+ haAdmin.createHAGroupStoreRecordInZooKeeper(initialRecord);
+ Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
+
+ // Set the HA group status to sync (ACTIVE)
+ haGroupStoreManager.setHAGroupStatusRecordToSync(haGroupName);
+ Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
+
+ // Verify the status was updated to ACTIVE
+ Optional<HAGroupStoreRecord> updatedRecordOpt = haGroupStoreManager.getHAGroupStoreRecord(haGroupName);
+ assertTrue(updatedRecordOpt.isPresent());
+ HAGroupStoreRecord updatedRecord = updatedRecordOpt.get();
+ assertEquals(ClusterRoleRecord.ClusterRole.ACTIVE, updatedRecord.getClusterRole());
+ }
+
+ @Test
+ public void testGetHAGroupNamesFiltersCorrectlyByZkUrl() throws Exception {
+ HAGroupStoreManager haGroupStoreManager = HAGroupStoreManager.getInstance(config);
+
+ List<String> initialHAGroupNames = haGroupStoreManager.getHAGroupNames();
+
+ // Create HA groups with current zkUrl as ZK_URL_1
+ String haGroupWithCurrentZkUrl = testName.getMethodName() + "_current_zk";
+ HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(haGroupWithCurrentZkUrl, zkUrl, this.peerZKUrl,
+ ClusterRoleRecord.ClusterRole.ACTIVE, ClusterRoleRecord.ClusterRole.STANDBY, null);
+
+ // Create HA group with current zkUrl as ZK_URL_2 (swapped)
+ String haGroupWithCurrentZkUrlAsPeer = testName.getMethodName() + "_current_as_peer";
+ HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(haGroupWithCurrentZkUrlAsPeer, this.peerZKUrl, zkUrl,
+ ClusterRoleRecord.ClusterRole.STANDBY, ClusterRoleRecord.ClusterRole.ACTIVE, zkUrl);
+
+ // Create HA group with different zkUrl (should not appear in results)
+ String differentZkUrl = "localhost:2182:/different";
+ String haGroupWithDifferentZkUrl = testName.getMethodName() + "_different_zk";
+ HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(haGroupWithDifferentZkUrl, differentZkUrl, "localhost:2183:/other",
+ ClusterRoleRecord.ClusterRole.ACTIVE, ClusterRoleRecord.ClusterRole.STANDBY, zkUrl);
+
+ // Get HA group names - should only return groups where current zkUrl matches ZK_URL_1 or ZK_URL_2
+ List<String> filteredHAGroupNames = haGroupStoreManager.getHAGroupNames();
+
+ // Extract only new groups from filteredHAGroupNames
+ List<String> newHAGroupNames = filteredHAGroupNames.stream()
+ .filter(name -> !initialHAGroupNames.contains(name))
+ .collect(Collectors.toList());
+
+ // Check size of filteredHAGroupNames
+ assertEquals(2, newHAGroupNames.size());
+
+
+ // Should contain groups where current zkUrl is involved
+ assertTrue("Should contain HA group with current zkUrl as ZK_URL_1",
+ newHAGroupNames.contains(haGroupWithCurrentZkUrl));
+ assertTrue("Should contain HA group with current zkUrl as ZK_URL_2",
+ newHAGroupNames.contains(haGroupWithCurrentZkUrlAsPeer));
+ // Should NOT contain HA group with different zkUrl
+ assertFalse("Should NOT contain HA group with different zkUrl",
+ newHAGroupNames.contains(haGroupWithDifferentZkUrl));
+
+
+ // Clean up
+ HAGroupStoreTestUtil.deleteHAGroupRecordInSystemTable(haGroupWithCurrentZkUrl, zkUrl);
+ HAGroupStoreTestUtil.deleteHAGroupRecordInSystemTable(haGroupWithCurrentZkUrlAsPeer, zkUrl);
+ HAGroupStoreTestUtil.deleteHAGroupRecordInSystemTable(haGroupWithDifferentZkUrl, zkUrl);
+ }
+
+ @Test
+ public void testGetHAGroupNamesWhenNoMatchingZkUrl() throws Exception {
+ HAGroupStoreManager haGroupStoreManager = HAGroupStoreManager.getInstance(config);
+
+ // Clean up existing HA group created in before()
+ String testHAGroupName = testName.getMethodName();
+ HAGroupStoreTestUtil.deleteHAGroupRecordInSystemTable(testHAGroupName, zkUrl);
+
+ // Create HA groups with completely different zkUrls
+ String differentZkUrl1 = "localhost:2182:/different1";
+ String differentZkUrl2 = "localhost:2183:/different2";
+ String haGroupWithDifferentZkUrls = testName.getMethodName() + "_different";
+ HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(haGroupWithDifferentZkUrls, differentZkUrl1, differentZkUrl2,
+ ClusterRoleRecord.ClusterRole.ACTIVE, ClusterRoleRecord.ClusterRole.STANDBY, zkUrl);
+
+ // Get HA group names - should not contain the group with different zkUrls
+ List<String> filteredHAGroupNames = haGroupStoreManager.getHAGroupNames();
+
+ // Should NOT contain the HA group with different zkUrls
+ assertFalse("Should NOT contain HA group with different zkUrls",
+ filteredHAGroupNames.contains(haGroupWithDifferentZkUrls));
+
+ // Clean up
+ HAGroupStoreTestUtil.deleteHAGroupRecordInSystemTable(haGroupWithDifferentZkUrls, zkUrl);
+
+ }
+
+}
\ No newline at end of file
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HighAvailabilityGroupTestIT.java b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HighAvailabilityGroupTestIT.java
index eb7ddc5505..e23dbdb260 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HighAvailabilityGroupTestIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HighAvailabilityGroupTestIT.java
@@ -302,8 +302,9 @@ public class HighAvailabilityGroupTestIT {
// Make ZK connectable and the cluster role record be missing
CuratorFramework curator = mock(CuratorFramework.class);
when(curator.blockUntilConnected(anyInt(), any(TimeUnit.class))).thenReturn(true);
- HighAvailabilityGroup.CURATOR_CACHE.put(ZK2, curator);
- HighAvailabilityGroup.CURATOR_CACHE.put(ZK1, curator);
+ // Use proper cache keys with default namespace
+ HighAvailabilityGroup.CURATOR_CACHE.put(HighAvailabilityGroup.generateCacheKey(ZK2, null), curator);
+ HighAvailabilityGroup.CURATOR_CACHE.put(HighAvailabilityGroup.generateCacheKey(ZK1, null), curator);
ExistsBuilder eb = mock(ExistsBuilder.class);
when(eb.forPath(anyString())).thenReturn(null);
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/ParallelPhoenixConnectionIT.java b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/ParallelPhoenixConnectionIT.java
index 32d011cd9c..9c66e1c956 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/ParallelPhoenixConnectionIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/ParallelPhoenixConnectionIT.java
@@ -36,6 +36,7 @@ import static org.apache.phoenix.monitoring.GlobalClientMetrics.GLOBAL_HA_PARALL
import static org.apache.phoenix.monitoring.GlobalClientMetrics.GLOBAL_HA_PARALLEL_POOL2_TASK_REJECTED_COUNTER;
import static org.apache.phoenix.query.BaseTest.extractThreadPoolExecutorFromCQSI;
import static org.apache.phoenix.query.QueryServices.AUTO_COMMIT_ATTRIB;
+import static org.apache.phoenix.query.QueryServices.CLUSTER_ROLE_BASED_MUTATION_BLOCK_ENABLED;
import static org.apache.phoenix.query.QueryServices.CQSI_THREAD_POOL_ALLOW_CORE_THREAD_TIMEOUT;
import static org.apache.phoenix.query.QueryServices.CQSI_THREAD_POOL_CORE_POOL_SIZE;
import static org.apache.phoenix.query.QueryServices.CQSI_THREAD_POOL_ENABLED;
@@ -51,14 +52,17 @@ import static org.junit.Assert.fail;
import static org.junit.Assume.assumeTrue;
import static org.mockito.Mockito.doAnswer;
-import java.lang.reflect.Field;
import java.sql.Connection;
+import java.lang.reflect.Field;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ThreadPoolExecutor;
@@ -66,9 +70,11 @@ import java.util.concurrent.ThreadPoolExecutor;
import org.apache.commons.lang3.RandomStringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.client.RetriesExhaustedWithDetailsException;
import org.apache.hadoop.hbase.util.VersionInfo;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.phoenix.end2end.NeedsOwnMiniClusterTest;
+import org.apache.phoenix.exception.MutationBlockedIOException;
import org.apache.phoenix.jdbc.ClusterRoleRecord.ClusterRole;
import org.apache.phoenix.jdbc.HighAvailabilityTestingUtility.HBaseTestingUtilityPair;
import org.apache.phoenix.jdbc.PhoenixStatement.Operation;
@@ -81,6 +87,7 @@ import org.apache.phoenix.monitoring.MetricType;
import org.apache.phoenix.query.ConnectionQueryServices;
import org.apache.phoenix.query.ConnectionQueryServicesImpl;
import org.apache.phoenix.query.QueryServices;
+import org.apache.phoenix.util.HAGroupStoreTestUtil;
import org.apache.phoenix.util.PhoenixRuntime;
import org.apache.phoenix.util.QueryUtil;
import org.junit.AfterClass;
@@ -118,6 +125,8 @@ public class ParallelPhoenixConnectionIT {
@BeforeClass
public static void setUpBeforeClass() throws Exception {
+ CLUSTERS.getHBaseCluster1().getConfiguration().setBoolean(CLUSTER_ROLE_BASED_MUTATION_BLOCK_ENABLED, true);
+ CLUSTERS.getHBaseCluster2().getConfiguration().setBoolean(CLUSTER_ROLE_BASED_MUTATION_BLOCK_ENABLED, true);
CLUSTERS.start();
DriverManager.registerDriver(PhoenixDriver.INSTANCE);
DriverManager.registerDriver(new PhoenixTestDriver());
@@ -131,6 +140,7 @@ public class ParallelPhoenixConnectionIT {
GLOBAL_PROPERTIES.setProperty(CQSI_THREAD_POOL_MAX_QUEUE, String.valueOf(23));
GLOBAL_PROPERTIES.setProperty(CQSI_THREAD_POOL_ALLOW_CORE_THREAD_TIMEOUT,
String.valueOf(true));
+ GLOBAL_PROPERTIES.setProperty("hbase.client.retries.number", "0");
GLOBAL_PROPERTIES.setProperty(CQSI_THREAD_POOL_METRICS_ENABLED, String.valueOf(true));
}
@@ -356,6 +366,77 @@ public class ParallelPhoenixConnectionIT {
}
}
+ /**
+ * Test Phoenix connection creation and basic operations with HBase both cluster is ACTIVE_TO_STANDBY role.
+ */
+ @Test
+ public void testBothClusterATSRole() throws Exception {
+ String zkUrl1 = CLUSTERS.getZkUrl1();
+ String zkUrl2 = CLUSTERS.getZkUrl2();
+ String haGroupName = testName.getMethodName();
+ HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(haGroupName, zkUrl1, zkUrl2,
+ ClusterRole.ACTIVE_TO_STANDBY, ClusterRole.ACTIVE_TO_STANDBY, null);
+ HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(haGroupName, zkUrl2, zkUrl1,
+ ClusterRole.ACTIVE_TO_STANDBY, ClusterRole.ACTIVE_TO_STANDBY, null);
+ try (ParallelPhoenixConnection conn = (ParallelPhoenixConnection) getParallelConnection()) {
+ PhoenixConnection conn1 = conn.futureConnection1.get();
+ PhoenixConnection conn2 = conn.futureConnection2.get();
+ conn1.setHAGroupName(haGroupName);
+ conn2.setHAGroupName(haGroupName);
+ doTestBasicOperationsWithConnection(conn, tableName, haGroupName);
+ fail("Expected MutationBlockedIOException to be thrown");
+ } catch (SQLException e) {
+ assertTrue(containsMutationBlockedException(e));
+ } finally {
+ CLUSTERS.transitClusterRole(haGroup, ClusterRole.ACTIVE, ClusterRole.STANDBY);
+ }
+ }
+
+ /**
+ * Test Phoenix connection creation and
+ * basic operations with HBase one cluster is ACTIVE_TO_STANDBY role
+ * and other in ACTIVE role.
+ */
+ @Test
+ public void testOneClusterATSRoleWithActive() throws Exception {
+ testOneClusterATSRole(ClusterRole.ACTIVE);
+ }
+
+ /**
+ * Test Phoenix connection creation and
+ * basic operations with HBase one cluster is ACTIVE_TO_STANDBY role
+ * and other in STANDBY role.
+ */
+ @Test
+ public void testOneClusterATSRoleWithStandby() throws Exception {
+ testOneClusterATSRole(ClusterRole.STANDBY);
+ }
+
+ private void testOneClusterATSRole(ClusterRole otherRole) throws Exception {
+ CLUSTERS.transitClusterRole(haGroup, ClusterRole.ACTIVE_TO_STANDBY, otherRole);
+ try (Connection conn = getParallelConnection()) {
+ doTestBasicOperationsWithConnection(conn, tableName, haGroupName);
+ } catch (SQLException e) {
+ fail("Expected no exception to be thrown as one cluster is "
+ + "in ACTIVE_TO_STANDBY and other in " + otherRole);
+ } finally {
+ CLUSTERS.transitClusterRole(haGroup, ClusterRole.ACTIVE, ClusterRole.STANDBY);
+ }
+ }
+
+ private boolean containsMutationBlockedException(SQLException e) {
+ Throwable cause = e.getCause();
+ // Recursively check for MutationBlockedIOException buried in exception stack.
+ while (cause != null) {
+ if (cause instanceof RetriesExhaustedWithDetailsException) {
+ RetriesExhaustedWithDetailsException re = (RetriesExhaustedWithDetailsException) cause;
+ return re.getCause(0) instanceof MutationBlockedIOException;
+ }
+ cause = cause.getCause();
+ }
+ return false;
+ }
+
/**
* Test Phoenix connection creation and basic operations.
*/
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/PhoenixHAAdminIT.java b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/PhoenixHAAdminIT.java
new file mode 100644
index 0000000000..5e2685d03a
--- /dev/null
+++ b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/PhoenixHAAdminIT.java
@@ -0,0 +1,481 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.jdbc;
+
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.phoenix.end2end.NeedsOwnMiniClusterTest;
+import org.apache.phoenix.exception.StaleHAGroupStoreRecordVersionException;
+import org.apache.phoenix.query.BaseTest;
+import org.apache.phoenix.thirdparty.com.google.common.collect.Maps;
+import org.apache.phoenix.util.ReadOnlyProps;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.data.Stat;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.rules.TestName;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static org.apache.phoenix.jdbc.HAGroupStoreClient.ZK_CONSISTENT_HA_NAMESPACE;
+import static org.apache.phoenix.jdbc.PhoenixHAAdmin.toPath;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+/**
+ * Integration tests for {@link PhoenixHAAdmin} HAGroupStoreRecord operations
+ */
+@Category(NeedsOwnMiniClusterTest.class)
+public class PhoenixHAAdminIT extends BaseTest {
+
+ private static final HighAvailabilityTestingUtility.HBaseTestingUtilityPair CLUSTERS = new HighAvailabilityTestingUtility.HBaseTestingUtilityPair();
+ private PhoenixHAAdmin haAdmin;
+ private PhoenixHAAdmin peerHaAdmin;
+
+ @Rule
+ public TestName testName = new TestName();
+
+ @BeforeClass
+ public static synchronized void doSetup() throws Exception {
+ Map<String, String> props = Maps.newHashMapWithExpectedSize(1);
+ setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator()));
+ CLUSTERS.start();
+ }
+
+ @Before
+ public void before() throws Exception {
+ haAdmin = new PhoenixHAAdmin(CLUSTERS.getHBaseCluster1().getConfiguration(), ZK_CONSISTENT_HA_NAMESPACE);
+ peerHaAdmin = new PhoenixHAAdmin(CLUSTERS.getHBaseCluster2().getConfiguration(), ZK_CONSISTENT_HA_NAMESPACE);
+ cleanupTestZnodes();
+ }
+
+ @After
+ public void after() throws Exception {
+ cleanupTestZnodes();
+ if (haAdmin != null) {
+ haAdmin.close();
+ }
+ if (peerHaAdmin != null) {
+ peerHaAdmin.close();
+ }
+ }
+
+ private void cleanupTestZnodes() throws Exception {
+ haAdmin.getCurator().delete().quietly().forPath(toPath(testName.getMethodName()));
+ peerHaAdmin.getCurator().delete().quietly().forPath(toPath(testName.getMethodName()));
+ }
+
+ @Test
+ public void testCreateHAGroupStoreRecordInZooKeeper() throws Exception {
+ String haGroupName = testName.getMethodName();
+ String peerZKUrl = CLUSTERS.getZkUrl2();
+
+ HAGroupStoreRecord record = new HAGroupStoreRecord(
+ "v1.0",
+ haGroupName,
+ HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC
+ );
+
+ // Create the record in ZooKeeper
+ haAdmin.createHAGroupStoreRecordInZooKeeper(record);
+
+ // Verify the record was created by reading it back
+ byte[] data = haAdmin.getCurator().getData().forPath(toPath(haGroupName));
+ HAGroupStoreRecord savedRecord = HAGroupStoreRecord.fromJson(data).get();
+
+ assertEquals(record.getHaGroupName(), savedRecord.getHaGroupName());
+ assertEquals(record.getClusterRole(), savedRecord.getClusterRole());
+ }
+
+ @Test
+ public void testCreateHAGroupStoreRecordInZooKeeperWithExistingNode() throws Exception {
+ String haGroupName = testName.getMethodName();
+
+ HAGroupStoreRecord record1 = new HAGroupStoreRecord(
+ "v1.0",
+ haGroupName,
+ HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC
+ );
+
+ // Create the first record
+ haAdmin.createHAGroupStoreRecordInZooKeeper(record1);
+
+ // Try to create again with different data
+ HAGroupStoreRecord record2 = new HAGroupStoreRecord(
+ "v1.0",
+ haGroupName,
+ HAGroupStoreRecord.HAGroupState.STANDBY
+ );
+
+ // This should throw an exception due to NodeExistsException handling
+ try {
+ haAdmin.createHAGroupStoreRecordInZooKeeper(record2);
+ fail("Expected NodeExistsException");
+ } catch (IOException e) {
+ // Expected exception
+ assertTrue(e.getCause() instanceof KeeperException.NodeExistsException);
+ assertTrue(e.getMessage().contains("Failed to create HAGroupStoreRecord for HA group"));
+ }
+
+ // Verify the original record is still there (not overwritten)
+ byte[] data = haAdmin.getCurator().getData().forPath(toPath(haGroupName));
+ HAGroupStoreRecord savedRecord = HAGroupStoreRecord.fromJson(data).get();
+
+ assertEquals(record1.getHAGroupState(), savedRecord.getHAGroupState());
+ assertEquals(record1.getProtocolVersion(), savedRecord.getProtocolVersion());
+ assertEquals(record1.getHaGroupName(), savedRecord.getHaGroupName());
+ }
+
+ @Test
+ public void testUpdateHAGroupStoreRecordInZooKeeper() throws Exception {
+ String haGroupName = testName.getMethodName();
+ String peerZKUrl = CLUSTERS.getZkUrl2();
+
+ // Create initial record
+ HAGroupStoreRecord initialRecord = new HAGroupStoreRecord(
+ "v1.0",
+ haGroupName,
+ HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC
+ );
+
+ haAdmin.createHAGroupStoreRecordInZooKeeper(initialRecord);
+
+ // Get the current stat for version checking
+ Stat stat = haAdmin.getCurator().checkExists().forPath(toPath(haGroupName));
+ assertNotNull(stat);
+ int currentVersion = stat.getVersion();
+
+ // Update the record
+ HAGroupStoreRecord updatedRecord = new HAGroupStoreRecord(
+ "v1.0",
+ haGroupName,
+ HAGroupStoreRecord.HAGroupState.STANDBY
+ );
+
+ haAdmin.updateHAGroupStoreRecordInZooKeeper(haGroupName, updatedRecord, currentVersion);
+
+ // Verify the record was updated
+ byte[] data = haAdmin.getCurator().getData().forPath(toPath(haGroupName));
+ HAGroupStoreRecord savedRecord = HAGroupStoreRecord.fromJson(data).get();
+
+ assertEquals(updatedRecord.getClusterRole(), savedRecord.getClusterRole());
+ assertEquals(updatedRecord.getHAGroupState(), savedRecord.getHAGroupState());
+ assertEquals(updatedRecord.getProtocolVersion(), savedRecord.getProtocolVersion());
+ assertEquals(updatedRecord.getHaGroupName(), savedRecord.getHaGroupName());
+ }
+
+ @Test
+ public void testUpdateHAGroupStoreRecordInZooKeeperWithStaleVersion() throws Exception {
+ String haGroupName = testName.getMethodName();
+ String peerZKUrl = CLUSTERS.getZkUrl2();
+
+ // Create initial record
+ HAGroupStoreRecord initialRecord = new HAGroupStoreRecord(
+ "v1.0",
+ haGroupName,
+ HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC
+ );
+
+ haAdmin.createHAGroupStoreRecordInZooKeeper(initialRecord);
+
+ // Get the current stat for version checking
+ Stat stat = haAdmin.getCurator().checkExists().forPath(toPath(haGroupName));
+ assertNotNull(stat);
+ int currentVersion = stat.getVersion();
+
+ // Update the record with current version (should succeed)
+ HAGroupStoreRecord updatedRecord = new HAGroupStoreRecord(
+ "v1.0",
+ haGroupName,
+ HAGroupStoreRecord.HAGroupState.STANDBY
+ );
+
+ haAdmin.updateHAGroupStoreRecordInZooKeeper(haGroupName, updatedRecord, currentVersion);
+
+ // Try to update again with the same (now stale) version - should fail
+ HAGroupStoreRecord anotherUpdate = new HAGroupStoreRecord(
+ "v1.0",
+ haGroupName,
+ HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY
+ );
+
+ try {
+ haAdmin.updateHAGroupStoreRecordInZooKeeper(haGroupName, anotherUpdate, currentVersion);
+ fail("Expected StaleHAGroupStoreRecordVersionException");
+ } catch (StaleHAGroupStoreRecordVersionException e) {
+ // Expected exception
+ assertTrue(e.getMessage().contains("with cached stat version"));
+ }
+ }
+
+ @Test
+ public void testGetHAGroupStoreRecordInZooKeeper() throws Exception {
+ String haGroupName = testName.getMethodName();
+ String peerZKUrl = CLUSTERS.getZkUrl2();
+
+ // Create initial record
+ HAGroupStoreRecord initialRecord = new HAGroupStoreRecord(
+ "v1.0",
+ haGroupName,
+ HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC
+ );
+
+ haAdmin.createHAGroupStoreRecordInZooKeeper(initialRecord);
+
+ // Get the record and stat
+ Pair<HAGroupStoreRecord, Stat> result = haAdmin.getHAGroupStoreRecordInZooKeeper(haGroupName);
+
+ assertNotNull(result);
+ assertNotNull(result.getLeft());
+ assertNotNull(result.getRight());
+
+ HAGroupStoreRecord retrievedRecord = result.getLeft();
+ Stat stat = result.getRight();
+
+ assertEquals(initialRecord.getHaGroupName(), retrievedRecord.getHaGroupName());
+ assertEquals(initialRecord.getClusterRole(), retrievedRecord.getClusterRole());
+ assertEquals(initialRecord.getHAGroupState(), retrievedRecord.getHAGroupState());
+ assertEquals(initialRecord.getProtocolVersion(), retrievedRecord.getProtocolVersion());
+
+ // Verify stat is valid
+ assertTrue(stat.getVersion() >= 0);
+ assertTrue(stat.getCtime() > 0);
+ assertTrue(stat.getMtime() > 0);
+ }
+
+ @Test
+ public void testGetHAGroupStoreRecordInZooKeeperNonExistentNode() throws Exception {
+ assertNull(haAdmin.getHAGroupStoreRecordInZooKeeper(testName.getMethodName()).getLeft());
+ assertNull(haAdmin.getHAGroupStoreRecordInZooKeeper(testName.getMethodName()).getLeft());
+ }
+
+ @Test
+ public void testCompleteWorkflowCreateUpdateGet() throws Exception {
+ String haGroupName = testName.getMethodName();
+ String peerZKUrl = CLUSTERS.getZkUrl2();
+
+ // Step 1: Create initial record
+ HAGroupStoreRecord initialRecord = new HAGroupStoreRecord(
+ "v1.0",
+ haGroupName,
+ HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC
+ );
+
+ haAdmin.createHAGroupStoreRecordInZooKeeper(initialRecord);
+
+ // Step 2: Get the record
+ Pair<HAGroupStoreRecord, Stat> result = haAdmin.getHAGroupStoreRecordInZooKeeper(haGroupName);
+ HAGroupStoreRecord retrievedRecord = result.getLeft();
+ Stat stat = result.getRight();
+
+ assertEquals(initialRecord.getHaGroupName(), retrievedRecord.getHaGroupName());
+ assertEquals(initialRecord.getClusterRole(), retrievedRecord.getClusterRole());
+ assertEquals(initialRecord.getHAGroupState(), retrievedRecord.getHAGroupState());
+ assertEquals(initialRecord.getProtocolVersion(), retrievedRecord.getProtocolVersion());
+
+ // Step 3: Update the record
+ HAGroupStoreRecord updatedRecord = new HAGroupStoreRecord(
+ "v1.0",
+ haGroupName,
+ HAGroupStoreRecord.HAGroupState.STANDBY
+ );
+
+ haAdmin.updateHAGroupStoreRecordInZooKeeper(haGroupName, updatedRecord, stat.getVersion());
+
+ // Step 4: Get the updated record
+ Pair<HAGroupStoreRecord, Stat> updatedResult = haAdmin.getHAGroupStoreRecordInZooKeeper(haGroupName);
+ HAGroupStoreRecord finalRecord = updatedResult.getLeft();
+ Stat finalStat = updatedResult.getRight();
+
+ assertEquals(updatedRecord.getHaGroupName(), finalRecord.getHaGroupName());
+ assertEquals(updatedRecord.getClusterRole(), finalRecord.getClusterRole());
+ assertEquals(updatedRecord.getHAGroupState(), finalRecord.getHAGroupState());
+ assertEquals(updatedRecord.getProtocolVersion(), finalRecord.getProtocolVersion());
+
+ // Verify stat version increased
+ assertTrue(finalStat.getVersion() > stat.getVersion());
+ }
+
+ @Test
+ public void testMultiThreadedUpdatesConcurrentVersionConflict() throws Exception {
+ String haGroupName = testName.getMethodName();
+
+ // Create initial record
+ HAGroupStoreRecord initialRecord = new HAGroupStoreRecord(
+ "v1.0",
+ haGroupName,
+ HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC
+ );
+
+ haAdmin.createHAGroupStoreRecordInZooKeeper(initialRecord);
+
+ // Get the current stat for version checking
+ Pair<HAGroupStoreRecord, Stat> result = haAdmin.getHAGroupStoreRecordInZooKeeper(haGroupName);
+ int currentVersion = result.getRight().getVersion();
+
+ // Number of threads to run concurrently
+ int threadCount = 5;
+ ExecutorService executorService = Executors.newFixedThreadPool(threadCount);
+ CountDownLatch startLatch = new CountDownLatch(1);
+ CountDownLatch finishLatch = new CountDownLatch(threadCount);
+
+ // Counters for tracking results
+ AtomicInteger successCount = new AtomicInteger(0);
+ AtomicInteger staleVersionExceptionCount = new AtomicInteger(0);
+ AtomicInteger otherExceptionCount = new AtomicInteger(0);
+
+ // Submit multiple threads that will all try to update with the same version
+ for (int i = 0; i < threadCount; i++) {
+ executorService.submit(() -> {
+ try {
+ // Wait for all threads to be ready
+ startLatch.await();
+
+ // Create a unique update record for this thread
+ HAGroupStoreRecord updatedRecord = new HAGroupStoreRecord(
+ "v1.0",
+ haGroupName,
+ HAGroupStoreRecord.HAGroupState.STANDBY
+ );
+
+ // All threads use the same currentVersion, causing conflicts
+ haAdmin.updateHAGroupStoreRecordInZooKeeper(haGroupName, updatedRecord, currentVersion);
+ successCount.incrementAndGet();
+
+ } catch (StaleHAGroupStoreRecordVersionException e) {
+ staleVersionExceptionCount.incrementAndGet();
+ } catch (Exception e) {
+ otherExceptionCount.incrementAndGet();
+ } finally {
+ finishLatch.countDown();
+ }
+ });
+ }
+
+ // Start all threads at the same time
+ startLatch.countDown();
+
+ // Wait for all threads to complete
+ assertTrue("Threads did not complete within timeout",
+ finishLatch.await(10, TimeUnit.SECONDS));
+
+ executorService.shutdown();
+ assertTrue("ExecutorService did not shutdown within timeout",
+ executorService.awaitTermination(5, TimeUnit.SECONDS));
+
+ // Verify results
+ assertEquals("No other exceptions should occur", 0, otherExceptionCount.get());
+ assertEquals("Exactly one thread should succeed", 1, successCount.get());
+ assertEquals("All other threads should get stale version exception",
+ threadCount - 1, staleVersionExceptionCount.get());
+
+ // Verify the final state - should contain the update from the successful thread
+ Pair<HAGroupStoreRecord, Stat> finalResult = haAdmin.getHAGroupStoreRecordInZooKeeper(haGroupName);
+ HAGroupStoreRecord finalRecord = finalResult.getLeft();
+ Stat finalStat = finalResult.getRight();
+
+ // The successful update should have changed the role to STANDBY
+ assertEquals(ClusterRoleRecord.ClusterRole.STANDBY, finalRecord.getClusterRole());
+
+ // The ZooKeeper version should have increased
+ assertTrue("ZooKeeper version should have increased", finalStat.getVersion() > currentVersion);
+ }
+
+ @Test
+ public void testMultiThreadedUpdatesWithDifferentVersions() throws Exception {
+ String haGroupName = testName.getMethodName();
+
+ // Create initial record
+ HAGroupStoreRecord initialRecord = new HAGroupStoreRecord(
+ "v1.0",
+ haGroupName,
+ HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC
+ );
+
+ haAdmin.createHAGroupStoreRecordInZooKeeper(initialRecord);
+
+ // Number of threads to run sequentially (each gets the latest version)
+ int threadCount = 10;
+ ExecutorService executorService = Executors.newFixedThreadPool(1); // Single thread pool for sequential execution
+ CountDownLatch finishLatch = new CountDownLatch(threadCount);
+
+ // Counter for tracking results
+ AtomicInteger successCount = new AtomicInteger(0);
+ AtomicInteger failureCount = new AtomicInteger(0);
+
+ // Submit multiple threads that will update sequentially
+ for (int i = 0; i < threadCount; i++) {
+ final int threadId = i;
+ executorService.submit(() -> {
+ try {
+ // Get the current version for this thread
+ Pair<HAGroupStoreRecord, Stat> currentResult = haAdmin.getHAGroupStoreRecordInZooKeeper(haGroupName);
+ int currentVersion = currentResult.getRight().getVersion();
+
+ // Create update record for this thread
+ HAGroupStoreRecord updatedRecord = new HAGroupStoreRecord(
+ "v1.0",
+ haGroupName,
+ threadId % 2 == 0 ? HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC : HAGroupStoreRecord.HAGroupState.STANDBY
+ );
+
+ // Update with the current version
+ haAdmin.updateHAGroupStoreRecordInZooKeeper(haGroupName, updatedRecord, currentVersion);
+ successCount.incrementAndGet();
+
+ } catch (Exception e) {
+ failureCount.incrementAndGet();
+ } finally {
+ finishLatch.countDown();
+ }
+ });
+ }
+
+ // Wait for all threads to complete
+ assertTrue("Threads did not complete within timeout",
+ finishLatch.await(15, TimeUnit.SECONDS));
+
+ executorService.shutdown();
+ assertTrue("ExecutorService did not shutdown within timeout",
+ executorService.awaitTermination(5, TimeUnit.SECONDS));
+
+ // Verify results - all should succeed since they each get the latest version
+ assertEquals("All threads should succeed when using correct versions",
+ threadCount, successCount.get());
+ assertEquals("No threads should fail", 0, failureCount.get());
+
+ // Verify the final state
+ Pair<HAGroupStoreRecord, Stat> finalResult = haAdmin.getHAGroupStoreRecordInZooKeeper(haGroupName);
+
+ // The final record should have the version from the last update
+ assertEquals(threadCount, finalResult.getRight().getVersion());
+ }
+
+}
\ No newline at end of file
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/jdbc/ClusterRoleRecordTest.java b/phoenix-core/src/test/java/org/apache/phoenix/jdbc/ClusterRoleRecordTest.java
index cb801f7d1d..24bdc2c631 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/jdbc/ClusterRoleRecordTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/jdbc/ClusterRoleRecordTest.java
@@ -275,6 +275,15 @@ public class ClusterRoleRecordTest {
assertNotEquals(record.toString(), record.toPrettyString());
}
+ @Test
+ public void testClusterRoleFromInvalidBytes() {
+ ClusterRole role = ClusterRole.from(new byte[0]);
+ assertEquals(ClusterRole.UNKNOWN, role);
+
+ role = ClusterRole.from("random".getBytes());
+ assertEquals(ClusterRole.UNKNOWN, role);
+ }
+
//Private Helper Methods
private ClusterRoleRecord getClusterRoleRecord(String name, HighAvailabilityPolicy policy,
@@ -322,4 +331,4 @@ public class ClusterRoleRecordTest {
}
return buffer.toByteArray();
}
-}
+}
\ No newline at end of file
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/jdbc/HAGroupStoreRecordTest.java b/phoenix-core/src/test/java/org/apache/phoenix/jdbc/HAGroupStoreRecordTest.java
new file mode 100644
index 0000000000..8661283c0f
--- /dev/null
+++ b/phoenix-core/src/test/java/org/apache/phoenix/jdbc/HAGroupStoreRecordTest.java
@@ -0,0 +1,359 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.jdbc;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Optional;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.phoenix.util.JacksonUtil;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TestName;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Unit test for {@link HAGroupStoreRecord}.
+ */
+public class HAGroupStoreRecordTest {
+ private static final Logger LOG = LoggerFactory.getLogger(HAGroupStoreRecordTest.class);
+ private static final String PROTOCOL_VERSION = "1.0";
+
+ @Rule
+ public final TestName testName = new TestName();
+
+ /**
+ * Helper method to create a temp JSON file with the given array of HA group store records.
+ */
+ public static String createJsonFileWithRecords(HAGroupStoreRecord record)
+ throws IOException {
+ File file = File.createTempFile("phoenix.ha.group.store.records", ".test.json");
+ file.deleteOnExit();
+ JacksonUtil.getObjectWriterPretty().writeValue(file, record);
+ LOG.info("Prepared the JSON file for testing, file:{}, content:\n{}", file,
+ FileUtils.readFileToString(file, "UTF-8"));
+ return file.getPath();
+ }
+
+ @Test
+ public void testReadWriteJsonToFile() throws IOException {
+ HAGroupStoreRecord record = getHAGroupStoreRecord(testName.getMethodName(),
+ PROTOCOL_VERSION, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+ String fileName = createJsonFileWithRecords(record);
+ String fileContent = FileUtils.readFileToString(new File(fileName), "UTF-8");
+ assertTrue(fileContent.contains(record.getHaGroupName()));
+ assertTrue(fileContent.contains(record.getProtocolVersion()));
+ assertTrue(fileContent.contains(record.getHAGroupState().toString()));
+ // Create a new record from file
+ Optional<HAGroupStoreRecord> record2 = HAGroupStoreRecord.fromJson(fileContent.getBytes());
+ assertTrue(record2.isPresent());
+ // Check if same info
+ assertTrue(record.hasSameInfo(record2.get()));
+ }
+
+ @Test
+ public void testToAndFromJson() throws IOException {
+ HAGroupStoreRecord record = getHAGroupStoreRecord(testName.getMethodName(),
+ PROTOCOL_VERSION, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+ byte[] bytes = HAGroupStoreRecord.toJson(record);
+ Optional<HAGroupStoreRecord> record2 = HAGroupStoreRecord.fromJson(bytes);
+ assertTrue(record2.isPresent());
+ assertEquals(record, record2.get());
+ }
+
+ @Test
+ public void testFromJsonWithNullBytes() {
+ Optional<HAGroupStoreRecord> record = HAGroupStoreRecord.fromJson(null);
+ assertFalse(record.isPresent());
+ }
+
+ @Test
+ public void testFromJsonWithInvalidJson() {
+ byte[] invalidJson = "invalid json".getBytes();
+ Optional<HAGroupStoreRecord> record = HAGroupStoreRecord.fromJson(invalidJson);
+ assertFalse(record.isPresent());
+ }
+
+ @Test
+ public void testHasSameInfo() {
+ String haGroupName = testName.getMethodName();
+ HAGroupStoreRecord record1 = getHAGroupStoreRecord(haGroupName,
+ PROTOCOL_VERSION, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+ HAGroupStoreRecord record2 = getHAGroupStoreRecord(haGroupName,
+ PROTOCOL_VERSION, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+
+ assertTrue(record1.hasSameInfo(record2)); // Same core info despite different state
+ assertTrue(record1.hasSameInfo(record1)); // reflexive
+ assertTrue(record2.hasSameInfo(record1)); // symmetric
+ }
+
+ @Test
+ public void testHasSameInfoNegative() {
+ String haGroupName = testName.getMethodName();
+ HAGroupStoreRecord record = getHAGroupStoreRecord(haGroupName,
+ PROTOCOL_VERSION, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+
+ // Different protocol version
+ HAGroupStoreRecord recordDifferentProtocol = getHAGroupStoreRecord(haGroupName,
+ "2.0", HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+ assertFalse(record.hasSameInfo(recordDifferentProtocol));
+ assertFalse(recordDifferentProtocol.hasSameInfo(record));
+
+ // Different HA group name
+ String haGroupName2 = haGroupName + RandomStringUtils.randomAlphabetic(2);
+ HAGroupStoreRecord record2 = getHAGroupStoreRecord(haGroupName2,
+ PROTOCOL_VERSION, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+ assertFalse(record.hasSameInfo(record2));
+ assertFalse(record2.hasSameInfo(record));
+
+ // Different HA group state
+ HAGroupStoreRecord recordDifferentState = getHAGroupStoreRecord(haGroupName,
+ PROTOCOL_VERSION, HAGroupStoreRecord.HAGroupState.STANDBY);
+ assertFalse(record.hasSameInfo(recordDifferentState));
+ assertFalse(recordDifferentState.hasSameInfo(record));
+ }
+
+ @Test
+ public void testGetters() {
+ String haGroupName = testName.getMethodName();
+ String protocolVersion = "1.5";
+ HAGroupStoreRecord.HAGroupState haGroupState = HAGroupStoreRecord.HAGroupState.STANDBY;
+
+ HAGroupStoreRecord record = getHAGroupStoreRecord(haGroupName, protocolVersion, haGroupState);
+
+ assertEquals(haGroupName, record.getHaGroupName());
+ assertEquals(protocolVersion, record.getProtocolVersion());
+ assertEquals(haGroupState, record.getHAGroupState());
+ assertEquals(haGroupState.getClusterRole(), record.getClusterRole());
+ }
+
+ @Test
+ public void testEqualsAndHashCode() {
+ String haGroupName = testName.getMethodName();
+ HAGroupStoreRecord record1 = getHAGroupStoreRecord(haGroupName,
+ PROTOCOL_VERSION, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+ HAGroupStoreRecord record2 = getHAGroupStoreRecord(haGroupName,
+ PROTOCOL_VERSION, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+ HAGroupStoreRecord record3 = getHAGroupStoreRecord(haGroupName,
+ PROTOCOL_VERSION, HAGroupStoreRecord.HAGroupState.STANDBY); // Different state
+
+ // Test equals
+ assertEquals(record1, record2); // symmetric
+ assertEquals(record2, record1); // symmetric
+ assertNotEquals(record1, record3); // different state
+ assertNotEquals(null, record1); // null comparison
+ assertNotEquals("not a record", record1); // different type
+
+ // Test hashCode
+ assertEquals(record1.hashCode(), record2.hashCode()); // equal objects have same hash
+ assertNotEquals(record1.hashCode(), record3.hashCode()); // different objects likely have different hash
+ }
+
+ @Test
+ public void testToString() {
+ HAGroupStoreRecord record = getHAGroupStoreRecord(testName.getMethodName(),
+ PROTOCOL_VERSION, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+ String toString = record.toString();
+
+ // Verify all fields are present in toString
+ assertTrue(toString.contains(record.getHaGroupName()));
+ assertTrue(toString.contains(record.getProtocolVersion()));
+ assertTrue(toString.contains(record.getHAGroupState().toString()));
+ }
+
+ @Test
+ public void testToPrettyString() {
+ HAGroupStoreRecord record = getHAGroupStoreRecord(testName.getMethodName(),
+ PROTOCOL_VERSION, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+ LOG.info("toString(): {}", record.toString());
+ LOG.info("toPrettyString:\n{}", record.toPrettyString());
+ assertNotEquals(record.toString(), record.toPrettyString());
+ assertTrue(record.toPrettyString().contains(record.getHaGroupName()));
+ }
+
+ @Test(expected = NullPointerException.class)
+ public void testConstructorWithNullHaGroupName() {
+ getHAGroupStoreRecord(null, PROTOCOL_VERSION, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+ }
+
+ @Test(expected = NullPointerException.class)
+ public void testConstructorWithNullHAGroupState() {
+ getHAGroupStoreRecord(testName.getMethodName(), PROTOCOL_VERSION, null);
+ }
+
+ // Tests for HAGroupState enum
+ @Test
+ public void testHAGroupStateGetClusterRole() {
+ // Test that each HAGroupState maps to the correct ClusterRole
+ assertEquals(ClusterRoleRecord.ClusterRole.ACTIVE,
+ HAGroupStoreRecord.HAGroupState.ABORT_TO_ACTIVE_IN_SYNC.getClusterRole());
+ assertEquals(ClusterRoleRecord.ClusterRole.STANDBY,
+ HAGroupStoreRecord.HAGroupState.ABORT_TO_STANDBY.getClusterRole());
+ assertEquals(ClusterRoleRecord.ClusterRole.ACTIVE,
+ HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC.getClusterRole());
+ assertEquals(ClusterRoleRecord.ClusterRole.ACTIVE,
+ HAGroupStoreRecord.HAGroupState.ACTIVE_NOT_IN_SYNC.getClusterRole());
+ assertEquals(ClusterRoleRecord.ClusterRole.ACTIVE_TO_STANDBY,
+ HAGroupStoreRecord.HAGroupState.ACTIVE_NOT_IN_SYNC_TO_STANDBY.getClusterRole());
+ assertEquals(ClusterRoleRecord.ClusterRole.ACTIVE,
+ HAGroupStoreRecord.HAGroupState.ACTIVE_NOT_IN_SYNC_WITH_OFFLINE_PEER.getClusterRole());
+ assertEquals(ClusterRoleRecord.ClusterRole.ACTIVE_TO_STANDBY,
+ HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY.getClusterRole());
+ assertEquals(ClusterRoleRecord.ClusterRole.ACTIVE,
+ HAGroupStoreRecord.HAGroupState.ACTIVE_WITH_OFFLINE_PEER.getClusterRole());
+ assertEquals(ClusterRoleRecord.ClusterRole.STANDBY,
+ HAGroupStoreRecord.HAGroupState.DEGRADED_STANDBY.getClusterRole());
+ assertEquals(ClusterRoleRecord.ClusterRole.STANDBY,
+ HAGroupStoreRecord.HAGroupState.DEGRADED_STANDBY_FOR_READER.getClusterRole());
+ assertEquals(ClusterRoleRecord.ClusterRole.STANDBY,
+ HAGroupStoreRecord.HAGroupState.DEGRADED_STANDBY_FOR_WRITER.getClusterRole());
+ assertEquals(ClusterRoleRecord.ClusterRole.OFFLINE,
+ HAGroupStoreRecord.HAGroupState.OFFLINE.getClusterRole());
+ assertEquals(ClusterRoleRecord.ClusterRole.STANDBY,
+ HAGroupStoreRecord.HAGroupState.STANDBY.getClusterRole());
+ assertEquals(ClusterRoleRecord.ClusterRole.STANDBY_TO_ACTIVE,
+ HAGroupStoreRecord.HAGroupState.STANDBY_TO_ACTIVE.getClusterRole());
+ assertEquals(ClusterRoleRecord.ClusterRole.UNKNOWN,
+ HAGroupStoreRecord.HAGroupState.UNKNOWN.getClusterRole());
+ }
+
+ @Test
+ public void testHAGroupStateValidTransitions() {
+ // Test valid transitions for ACTIVE_NOT_IN_SYNC
+ assertTrue(HAGroupStoreRecord.HAGroupState.ACTIVE_NOT_IN_SYNC
+ .isTransitionAllowed(HAGroupStoreRecord.HAGroupState.ACTIVE_NOT_IN_SYNC));
+ assertTrue(HAGroupStoreRecord.HAGroupState.ACTIVE_NOT_IN_SYNC
+ .isTransitionAllowed(HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC));
+ assertTrue(HAGroupStoreRecord.HAGroupState.ACTIVE_NOT_IN_SYNC
+ .isTransitionAllowed(HAGroupStoreRecord.HAGroupState.ACTIVE_NOT_IN_SYNC_TO_STANDBY));
+ assertTrue(HAGroupStoreRecord.HAGroupState.ACTIVE_NOT_IN_SYNC
+ .isTransitionAllowed(HAGroupStoreRecord.HAGroupState.ACTIVE_NOT_IN_SYNC_WITH_OFFLINE_PEER));
+
+ // Test valid transitions for ACTIVE
+ assertTrue(HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC
+ .isTransitionAllowed(HAGroupStoreRecord.HAGroupState.ACTIVE_NOT_IN_SYNC));
+ assertTrue(HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC
+ .isTransitionAllowed(HAGroupStoreRecord.HAGroupState.ACTIVE_WITH_OFFLINE_PEER));
+ assertTrue(HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC
+ .isTransitionAllowed(HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY));
+
+ // Test valid transitions for STANDBY
+ assertTrue(HAGroupStoreRecord.HAGroupState.STANDBY
+ .isTransitionAllowed(HAGroupStoreRecord.HAGroupState.STANDBY_TO_ACTIVE));
+ assertTrue(HAGroupStoreRecord.HAGroupState.STANDBY
+ .isTransitionAllowed(HAGroupStoreRecord.HAGroupState.DEGRADED_STANDBY_FOR_READER));
+ assertTrue(HAGroupStoreRecord.HAGroupState.STANDBY
+ .isTransitionAllowed(HAGroupStoreRecord.HAGroupState.DEGRADED_STANDBY_FOR_WRITER));
+
+ // Test valid transitions for ACTIVE_TO_STANDBY
+ assertTrue(HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY
+ .isTransitionAllowed(HAGroupStoreRecord.HAGroupState.ABORT_TO_ACTIVE_IN_SYNC));
+ assertTrue(HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY
+ .isTransitionAllowed(HAGroupStoreRecord.HAGroupState.STANDBY));
+
+ // Test valid transitions for STANDBY_TO_ACTIVE
+ assertTrue(HAGroupStoreRecord.HAGroupState.STANDBY_TO_ACTIVE
+ .isTransitionAllowed(HAGroupStoreRecord.HAGroupState.ABORT_TO_STANDBY));
+ assertTrue(HAGroupStoreRecord.HAGroupState.STANDBY_TO_ACTIVE
+ .isTransitionAllowed(HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC));
+ }
+
+ @Test
+ public void testHAGroupStateInvalidTransitions() {
+ // Test invalid transitions - ACTIVE cannot directly go to STANDBY
+ assertFalse(HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC
+ .isTransitionAllowed(HAGroupStoreRecord.HAGroupState.STANDBY));
+
+ // Test invalid transitions - STANDBY cannot directly go to ACTIVE
+ assertFalse(HAGroupStoreRecord.HAGroupState.STANDBY
+ .isTransitionAllowed(HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC));
+
+ // Test invalid transitions - OFFLINE has no allowed transitions (manual recovery needed)
+ assertFalse(HAGroupStoreRecord.HAGroupState.OFFLINE
+ .isTransitionAllowed(HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC));
+ assertFalse(HAGroupStoreRecord.HAGroupState.OFFLINE
+ .isTransitionAllowed(HAGroupStoreRecord.HAGroupState.STANDBY));
+ assertFalse(HAGroupStoreRecord.HAGroupState.OFFLINE
+ .isTransitionAllowed(HAGroupStoreRecord.HAGroupState.UNKNOWN));
+
+ // Test invalid transitions - UNKNOWN has no allowed transitions (manual recovery needed)
+ assertFalse(HAGroupStoreRecord.HAGroupState.UNKNOWN
+ .isTransitionAllowed(HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC));
+ assertFalse(HAGroupStoreRecord.HAGroupState.UNKNOWN
+ .isTransitionAllowed(HAGroupStoreRecord.HAGroupState.STANDBY));
+ assertFalse(HAGroupStoreRecord.HAGroupState.UNKNOWN
+ .isTransitionAllowed(HAGroupStoreRecord.HAGroupState.OFFLINE));
+ }
+
+ @Test
+ public void testHAGroupStateFromBytesValidValues() {
+ // Test valid enum names (case insensitive)
+ assertEquals(HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC,
+ HAGroupStoreRecord.HAGroupState.from("ACTIVE_IN_SYNC".getBytes()));
+ assertEquals(HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC,
+ HAGroupStoreRecord.HAGroupState.from("active_in_sync".getBytes()));
+ assertEquals(HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC,
+ HAGroupStoreRecord.HAGroupState.from("Active_in_Sync".getBytes()));
+
+ assertEquals(HAGroupStoreRecord.HAGroupState.STANDBY,
+ HAGroupStoreRecord.HAGroupState.from("STANDBY".getBytes()));
+ assertEquals(HAGroupStoreRecord.HAGroupState.STANDBY,
+ HAGroupStoreRecord.HAGroupState.from("standby".getBytes()));
+
+ assertEquals(HAGroupStoreRecord.HAGroupState.OFFLINE,
+ HAGroupStoreRecord.HAGroupState.from("OFFLINE".getBytes()));
+
+ assertEquals(HAGroupStoreRecord.HAGroupState.ACTIVE_NOT_IN_SYNC,
+ HAGroupStoreRecord.HAGroupState.from("ACTIVE_NOT_IN_SYNC".getBytes()));
+
+ assertEquals(HAGroupStoreRecord.HAGroupState.DEGRADED_STANDBY_FOR_READER,
+ HAGroupStoreRecord.HAGroupState.from("DEGRADED_STANDBY_FOR_READER".getBytes()));
+ }
+
+ @Test
+ public void testHAGroupStateFromBytesInvalidValues() {
+ // Test invalid enum names return UNKNOWN
+ assertEquals(HAGroupStoreRecord.HAGroupState.UNKNOWN,
+ HAGroupStoreRecord.HAGroupState.from("INVALID_STATE".getBytes()));
+ assertEquals(HAGroupStoreRecord.HAGroupState.UNKNOWN,
+ HAGroupStoreRecord.HAGroupState.from("".getBytes()));
+ assertEquals(HAGroupStoreRecord.HAGroupState.UNKNOWN,
+ HAGroupStoreRecord.HAGroupState.from("null".getBytes()));
+ assertEquals(HAGroupStoreRecord.HAGroupState.UNKNOWN,
+ HAGroupStoreRecord.HAGroupState.from("ACTIV".getBytes())); // typo
+ assertEquals(HAGroupStoreRecord.HAGroupState.UNKNOWN,
+ HAGroupStoreRecord.HAGroupState.from("ACTIVE_EXTRA".getBytes())); // extra text
+ assertEquals(HAGroupStoreRecord.HAGroupState.UNKNOWN,
+ HAGroupStoreRecord.HAGroupState.from(null));
+
+ }
+
+ // Private Helper Methods
+ private HAGroupStoreRecord getHAGroupStoreRecord(String haGroupName, String protocolVersion,
+ HAGroupStoreRecord.HAGroupState haGroupState) {
+ return new HAGroupStoreRecord(protocolVersion, haGroupName, haGroupState);
+ }
+}
\ No newline at end of file
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/jdbc/PhoenixHAAdminToolTest.java b/phoenix-core/src/test/java/org/apache/phoenix/jdbc/PhoenixHAAdminToolTest.java
index fa88b5909a..0e78da74d3 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/jdbc/PhoenixHAAdminToolTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/jdbc/PhoenixHAAdminToolTest.java
@@ -87,7 +87,7 @@ public class PhoenixHAAdminToolTest {
@Before
public void setup() throws Exception {
- when(mockHighAvailibilityCuratorProvider.getCurator(Mockito.anyString(), any(Properties.class))).thenReturn(curator);
+ when(mockHighAvailibilityCuratorProvider.getCurator(Mockito.anyString(), any(Properties.class), anyString())).thenReturn(curator);
haGroupName = testName.getMethodName();
recordV1 = new ClusterRoleRecord(
haGroupName, HighAvailabilityPolicy.FAILOVER,
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/util/HAGroupStoreTestUtil.java b/phoenix-core/src/test/java/org/apache/phoenix/util/HAGroupStoreTestUtil.java
new file mode 100644
index 0000000000..d15e73aca2
--- /dev/null
+++ b/phoenix-core/src/test/java/org/apache/phoenix/util/HAGroupStoreTestUtil.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.util;
+
+import org.apache.phoenix.jdbc.ClusterRoleRecord;
+import org.apache.phoenix.jdbc.PhoenixConnection;
+
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.sql.Statement;
+
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_HA_GROUP_NAME;
+import static org.apache.phoenix.util.PhoenixRuntime.JDBC_PROTOCOL_SEPARATOR;
+import static org.apache.phoenix.util.PhoenixRuntime.JDBC_PROTOCOL_ZK;
+
+/**
+ * Utility class for HA Group Store testing operations.
+ */
+public class HAGroupStoreTestUtil {
+
+ /**
+ * Upserts an HA group record into the system table for testing purposes.
+ *
+ * @param haGroupName the HA group name
+ * @param zkUrl the ZooKeeper URL for the local cluster
+ * @param peerZKUrl the ZooKeeper URL for the peer cluster
+ * @param localClusterRole the role of the local cluster
+ * @param peerClusterRole the role of the peer cluster
+ * @param overrideConnZkUrl optional override for the connection ZK URL
+ * @throws SQLException if the database operation fails
+ */
+ public static void upsertHAGroupRecordInSystemTable(String haGroupName, String zkUrl, String peerZKUrl,
+ ClusterRoleRecord.ClusterRole localClusterRole,
+ ClusterRoleRecord.ClusterRole peerClusterRole, String overrideConnZkUrl) throws SQLException {
+ try (PhoenixConnection conn = (PhoenixConnection) DriverManager.getConnection(JDBC_PROTOCOL_ZK + JDBC_PROTOCOL_SEPARATOR + (overrideConnZkUrl != null ? overrideConnZkUrl : zkUrl));
+ Statement stmt = conn.createStatement()) {
+ // Only insert values that are not null
+ StringBuilder queryBuilder = new StringBuilder("UPSERT INTO " + SYSTEM_HA_GROUP_NAME + " (HA_GROUP_NAME, ");
+ if (zkUrl != null) {
+ queryBuilder.append("ZK_URL_1, ");
+ }
+ if (peerZKUrl != null) {
+ queryBuilder.append("ZK_URL_2, ");
+ }
+ if (localClusterRole != null) {
+ queryBuilder.append("CLUSTER_ROLE_1, ");
+ }
+ if (peerClusterRole != null) {
+ queryBuilder.append("CLUSTER_ROLE_2, ");
+ }
+ if (zkUrl != null) {
+ queryBuilder.append("CLUSTER_URL_1, ");
+ }
+ if (peerZKUrl != null) {
+ queryBuilder.append("CLUSTER_URL_2, ");
+ }
+ queryBuilder.append("POLICY, VERSION) ");
+ queryBuilder.append("VALUES ('" + haGroupName + "', ");
+ if (zkUrl != null) {
+ queryBuilder.append("'" + zkUrl + "', ");
+ }
+ if (peerZKUrl != null) {
+ queryBuilder.append("'" + peerZKUrl + "', ");
+ }
+ if (localClusterRole != null) {
+ queryBuilder.append("'" + localClusterRole + "', ");
+ }
+ if (peerClusterRole != null) {
+ queryBuilder.append("'" + peerClusterRole + "', ");
+ }
+ if (zkUrl != null) {
+ queryBuilder.append("'" + zkUrl + "', ");
+ }
+ if (peerZKUrl != null) {
+ queryBuilder.append("'" + peerZKUrl + "', ");
+ }
+ queryBuilder.append("'FAILOVER', 1)");
+ stmt.executeUpdate(queryBuilder.toString());
+ conn.commit();
+ }
+ }
+
+ /**
+ * Deletes an HA group record from the system table for testing purposes.
+ *
+ * @param haGroupName the HA group name to delete
+ * @param zkUrl the ZooKeeper URL to connect to
+ * @throws SQLException if the database operation fails
+ */
+ public static void deleteHAGroupRecordInSystemTable(String haGroupName, String zkUrl) throws SQLException {
+ // Delete the record from System Table
+ try (PhoenixConnection conn = (PhoenixConnection) DriverManager.getConnection(JDBC_PROTOCOL_ZK + JDBC_PROTOCOL_SEPARATOR + zkUrl);
+ Statement stmt = conn.createStatement()) {
+ stmt.execute("DELETE FROM " + SYSTEM_HA_GROUP_NAME + " WHERE HA_GROUP_NAME = '" + haGroupName + "'");
+ conn.commit();
+ }
+ }
+}
\ No newline at end of file