You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by ka...@apache.org on 2025/08/11 17:32:18 UTC

(phoenix) branch PHOENIX-7562-feature updated: PHOENIX-7566 [1/n] HAGroupStoreClient/Manager changes for Consistent HA (#2244)

This is an automated email from the ASF dual-hosted git repository.

kadir pushed a commit to branch PHOENIX-7562-feature
in repository https://gitbox.apache.org/repos/asf/phoenix.git


The following commit(s) were added to refs/heads/PHOENIX-7562-feature by this push:
     new 6195138285 PHOENIX-7566 [1/n] HAGroupStoreClient/Manager changes for Consistent HA (#2244)
6195138285 is described below

commit 619513828509cf7ddf286891acdb06387ce21e46
Author: ritegarg <58...@users.noreply.github.com>
AuthorDate: Mon Aug 11 10:32:13 2025 -0700

    PHOENIX-7566 [1/n] HAGroupStoreClient/Manager changes for Consistent HA (#2244)
    
    Co-authored-by: Ritesh Garg <ri...@riteshg-ltmd34g.internal.salesforce.com>
---
 .../BaseScannerRegionObserverConstants.java        |   5 +
 .../InvalidClusterRoleTransitionException.java     |  41 +
 .../exception/MutationBlockedIOException.java      |  36 +
 .../StaleHAGroupStoreRecordVersionException.java   |  42 +
 .../org/apache/phoenix/execute/MutationState.java  |   8 +-
 .../org/apache/phoenix/jdbc/ClusterRoleRecord.java |  31 +-
 .../apache/phoenix/jdbc/HAGroupStoreClient.java    | 666 ++++++++++++---
 .../apache/phoenix/jdbc/HAGroupStoreManager.java   | 167 +++-
 .../apache/phoenix/jdbc/HAGroupStoreRecord.java    | 264 ++++++
 .../apache/phoenix/jdbc/HighAvailabilityGroup.java |  57 +-
 .../org/apache/phoenix/jdbc/PhoenixConnection.java |  13 +
 .../phoenix/jdbc/PhoenixDatabaseMetaData.java      |  16 +
 .../org/apache/phoenix/jdbc/PhoenixHAAdmin.java    | 127 ++-
 .../phoenix/query/ConnectionQueryServicesImpl.java |  90 +-
 .../query/ConnectionlessQueryServicesImpl.java     |  63 +-
 .../org/apache/phoenix/query/QueryConstants.java   |  30 +
 .../org/apache/phoenix/query/QueryServices.java    |   1 -
 .../org/apache/phoenix/util/WALAnnotationUtil.java |   6 +-
 .../protobuf/RegionServerEndpointService.proto     |  20 +
 .../coprocessor/PhoenixRegionServerEndpoint.java   |  58 +-
 .../UngroupedAggregateRegionObserver.java          |  18 -
 .../UngroupedAggregateRegionScanner.java           |   5 +-
 .../phoenix/hbase/index/IndexRegionObserver.java   |  37 +-
 .../MigrateSystemTablesToSystemNamespaceIT.java    |   4 +-
 .../end2end/PhoenixRegionServerEndpointIT.java     |  12 -
 ...gionServerEndpointITWithConsistentFailover.java | 158 ++++
 .../phoenix/end2end/QueryDatabaseMetaDataIT.java   |   6 +-
 .../SystemTablesCreationOnConnectionIT.java        |   4 +-
 .../phoenix/end2end/TenantSpecificTablesDDLIT.java |   4 +-
 .../IndexRegionObserverMutationBlockingIT.java     | 217 +++++
 .../apache/phoenix/jdbc/HAGroupStoreClientIT.java  | 916 ++++++++++++++++-----
 .../apache/phoenix/jdbc/HAGroupStoreManagerIT.java | 323 +++++++-
 .../phoenix/jdbc/HighAvailabilityGroupTestIT.java  |   5 +-
 .../phoenix/jdbc/ParallelPhoenixConnectionIT.java  |  85 +-
 .../org/apache/phoenix/jdbc/PhoenixHAAdminIT.java  | 481 +++++++++++
 .../apache/phoenix/jdbc/ClusterRoleRecordTest.java |  11 +-
 .../phoenix/jdbc/HAGroupStoreRecordTest.java       | 359 ++++++++
 .../phoenix/jdbc/PhoenixHAAdminToolTest.java       |   2 +-
 .../apache/phoenix/util/HAGroupStoreTestUtil.java  | 113 +++
 39 files changed, 3994 insertions(+), 507 deletions(-)

diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/coprocessorclient/BaseScannerRegionObserverConstants.java b/phoenix-core-client/src/main/java/org/apache/phoenix/coprocessorclient/BaseScannerRegionObserverConstants.java
index 74f6c67026..41becf9374 100644
--- a/phoenix-core-client/src/main/java/org/apache/phoenix/coprocessorclient/BaseScannerRegionObserverConstants.java
+++ b/phoenix-core-client/src/main/java/org/apache/phoenix/coprocessorclient/BaseScannerRegionObserverConstants.java
@@ -190,4 +190,9 @@ public class BaseScannerRegionObserverConstants {
 
     /** Exposed for testing */
     public static final String SCANNER_OPENED_TRACE_INFO = "Scanner opened on server";
+
+    /**
+     * Attribute name used to pass HAGroupName in Mutations.
+     */
+    public static final String HA_GROUP_NAME_ATTRIB = "_HAGroupName";
 }
diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/exception/InvalidClusterRoleTransitionException.java b/phoenix-core-client/src/main/java/org/apache/phoenix/exception/InvalidClusterRoleTransitionException.java
new file mode 100644
index 0000000000..e5fa01e958
--- /dev/null
+++ b/phoenix-core-client/src/main/java/org/apache/phoenix/exception/InvalidClusterRoleTransitionException.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.exception;
+
+/**
+ * Exception thrown when attempting an invalid cluster role transition.
+ */
+public class InvalidClusterRoleTransitionException extends Exception {
+    private static final long serialVersionUID = 1L;
+
+    /**
+     * @param msg reason for the exception
+     */
+    public InvalidClusterRoleTransitionException(String msg) {
+        super(msg);
+    }
+
+    /**
+     * @param msg reason for the exception
+     * @param cause the underlying cause
+     */
+    public InvalidClusterRoleTransitionException(String msg, Throwable cause) {
+        super(msg, cause);
+    }
+
+}
diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/exception/MutationBlockedIOException.java b/phoenix-core-client/src/main/java/org/apache/phoenix/exception/MutationBlockedIOException.java
new file mode 100644
index 0000000000..590c3615d6
--- /dev/null
+++ b/phoenix-core-client/src/main/java/org/apache/phoenix/exception/MutationBlockedIOException.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.exception;
+
+import java.io.IOException;
+
+/**
+ * Exception thrown when CLUSTER_ROLE_BASED_MUTATION_BLOCK_ENABLED is set
+ * and the current cluster role is ACTIVE_TO_STANDBY.
+ */
+public class MutationBlockedIOException extends IOException {
+    private static final long serialVersionUID = 1L;
+
+    /**
+     * @param msg reason for the exception
+     */
+    public MutationBlockedIOException(String msg) {
+        super(msg);
+    }
+
+}
diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/exception/StaleHAGroupStoreRecordVersionException.java b/phoenix-core-client/src/main/java/org/apache/phoenix/exception/StaleHAGroupStoreRecordVersionException.java
new file mode 100644
index 0000000000..ca55305917
--- /dev/null
+++ b/phoenix-core-client/src/main/java/org/apache/phoenix/exception/StaleHAGroupStoreRecordVersionException.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.exception;
+
+/**
+ * Exception thrown when attempting to update HAGroupStoreRecord with a stale ZK stat version,
+ * indicating that the record has been modified by another process.
+ */
+public class StaleHAGroupStoreRecordVersionException extends Exception {
+    private static final long serialVersionUID = 1L;
+
+    /**
+     * @param msg reason for the exception
+     */
+    public StaleHAGroupStoreRecordVersionException(String msg) {
+        super(msg);
+    }
+
+    /**
+     * @param msg reason for the exception
+     * @param cause the underlying cause
+     */
+    public StaleHAGroupStoreRecordVersionException(String msg, Throwable cause) {
+        super(msg, cause);
+    }
+
+}
diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/execute/MutationState.java b/phoenix-core-client/src/main/java/org/apache/phoenix/execute/MutationState.java
index b92d699867..2a52426ffd 100644
--- a/phoenix-core-client/src/main/java/org/apache/phoenix/execute/MutationState.java
+++ b/phoenix-core-client/src/main/java/org/apache/phoenix/execute/MutationState.java
@@ -55,6 +55,7 @@ import java.util.Set;
 import javax.annotation.Nonnull;
 import javax.annotation.concurrent.Immutable;
 
+import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.hbase.Cell;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.TableName;
@@ -640,8 +641,8 @@ public class MutationState implements SQLCloseable {
             final MultiRowMutationState values, final long mutationTimestamp, final long serverTimestamp,
             boolean includeAllIndexes, final boolean sendAll) {
         final PTable table = tableRef.getTable();
-        final List<PTable> indexList = includeAllIndexes ? 
-                Lists.newArrayList(IndexMaintainer.maintainedIndexes(table.getIndexes().iterator())) : 
+        final List<PTable> indexList = includeAllIndexes ?
+                Lists.newArrayList(IndexMaintainer.maintainedIndexes(table.getIndexes().iterator())) :
                     IndexUtil.getClientMaintainedIndexes(table);
         final Iterator<PTable> indexes = indexList.iterator();
         final List<Mutation> mutationList = Lists.newArrayListWithExpectedSize(values.size());
@@ -864,7 +865,8 @@ public class MutationState implements SQLCloseable {
                 Bytes.toBytes(table.getExternalSchemaId()) : null;
         byte[] lastDDLTimestamp =
                 table.getLastDDLTimestamp() != null ? Bytes.toBytes(table.getLastDDLTimestamp()) : null;
-        WALAnnotationUtil.annotateMutation(mutation, tenantId, schemaName, tableName, tableType, lastDDLTimestamp);
+        byte[] haGroupName = StringUtils.isNotBlank(connection.getHAGroupName()) ? Bytes.toBytes(connection.getHAGroupName()) : null;
+        WALAnnotationUtil.annotateMutation(mutation, tenantId, schemaName, tableName, tableType, lastDDLTimestamp, haGroupName);
     }
 
     /**
diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/ClusterRoleRecord.java b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/ClusterRoleRecord.java
index cc463b7972..3b71f8f9cd 100644
--- a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/ClusterRoleRecord.java
+++ b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/ClusterRoleRecord.java
@@ -60,13 +60,14 @@ public class ClusterRoleRecord {
      * take traffic, standby and offline do not, and unknown is used if the state cannot be determined.
      */
     public enum ClusterRole {
-        ACTIVE, STANDBY, OFFLINE, UNKNOWN, ACTIVE_TO_STANDBY;
+        ACTIVE, STANDBY, OFFLINE, UNKNOWN, ACTIVE_TO_STANDBY, STANDBY_TO_ACTIVE;
 
         /**
          * @return true if a cluster with this role can be connected, otherwise false
          */
         public boolean canConnect() {
-            return this == ACTIVE || this == STANDBY || this == ACTIVE_TO_STANDBY;
+            return this == ACTIVE || this == STANDBY
+                    || this == ACTIVE_TO_STANDBY || this == STANDBY_TO_ACTIVE;
         }
 
         public static ClusterRole from(byte[] bytes) {
@@ -76,6 +77,32 @@ public class ClusterRoleRecord {
                     .findFirst()
                     .orElse(UNKNOWN);
         }
+
+        public boolean isMutationBlocked() {
+            return this == ACTIVE_TO_STANDBY;
+        }
+
+        /**
+         * @return the default HAGroupState for this ClusterRole
+         */
+        @JsonIgnore
+        public HAGroupStoreRecord.HAGroupState getDefaultHAGroupState() {
+            switch (this) {
+                case ACTIVE:
+                    return HAGroupStoreRecord.HAGroupState.ACTIVE_NOT_IN_SYNC;
+                case STANDBY:
+                    return HAGroupStoreRecord.HAGroupState.DEGRADED_STANDBY;
+                case OFFLINE:
+                    return HAGroupStoreRecord.HAGroupState.OFFLINE;
+                case ACTIVE_TO_STANDBY:
+                    return HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY;
+                case STANDBY_TO_ACTIVE:
+                    return HAGroupStoreRecord.HAGroupState.STANDBY_TO_ACTIVE;
+                case UNKNOWN:
+                default:
+                    return HAGroupStoreRecord.HAGroupState.UNKNOWN;
+            }
+        }
     }
 
     /**
diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HAGroupStoreClient.java b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HAGroupStoreClient.java
index c3606ddad2..123dc28d5c 100644
--- a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HAGroupStoreClient.java
+++ b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HAGroupStoreClient.java
@@ -17,191 +17,625 @@
  */
 package org.apache.phoenix.jdbc;
 
+import java.io.Closeable;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
 
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.tuple.Pair;
 import org.apache.curator.framework.recipes.cache.ChildData;
 import org.apache.curator.framework.recipes.cache.PathChildrenCache;
 import org.apache.curator.framework.recipes.cache.PathChildrenCacheListener;
 import org.apache.curator.utils.ZKPaths;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.phoenix.exception.InvalidClusterRoleTransitionException;
+import org.apache.phoenix.exception.StaleHAGroupStoreRecordVersionException;
+import org.apache.phoenix.jdbc.ClusterRoleRecord.ClusterRole;
+import org.apache.phoenix.jdbc.ClusterRoleRecord.RegistryType;
+import org.apache.phoenix.jdbc.HAGroupStoreRecord.HAGroupState;
 import org.apache.phoenix.thirdparty.com.google.common.annotations.VisibleForTesting;
-import org.apache.phoenix.thirdparty.com.google.common.collect.ImmutableList;
+import org.apache.phoenix.thirdparty.com.google.common.base.Preconditions;
+import org.apache.phoenix.util.JDBCUtil;
+import org.apache.zookeeper.data.Stat;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.List;
-import java.util.Objects;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.TimeUnit;
-import java.util.stream.Collectors;
+import static org.apache.hadoop.hbase.HConstants.DEFAULT_ZK_SESSION_TIMEOUT;
+import static org.apache.hadoop.hbase.HConstants.ZK_SESSION_TIMEOUT;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.CLUSTER_ROLE_1;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.CLUSTER_ROLE_2;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.CLUSTER_URL_1;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.CLUSTER_URL_2;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.HA_GROUP_NAME;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.POLICY;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_HA_GROUP_NAME;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VERSION;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.ZK_URL_1;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.ZK_URL_2;
+import static org.apache.phoenix.jdbc.PhoenixHAAdmin.getLocalZkUrl;
+import static org.apache.phoenix.jdbc.PhoenixHAAdmin.toPath;
+import static org.apache.phoenix.util.PhoenixRuntime.JDBC_PROTOCOL_SEPARATOR;
+import static org.apache.phoenix.util.PhoenixRuntime.JDBC_PROTOCOL_ZK;
 
 
 /**
- * Write-through cache for HAGroupStore.
+ * Main implementation of HAGroupStoreClient with peer support.
+ * Write-through cache for HAGroupStore based on {@link HAGroupStoreRecord}.
  * Uses {@link PathChildrenCache} from {@link org.apache.curator.framework.CuratorFramework}.
  */
 public class HAGroupStoreClient implements Closeable {
 
+    public static final String ZK_CONSISTENT_HA_NAMESPACE =
+            "phoenix" + ZKPaths.PATH_SEPARATOR + "consistentHA";
     private static final long HA_GROUP_STORE_CLIENT_INITIALIZATION_TIMEOUT_MS = 30000L;
-    private static volatile HAGroupStoreClient haGroupStoreClientInstance;
+    // Multiplier for ZK session timeout to account for time it will take for HMaster to abort
+    // the region server in case ZK connection is lost from the region server.
+    private static final double ZK_SESSION_TIMEOUT_MULTIPLIER = 1.1;
+    private static final String CACHE_TYPE_LOCAL = "LOCAL";
+    private static final String CACHE_TYPE_PEER = "PEER";
     private PhoenixHAAdmin phoenixHaAdmin;
+    private PhoenixHAAdmin peerPhoenixHaAdmin;
     private static final Logger LOGGER = LoggerFactory.getLogger(HAGroupStoreClient.class);
-    // Map contains <ClusterRole, Map<HAGroupName(String), ClusterRoleRecord>>
-    private final ConcurrentHashMap<ClusterRoleRecord.ClusterRole, ConcurrentHashMap<String, ClusterRoleRecord>> clusterRoleToCRRMap
-            = new ConcurrentHashMap<>();
-    private PathChildrenCache pathChildrenCache;
-    private volatile boolean isHealthy;
+    // Map of <ZKUrl, <HAGroupName, HAGroupStoreClientInstance>>
+    private static final Map<String, ConcurrentHashMap<String, HAGroupStoreClient>> instances =
+            new ConcurrentHashMap<>();
+    // HAGroupName for this instance
+    private final String haGroupName;
+    // PathChildrenCache for current cluster and HAGroupName
+    private PathChildrenCache pathChildrenCache = null;
+    // PathChildrenCache for peer cluster and HAGroupName
+    private PathChildrenCache peerPathChildrenCache = null;
+    // Whether the client is healthy
+    private volatile boolean isHealthy = false;
+    // Configuration
+    private final Configuration conf;
+    // ZK URL for the current cluster and HAGroupName
+    private String zkUrl;
+    // Peer ZK URL for peer cluster and HAGroupName
+    private String peerZKUrl = null;
+    // Peer Custom Event Listener
+    private final PathChildrenCacheListener peerCustomPathChildrenCacheListener;
+    // Wait time for sync mode
+    private final long waitTimeForSyncModeInMs;
+    // Policy for the HA group
+    private HighAvailabilityPolicy policy;
+    private ClusterRole clusterRole;
+    private ClusterRole peerClusterRole;
+    private String clusterUrl;
+    private String peerClusterUrl;
+    private long clusterRoleRecordVersion;
+
+    public static HAGroupStoreClient getInstance(Configuration conf, String haGroupName) throws SQLException {
+        return getInstanceForZkUrl(conf, haGroupName, null);
+    }
 
     /**
      * Creates/gets an instance of HAGroupStoreClient.
      * Can return null instance if unable to initialize.
      *
      * @param conf configuration
+     * @param haGroupName name of the HA group. Only specified HA group is tracked.
+     * @param zkUrl zkUrl to use for the client. Prefer providing this parameter to avoid
+     *             the overhead of getting the local zkUrl from the configuration.
      * @return HAGroupStoreClient instance
      */
-    public static HAGroupStoreClient getInstance(Configuration conf) {
-        if (haGroupStoreClientInstance == null || !haGroupStoreClientInstance.isHealthy) {
+    public static HAGroupStoreClient getInstanceForZkUrl(Configuration conf, String haGroupName,
+            String zkUrl) throws SQLException {
+        Preconditions.checkNotNull(haGroupName, "haGroupName cannot be null");
+        String localZkUrl = Objects.toString(zkUrl, getLocalZkUrl(conf));
+        Preconditions.checkNotNull(localZkUrl, "zkUrl cannot be null");
+        HAGroupStoreClient result = instances.getOrDefault(localZkUrl, new ConcurrentHashMap<>())
+                .getOrDefault(haGroupName, null);
+        if (result == null || !result.isHealthy) {
             synchronized (HAGroupStoreClient.class) {
-                if (haGroupStoreClientInstance == null || !haGroupStoreClientInstance.isHealthy) {
-                    haGroupStoreClientInstance = new HAGroupStoreClient(conf, null);
-                    if (!haGroupStoreClientInstance.isHealthy) {
-                        haGroupStoreClientInstance.close();
-                        haGroupStoreClientInstance = null;
+                result = instances.getOrDefault(localZkUrl, new ConcurrentHashMap<>())
+                        .getOrDefault(haGroupName, null);
+                if (result == null || !result.isHealthy) {
+                    result = new HAGroupStoreClient(conf, null, null, haGroupName, zkUrl);
+                    if (!result.isHealthy) {
+                        result.close();
+                        result = null;
+                    } else {
+                        instances.putIfAbsent(localZkUrl, new ConcurrentHashMap<>());
+                        instances.get(localZkUrl).put(haGroupName, result);
                     }
                 }
             }
         }
-        return haGroupStoreClientInstance;
+        return result;
+    }
+
+    /**
+     * Get the list of HAGroupNames from system table.
+     * We can also get the list of HAGroupNames from the system table by providing the zkUrl in
+     * where clause but we need to match the formatted zkUrl with the zkUrl in the system table so
+     * that matching is done correctly.
+     *
+     * @param zkUrl for connecting to Table
+     * @return the list of HAGroupNames
+     * @throws SQLException in case of unexpected error
+     */
+    public static List<String> getHAGroupNames(String zkUrl) throws SQLException {
+        List<String> result = new ArrayList<>();
+        String queryString = String.format("SELECT %s,%s,%s FROM %s", HA_GROUP_NAME, ZK_URL_1,
+                ZK_URL_2, SYSTEM_HA_GROUP_NAME);
+        try (PhoenixConnection conn = (PhoenixConnection) DriverManager.getConnection(
+                JDBC_PROTOCOL_ZK + JDBC_PROTOCOL_SEPARATOR + zkUrl);
+             Statement stmt = conn.createStatement();
+             ResultSet rs = stmt.executeQuery(queryString)) {
+            while (rs.next()) {
+                String zkUrl1 = rs.getString(ZK_URL_1);
+                String zkUrl2 = rs.getString(ZK_URL_2);
+                String formattedZkUrl1 = JDBCUtil.formatUrl(zkUrl1, RegistryType.ZK);
+                String formattedZkUrl2 = JDBCUtil.formatUrl(zkUrl2, RegistryType.ZK);
+                String formattedZkUrl = JDBCUtil.formatUrl(zkUrl, RegistryType.ZK);
+                if (StringUtils.equals(formattedZkUrl1, formattedZkUrl) ||
+                        StringUtils.equals(formattedZkUrl2, formattedZkUrl)) {
+                    result.add(rs.getString(HA_GROUP_NAME));
+                }
+            }
+        }
+        return result;
     }
 
     @VisibleForTesting
-    HAGroupStoreClient(final Configuration conf, final PathChildrenCacheListener pathChildrenCacheListener) {
+    HAGroupStoreClient(final Configuration conf,
+            final PathChildrenCacheListener pathChildrenCacheListener,
+            final PathChildrenCacheListener peerPathChildrenCacheListener,
+            final String haGroupName,
+            final String zkUrl) {
+        this.conf = conf;
+        this.haGroupName = haGroupName;
+        this.zkUrl = zkUrl;
+        this.waitTimeForSyncModeInMs =  (long) Math.ceil(conf.getLong(ZK_SESSION_TIMEOUT, DEFAULT_ZK_SESSION_TIMEOUT)
+                * ZK_SESSION_TIMEOUT_MULTIPLIER);
+        // Custom Event Listener
+        this.peerCustomPathChildrenCacheListener = peerPathChildrenCacheListener;
         try {
-            this.phoenixHaAdmin = new PhoenixHAAdmin(conf);
-            final PathChildrenCache pathChildrenCache;
-                pathChildrenCache = new PathChildrenCache(phoenixHaAdmin.getCurator(), ZKPaths.PATH_SEPARATOR, true);
-            final CountDownLatch latch = new CountDownLatch(1);
-            if (pathChildrenCacheListener != null) {
-                pathChildrenCache.getListenable().addListener(pathChildrenCacheListener);
-            } else {
-                pathChildrenCache.getListenable().addListener((client, event) -> {
-                    LOGGER.info("HAGroupStoreClient PathChildrenCache Received event for type {}", event.getType());
-                    final ChildData childData = event.getData();
-                    ClusterRoleRecord eventCRR = extractCRROrNull(childData);
-                    switch (event.getType()) {
-                        case CHILD_ADDED:
-                        case CHILD_UPDATED:
-                            if (eventCRR != null && eventCRR.getHaGroupName() != null) {
-                                updateClusterRoleRecordMap(eventCRR);
-                            }
-                            break;
-                        case CHILD_REMOVED:
-                            // In case of CHILD_REMOVED, we get the old version of data that was just deleted in event.
-                            if (eventCRR != null && eventCRR.getHaGroupName() != null
-                                    && !eventCRR.getHaGroupName().isEmpty()
-                                    && eventCRR.getRole(phoenixHaAdmin.getZkUrl()) != null) {
-                                LOGGER.info("Received CHILD_REMOVED event, Removing CRR {} from existing CRR Map {}", eventCRR, clusterRoleToCRRMap);
-                                final ClusterRoleRecord.ClusterRole role = eventCRR.getRole(phoenixHaAdmin.getZkUrl());
-                                clusterRoleToCRRMap.putIfAbsent(role, new ConcurrentHashMap<>());
-                                clusterRoleToCRRMap.get(role).remove(eventCRR.getHaGroupName());
-                            }
-                            break;
-                        case INITIALIZED:
-                            latch.countDown();
-                            break;
-                        case CONNECTION_LOST:
-                        case CONNECTION_SUSPENDED:
-                            isHealthy = false;
-                            break;
-                        case CONNECTION_RECONNECTED:
-                            isHealthy = true;
-                            break;
-                        default:
-                            LOGGER.warn("Unexpected event type {}, complete event {}", event.getType(), event);
-                    }
-                });
+            // Initialize HAGroupStoreClient attributes
+            initializeHAGroupStoreClientAttributes(haGroupName);
+            // Initialize Phoenix HA Admin
+            this.phoenixHaAdmin = new PhoenixHAAdmin(this.zkUrl, conf, ZK_CONSISTENT_HA_NAMESPACE);
+            // Initialize local cache
+            this.pathChildrenCache = initializePathChildrenCache(phoenixHaAdmin,
+                    pathChildrenCacheListener, CACHE_TYPE_LOCAL);
+            // Initialize ZNode if not present in ZK
+            initializeZNodeIfNeeded();
+            if (this.pathChildrenCache != null) {
+                this.isHealthy = true;
+                // Initialize peer cache
+                maybeInitializePeerPathChildrenCache();
             }
-            pathChildrenCache.start(PathChildrenCache.StartMode.POST_INITIALIZED_EVENT);
-            this.pathChildrenCache = pathChildrenCache;
-            isHealthy = latch.await(HA_GROUP_STORE_CLIENT_INITIALIZATION_TIMEOUT_MS, TimeUnit.MILLISECONDS);
-            buildClusterRoleToCRRMap();
+
         } catch (Exception e) {
-            isHealthy = false;
-            LOGGER.error("Unexpected error occurred while initializing HAGroupStoreClient, marking cache as unhealthy", e);
+            this.isHealthy = false;
+            close();
+            LOGGER.error("Unexpected error occurred while initializing HAGroupStoreClient, "
+                    + "marking cache as unhealthy", e);
         }
     }
 
-    private ClusterRoleRecord extractCRROrNull(final ChildData childData) {
-        if (childData != null) {
-            byte[] data = childData.getData();
-            return ClusterRoleRecord.fromJson(data).orElse(null);
+    /**
+     * Rebuilds the internal cache by querying for all needed data.
+     * This is a blocking operation that does not generate events to listeners.
+     *
+     * @throws Exception if rebuild fails or client is not healthy
+     */
+    public void rebuild() throws Exception {
+        if (!isHealthy) {
+            throw new IOException("HAGroupStoreClient is not healthy");
         }
-        return null;
+        LOGGER.info("Rebuilding HAGroupStoreClient for HA group {}", haGroupName);
+        initializeHAGroupStoreClientAttributes(haGroupName);
+        initializeZNodeIfNeeded();
+        maybeInitializePeerPathChildrenCache();
+
+        // NOTE: this is a BLOCKING method.
+        // Completely rebuild the internal cache by querying for all needed data
+        // WITHOUT generating any events to send to listeners.
+        if (pathChildrenCache != null) {
+            pathChildrenCache.rebuild();
+        }
+        if (peerPathChildrenCache != null) {
+            peerPathChildrenCache.rebuild();
+        }
+        LOGGER.info("Rebuild Complete for HAGroupStoreClient for HA group {}", haGroupName);
     }
 
-    private void updateClusterRoleRecordMap(final ClusterRoleRecord crr) {
-        if (crr != null && crr.getHaGroupName() != null && crr.getRole(phoenixHaAdmin.getZkUrl()) != null) {
-            ClusterRoleRecord.ClusterRole role = crr.getRole(phoenixHaAdmin.getZkUrl());
-            LOGGER.info("Updating Existing CRR Map {} with new CRR {}", clusterRoleToCRRMap, crr);
-            clusterRoleToCRRMap.putIfAbsent(role, new ConcurrentHashMap<>());
-            clusterRoleToCRRMap.get(role).put(crr.getHaGroupName(), crr);
-            LOGGER.info("Added new CRR {} to CRR Map", crr);
-            // Remove any pre-existing mapping with any other role for this HAGroupName
-            for (ClusterRoleRecord.ClusterRole mapRole : clusterRoleToCRRMap.keySet()) {
-                if (mapRole != role) {
-                    ConcurrentHashMap<String, ClusterRoleRecord> roleWiseMap = clusterRoleToCRRMap.get(mapRole);
-                    if (roleWiseMap.containsKey(crr.getHaGroupName())) {
-                        LOGGER.info("Removing any pre-existing mapping with role {} for HAGroupName {}", mapRole, crr.getHaGroupName());
-                        roleWiseMap.remove(crr.getHaGroupName());
-                    }
-                }
-            }
-            LOGGER.info("Final Updated CRR Map {}", clusterRoleToCRRMap);
+    /**
+     * Get HAGroupStoreRecord from local quorum.
+     *
+     * @return HAGroupStoreRecord for the specified HA group name, or null if not found
+     * @throws IOException if the client is not healthy
+     */
+    public HAGroupStoreRecord getHAGroupStoreRecord() throws IOException {
+        if (!isHealthy) {
+            throw new IOException("HAGroupStoreClient is not healthy");
         }
+        return fetchCacheRecord(this.pathChildrenCache, CACHE_TYPE_LOCAL).getLeft();
     }
 
-    private void buildClusterRoleToCRRMap() {
-        List<ClusterRoleRecord> clusterRoleRecords = pathChildrenCache.getCurrentData().stream().map(this::extractCRROrNull)
-                .filter(Objects::nonNull).collect(Collectors.toList());
-        for (ClusterRoleRecord crr : clusterRoleRecords) {
-            updateClusterRoleRecordMap(crr);
+    /**
+     * Set the HA group status for the specified HA group name.
+     * Checks if the status is needed to be updated based on logic in isUpdateNeeded function.
+     *
+     * @param haGroupState the HA group state to set
+     * @throws IOException if the client is not healthy or the operation fails
+     * @throws StaleHAGroupStoreRecordVersionException if the version is stale
+     * @throws InvalidClusterRoleTransitionException when transition is not valid
+     */
+    public void setHAGroupStatusIfNeeded(HAGroupStoreRecord.HAGroupState haGroupState)
+            throws IOException, StaleHAGroupStoreRecordVersionException,
+            InvalidClusterRoleTransitionException {
+        Preconditions.checkNotNull(haGroupState, "haGroupState cannot be null");
+        if (!isHealthy) {
+            throw new IOException("HAGroupStoreClient is not healthy");
+        }
+        Pair<HAGroupStoreRecord, Stat> cacheRecord = fetchCacheRecord(
+                this.pathChildrenCache, CACHE_TYPE_LOCAL);
+        HAGroupStoreRecord currentHAGroupStoreRecord = cacheRecord.getLeft();
+        Stat currentHAGroupStoreRecordStat = cacheRecord.getRight();
+        if (currentHAGroupStoreRecord == null) {
+            throw new IOException("Current HAGroupStoreRecordStat in cache is null, "
+                    + "cannot update HAGroupStoreRecord, the record should be initialized "
+                    + "in System Table first" + haGroupName);
+        }
+        if (isUpdateNeeded(currentHAGroupStoreRecord.getHAGroupState(),
+                currentHAGroupStoreRecordStat.getMtime(), haGroupState)) {
+                HAGroupStoreRecord newHAGroupStoreRecord = new HAGroupStoreRecord(
+                        currentHAGroupStoreRecord.getProtocolVersion(),
+                        currentHAGroupStoreRecord.getHaGroupName(),
+                        haGroupState
+                );
+                // TODO: Check if cluster role is changing, if so, we need to update
+                // the system table first
+                // Lock the row in System Table and make sure update is reflected
+                // in all regionservers
+                // It should automatically update the ZK record as well.
+                phoenixHaAdmin.updateHAGroupStoreRecordInZooKeeper(haGroupName,
+                        newHAGroupStoreRecord, currentHAGroupStoreRecordStat.getVersion());
+        } else {
+            LOGGER.info("Not updating HAGroupStoreRecord for HA group {} with state {}",
+                    haGroupName, haGroupState);
         }
     }
 
-    public void rebuild() throws Exception {
+    /**
+     * Returns the ClusterRoleRecord for the cluster pair.
+     * Information in System Table for peer cluster might be stale,
+     * so we need to get the latest information from ZK.
+     *
+     * @return ClusterRoleRecord for the cluster pair
+     * @throws IOException
+     */
+    public ClusterRoleRecord getClusterRoleRecord() throws IOException {
+        HAGroupStoreRecord peerHAGroupStoreRecord = getHAGroupStoreRecordFromPeer();
+        ClusterRoleRecord.ClusterRole peerClusterRole = peerHAGroupStoreRecord != null
+                ? peerHAGroupStoreRecord.getClusterRole()
+                : ClusterRole.UNKNOWN;
+        return new ClusterRoleRecord(this.haGroupName,
+                                    this.policy,
+                                    this.clusterUrl,
+                                    this.clusterRole,
+                                    this.peerClusterUrl,
+                                    peerClusterRole,
+                                    this.clusterRoleRecordVersion);
+    }
+
+    /**
+     * Get HAGroupStoreRecord from peer cluster.
+     *
+     * @return HAGroupStoreRecord for the specified HA group name, or null if not found
+     * @throws IOException if the client is not healthy
+     */
+    private HAGroupStoreRecord getHAGroupStoreRecordFromPeer() throws IOException {
         if (!isHealthy) {
             throw new IOException("HAGroupStoreClient is not healthy");
         }
-        LOGGER.info("Rebuilding HAGroupStoreClient for HA groups");
-        // NOTE: this is a BLOCKING method.
-        // Completely rebuild the internal cache by querying for all needed data
-        // WITHOUT generating any events to send to listeners.
-        pathChildrenCache.rebuild();
-        buildClusterRoleToCRRMap();
-        LOGGER.info("Rebuild Complete for HAGroupStoreClient");
+        return fetchCacheRecord(this.peerPathChildrenCache, CACHE_TYPE_PEER).getLeft();
+    }
+
+    private void initializeZNodeIfNeeded() throws IOException,
+            StaleHAGroupStoreRecordVersionException {
+        // Sometimes the ZNode might not be available in local cache yet, so better to check
+        // in ZK directly if we need to initialize
+        Pair<HAGroupStoreRecord, Stat> cacheRecordFromZK =
+                phoenixHaAdmin.getHAGroupStoreRecordInZooKeeper(this.haGroupName);
+        HAGroupStoreRecord haGroupStoreRecord = cacheRecordFromZK.getLeft();
+        HAGroupStoreRecord newHAGroupStoreRecord = new HAGroupStoreRecord(
+            HAGroupStoreRecord.DEFAULT_PROTOCOL_VERSION,
+            haGroupName,
+            this.clusterRole.getDefaultHAGroupState()
+        );
+        // Only update current ZNode if it doesn't have the same role as present in System Table.
+        // If not exists, then create ZNode
+        if (haGroupStoreRecord != null &&
+                !haGroupStoreRecord.getClusterRole().equals(this.clusterRole)) {
+            phoenixHaAdmin.updateHAGroupStoreRecordInZooKeeper(haGroupName,
+                    newHAGroupStoreRecord, cacheRecordFromZK.getRight().getVersion());
+        } else if (haGroupStoreRecord == null) {
+            phoenixHaAdmin.createHAGroupStoreRecordInZooKeeper(newHAGroupStoreRecord);
+        }
+    }
+
+    private void initializeHAGroupStoreClientAttributes(String haGroupName) throws SQLException {
+        String queryString = String.format("SELECT * FROM %s WHERE %s = '%s'",
+                SYSTEM_HA_GROUP_NAME, HA_GROUP_NAME, haGroupName);
+        try (PhoenixConnection conn = (PhoenixConnection) DriverManager.getConnection(
+                JDBC_PROTOCOL_ZK + JDBC_PROTOCOL_SEPARATOR + zkUrl);
+             Statement stmt = conn.createStatement();
+             ResultSet rs = stmt.executeQuery(queryString)) {
+            if (rs.next()) {
+                this.policy = HighAvailabilityPolicy.valueOf(rs.getString(POLICY));
+                String zkUrl1 = rs.getString(ZK_URL_1);
+                String zkUrl2 = rs.getString(ZK_URL_2);
+                String clusterRole1 = rs.getString(CLUSTER_ROLE_1);
+                String clusterRole2 = rs.getString(CLUSTER_ROLE_2);
+                String clusterUrl1 = rs.getString(CLUSTER_URL_1);
+                String clusterUrl2 = rs.getString(CLUSTER_URL_2);
+                this.clusterRoleRecordVersion = rs.getLong(VERSION);
+                Preconditions.checkNotNull(zkUrl1, "ZK_URL_1 in System Table cannot be null");
+                Preconditions.checkNotNull(zkUrl2, "ZK_URL_2 in System Table cannot be null");
+                String formattedZkUrl1 = JDBCUtil.formatUrl(zkUrl1, RegistryType.ZK);
+                String formattedZkUrl2 = JDBCUtil.formatUrl(zkUrl2, RegistryType.ZK);
+                String formattedZkUrl = JDBCUtil.formatUrl(this.zkUrl, RegistryType.ZK);
+                if (StringUtils.equals(formattedZkUrl1, formattedZkUrl)) {
+                    this.peerZKUrl = formattedZkUrl2;
+                    this.clusterRole = ClusterRoleRecord.ClusterRole.from(
+                            clusterRole1.getBytes(StandardCharsets.UTF_8));
+                    this.peerClusterRole = ClusterRoleRecord.ClusterRole.from(
+                            clusterRole2.getBytes(StandardCharsets.UTF_8));
+                    this.clusterUrl = clusterUrl1;
+                    this.peerClusterUrl = clusterUrl2;
+                } else if (StringUtils.equals(formattedZkUrl2, formattedZkUrl)) {
+                    this.peerZKUrl = JDBCUtil.formatUrl(zkUrl1, RegistryType.ZK);
+                    this.clusterRole = ClusterRoleRecord.ClusterRole.from(
+                            clusterRole2.getBytes(StandardCharsets.UTF_8));
+                    this.peerClusterRole = ClusterRoleRecord.ClusterRole.from(
+                            clusterRole1.getBytes(StandardCharsets.UTF_8));
+                    this.clusterUrl = clusterUrl2;
+                    this.peerClusterUrl = clusterUrl1;
+                }
+            } else {
+                throw new SQLException("HAGroupStoreRecord not found for HA group name: " +
+                        haGroupName + " in System Table " + SYSTEM_HA_GROUP_NAME);
+            }
+        }
+        Preconditions.checkNotNull(this.clusterRole,
+                "Cluster role in System Table cannot be null");
+        Preconditions.checkNotNull(this.peerClusterRole,
+                "Peer cluster role in System Table cannot be null");
+        Preconditions.checkNotNull(this.clusterUrl,
+                "Cluster URL in System Table cannot be null");
+        Preconditions.checkNotNull(this.peerZKUrl,
+                "Peer ZK URL in System Table cannot be null");
+        Preconditions.checkNotNull(this.peerClusterUrl,
+                "Peer Cluster URL in System Table cannot be null");
+        Preconditions.checkNotNull(this.clusterRoleRecordVersion,
+                "Cluster role record version in System Table cannot be null");
+    }
+
+    private void maybeInitializePeerPathChildrenCache() {
+        if (StringUtils.isNotBlank(this.peerZKUrl)) {
+            try {
+                // Setup peer connection if needed (first time or ZK Url changed)
+                if (peerPathChildrenCache == null
+                    || (peerPhoenixHaAdmin != null &&
+                            !StringUtils.equals(this.peerZKUrl, peerPhoenixHaAdmin.getZkUrl()))) {
+                    // Clean up existing peer connection if it exists
+                    closePeerConnection();
+                    // Setup new peer connection
+                    this.peerPhoenixHaAdmin
+                            = new PhoenixHAAdmin(this.peerZKUrl, conf, ZK_CONSISTENT_HA_NAMESPACE);
+                    // Create new PeerPathChildrenCache
+                    this.peerPathChildrenCache = initializePathChildrenCache(peerPhoenixHaAdmin,
+                            this.peerCustomPathChildrenCacheListener, CACHE_TYPE_PEER);
+                }
+            } catch (Exception e) {
+                closePeerConnection();
+                LOGGER.error("Unable to initialize PeerPathChildrenCache for HAGroupStoreClient",
+                        e);
+                // Don't think we should mark HAGroupStoreClient as unhealthy if
+                // peerCache is unhealthy, if needed we can introduce a config to control behavior.
+            }
+        } else {
+            // Close Peer Cache for this HAGroupName if currentClusterRecord is null
+            // or peerZKUrl is blank
+            closePeerConnection();
+            LOGGER.error("Not initializing PeerPathChildrenCache for HAGroupStoreClient "
+                    + "with HAGroupName {} as peerZKUrl is blank", haGroupName);
+        }
     }
 
+    private PathChildrenCache initializePathChildrenCache(PhoenixHAAdmin admin,
+            PathChildrenCacheListener customListener, String cacheType) {
+        LOGGER.info("Initializing {} PathChildrenCache with URL {}", cacheType, admin.getZkUrl());
+        PathChildrenCache newPathChildrenCache = null;
+        try {
+            newPathChildrenCache = new PathChildrenCache(admin.getCurator(),
+                    ZKPaths.PATH_SEPARATOR, true);
+            final CountDownLatch latch = new CountDownLatch(1);
+            newPathChildrenCache.getListenable().addListener(customListener != null
+                    ? customListener
+                    : createCacheListener(latch, cacheType));
+            newPathChildrenCache.start(PathChildrenCache.StartMode.POST_INITIALIZED_EVENT);
+            boolean initialized = latch.await(HA_GROUP_STORE_CLIENT_INITIALIZATION_TIMEOUT_MS,
+                    TimeUnit.MILLISECONDS);
+            return initialized ? newPathChildrenCache : null;
+        } catch (Exception e) {
+            if (newPathChildrenCache != null) {
+                try {
+                    newPathChildrenCache.close();
+                } catch (IOException ioe) {
+                    LOGGER.error("Failed to close {} PathChildrenCache with ZKUrl",
+                            cacheType, ioe);
+                }
+            }
+            LOGGER.error("Failed to initialize {} PathChildrenCache", cacheType, e);
+            return null;
+        }
+    }
+
+    private PathChildrenCacheListener createCacheListener(CountDownLatch latch, String cacheType) {
+        return (client, event) -> {
+            final ChildData childData = event.getData();
+            HAGroupStoreRecord eventRecord = extractHAGroupStoreRecordOrNull(childData);
+            LOGGER.info("HAGroupStoreClient Cache {} received event {} type {} at {}",
+                    cacheType, eventRecord, event.getType(), System.currentTimeMillis());
+            switch (event.getType()) {
+                // TODO: Add support for event watcher for HAGroupStoreRecord
+                // case CHILD_ADDED:
+                // case CHILD_UPDATED:
+                // case CHILD_REMOVED:
+                case INITIALIZED:
+                    latch.countDown();
+                    break;
+                case CONNECTION_LOST:
+                case CONNECTION_SUSPENDED:
+                    if (CACHE_TYPE_LOCAL.equals(cacheType)) {
+                        isHealthy = false;
+                    }
+                    LOGGER.warn("{} HAGroupStoreClient cache connection lost/suspended",
+                            cacheType);
+                    break;
+                case CONNECTION_RECONNECTED:
+                    if (CACHE_TYPE_LOCAL.equals(cacheType)) {
+                        isHealthy = true;
+                    }
+                    LOGGER.info("{} HAGroupStoreClient cache connection reconnected", cacheType);
+                    break;
+                default:
+                    LOGGER.warn("Unexpected {} event type {}, complete event {}",
+                            cacheType, event.getType(), event);
+            }
+        };
+    }
+
+
+    private Pair<HAGroupStoreRecord, Stat> fetchCacheRecord(PathChildrenCache cache,
+            String cacheType) {
+        if (cache == null) {
+            LOGGER.warn("{} HAGroupStoreClient cache is null, returning null", cacheType);
+            return Pair.of(null, null);
+        }
+
+        String targetPath = toPath(this.haGroupName);
+        // Try to get record from current cache data
+        Pair<HAGroupStoreRecord, Stat> result = extractRecordAndStat(cache, targetPath, cacheType);
+        if (result.getLeft() != null) {
+            return result;
+        }
+
+        if (cacheType.equals(CACHE_TYPE_PEER)) {
+            return Pair.of(null, null);
+        }
+        // If no record found, try to rebuild and fetch again
+        LOGGER.info("No record found at path {} for {} cluster, trying to initialize ZNode "
+                + "from System Table in case it might have been deleted",
+                    targetPath, cacheType);
+        try {
+            rebuild();
+            return extractRecordAndStat(cache, targetPath, cacheType);
+        } catch (Exception e) {
+            LOGGER.error("Failed to initialize ZNode from System Table, giving up "
+                    + "and returning null", e);
+            return Pair.of(null, null);
+        }
+    }
+
+    private Pair<HAGroupStoreRecord, Stat> extractRecordAndStat(PathChildrenCache cache,
+            String targetPath, String cacheType) {
+        ChildData childData = cache.getCurrentData(targetPath);
+        if (childData != null) {
+            HAGroupStoreRecord record = extractHAGroupStoreRecordOrNull(childData);
+            Stat currentStat = childData.getStat();
+            LOGGER.info("Built {} cluster record: {}", cacheType, record);
+            return Pair.of(record, currentStat);
+        }
+        return Pair.of(null, null);
+    }
+
+    private HAGroupStoreRecord extractHAGroupStoreRecordOrNull(final ChildData childData) {
+        if (childData != null) {
+            byte[] data = childData.getData();
+            return HAGroupStoreRecord.fromJson(data).orElse(null);
+        }
+        return null;
+    }
+
+
+    /**
+     * Closes the peer connection and cleans up peer-related resources.
+     */
+    private void closePeerConnection() {
+        try {
+            if (peerPathChildrenCache != null) {
+                peerPathChildrenCache.close();
+                peerPathChildrenCache = null;
+            }
+            if (peerPhoenixHaAdmin != null) {
+                peerPhoenixHaAdmin.close();
+                peerPhoenixHaAdmin = null;
+            }
+        } catch (Exception e) {
+            LOGGER.warn("Failed to close peer connection", e);
+        }
+    }
 
     @Override
     public void close() {
         try {
             LOGGER.info("Closing HAGroupStoreClient");
-            clusterRoleToCRRMap.clear();
             if (pathChildrenCache != null) {
                 pathChildrenCache.close();
+                pathChildrenCache = null;
             }
+            closePeerConnection();
             LOGGER.info("Closed HAGroupStoreClient");
         } catch (IOException e) {
             LOGGER.error("Exception closing HAGroupStoreClient", e);
         }
     }
 
-    public List<ClusterRoleRecord> getCRRsByClusterRole(ClusterRoleRecord.ClusterRole clusterRole) throws IOException {
-        if (!isHealthy) {
-            throw new IOException("HAGroupStoreClient is not healthy");
+    /**
+     * Checks if the HAGroupStoreRecord needs to be updated.
+     * If the cluster role is allowed to transition to the new state and the status refresh
+     * interval has expired, the HAGroupStoreRecord needs to be updated.
+     * If the transition is not allowed, an exception is thrown.
+     *
+     * @param currentHAGroupState the current HAGroupState of the HAGroupStoreRecord
+     * @param currentHAGroupStoreRecordMtime the last modified time of the current
+     *                                      HAGroupStoreRecord
+     * @param newHAGroupState the cluster state to check
+     * @return true if the HAGroupStoreRecord needs to be updated, false otherwise
+     * @throws InvalidClusterRoleTransitionException if the cluster role transition is invalid
+     */
+    private boolean isUpdateNeeded(HAGroupStoreRecord.HAGroupState currentHAGroupState,
+                                   long currentHAGroupStoreRecordMtime,
+                                   HAGroupStoreRecord.HAGroupState newHAGroupState)
+            throws InvalidClusterRoleTransitionException {
+        long waitTime = 0L;
+        if (currentHAGroupState.isTransitionAllowed(newHAGroupState)) {
+            if (currentHAGroupState == HAGroupState.ACTIVE_NOT_IN_SYNC
+                    && newHAGroupState == HAGroupState.ACTIVE_IN_SYNC) {
+                waitTime = waitTimeForSyncModeInMs;
+            }
+        } else {
+            throw new InvalidClusterRoleTransitionException("Cannot transition from "
+                    + currentHAGroupState + " to " + newHAGroupState);
         }
-        return ImmutableList.copyOf(clusterRoleToCRRMap.getOrDefault(clusterRole, new ConcurrentHashMap<>()).values());
+        return ((System.currentTimeMillis() - currentHAGroupStoreRecordMtime) > waitTime);
     }
-}
+
+}
\ No newline at end of file
diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HAGroupStoreManager.java b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HAGroupStoreManager.java
index 690953a7b0..e04d1a2dac 100644
--- a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HAGroupStoreManager.java
+++ b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HAGroupStoreManager.java
@@ -18,13 +18,31 @@
 package org.apache.phoenix.jdbc;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.phoenix.exception.InvalidClusterRoleTransitionException;
+import org.apache.phoenix.exception.StaleHAGroupStoreRecordVersionException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import java.io.IOException;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+
+import static org.apache.phoenix.jdbc.PhoenixHAAdmin.getLocalZkUrl;
 import static org.apache.phoenix.query.QueryServices.CLUSTER_ROLE_BASED_MUTATION_BLOCK_ENABLED;
-import static org.apache.phoenix.query.QueryServicesOptions.DEFAULT_CLUSTER_ROLE_BASED_MUTATION_BLOCK_ENABLED;
+import static org.apache.phoenix.query.QueryServicesOptions
+        .DEFAULT_CLUSTER_ROLE_BASED_MUTATION_BLOCK_ENABLED;
 
+/**
+ * Implementation of HAGroupStoreManager that uses HAGroupStoreClient.
+ * Manages all HAGroupStoreClient instances.
+ */
 public class HAGroupStoreManager {
     private static volatile HAGroupStoreManager haGroupStoreManagerInstance;
+    private static final Logger LOGGER = LoggerFactory.getLogger(HAGroupStoreManager.class);
     private final boolean mutationBlockEnabled;
+    private final String zkUrl;
     private final Configuration conf;
 
     /**
@@ -33,7 +51,7 @@ public class HAGroupStoreManager {
      * @param conf configuration
      * @return HAGroupStoreManager instance
      */
-    public static HAGroupStoreManager getInstance(Configuration conf) {
+    public static HAGroupStoreManager getInstance(final Configuration conf) {
         if (haGroupStoreManagerInstance == null) {
             synchronized (HAGroupStoreManager.class) {
                 if (haGroupStoreManagerInstance == null) {
@@ -47,18 +65,37 @@ public class HAGroupStoreManager {
     private HAGroupStoreManager(final Configuration conf) {
         this.mutationBlockEnabled = conf.getBoolean(CLUSTER_ROLE_BASED_MUTATION_BLOCK_ENABLED,
                 DEFAULT_CLUSTER_ROLE_BASED_MUTATION_BLOCK_ENABLED);
+        this.zkUrl = getLocalZkUrl(conf);
         this.conf = conf;
     }
 
     /**
-     * Checks whether mutation is blocked or not.
+     * Returns the list of all HA group names.
+     * @return list of all HA group names.
+     * @throws SQLException if there is an error with querying the table.
+     */
+    public List<String> getHAGroupNames() throws SQLException {
+        return HAGroupStoreClient.getHAGroupNames(this.zkUrl);
+    }
+
+
+    /**
+     * Checks whether mutation is blocked or not for a specific HA group.
+     *
+     * @param haGroupName name of the HA group, null for default HA group which tracks
+     *                   all HA groups.
+     * @return true if mutation is blocked, false otherwise.
      * @throws IOException when HAGroupStoreClient is not healthy.
      */
-    public boolean isMutationBlocked() throws IOException {
+    public boolean isMutationBlocked(String haGroupName) throws IOException, SQLException {
         if (mutationBlockEnabled) {
-            HAGroupStoreClient haGroupStoreClient = HAGroupStoreClient.getInstance(conf);
+            HAGroupStoreClient haGroupStoreClient = HAGroupStoreClient.getInstanceForZkUrl(conf,
+                    haGroupName, zkUrl);
             if (haGroupStoreClient != null) {
-                return !haGroupStoreClient.getCRRsByClusterRole(ClusterRoleRecord.ClusterRole.ACTIVE_TO_STANDBY).isEmpty();
+                return haGroupStoreClient.getHAGroupStoreRecord() != null
+                        && haGroupStoreClient.getHAGroupStoreRecord().getClusterRole() != null
+                        && haGroupStoreClient.getHAGroupStoreRecord().getClusterRole()
+                        .isMutationBlocked();
             }
             throw new IOException("HAGroupStoreClient is not initialized");
         }
@@ -66,15 +103,123 @@ public class HAGroupStoreManager {
     }
 
     /**
-     * Force rebuilds the HAGroupStoreClient
-     * @throws Exception
+     * Force rebuilds the HAGroupStoreClient instance for all HA groups.
+     * If any HAGroupStoreClient instance is not created, it will be created.
+     * @param broadcastUpdate if true, the update will be broadcasted to all
+     *                       regionserver endpoints.
+     * @throws Exception in case of an error with dependencies or table.
      */
-    public void invalidateHAGroupStoreClient() throws Exception {
-        HAGroupStoreClient haGroupStoreClient = HAGroupStoreClient.getInstance(conf);
+    public void invalidateHAGroupStoreClient(boolean broadcastUpdate) throws Exception {
+        List<String> haGroupNames = HAGroupStoreClient.getHAGroupNames(this.zkUrl);
+        List<String> failedHAGroupNames = new ArrayList<>();
+        for (String haGroupName : haGroupNames) {
+            try {
+                invalidateHAGroupStoreClient(haGroupName, broadcastUpdate);
+            } catch (Exception e) {
+                failedHAGroupNames.add(haGroupName);
+                LOGGER.error("Failed to invalidate HAGroupStoreClient for " + haGroupName, e);
+            }
+        }
+        if (!failedHAGroupNames.isEmpty()) {
+            throw new IOException("Failed to invalidate HAGroupStoreClient for " +
+                    failedHAGroupNames
+                    + "successfully invalidated HAGroupStoreClient instance for "
+                    + haGroupNames.removeAll(failedHAGroupNames) + " HA groups");
+        }
+    }
+
+    /**
+     * Force rebuilds the HAGroupStoreClient for a specific HA group.
+     *
+     * @param haGroupName name of the HA group, null for default HA group and tracks all HA groups.
+     * @param broadcastUpdate if true, the update will be broadcasted to all
+     *                       regionserver endpoints.
+     * @throws Exception in case of an error with dependencies or table.
+     */
+    public void invalidateHAGroupStoreClient(final String haGroupName,
+            boolean broadcastUpdate) throws Exception {
+        HAGroupStoreClient haGroupStoreClient = HAGroupStoreClient.getInstanceForZkUrl(conf,
+                haGroupName, zkUrl);
         if (haGroupStoreClient != null) {
             haGroupStoreClient.rebuild();
         } else {
             throw new IOException("HAGroupStoreClient is not initialized");
         }
     }
-}
+
+    /**
+     * Returns the HAGroupStoreRecord for a specific HA group.
+     *
+     * @param haGroupName name of the HA group
+     * @return Optional HAGroupStoreRecord for the HA group, can be empty if the HA group
+     *        is not found.
+     * @throws IOException when HAGroupStoreClient is not healthy.
+     */
+    public Optional<HAGroupStoreRecord> getHAGroupStoreRecord(final String haGroupName)
+            throws IOException, SQLException {
+        HAGroupStoreClient haGroupStoreClient = HAGroupStoreClient.getInstanceForZkUrl(conf,
+                haGroupName, zkUrl);
+        if (haGroupStoreClient != null) {
+            return Optional.ofNullable(haGroupStoreClient.getHAGroupStoreRecord());
+        }
+        throw new IOException("HAGroupStoreClient is not initialized");
+    }
+
+    /**
+     * Sets the HAGroupStoreRecord to StoreAndForward mode in local cluster.
+     *
+     * @param haGroupName name of the HA group
+     * @throws IOException when HAGroupStoreClient is not healthy.
+     */
+    public void setHAGroupStatusToStoreAndForward(final String haGroupName)
+            throws IOException, StaleHAGroupStoreRecordVersionException,
+            InvalidClusterRoleTransitionException, SQLException {
+        HAGroupStoreClient haGroupStoreClient = HAGroupStoreClient.getInstanceForZkUrl(conf,
+                haGroupName, zkUrl);
+        if (haGroupStoreClient != null) {
+            haGroupStoreClient.setHAGroupStatusIfNeeded(
+                    HAGroupStoreRecord.HAGroupState.ACTIVE_NOT_IN_SYNC);
+        } else {
+            throw new IOException("HAGroupStoreClient is not initialized");
+        }
+    }
+
+    /**
+     * Sets the HAGroupStoreRecord to Sync mode in local cluster.
+     *
+     * @param haGroupName name of the HA group
+     * @throws IOException when HAGroupStoreClient is not healthy.
+     */
+    public void setHAGroupStatusRecordToSync(final String haGroupName)
+            throws IOException, StaleHAGroupStoreRecordVersionException,
+            InvalidClusterRoleTransitionException, SQLException {
+        HAGroupStoreClient haGroupStoreClient = HAGroupStoreClient.getInstanceForZkUrl(conf,
+                haGroupName, zkUrl);
+        if (haGroupStoreClient != null) {
+            haGroupStoreClient.setHAGroupStatusIfNeeded(
+                    HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+        } else {
+            throw new IOException("HAGroupStoreClient is not initialized");
+        }
+    }
+
+
+    /**
+     * Returns the ClusterRoleRecord for the cluster pair.
+     * If the peer cluster is not connected or peer cluster is not configured, it will
+     * return UNKNOWN for peer cluster.
+     * Only implemented by HAGroupStoreManagerImpl.
+     *
+     * @return ClusterRoleRecord for the cluster pair
+     * @throws IOException when HAGroupStoreClient is not healthy.
+     */
+    public ClusterRoleRecord getClusterRoleRecord(String haGroupName)
+            throws IOException, SQLException {
+        HAGroupStoreClient haGroupStoreClient = HAGroupStoreClient.getInstanceForZkUrl(conf,
+                haGroupName, zkUrl);
+        if (haGroupStoreClient != null) {
+            return haGroupStoreClient.getClusterRoleRecord();
+        }
+        throw new IOException("HAGroupStoreClient is not initialized");
+    }
+}
\ No newline at end of file
diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HAGroupStoreRecord.java b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HAGroupStoreRecord.java
new file mode 100644
index 0000000000..aa49a22eb2
--- /dev/null
+++ b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HAGroupStoreRecord.java
@@ -0,0 +1,264 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.phoenix.jdbc;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import org.apache.commons.lang3.builder.EqualsBuilder;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
+import org.apache.phoenix.thirdparty.com.google.common.base.Preconditions;
+import org.apache.phoenix.thirdparty.com.google.common.collect.ImmutableSet;
+import org.apache.phoenix.util.JacksonUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+
+/**
+ * Immutable class representing an HA group store record with simplified fields.
+ * This is a simplified version of ClusterRoleRecord that contains essential
+ * information about an HA group only for a single cluster.
+ */
+public class HAGroupStoreRecord {
+
+    private static final Logger LOG = LoggerFactory.getLogger(HAGroupStoreRecord.class);
+    public static final String DEFAULT_PROTOCOL_VERSION = "1.0";
+
+    /**
+     * Enum representing the HA group state with each state having a corresponding ClusterRole.
+     */
+    public enum HAGroupState {
+        ABORT_TO_ACTIVE_IN_SYNC,
+        ABORT_TO_ACTIVE_NOT_IN_SYNC,
+        ABORT_TO_STANDBY,
+        ACTIVE_IN_SYNC,
+        ACTIVE_NOT_IN_SYNC,
+        ACTIVE_NOT_IN_SYNC_TO_STANDBY,
+        ACTIVE_NOT_IN_SYNC_WITH_OFFLINE_PEER,
+        ACTIVE_IN_SYNC_TO_STANDBY,
+        ACTIVE_WITH_OFFLINE_PEER,
+        DEGRADED_STANDBY,
+        DEGRADED_STANDBY_FOR_READER,
+        DEGRADED_STANDBY_FOR_WRITER,
+        OFFLINE,
+        STANDBY,
+        STANDBY_TO_ACTIVE,
+        UNKNOWN;
+
+        private Set<HAGroupState> allowedTransitions;
+
+        /**
+         * Gets the corresponding ClusterRole for this HAGroupState.
+         * @return the corresponding ClusterRole
+         */
+        public ClusterRoleRecord.ClusterRole getClusterRole() {
+            switch (this) {
+                case ABORT_TO_ACTIVE_IN_SYNC:
+                case ABORT_TO_ACTIVE_NOT_IN_SYNC:
+                case ACTIVE_IN_SYNC:
+                case ACTIVE_NOT_IN_SYNC:
+                case ACTIVE_NOT_IN_SYNC_WITH_OFFLINE_PEER:
+                case ACTIVE_WITH_OFFLINE_PEER:
+                    return ClusterRoleRecord.ClusterRole.ACTIVE;
+                case ACTIVE_IN_SYNC_TO_STANDBY:
+                case ACTIVE_NOT_IN_SYNC_TO_STANDBY:
+                    return ClusterRoleRecord.ClusterRole.ACTIVE_TO_STANDBY;
+                case ABORT_TO_STANDBY:
+                case DEGRADED_STANDBY:
+                case DEGRADED_STANDBY_FOR_READER:
+                case DEGRADED_STANDBY_FOR_WRITER:
+                case STANDBY:
+                    return ClusterRoleRecord.ClusterRole.STANDBY;
+                case STANDBY_TO_ACTIVE:
+                    return ClusterRoleRecord.ClusterRole.STANDBY_TO_ACTIVE;
+                case OFFLINE:
+                    return ClusterRoleRecord.ClusterRole.OFFLINE;
+                case UNKNOWN:
+                default:
+                    return ClusterRoleRecord.ClusterRole.UNKNOWN;
+            }
+        }
+
+        static {
+            // Initialize allowed transitions
+            ACTIVE_NOT_IN_SYNC.allowedTransitions = ImmutableSet.of(
+                    ACTIVE_NOT_IN_SYNC, ACTIVE_IN_SYNC,
+                    ACTIVE_NOT_IN_SYNC_TO_STANDBY, ACTIVE_NOT_IN_SYNC_WITH_OFFLINE_PEER
+            );
+
+            ACTIVE_IN_SYNC.allowedTransitions = ImmutableSet.of(
+                    ACTIVE_NOT_IN_SYNC, ACTIVE_WITH_OFFLINE_PEER, ACTIVE_IN_SYNC_TO_STANDBY
+            );
+
+            STANDBY.allowedTransitions = ImmutableSet.of(STANDBY_TO_ACTIVE,
+                    DEGRADED_STANDBY_FOR_READER, DEGRADED_STANDBY_FOR_WRITER);
+            // This needs to be manually recovered by operator
+            OFFLINE.allowedTransitions = ImmutableSet.of();
+            // This needs to be manually recovered by operator
+            UNKNOWN.allowedTransitions = ImmutableSet.of();
+            ACTIVE_NOT_IN_SYNC_TO_STANDBY.allowedTransitions = ImmutableSet.of(ABORT_TO_ACTIVE_NOT_IN_SYNC, 
+                    ACTIVE_IN_SYNC_TO_STANDBY);
+            ACTIVE_IN_SYNC_TO_STANDBY.allowedTransitions = ImmutableSet.of(ABORT_TO_ACTIVE_IN_SYNC, STANDBY);
+            STANDBY_TO_ACTIVE.allowedTransitions = ImmutableSet.of(ABORT_TO_STANDBY,
+                    ACTIVE_IN_SYNC);
+            DEGRADED_STANDBY.allowedTransitions
+                    = ImmutableSet.of(DEGRADED_STANDBY_FOR_READER, DEGRADED_STANDBY_FOR_WRITER);
+            DEGRADED_STANDBY_FOR_WRITER.allowedTransitions = ImmutableSet.of(STANDBY,
+                    DEGRADED_STANDBY);
+            DEGRADED_STANDBY_FOR_READER.allowedTransitions = ImmutableSet.of(STANDBY,
+                    DEGRADED_STANDBY);
+            ACTIVE_WITH_OFFLINE_PEER.allowedTransitions = ImmutableSet.of(ACTIVE_NOT_IN_SYNC);
+            ABORT_TO_ACTIVE_IN_SYNC.allowedTransitions = ImmutableSet.of(ACTIVE_IN_SYNC);
+            ABORT_TO_ACTIVE_NOT_IN_SYNC.allowedTransitions = ImmutableSet.of(ACTIVE_NOT_IN_SYNC);
+            ABORT_TO_STANDBY.allowedTransitions = ImmutableSet.of(STANDBY);
+            ACTIVE_NOT_IN_SYNC_WITH_OFFLINE_PEER.allowedTransitions =
+                    ImmutableSet.of(ACTIVE_NOT_IN_SYNC);
+        }
+
+        /**
+         * Checks if the transition from this state to the target state is allowed.
+         * @param targetState the state to transition to
+         * @return true if the transition is allowed, false otherwise
+         */
+        public boolean isTransitionAllowed(HAGroupState targetState) {
+            return allowedTransitions.contains(targetState);
+        }
+
+        public static HAGroupState from(byte[] bytes) {
+            if (bytes == null) {
+                return UNKNOWN;
+            }
+            String value = new String(bytes, StandardCharsets.UTF_8);
+            return Arrays.stream(HAGroupState.values())
+                    .filter(r -> r.name().equalsIgnoreCase(value))
+                    .findFirst()
+                    .orElse(UNKNOWN);
+        }
+    }
+
+    private final String protocolVersion;
+    private final String haGroupName;
+    private final HAGroupState haGroupState;
+
+    @JsonCreator
+    public HAGroupStoreRecord(@JsonProperty("protocolVersion") String protocolVersion,
+                              @JsonProperty("haGroupName") String haGroupName,
+                              @JsonProperty("haGroupState") HAGroupState haGroupState) {
+        Preconditions.checkNotNull(haGroupName, "HA group name cannot be null!");
+        Preconditions.checkNotNull(haGroupState, "HA group state cannot be null!");
+
+        this.protocolVersion = Objects.toString(protocolVersion, DEFAULT_PROTOCOL_VERSION);
+        this.haGroupName = haGroupName;
+        this.haGroupState = haGroupState;
+    }
+
+    public static Optional<HAGroupStoreRecord> fromJson(byte[] bytes) {
+        if (bytes == null) {
+            return Optional.empty();
+        }
+        try {
+            return Optional.of(JacksonUtil.getObjectReader(HAGroupStoreRecord.class)
+                    .readValue(bytes));
+        } catch (Exception e) {
+            LOG.error("Fail to deserialize data to an HA group store record", e);
+            return Optional.empty();
+        }
+    }
+
+    public static byte[] toJson(HAGroupStoreRecord record) throws IOException {
+        return JacksonUtil.getObjectWriter().withoutAttribute("clusterRole")
+                .writeValueAsBytes(record);
+    }
+
+    public boolean hasSameInfo(HAGroupStoreRecord other) {
+        return haGroupName.equals(other.haGroupName) &&
+                haGroupState.equals(other.haGroupState) &&
+                protocolVersion.equals(other.protocolVersion);
+    }
+
+    public String getProtocolVersion() {
+        return protocolVersion;
+    }
+
+    public String getHaGroupName() {
+        return haGroupName;
+    }
+
+    @JsonProperty("haGroupState")
+    public HAGroupState getHAGroupState() {
+        return haGroupState;
+    }
+
+    @JsonIgnore
+    public ClusterRoleRecord.ClusterRole getClusterRole() {
+        return haGroupState.getClusterRole();
+    }
+
+    @Override
+    public int hashCode() {
+        return new HashCodeBuilder()
+                .append(protocolVersion)
+                .append(haGroupName)
+                .append(haGroupState)
+                .hashCode();
+    }
+
+    @Override
+    public boolean equals(Object other) {
+        if (other == this) {
+            return true;
+        } else if (other == null) {
+            return false;
+        } else if (!(other instanceof HAGroupStoreRecord)) {
+            return false;
+        } else {
+            HAGroupStoreRecord otherRecord = (HAGroupStoreRecord) other;
+            return new EqualsBuilder()
+                    .append(protocolVersion, otherRecord.protocolVersion)
+                    .append(haGroupName, otherRecord.haGroupName)
+                    .append(haGroupState, otherRecord.haGroupState)
+                    .isEquals();
+        }
+    }
+
+    @Override
+    public String toString() {
+        return "HAGroupStoreRecord{"
+                + "protocolVersion='" + protocolVersion + '\''
+                + ", haGroupName='" + haGroupName + '\''
+                + ", haGroupState=" + haGroupState
+                + '}';
+    }
+
+    public String toPrettyString() {
+        try {
+            return JacksonUtil.getObjectWriterPretty().writeValueAsString(this);
+        } catch (Exception e) {
+            LOG.error("Fail to wrap this object as JSON, returning the oneliner "
+                    + "using toString", e);
+            return toString();
+        }
+    }
+}
\ No newline at end of file
diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HighAvailabilityGroup.java b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HighAvailabilityGroup.java
index 9b704973c4..3d743315e8 100644
--- a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HighAvailabilityGroup.java
+++ b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HighAvailabilityGroup.java
@@ -148,7 +148,7 @@ public class HighAvailabilityGroup {
             .expireAfterWrite(PHOENIX_HA_TRANSITION_TIMEOUT_MS_DEFAULT, TimeUnit.MILLISECONDS)
             .build();
     /**
-     * The Curator client cache, one client instance per cluster.
+     * The Curator client cache, one client instance per cluster and namespace combination.
      */
     @VisibleForTesting
     static final Cache<String, CuratorFramework> CURATOR_CACHE = CacheBuilder.newBuilder()
@@ -354,8 +354,8 @@ public class HighAvailabilityGroup {
             GROUPS.remove(info);
             haGroup.close();
             try {
-                CuratorFramework curator1 = CURATOR_CACHE.getIfPresent(info.getUrl1());
-                CuratorFramework curator2 = CURATOR_CACHE.getIfPresent(info.getUrl2());
+                CuratorFramework curator1 = CURATOR_CACHE.getIfPresent(generateCacheKey(info.getUrl1(), null));
+                CuratorFramework curator2 = CURATOR_CACHE.getIfPresent(generateCacheKey(info.getUrl2(), null));
                 if (curator1 != null && curator2 != null) {
                     Stat node1 = curator1.checkExists().forPath(info.getZkPath());
                     Stat node2 = curator2.checkExists().forPath(info.getZkPath());
@@ -423,6 +423,19 @@ public class HighAvailabilityGroup {
         return Optional.of(fallbackCluster);
     }
 
+    /**
+     * Generate cache key for curator cache based on URL and namespace.
+     *
+     * @param jdbcUrl the ZK endpoint host:port or the JDBC connection String host:port:/hbase
+     * @param namespace the ZooKeeper namespace, uses default if null
+     * @return cache key string
+     */
+    @VisibleForTesting
+    static String generateCacheKey(String jdbcUrl, String namespace) {
+        String effectiveNamespace = namespace != null ? namespace : PHOENIX_HA_ZOOKEEPER_ZNODE_NAMESPACE;
+        return jdbcUrl + ":" + effectiveNamespace;
+    }
+
     /**
      * Get an active curator ZK client for the given properties and ZK endpoint.
      * <p>
@@ -435,9 +448,28 @@ public class HighAvailabilityGroup {
     @SuppressWarnings("UnstableApiUsage")
     public static CuratorFramework getCurator(String jdbcUrl, Properties properties)
             throws IOException {
+        return getCurator(jdbcUrl, properties, PHOENIX_HA_ZOOKEEPER_ZNODE_NAMESPACE);
+    }
+
+    /**
+     * Get an active curator ZK client for the given properties, ZK endpoint and namespace.
+     * <p>
+     * This can be from cached object since Curator should be shared per cluster.
+     *
+     * @param jdbcUrl    the ZK endpoint host:port or the JDBC connection String host:port:/hbase
+     * @param properties the properties defining time out values and retry count
+     * @param namespace  the ZooKeeper namespace to use, defaults to PHOENIX_HA_ZOOKEEPER_ZNODE_NAMESPACE if null
+     * @return a new Curator framework client
+     */
+    @SuppressWarnings("UnstableApiUsage")
+    public static CuratorFramework getCurator(String jdbcUrl, Properties properties, String namespace)
+            throws IOException {
+        // Use namespace as part of cache key to avoid conflicts between different namespaces
+        String effectiveNamespace = namespace != null ? namespace : PHOENIX_HA_ZOOKEEPER_ZNODE_NAMESPACE;
+        String cacheKey = generateCacheKey(jdbcUrl, namespace);
         try {
-            return CURATOR_CACHE.get(jdbcUrl, () -> {
-                CuratorFramework curator = createCurator(jdbcUrl, properties);
+            return CURATOR_CACHE.get(cacheKey, () -> {
+                CuratorFramework curator = createCurator(jdbcUrl, properties, effectiveNamespace);
                 try {
                     if (!curator.blockUntilConnected(PHOENIX_HA_ZK_CONNECTION_TIMEOUT_MS_DEFAULT,
                             TimeUnit.MILLISECONDS)) {
@@ -458,17 +490,22 @@ public class HighAvailabilityGroup {
         } catch (Exception e) {
             LOG.error("Fail to get an active curator for url {}", jdbcUrl, e);
             // invalidate the cache when getting/creating throws exception
-            CURATOR_CACHE.invalidate(jdbcUrl);
+            CURATOR_CACHE.invalidate(cacheKey);
             throw new IOException(e);
         }
     }
 
     /**
-     * Create a curator ZK client for the given properties and ZK endpoint.
+     * Create a curator ZK client for the given properties, ZK endpoint and namespace.
      * <p>
      * Unless caller needs a new curator, it should use {@link #getCurator(String, Properties)}.
+     *
+     * @param jdbcUrl the ZK endpoint host:port or the JDBC connection String host:port:/hbase
+     * @param properties the properties defining time out values and retry count
+     * @param namespace the ZooKeeper namespace to use, defaults to PHOENIX_HA_ZOOKEEPER_ZNODE_NAMESPACE if null
+     * @return a new Curator framework client
      */
-    private static CuratorFramework createCurator(String jdbcUrl, Properties properties) {
+    private static CuratorFramework createCurator(String jdbcUrl, Properties properties, String namespace) {
         // Get the ZK endpoint in host:port format by removing JDBC protocol and HBase root node
         final String zkUrl;
         if (jdbcUrl.startsWith(PhoenixRuntime.JDBC_PROTOCOL)) {
@@ -498,7 +535,7 @@ public class HighAvailabilityGroup {
         CuratorFramework curator = CuratorFrameworkFactory
                 .builder()
                 .connectString(zkUrl)
-                .namespace(PHOENIX_HA_ZOOKEEPER_ZNODE_NAMESPACE)
+                .namespace(namespace != null ? namespace : PHOENIX_HA_ZOOKEEPER_ZNODE_NAMESPACE)
                 .connectionTimeoutMs(connectionTimeoutMs)
                 .sessionTimeoutMs(sessionTimeoutMs)
                 .retryPolicy(retryPolicy)
@@ -1073,4 +1110,4 @@ public class HighAvailabilityGroup {
         }
 
     }
-}
+}
\ No newline at end of file
diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/PhoenixConnection.java b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/PhoenixConnection.java
index e2830f19a8..53da422dba 100644
--- a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/PhoenixConnection.java
+++ b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/PhoenixConnection.java
@@ -189,6 +189,7 @@ public class PhoenixConnection implements MetaDataMutated, SQLCloseable, Phoenix
     private LogLevel auditLogLevel;
     private Double logSamplingRate;
     private String sourceOfOperation;
+    private String haGroupName;
     private volatile SQLException reasonForClose;
     private static final String[] CONNECTION_PROPERTIES;
 
@@ -1252,6 +1253,18 @@ public class PhoenixConnection implements MetaDataMutated, SQLCloseable, Phoenix
         this.consistency = val;
     }
 
+    /**
+     * This is temporary method to set HAGroupName for the connection.
+     * This will be removed once we have a proper way to set HAGroupName in the connection.
+     */
+    public void setHAGroupName(String haGroupName) {
+        this.haGroupName = haGroupName;
+    }
+
+    public String getHAGroupName() {
+        return this.haGroupName;
+    }
+
     @Override
     public void setCatalog(String catalog) throws SQLException {
         checkOpen();
diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
index 3625dd006e..c40f4521e1 100644
--- a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
+++ b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
@@ -458,6 +458,13 @@ public class PhoenixDatabaseMetaData implements DatabaseMetaData {
     public static final String SYSTEM_CDC_STREAM_TABLE = "CDC_STREAM";
     public static final String SYSTEM_CDC_STREAM_NAME =
             SchemaUtil.getTableName(SYSTEM_CATALOG_SCHEMA, SYSTEM_CDC_STREAM_TABLE);
+
+    public static final String SYSTEM_HA_GROUP_TABLE = "HA_GROUP";
+    public static final String SYSTEM_HA_GROUP_NAME =
+            SchemaUtil.getTableName(SYSTEM_CATALOG_SCHEMA, SYSTEM_HA_GROUP_TABLE);
+    public static final byte[] SYSTEM_HA_GROUP_NAME_BYTES = Bytes.toBytes(SYSTEM_HA_GROUP_NAME);
+    public static final TableName SYSTEM_HA_GROUP_HBASE_TABLE_NAME = TableName.valueOf(SYSTEM_HA_GROUP_NAME);
+
     public static final String STREAM_NAME = "STREAM_NAME";
     public static final String STREAM_STATUS = "STREAM_STATUS";
     public static final String STREAM_TYPE = "STREAM_TYPE";
@@ -468,6 +475,15 @@ public class PhoenixDatabaseMetaData implements DatabaseMetaData {
     public static final String PARTITION_START_KEY = "PARTITION_START_KEY";
     public static final String PARTITION_END_KEY = "PARTITION_END_KEY";
     public static final String PARENT_PARTITION_START_TIME = "PARENT_PARTITION_START_TIME";
+    public static final String HA_GROUP_NAME = "HA_GROUP_NAME";
+    public static final String POLICY = "POLICY";
+    public static final String VERSION = "VERSION";
+    public static final String ZK_URL_1 = "ZK_URL_1";
+    public static final String ZK_URL_2 = "ZK_URL_2";
+    public static final String CLUSTER_URL_1 = "CLUSTER_URL_1";
+    public static final String CLUSTER_URL_2 = "CLUSTER_URL_2";
+    public static final String CLUSTER_ROLE_1 = "CLUSTER_ROLE_1";
+    public static final String CLUSTER_ROLE_2 = "CLUSTER_ROLE_2";
 
     public static final String QUERY_ID = "QUERY_ID";
     public static final String USER = "USER";
diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/PhoenixHAAdmin.java b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/PhoenixHAAdmin.java
index 0ef78372e0..6735288734 100644
--- a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/PhoenixHAAdmin.java
+++ b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/PhoenixHAAdmin.java
@@ -18,6 +18,7 @@
 package org.apache.phoenix.jdbc;
 
 import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.tuple.Pair;
 import org.apache.curator.RetryPolicy;
 import org.apache.curator.framework.CuratorFramework;
 import org.apache.curator.framework.recipes.atomic.AtomicValue;
@@ -26,10 +27,12 @@ import org.apache.curator.utils.ZKPaths;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.util.PairOfSameType;
+import org.apache.phoenix.exception.StaleHAGroupStoreRecordVersionException;
 import org.apache.phoenix.thirdparty.com.google.common.base.Preconditions;
 import org.apache.phoenix.util.JDBCUtil;
 import org.apache.zookeeper.CreateMode;
 import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.data.Stat;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -40,9 +43,12 @@ import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Optional;
 import java.util.Properties;
 
+import static org.apache.phoenix.jdbc.HighAvailabilityGroup.PHOENIX_HA_ZOOKEEPER_ZNODE_NAMESPACE;
+
 /**
  * Helper class to update cluster role record for a ZK cluster.
  * The ZK client this accessor has is confined to a single ZK cluster, but it can be used to operate
@@ -63,6 +69,13 @@ public class PhoenixHAAdmin implements Closeable {
         public CuratorFramework getCurator(String zkUrl, Properties properties) throws IOException {
             return HighAvailabilityGroup.getCurator(zkUrl, properties);
         }
+
+        /**
+         * Gets curator with specific namespace blocking if necessary to create it
+         */
+        public CuratorFramework getCurator(String zkUrl, Properties properties, String namespace) throws IOException {
+            return HighAvailabilityGroup.getCurator(zkUrl, properties, namespace);
+        }
     }
 
     private static final Logger LOG = LoggerFactory.getLogger(PhoenixHAAdmin.class);
@@ -76,17 +89,29 @@ public class PhoenixHAAdmin implements Closeable {
     /** Curator Provider **/
     private final HighAvailibilityCuratorProvider
             highAvailibilityCuratorProvider;
+    /** Namespace for Curator **/
+    private final String namespace;
 
     public PhoenixHAAdmin(Configuration conf) {
-        this(getLocalZkUrl(conf), conf, HighAvailibilityCuratorProvider.INSTANCE);
+        this(getLocalZkUrl(conf), conf, HighAvailibilityCuratorProvider.INSTANCE, null);
+    }
+
+    public PhoenixHAAdmin(Configuration conf, String namespace) {
+        this(getLocalZkUrl(conf), conf, HighAvailibilityCuratorProvider.INSTANCE, namespace);
     }
 
-    public PhoenixHAAdmin(String zkUrl, Configuration conf) {
-        this(zkUrl, conf, HighAvailibilityCuratorProvider.INSTANCE);
+    public PhoenixHAAdmin(String zkUrl, Configuration conf, String namespace) {
+        this(zkUrl, conf, HighAvailibilityCuratorProvider.INSTANCE, namespace);
     }
 
     public PhoenixHAAdmin(String zkUrl, Configuration conf,
-            HighAvailibilityCuratorProvider highAvailibilityCuratorProvider) {
+                          HighAvailibilityCuratorProvider highAvailibilityCuratorProvider) {
+        this(zkUrl, conf, highAvailibilityCuratorProvider, null);
+    }
+
+    public PhoenixHAAdmin(String zkUrl, Configuration conf,
+                          HighAvailibilityCuratorProvider highAvailibilityCuratorProvider,
+                          String namespace) {
         Preconditions.checkNotNull(zkUrl);
         Preconditions.checkNotNull(conf);
         Preconditions.checkNotNull(highAvailibilityCuratorProvider);
@@ -94,6 +119,7 @@ public class PhoenixHAAdmin implements Closeable {
         this.conf = conf;
         conf.iterator().forEachRemaining(k -> properties.setProperty(k.getKey(), k.getValue()));
         this.highAvailibilityCuratorProvider = highAvailibilityCuratorProvider;
+        this.namespace = Objects.toString(namespace, PHOENIX_HA_ZOOKEEPER_ZNODE_NAMESPACE);
     }
 
     /**
@@ -129,7 +155,7 @@ public class PhoenixHAAdmin implements Closeable {
      * Gets curator from the cache if available otherwise calls into getCurator to make it.
      */
     public CuratorFramework getCurator() throws IOException {
-        return highAvailibilityCuratorProvider.getCurator(zkUrl, properties);
+        return highAvailibilityCuratorProvider.getCurator(zkUrl, properties, namespace);
     }
 
 
@@ -160,20 +186,22 @@ public class PhoenixHAAdmin implements Closeable {
         }
     }
 
-    /**
-     * This lists all cluster role records stored in the zookeeper nodes.
-     * This read-only operation and hence no side effect on the ZK cluster.
-     */
-    public List<ClusterRoleRecord> listAllClusterRoleRecordsOnZookeeper() throws IOException {
-        List<String> haGroupNames;
+    public List<String> getHAGroupNames() throws IOException {
         try {
-            haGroupNames = getCurator().getChildren().forPath(ZKPaths.PATH_SEPARATOR);
+            return getCurator().getChildren().forPath(ZKPaths.PATH_SEPARATOR);
         } catch (Exception e) {
             String msg = String.format("Got exception when listing all HA groups in %s", zkUrl);
             LOG.error(msg);
             throw new IOException(msg, e);
         }
+    }
 
+    /**
+     * This lists all cluster role records stored in the zookeeper nodes.
+     * This read-only operation and hence no side effect on the ZK cluster.
+     */
+    public List<ClusterRoleRecord> listAllClusterRoleRecordsOnZookeeper() throws IOException {
+        List<String> haGroupNames = getHAGroupNames();
         List<ClusterRoleRecord> records = new ArrayList<>();
         List<String> failedHaGroups = new ArrayList<>();
         for (String haGroupName : haGroupNames) {
@@ -327,6 +355,15 @@ public class PhoenixHAAdmin implements Closeable {
         return inconsistentHaGroups;
     }
 
+    public void deleteHAGroupStoreRecordInZooKeeper(String haGroupName) throws IOException {
+        try {
+            getCurator().delete().quietly().deletingChildrenIfNeeded().forPath(toPath(haGroupName));
+        } catch (Exception e) {
+            LOG.error("Failed to delete HAGroupStoreRecord for HA group {}", haGroupName, e);
+            throw new IOException("Failed to delete HAGroupStoreRecord for HA group " + haGroupName, e);
+        }
+    }
+
     /**
      * This updates the cluster role data on the zookeeper it connects to.
      * To avoid conflicts, it does CAS (compare-and-set) when updating.  The constraint is that the
@@ -460,6 +497,72 @@ public class PhoenixHAAdmin implements Closeable {
         }
     }
 
+    /**
+     * Creates a new HAGroupStoreRecord in ZooKeeper.
+     *
+     * @param newHAGroupStoreRecord the new record to create
+     * @throws IOException if any error occurs during the creation
+     */
+    public void createHAGroupStoreRecordInZooKeeper(HAGroupStoreRecord newHAGroupStoreRecord)
+            throws IOException {
+        try {
+            getCurator()
+                    .create()
+                    .creatingParentsIfNeeded()
+                    .forPath(toPath(newHAGroupStoreRecord.getHaGroupName()), HAGroupStoreRecord.toJson(newHAGroupStoreRecord));
+        } catch (Exception e) {
+            throw new IOException("Failed to create HAGroupStoreRecord for HA group " + newHAGroupStoreRecord.getHaGroupName(), e);
+        }
+    }
+
+    /**
+     * Updates the HAGroupStoreRecord in ZooKeeper with version checking.
+     *
+     * @param haGroupName the HA group name
+     * @param newHAGroupStoreRecord the new record to store
+     * @param currentStatVersion the expected stat version for optimistic locking
+     * @throws StaleHAGroupStoreRecordVersionException if the version is stale
+     * @throws IOException if any other error occurs during the update
+     */
+    public void updateHAGroupStoreRecordInZooKeeper(String haGroupName, HAGroupStoreRecord newHAGroupStoreRecord, int currentStatVersion)
+            throws IOException, StaleHAGroupStoreRecordVersionException {
+        try {
+            getCurator()
+                    .setData()
+                    .withVersion(currentStatVersion)
+                    .forPath(toPath(haGroupName), HAGroupStoreRecord.toJson(newHAGroupStoreRecord));
+        } catch (KeeperException.BadVersionException e) {
+            LOG.error("Failed to set HAGroupStoreRecord for HA group {}, stale stat version", haGroupName, e);
+            throw new StaleHAGroupStoreRecordVersionException("Failed to set HAGroupStoreRecord for HA group "
+                    + haGroupName + " with cached stat version " + currentStatVersion, e);
+        } catch (Exception e) {
+            LOG.error("Failed to set HAGroupStoreRecord for HA group {}, unexpected error", haGroupName, e);
+            throw new IOException("Failed to set HAGroupStoreRecord for HA group " + haGroupName, e);
+        }
+    }
+
+    /**
+     * Gets the HAGroupStoreRecord and Stat from ZooKeeper.
+     *
+     * @param haGroupName the HA group name
+     * @return a pair of HAGroupStoreRecord and Stat
+     * @throws IOException if any error occurs during the retrieval
+     */
+    public Pair<HAGroupStoreRecord, Stat> getHAGroupStoreRecordInZooKeeper(String haGroupName) throws IOException {
+        try {
+            byte[] data = getCurator().getData().forPath(toPath(haGroupName));
+            HAGroupStoreRecord record = HAGroupStoreRecord.fromJson(data).orElse(null);
+            Stat stat = getCurator().checkExists().forPath(toPath(haGroupName));
+            return Pair.of(record, stat);
+        } catch (KeeperException.NoNodeException nne) {
+            LOG.warn("No HAGroupStoreRecord for HA group {} in ZK", haGroupName, nne);
+            return Pair.of(null, null);
+        } catch (Exception e) {
+            LOG.error("Failed to get HAGroupStoreRecord for HA group {}", haGroupName, e);
+            throw new IOException("Failed to get HAGroupStoreRecord for HA group " + haGroupName, e);
+        }
+    }
+
     /**
      * Helper method to get ZK path for an HA group given the HA group name.
      * It assumes the ZK namespace (prefix) has been set.
diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java b/phoenix-core-client/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
index b2c7f3b4e3..d2efaf1460 100644
--- a/phoenix-core-client/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
+++ b/phoenix-core-client/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
@@ -1698,7 +1698,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
 
     private void pollForUpdatedTableDescriptor(final Admin admin, final TableDescriptor newTableDescriptor,
             final byte[] tableName) throws InterruptedException, TimeoutException {
-        checkAndRetry(new RetriableOperation() {
+        checkAndRetry (new RetriableOperation() {
 
             @Override
             public String getOperationName() {
@@ -1713,7 +1713,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
         });
     }
 
-    private void checkAndRetry(RetriableOperation op) throws InterruptedException, TimeoutException {
+    private void checkAndRetry (RetriableOperation op) throws InterruptedException, TimeoutException {
         int maxRetries = ConnectionQueryServicesImpl.this.props.getInt(
                 QueryServices.NUM_RETRIES_FOR_SCHEMA_UPDATE_CHECK,
                 QueryServicesOptions.DEFAULT_RETRIES_FOR_SCHEMA_UPDATE_CHECK);
@@ -3875,6 +3875,10 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
         return setSystemDDLProperties(QueryConstants.CREATE_CDC_STREAM_METADATA);
     }
 
+    protected String getHAGroupDDL() {
+        return setSystemDDLProperties(QueryConstants.CREATE_HA_GROUP_METADATA);
+    }
+
     private String setSystemDDLProperties(String ddl) {
         return String.format(ddl,
           props.getInt(DEFAULT_SYSTEM_MAX_VERSIONS_ATTRIB, QueryServicesOptions.DEFAULT_SYSTEM_MAX_VERSIONS),
@@ -4140,8 +4144,8 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
     }
 
     private void createOtherSystemTables(PhoenixConnection metaConnection) throws SQLException, IOException {
-        try {
-            metaConnection.createStatement().execute(getSystemSequenceTableDDL(nSequenceSaltBuckets));
+        try (Statement stmt = metaConnection.createStatement()) {
+            stmt.execute(getSystemSequenceTableDDL(nSequenceSaltBuckets));
             // When creating the table above, DDL statements are
             // used. However, the CFD level properties are not set
             // via DDL commands, hence we are explicitly setting
@@ -4150,33 +4154,46 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
         } catch (TableAlreadyExistsException e) {
             nSequenceSaltBuckets = getSaltBuckets(e);
         }
-        try {
-            metaConnection.createStatement().execute(QueryConstants.CREATE_STATS_TABLE_METADATA);
-        } catch (TableAlreadyExistsException ignore) {}
-        try {
-            metaConnection.createStatement().execute(getFunctionTableDDL());
-        } catch (TableAlreadyExistsException ignore) {}
-        try {
-            metaConnection.createStatement().execute(getLogTableDDL());
-        } catch (TableAlreadyExistsException ignore) {}
-        try {
-            metaConnection.createStatement().executeUpdate(getChildLinkDDL());
-        } catch (TableAlreadyExistsException ignore) {}
-        try {
-            metaConnection.createStatement().executeUpdate(getMutexDDL());
-        } catch (TableAlreadyExistsException ignore) {}
-        try {
-            metaConnection.createStatement().executeUpdate(getTaskDDL());
-        } catch (TableAlreadyExistsException ignore) {}
-        try {
-            metaConnection.createStatement().executeUpdate(getTransformDDL());
-        } catch (TableAlreadyExistsException ignore) {}
-        try {
-            metaConnection.createStatement().executeUpdate(getCDCStreamStatusDDL());
-        } catch (TableAlreadyExistsException ignore) {}
-        try {
-            metaConnection.createStatement().executeUpdate(getCDCStreamDDL());
-        } catch (TableAlreadyExistsException ignore) {}
+        try (Statement stmt = metaConnection.createStatement()) {
+            stmt.execute(QueryConstants.CREATE_STATS_TABLE_METADATA);
+        } catch (TableAlreadyExistsException ignore) {
+        }
+        try (Statement stmt = metaConnection.createStatement()) {
+            stmt.execute(getFunctionTableDDL());
+        } catch (TableAlreadyExistsException ignore) {
+        }
+        try (Statement stmt = metaConnection.createStatement()) {
+            stmt.execute(getLogTableDDL());
+        } catch (TableAlreadyExistsException ignore) {
+        }
+        try (Statement stmt = metaConnection.createStatement()) {
+            stmt.execute(getChildLinkDDL());
+        } catch (TableAlreadyExistsException ignore) {
+        }
+        try (Statement stmt = metaConnection.createStatement()) {
+            stmt.executeUpdate(getMutexDDL());
+        } catch (TableAlreadyExistsException ignore) {
+        }
+        try (Statement stmt = metaConnection.createStatement()) {
+            stmt.executeUpdate(getTaskDDL());
+        } catch (TableAlreadyExistsException ignore) {
+        }
+        try (Statement stmt = metaConnection.createStatement()) {
+            stmt.executeUpdate(getTransformDDL());
+        } catch (TableAlreadyExistsException ignore) {
+        }
+        try (Statement stmt = metaConnection.createStatement()) {
+            stmt.executeUpdate(getCDCStreamStatusDDL());
+        } catch (TableAlreadyExistsException ignore) {
+        }
+        try (Statement stmt = metaConnection.createStatement()) {
+            stmt.executeUpdate(getCDCStreamDDL());
+        } catch (TableAlreadyExistsException ignore) {
+        }
+        try (Statement stmt = metaConnection.createStatement()) {
+            stmt.executeUpdate(getHAGroupDDL());
+        } catch (TableAlreadyExistsException ignore) {
+        }
     }
 
     /**
@@ -4811,7 +4828,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
         metaConnection = upgradeSystemMutex(metaConnection);
         metaConnection = upgradeSystemCDCStreamStatus(metaConnection);
         metaConnection = upgradeSystemCDCStream(metaConnection);
-
+        metaConnection = upgradeSystemHAGroup(metaConnection);
         // As this is where the most time will be spent during an upgrade,
         // especially when there are large number of views.
         // Upgrade the SYSTEM.CHILD_LINK towards the end,
@@ -5164,6 +5181,15 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
         return metaConnection;
     }
 
+    private PhoenixConnection upgradeSystemHAGroup(PhoenixConnection metaConnection)
+            throws SQLException {
+        try (Statement stmt = metaConnection.createStatement()) {
+            stmt.executeUpdate(getHAGroupDDL());
+        } catch (TableAlreadyExistsException ignored) {
+        }
+        return metaConnection;
+    }
+
 
     // Special method for adding the column qualifier column for 4.10. 
     private PhoenixConnection addColumnQualifierColumn(PhoenixConnection oldMetaConnection, Long timestamp) throws SQLException {
diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java b/phoenix-core-client/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
index e2de008e73..cfce561b0a 100644
--- a/phoenix-core-client/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
+++ b/phoenix-core-client/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
@@ -21,6 +21,7 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.INDEX_STATE_BYTES;
 import static org.apache.phoenix.schema.PTableImpl.getColumnsToClone;
 
 import java.sql.SQLException;
+import java.sql.Statement;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
@@ -216,6 +217,10 @@ public class ConnectionlessQueryServicesImpl extends DelegateQueryServices imple
         return setSystemDDLProperties(QueryConstants.CREATE_CDC_STREAM_METADATA);
     }
 
+    protected String getHAGroupDDL() {
+        return setSystemDDLProperties(QueryConstants.CREATE_HA_GROUP_METADATA);
+    }
+
     private String setSystemDDLProperties(String ddl) {
         return String.format(ddl,
           props.getInt(DEFAULT_SYSTEM_MAX_VERSIONS_ATTRIB, QueryServicesOptions.DEFAULT_SYSTEM_MAX_VERSIONS),
@@ -430,22 +435,22 @@ public class ConnectionlessQueryServicesImpl extends DelegateQueryServices imple
                 String globalUrl = JDBCUtil.removeProperty(url, PhoenixRuntime.TENANT_ID_ATTRIB);
                 metaConnection = new PhoenixConnection(this, globalUrl, scnProps);
                 metaConnection.setRunningUpgrade(true);
-                try {
-                    metaConnection.createStatement().executeUpdate(getSystemCatalogTableDDL());
+                try (Statement stmt = metaConnection.createStatement()) {
+                    stmt.executeUpdate(getSystemCatalogTableDDL());
                 } catch (TableAlreadyExistsException ignore) {
                     // Ignore, as this will happen if the SYSTEM.TABLE already exists at this fixed timestamp.
                     // A TableAlreadyExistsException is not thrown, since the table only exists *after* this fixed timestamp.
                 }
-                try {
+                try (Statement stmt = metaConnection.createStatement()) {
                     int nSaltBuckets = getSequenceSaltBuckets();
                     String createTableStatement = getSystemSequenceTableDDL(nSaltBuckets);
-                   metaConnection.createStatement().executeUpdate(createTableStatement);
+                    stmt.executeUpdate(createTableStatement);
                 } catch (NewerTableAlreadyExistsException ignore) {
                     // Ignore, as this will happen if the SYSTEM.SEQUENCE already exists at this fixed timestamp.
                     // A TableAlreadyExistsException is not thrown, since the table only exists *after* this fixed timestamp.
                 }
-                try {
-                    metaConnection.createStatement().executeUpdate(QueryConstants.CREATE_STATS_TABLE_METADATA);
+                try (Statement stmt = metaConnection.createStatement()) {
+                    stmt.executeUpdate(QueryConstants.CREATE_STATS_TABLE_METADATA);
                 } catch (NewerTableAlreadyExistsException ignore) {
                     // Ignore, as this will happen if the SYSTEM.SEQUENCE already exists at this fixed
                     // timestamp.
@@ -453,39 +458,41 @@ public class ConnectionlessQueryServicesImpl extends DelegateQueryServices imple
                     // fixed timestamp.
                 }
                 
-                try {
-                    metaConnection.createStatement().executeUpdate(getFunctionTableDDL());
+                try (Statement stmt = metaConnection.createStatement()) {
+                    stmt.executeUpdate(getFunctionTableDDL());
                 } catch (NewerTableAlreadyExistsException ignore) {
                 }
-                try {
-                    metaConnection.createStatement().executeUpdate(getLogTableDDL());
+                try (Statement stmt = metaConnection.createStatement()) {
+                    stmt.executeUpdate(getLogTableDDL());
                 } catch (NewerTableAlreadyExistsException ignore) {}
-                try {
-                    metaConnection.createStatement()
-                            .executeUpdate(getChildLinkDDL());
+                try (Statement stmt = metaConnection.createStatement()) {
+                    stmt.executeUpdate(getChildLinkDDL());
                 } catch (NewerTableAlreadyExistsException ignore) {
                 }
-                try {
-                    metaConnection.createStatement()
-                            .executeUpdate(getMutexDDL());
+                try (Statement stmt = metaConnection.createStatement()) {
+                    stmt.executeUpdate(getMutexDDL());
                 } catch (NewerTableAlreadyExistsException ignore) {
                 }
-                try {
-                    metaConnection.createStatement()
-                            .executeUpdate(getTaskDDL());
+                try (Statement stmt = metaConnection.createStatement()) {
+                    stmt.executeUpdate(getTaskDDL());
                 } catch (NewerTableAlreadyExistsException ignore) {
                 }
-                try {
-                    metaConnection.createStatement()
-                            .executeUpdate(getTransformDDL());
+                try (Statement stmt = metaConnection.createStatement()) {
+                    stmt.executeUpdate(getTransformDDL());
                 } catch (NewerTableAlreadyExistsException ignore) {
                 }
-                try {
-                    metaConnection.createStatement().executeUpdate(getCDCStreamStatusDDL());
-                } catch (TableAlreadyExistsException ignore) {}
-                try {
-                    metaConnection.createStatement().executeUpdate(getCDCStreamDDL());
-                } catch (TableAlreadyExistsException ignore) {}
+                try (Statement stmt = metaConnection.createStatement()) {
+                    stmt.executeUpdate(getCDCStreamStatusDDL());
+                } catch (TableAlreadyExistsException ignore) {
+                }
+                try (Statement stmt = metaConnection.createStatement()) {
+                    stmt.executeUpdate(getCDCStreamDDL());
+                } catch (TableAlreadyExistsException ignore) {
+                }
+                try (Statement stmt = metaConnection.createStatement()) {
+                    stmt.executeUpdate(getHAGroupDDL());
+                } catch (TableAlreadyExistsException ignore) {
+                }
             } catch (SQLException e) {
                 sqlE = e;
             } finally {
diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/query/QueryConstants.java b/phoenix-core-client/src/main/java/org/apache/phoenix/query/QueryConstants.java
index 9911f5f0a3..f7011aabc0 100644
--- a/phoenix-core-client/src/main/java/org/apache/phoenix/query/QueryConstants.java
+++ b/phoenix-core-client/src/main/java/org/apache/phoenix/query/QueryConstants.java
@@ -48,6 +48,10 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.CHANGE_DETECTION_E
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.CHAR_OCTET_LENGTH;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.CLASS_NAME;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.CLIENT_IP;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.CLUSTER_ROLE_1;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.CLUSTER_ROLE_2;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.CLUSTER_URL_1;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.CLUSTER_URL_2;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_COUNT;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_DEF;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_FAMILY;
@@ -72,6 +76,7 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.GLOBAL_SCAN_DETAIL
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.GUIDE_POSTS_ROW_COUNT;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.GUIDE_POST_KEY;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.HA_GROUP_NAME;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.IMMUTABLE_ROWS;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.IMMUTABLE_STORAGE_SCHEME;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.INCREMENT_BY;
@@ -117,6 +122,7 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.PHOENIX_TTL_HWM;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.PHYSICAL_NAME;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.PHYSICAL_TABLE_NAME;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.PK_NAME;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.POLICY;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.QUERY;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.QUERY_ID;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.QUERY_STATUS;
@@ -150,6 +156,7 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_CDC_STREAM_
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_CDC_STREAM_TABLE;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_CHILD_LINK_TABLE;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_FUNCTION_TABLE;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_HA_GROUP_TABLE;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_LOG_TABLE;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_MUTEX_TABLE_NAME;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_STATS_TABLE;
@@ -185,11 +192,14 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TYPE_SEQUENCE;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.UPDATE_CACHE_FREQUENCY;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.USER;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.USE_STATS_FOR_PARALLELIZATION;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VERSION;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_CONSTANT;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_INDEX_ID;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_INDEX_ID_DATA_TYPE;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_STATEMENT;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_TYPE;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.ZK_URL_1;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.ZK_URL_2;
 
 /**
  *
@@ -685,4 +695,24 @@ public interface QueryConstants {
             ColumnFamilyDescriptorBuilder.KEEP_DELETED_CELLS + "=%s,\n" +
             TRANSACTIONAL + "=" + Boolean.FALSE + ",\n" +
             UPDATE_CACHE_FREQUENCY + "=" + "7200000";
+
+    String CREATE_HA_GROUP_METADATA = "CREATE TABLE " + SYSTEM_CATALOG_SCHEMA + ".\"" +
+            SYSTEM_HA_GROUP_TABLE + "\"(\n" +
+            // PK column
+            HA_GROUP_NAME + " VARCHAR NOT NULL," + // This is the unique identifier for the HA group
+            // Non-PK columns
+            ZK_URL_1 + " VARCHAR," + // This will be used to associate the ClusterRole and URL with local or peer cluster
+            ZK_URL_2 + " VARCHAR," + // This will be used to associate the ClusterRole and URL with local or peer cluster
+            CLUSTER_URL_1 + " VARCHAR," + // This will be returned to the client
+            CLUSTER_URL_2 + " VARCHAR," + // This will be returned to the client
+            CLUSTER_ROLE_1 + " VARCHAR," + // Role for peer cluster might not be accurate, we will use only local role for recovery if needed
+            CLUSTER_ROLE_2 + " VARCHAR," + // Role for peer cluster might not be accurate, we will use only local role for recovery if needed
+            POLICY + " VARCHAR," +  // There should be only one policy for an HA group
+            VERSION + " BIGINT," +  // Version should be incremented for Admin Updates, only for CLUSTER_URLs and REGISTRY_TYPE
+            "CONSTRAINT " + SYSTEM_TABLE_PK_NAME +
+            " PRIMARY KEY (" + HA_GROUP_NAME + "))\n" +
+            HConstants.VERSIONS + "=%s,\n" +
+            ColumnFamilyDescriptorBuilder.KEEP_DELETED_CELLS + "=%s,\n" +
+            TRANSACTIONAL + "=" + Boolean.FALSE + ",\n" +
+            UPDATE_CACHE_FREQUENCY + "=" + "7200000";
 }
diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/query/QueryServices.java b/phoenix-core-client/src/main/java/org/apache/phoenix/query/QueryServices.java
index 4f6926144a..0ba347918c 100644
--- a/phoenix-core-client/src/main/java/org/apache/phoenix/query/QueryServices.java
+++ b/phoenix-core-client/src/main/java/org/apache/phoenix/query/QueryServices.java
@@ -140,7 +140,6 @@ public interface QueryServices extends SQLCloseable {
     public static final String MAX_SERVER_METADATA_CACHE_TIME_TO_LIVE_MS_ATTRIB = "phoenix.coprocessor.maxMetaDataCacheTimeToLiveMs";
     public static final String MAX_SERVER_METADATA_CACHE_SIZE_ATTRIB = "phoenix.coprocessor.maxMetaDataCacheSize";
     public static final String MAX_CLIENT_METADATA_CACHE_SIZE_ATTRIB = "phoenix.client.maxMetaDataCacheSize";
-    public static final String HA_GROUP_NAME_ATTRIB = "phoenix.ha.group";
     public static final String AUTO_UPGRADE_WHITELIST_ATTRIB = "phoenix.client.autoUpgradeWhiteList";
     // Mainly for testing to force spilling
     public static final String MAX_MEMORY_SIZE_ATTRIB = "phoenix.query.maxGlobalMemorySize";
diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/util/WALAnnotationUtil.java b/phoenix-core-client/src/main/java/org/apache/phoenix/util/WALAnnotationUtil.java
index 69afe0eb72..3dfb9f996f 100644
--- a/phoenix-core-client/src/main/java/org/apache/phoenix/util/WALAnnotationUtil.java
+++ b/phoenix-core-client/src/main/java/org/apache/phoenix/util/WALAnnotationUtil.java
@@ -17,6 +17,7 @@ package org.apache.phoenix.util;
 
 import org.apache.hadoop.hbase.client.Durability;
 import org.apache.hadoop.hbase.client.Mutation;
+import org.apache.phoenix.coprocessorclient.BaseScannerRegionObserverConstants;
 import org.apache.phoenix.execute.MutationState;
 
 /**
@@ -42,7 +43,7 @@ public class WALAnnotationUtil {
     }
 
     public static void annotateMutation(Mutation m, byte[] tenantId, byte[] schemaName,
-                                        byte[] logicalTableName, byte[] tableType, byte[] ddlTimestamp) {
+                                        byte[] logicalTableName, byte[] tableType, byte[] ddlTimestamp, byte[] haGroupName) {
         if (!m.getDurability().equals(Durability.SKIP_WAL)) {
             if (tenantId != null) {
                 m.setAttribute(MutationState.MutationMetadataType.TENANT_ID.toString(), tenantId);
@@ -52,6 +53,9 @@ public class WALAnnotationUtil {
                     logicalTableName);
             m.setAttribute(MutationState.MutationMetadataType.TABLE_TYPE.toString(), tableType);
             m.setAttribute(MutationState.MutationMetadataType.TIMESTAMP.toString(), ddlTimestamp);
+            if (haGroupName != null) {
+                m.setAttribute(BaseScannerRegionObserverConstants.HA_GROUP_NAME_ATTRIB, haGroupName);
+            }
         }
     }
 
diff --git a/phoenix-core-client/src/main/protobuf/RegionServerEndpointService.proto b/phoenix-core-client/src/main/protobuf/RegionServerEndpointService.proto
index 725f3fc763..3df27aebaf 100644
--- a/phoenix-core-client/src/main/protobuf/RegionServerEndpointService.proto
+++ b/phoenix-core-client/src/main/protobuf/RegionServerEndpointService.proto
@@ -51,11 +51,28 @@ message InvalidateServerMetadataCacheRequest {
 }
 
 message InvalidateHAGroupStoreClientRequest {
+  required bytes haGroupName = 1;
+  required bool broadcastUpdate = 2;
 }
 
 message InvalidateHAGroupStoreClientResponse {
 }
 
+message GetClusterRoleRecordRequest {
+  required bytes haGroupName = 1;
+}
+
+message GetClusterRoleRecordResponse {
+  // In case of single cluster configuration, one of the urls and roles can be empty
+  required bytes haGroupName = 1;
+  required bytes policy = 2;
+  optional bytes url1 = 3;
+  optional bytes role1 = 4;
+  optional bytes url2 = 5;
+  optional bytes role2 = 6;
+  optional int64 version = 7;
+}
+
 service RegionServerEndpointService {
   rpc validateLastDDLTimestamp(ValidateLastDDLTimestampRequest)
       returns (ValidateLastDDLTimestampResponse);
@@ -65,4 +82,7 @@ service RegionServerEndpointService {
 
   rpc invalidateHAGroupStoreClient(InvalidateHAGroupStoreClientRequest)
       returns (InvalidateHAGroupStoreClientResponse);
+
+  rpc getClusterRoleRecord(GetClusterRoleRecordRequest)
+      returns (GetClusterRoleRecordResponse);
 }
\ No newline at end of file
diff --git a/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/PhoenixRegionServerEndpoint.java b/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/PhoenixRegionServerEndpoint.java
index 97f71d0862..ce314020eb 100644
--- a/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/PhoenixRegionServerEndpoint.java
+++ b/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/PhoenixRegionServerEndpoint.java
@@ -17,6 +17,7 @@
  */
 package org.apache.phoenix.coprocessor;
 
+import com.google.protobuf.ByteString;
 import com.google.protobuf.RpcCallback;
 import com.google.protobuf.RpcController;
 import com.google.protobuf.Service;
@@ -33,6 +34,7 @@ import org.apache.phoenix.cache.ServerMetadataCacheImpl;
 import org.apache.phoenix.coprocessor.generated.RegionServerEndpointProtos;
 import org.apache.phoenix.coprocessorclient.metrics.MetricsMetadataCachingSource;
 import org.apache.phoenix.coprocessorclient.metrics.MetricsPhoenixCoprocessorSourceFactory;
+import org.apache.phoenix.jdbc.ClusterRoleRecord;
 import org.apache.phoenix.jdbc.HAGroupStoreManager;
 import org.apache.phoenix.protobuf.ProtobufUtil;
 import org.apache.phoenix.util.ClientUtil;
@@ -41,6 +43,8 @@ import org.apache.phoenix.util.ServerUtil;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import static org.apache.phoenix.jdbc.PhoenixHAAdmin.getLocalZkUrl;
+
 /**
  * This is first implementation of RegionServer coprocessor introduced by Phoenix.
  */
@@ -50,12 +54,14 @@ public class PhoenixRegionServerEndpoint
     private static final Logger LOGGER = LoggerFactory.getLogger(PhoenixRegionServerEndpoint.class);
     private MetricsMetadataCachingSource metricsSource;
     protected Configuration conf;
+    private String zkUrl;
 
     @Override
     public void start(CoprocessorEnvironment env) throws IOException {
         this.conf = env.getConfiguration();
         this.metricsSource = MetricsPhoenixCoprocessorSourceFactory
                                 .getInstance().getMetadataCachingSource();
+        this.zkUrl = getLocalZkUrl(conf);
     }
 
     @Override
@@ -119,10 +125,17 @@ public class PhoenixRegionServerEndpoint
             RegionServerEndpointProtos.InvalidateHAGroupStoreClientRequest request,
             RpcCallback<RegionServerEndpointProtos.InvalidateHAGroupStoreClientResponse> done) {
         LOGGER.info("PhoenixRegionServerEndpoint invalidating HAGroupStoreClient");
-        HAGroupStoreManager haGroupStoreManager;
         try {
-            haGroupStoreManager = HAGroupStoreManager.getInstance(conf);
-            haGroupStoreManager.invalidateHAGroupStoreClient();
+            HAGroupStoreManager haGroupStoreManager
+                    = HAGroupStoreManager.getInstance(conf);
+            if (haGroupStoreManager != null) {
+                haGroupStoreManager
+                        .invalidateHAGroupStoreClient(request.getHaGroupName().toStringUtf8(),
+                        request.getBroadcastUpdate());
+            } else {
+                throw new IOException("HAGroupStoreManager is null for "
+                        + "current cluster, check configuration");
+            }
         } catch (Throwable t) {
             String errorMsg = "Invalidating HAGroupStoreClient FAILED, check exception for "
                     + "specific details";
@@ -132,6 +145,43 @@ public class PhoenixRegionServerEndpoint
         }
     }
 
+    @Override
+    public void getClusterRoleRecord(RpcController controller,
+            RegionServerEndpointProtos.GetClusterRoleRecordRequest request,
+            RpcCallback<RegionServerEndpointProtos.GetClusterRoleRecordResponse> done) {
+        try {
+            HAGroupStoreManager haGroupStoreManager
+                    = HAGroupStoreManager.getInstance(conf);
+            if (haGroupStoreManager != null) {
+                ClusterRoleRecord clusterRoleRecord = haGroupStoreManager
+                        .getClusterRoleRecord(request.getHaGroupName().toStringUtf8());
+                RegionServerEndpointProtos.GetClusterRoleRecordResponse.Builder responseBuilder
+                        = RegionServerEndpointProtos.GetClusterRoleRecordResponse.newBuilder();
+                responseBuilder.setHaGroupName(request.getHaGroupName());
+                responseBuilder
+                        .setPolicy(ByteString.copyFromUtf8(clusterRoleRecord.getPolicy().name()));
+                responseBuilder.setUrl1(ByteString.copyFromUtf8(clusterRoleRecord.getUrl1()));
+                responseBuilder
+                        .setRole1(ByteString.copyFromUtf8(clusterRoleRecord.getRole1().name()));
+                responseBuilder
+                        .setUrl2(ByteString.copyFromUtf8(clusterRoleRecord.getUrl2()));
+                responseBuilder.
+                        setRole2(ByteString.copyFromUtf8(clusterRoleRecord.getRole2().name()));
+                responseBuilder.setVersion(clusterRoleRecord.getVersion());
+                done.run(responseBuilder.build());
+            } else {
+                throw new IOException("HAGroupStoreManager is null for "
+                        + "current cluster, check configuration");
+            }
+        } catch (Throwable t) {
+            String errorMsg = "Getting ClusterRoleRecord FAILED, check exception for "
+                    + "specific details";
+            LOGGER.error(errorMsg,  t);
+            IOException ioe = ClientUtil.createIOException(errorMsg, t);
+            ProtobufUtil.setControllerException(controller, ioe);
+        }
+    }
+
     @Override
     public Iterable<Service> getServices() {
         return Collections.singletonList(this);
@@ -141,4 +191,4 @@ public class PhoenixRegionServerEndpoint
         return ServerMetadataCacheImpl.getInstance(conf);
     }
 
-}
\ No newline at end of file
+}
diff --git a/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java b/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
index cf8797ecfe..4b0ba455a1 100644
--- a/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
+++ b/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
@@ -61,7 +61,6 @@ import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
 import org.apache.hadoop.hbase.ipc.RpcControllerFactory;
 import org.apache.hadoop.hbase.ipc.controller.InterRegionServerIndexRpcControllerFactory;
 import org.apache.hadoop.hbase.regionserver.InternalScanner;
-import org.apache.hadoop.hbase.regionserver.MiniBatchOperationInProgress;
 import org.apache.hadoop.hbase.regionserver.Region;
 import org.apache.hadoop.hbase.regionserver.RegionScanner;
 import org.apache.hadoop.hbase.regionserver.ScanType;
@@ -89,7 +88,6 @@ import org.apache.phoenix.index.PhoenixIndexCodec;
 import org.apache.phoenix.index.PhoenixIndexFailurePolicyHelper;
 import org.apache.phoenix.index.PhoenixIndexFailurePolicyHelper.MutateCommand;
 import org.apache.phoenix.index.PhoenixIndexMetaData;
-import org.apache.phoenix.jdbc.HAGroupStoreManager;
 import org.apache.phoenix.jdbc.PhoenixConnection;
 import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
 import org.apache.phoenix.join.HashJoinInfo;
@@ -1055,20 +1053,4 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver
     protected boolean isRegionObserverFor(Scan scan) {
         return scan.getAttribute(BaseScannerRegionObserverConstants.UNGROUPED_AGG) != null;
     }
-
-    @Override
-    public void preBatchMutate(ObserverContext<RegionCoprocessorEnvironment> c,
-                               MiniBatchOperationInProgress<Mutation> miniBatchOp)
-            throws IOException {
-        final Configuration conf = c.getEnvironment().getConfiguration();
-        try {
-            final HAGroupStoreManager haGroupStoreManager = HAGroupStoreManager.getInstance(conf);
-            if (haGroupStoreManager.isMutationBlocked()) {
-                throw new IOException("Blocking Mutation as Some CRRs are in ACTIVE_TO_STANDBY "
-                        + "state and CLUSTER_ROLE_BASED_MUTATION_BLOCK_ENABLED is true");
-            }
-        } catch (Exception e) {
-            throw new IOException(e);
-        }
-    }
 }
diff --git a/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionScanner.java b/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionScanner.java
index 32a837c080..951e541893 100644
--- a/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionScanner.java
+++ b/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionScanner.java
@@ -793,9 +793,12 @@ public class UngroupedAggregateRegionScanner extends BaseRegionScanner {
                 scan.getAttribute(MutationState.MutationMetadataType.TABLE_TYPE.toString());
         byte[] ddlTimestamp =
                 scan.getAttribute(MutationState.MutationMetadataType.TIMESTAMP.toString());
+        byte[] haGroupName =
+                scan.getAttribute(BaseScannerRegionObserverConstants.HA_GROUP_NAME_ATTRIB);
 
         for (Mutation m : mutationsList) {
-            annotateMutation(m, tenantId, schemaName, logicalTableName, tableType, ddlTimestamp);
+            annotateMutation(m, tenantId, schemaName, logicalTableName,
+                    tableType, ddlTimestamp, haGroupName);
         }
     }
 
diff --git a/phoenix-core-server/src/main/java/org/apache/phoenix/hbase/index/IndexRegionObserver.java b/phoenix-core-server/src/main/java/org/apache/phoenix/hbase/index/IndexRegionObserver.java
index 7ade7ca48e..1aeb7787ce 100644
--- a/phoenix-core-server/src/main/java/org/apache/phoenix/hbase/index/IndexRegionObserver.java
+++ b/phoenix-core-server/src/main/java/org/apache/phoenix/hbase/index/IndexRegionObserver.java
@@ -17,6 +17,7 @@
  */
 package org.apache.phoenix.hbase.index;
 
+import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.Cell;
 import org.apache.hadoop.hbase.CellComparator;
@@ -59,6 +60,7 @@ import org.apache.phoenix.coprocessor.DelegateRegionCoprocessorEnvironment;
 import org.apache.phoenix.coprocessor.generated.PTableProtos;
 import org.apache.phoenix.coprocessorclient.BaseScannerRegionObserverConstants;
 import org.apache.phoenix.exception.DataExceedsCapacityException;
+import org.apache.phoenix.exception.MutationBlockedIOException;
 import org.apache.phoenix.execute.MutationState;
 import org.apache.phoenix.expression.CaseExpression;
 import org.apache.phoenix.expression.Expression;
@@ -126,6 +128,7 @@ import java.io.ByteArrayInputStream;
 import java.io.DataInputStream;
 import java.io.EOFException;
 import java.io.IOException;
+import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
@@ -621,10 +624,22 @@ public class IndexRegionObserver implements RegionCoprocessor, RegionObserver {
       }
       try {
           final Configuration conf = c.getEnvironment().getConfiguration();
-          final HAGroupStoreManager haGroupStoreManager = HAGroupStoreManager.getInstance(conf);
-          if (haGroupStoreManager.isMutationBlocked()) {
-              throw new IOException("Blocking Mutation as Some CRRs are in ACTIVE_TO_STANDBY "
-                      + "state and CLUSTER_ROLE_BASED_MUTATION_BLOCK_ENABLED is true");
+          final HAGroupStoreManager haGroupStoreManager
+                  = HAGroupStoreManager.getInstance(conf);
+          if (haGroupStoreManager == null) {
+              throw new IOException("HAGroupStoreManager is null "
+                      + "for current cluster, check configuration");
+          }
+          // Extract HAGroupName from the mutations
+          final Set<String> haGroupNames = extractHAGroupNameAttribute(miniBatchOp);
+          // Check if mutation is blocked for any of the HAGroupNames
+          for (String haGroupName : haGroupNames) {
+              if (StringUtils.isNotBlank(haGroupName)
+                      && haGroupStoreManager.isMutationBlocked(haGroupName)) {
+                  throw new MutationBlockedIOException("Blocking Mutation as Some CRRs are in "
+                          + "ACTIVE_TO_STANDBY state and "
+                          + "CLUSTER_ROLE_BASED_MUTATION_BLOCK_ENABLED is true");
+              }
           }
           preBatchMutateWithExceptions(c, miniBatchOp);
           return;
@@ -635,6 +650,20 @@ public class IndexRegionObserver implements RegionCoprocessor, RegionObserver {
         "Somehow didn't return an index update but also didn't propagate the failure to the client!");
   }
 
+  private Set<String> extractHAGroupNameAttribute(
+          MiniBatchOperationInProgress<Mutation> miniBatchOp) {
+      Set<String> haGroupNames = new HashSet<>();
+      for (int i = 0; i < miniBatchOp.size(); i++) {
+          Mutation m = miniBatchOp.getOperation(i);
+          byte[] haGroupName = m.getAttribute(
+                  BaseScannerRegionObserverConstants.HA_GROUP_NAME_ATTRIB);
+          if (haGroupName != null) {
+              haGroupNames.add(new String(haGroupName, StandardCharsets.UTF_8));
+          }
+      }
+      return haGroupNames;
+    }
+
     @Override
     public void preWALRestore(
             org.apache.hadoop.hbase.coprocessor.ObserverContext<? extends RegionCoprocessorEnvironment> ctx,
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/MigrateSystemTablesToSystemNamespaceIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/MigrateSystemTablesToSystemNamespaceIT.java
index 6aaba958ff..31fb30ee40 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/MigrateSystemTablesToSystemNamespaceIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/MigrateSystemTablesToSystemNamespaceIT.java
@@ -67,11 +67,11 @@ public class MigrateSystemTablesToSystemNamespaceIT extends BaseTest {
     private static final Set<String> PHOENIX_SYSTEM_TABLES = new HashSet<>(Arrays.asList(
             "SYSTEM.CATALOG", "SYSTEM.SEQUENCE", "SYSTEM.STATS", "SYSTEM.FUNCTION",
             "SYSTEM.MUTEX","SYSTEM.LOG", "SYSTEM.CHILD_LINK", "SYSTEM.TASK", "SYSTEM.TRANSFORM",
-            "SYSTEM.CDC_STREAM_STATUS", "SYSTEM.CDC_STREAM"));
+            "SYSTEM.CDC_STREAM_STATUS", "SYSTEM.CDC_STREAM", "SYSTEM.HA_GROUP"));
     private static final Set<String> PHOENIX_NAMESPACE_MAPPED_SYSTEM_TABLES = new HashSet<>(
             Arrays.asList("SYSTEM:CATALOG", "SYSTEM:SEQUENCE", "SYSTEM:STATS", "SYSTEM:FUNCTION",
                     "SYSTEM:MUTEX","SYSTEM:LOG", "SYSTEM:CHILD_LINK", "SYSTEM:TASK", "SYSTEM:TRANSFORM",
-                    "SYSTEM:CDC_STREAM_STATUS", "SYSTEM:CDC_STREAM"));
+                    "SYSTEM:CDC_STREAM_STATUS", "SYSTEM:CDC_STREAM", "SYSTEM:HA_GROUP"));
     private static final String SCHEMA_NAME = "MIGRATETEST";
     private static final String TABLE_NAME =
             SCHEMA_NAME + "." + MigrateSystemTablesToSystemNamespaceIT.class.getSimpleName().toUpperCase();
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/PhoenixRegionServerEndpointIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/PhoenixRegionServerEndpointIT.java
index 670aba7f15..7fe8dd48c7 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/PhoenixRegionServerEndpointIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/PhoenixRegionServerEndpointIT.java
@@ -157,18 +157,6 @@ public class PhoenixRegionServerEndpointIT extends BaseTest {
         }
     }
 
-    @Test
-    public void testInvalidateHAGroupStoreClient() {
-        HRegionServer regionServer = utility.getMiniHBaseCluster().getRegionServer(0);
-        PhoenixRegionServerEndpoint coprocessor = getPhoenixRegionServerEndpoint(regionServer);
-        assertNotNull(coprocessor);
-        ServerRpcController controller = new ServerRpcController();
-        RegionServerEndpointProtos.InvalidateHAGroupStoreClientRequest request
-                = RegionServerEndpointProtos.InvalidateHAGroupStoreClientRequest.newBuilder().build();
-        coprocessor.invalidateHAGroupStoreClient(controller, request, null);
-        assertFalse(controller.failed());
-    }
-
     private String getCreateTableStmt(String tableName) {
         return   "CREATE TABLE " + tableName +
                 "  (a_string varchar not null, col1 integer" +
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/PhoenixRegionServerEndpointITWithConsistentFailover.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/PhoenixRegionServerEndpointITWithConsistentFailover.java
new file mode 100644
index 0000000000..9aa6173fd1
--- /dev/null
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/PhoenixRegionServerEndpointITWithConsistentFailover.java
@@ -0,0 +1,158 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.end2end;
+
+import com.google.protobuf.RpcCallback;
+import org.apache.hadoop.hbase.ipc.ServerRpcController;
+import org.apache.hadoop.hbase.regionserver.HRegionServer;
+import org.apache.hadoop.hbase.util.ByteStringer;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.phoenix.coprocessor.PhoenixRegionServerEndpoint;
+import org.apache.phoenix.coprocessor.generated.RegionServerEndpointProtos;
+import org.apache.phoenix.jdbc.ClusterRoleRecord;
+import org.apache.phoenix.jdbc.HAGroupStoreRecord;
+import org.apache.phoenix.jdbc.HighAvailabilityPolicy;
+import org.apache.phoenix.jdbc.HighAvailabilityTestingUtility;
+import org.apache.phoenix.jdbc.PhoenixHAAdmin;
+import org.apache.phoenix.jdbc.HAGroupStoreRecord.HAGroupState;
+import org.apache.phoenix.query.BaseTest;
+import org.apache.phoenix.thirdparty.com.google.common.collect.Maps;
+import org.apache.phoenix.util.HAGroupStoreTestUtil;
+import org.apache.phoenix.util.ReadOnlyProps;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.rules.TestName;
+
+import java.util.Map;
+
+import static org.apache.phoenix.jdbc.HAGroupStoreClient.ZK_CONSISTENT_HA_NAMESPACE;
+import static org.apache.phoenix.jdbc.PhoenixHAAdmin.getLocalZkUrl;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+
+@Category({NeedsOwnMiniClusterTest.class })
+public class PhoenixRegionServerEndpointITWithConsistentFailover extends BaseTest {
+
+    private static final Long ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS = 5000L;
+    private static final HighAvailabilityTestingUtility.HBaseTestingUtilityPair CLUSTERS = new HighAvailabilityTestingUtility.HBaseTestingUtilityPair();
+    private String zkUrl;
+    private String peerZkUrl;
+
+    @Rule
+    public TestName testName = new TestName();
+
+    @BeforeClass
+    public static synchronized void doSetup() throws Exception {
+        Map<String, String> props = Maps.newHashMapWithExpectedSize(1);
+        setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator()));
+        CLUSTERS.start();
+    }
+
+    @Before
+    public void setUp() throws Exception {
+        zkUrl = getLocalZkUrl(config);
+        peerZkUrl = CLUSTERS.getZkUrl2();
+        HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(testName.getMethodName(), zkUrl, peerZkUrl,
+                ClusterRoleRecord.ClusterRole.ACTIVE, ClusterRoleRecord.ClusterRole.STANDBY, null);
+    }
+
+    @Test
+    public void testGetClusterRoleRecordAndInvalidate() throws Exception {
+        String haGroupName = testName.getMethodName();
+        HRegionServer regionServer = utility.getMiniHBaseCluster().getRegionServer(0);
+        PhoenixRegionServerEndpoint coprocessor = getPhoenixRegionServerEndpoint(regionServer);
+        assertNotNull(coprocessor);
+        ServerRpcController controller = new ServerRpcController();
+
+        try (PhoenixHAAdmin peerHAAdmin = new PhoenixHAAdmin(CLUSTERS.getHBaseCluster2().getConfiguration(), ZK_CONSISTENT_HA_NAMESPACE)) {
+            HAGroupStoreRecord peerHAGroupStoreRecord = new HAGroupStoreRecord(HAGroupStoreRecord.DEFAULT_PROTOCOL_VERSION, haGroupName, HAGroupState.STANDBY);
+            peerHAAdmin.createHAGroupStoreRecordInZooKeeper(peerHAGroupStoreRecord);
+        }
+        Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
+
+        // First getClusterRoleRecord to check if HAGroupStoreClient is working as expected
+        ClusterRoleRecord expectedRecord = buildExpectedClusterRoleRecord(haGroupName, ClusterRoleRecord.ClusterRole.ACTIVE, ClusterRoleRecord.ClusterRole.STANDBY);
+        executeGetClusterRoleRecordAndVerify(coprocessor, controller, haGroupName, expectedRecord);
+
+        // Change the role of local cluster to ACTIVE_TO_STANDBY in System Table
+        HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(testName.getMethodName(), zkUrl, peerZkUrl,
+        ClusterRoleRecord.ClusterRole.ACTIVE_TO_STANDBY, ClusterRoleRecord.ClusterRole.STANDBY, null);
+
+        // Cluster Role will still be same as before as cache is not invalidated yet
+        executeGetClusterRoleRecordAndVerify(coprocessor, controller, haGroupName, expectedRecord);
+
+        // Now Invalidate the Cache
+        coprocessor.invalidateHAGroupStoreClient(controller, getInvalidateHAGroupStoreClientRequest(haGroupName), null);
+        assertFalse(controller.failed());
+
+        // Local Cluster Role will be updated to ACTIVE_TO_STANDBY as cache is invalidated
+        ClusterRoleRecord expectedRecordAfterInvalidation = buildExpectedClusterRoleRecord(haGroupName, ClusterRoleRecord.ClusterRole.ACTIVE_TO_STANDBY, ClusterRoleRecord.ClusterRole.STANDBY);
+        executeGetClusterRoleRecordAndVerify(coprocessor, controller, haGroupName, expectedRecordAfterInvalidation);
+    }
+
+    private ClusterRoleRecord buildExpectedClusterRoleRecord(String haGroupName, ClusterRoleRecord.ClusterRole localRole, ClusterRoleRecord.ClusterRole peerRole) {
+        return new ClusterRoleRecord(haGroupName, HighAvailabilityPolicy.FAILOVER, zkUrl, localRole, peerZkUrl, peerRole, 1);
+    }
+
+    private void executeGetClusterRoleRecordAndVerify(PhoenixRegionServerEndpoint coprocessor, ServerRpcController controller,
+                                                      String haGroupName, ClusterRoleRecord expectedRecord) {
+        RpcCallback<RegionServerEndpointProtos.GetClusterRoleRecordResponse> rpcCallback = createValidationCallback(haGroupName, expectedRecord);
+        coprocessor.getClusterRoleRecord(controller, getClusterRoleRecordRequest(haGroupName), rpcCallback);
+        assertFalse(controller.failed());
+    }
+
+    private RpcCallback<RegionServerEndpointProtos.GetClusterRoleRecordResponse> createValidationCallback(String haGroupName, ClusterRoleRecord expectedRecord) {
+        return response -> {
+            assertNotNull(response);
+            ClusterRoleRecord actual = new ClusterRoleRecord(haGroupName,
+                    HighAvailabilityPolicy.valueOf(response.getPolicy().toStringUtf8()),
+                    response.getUrl1().toStringUtf8(),
+                    ClusterRoleRecord.ClusterRole.valueOf(response.getRole1().toStringUtf8()),
+                    response.getUrl2().toStringUtf8(),
+                    ClusterRoleRecord.ClusterRole.valueOf(response.getRole2().toStringUtf8()),
+                    response.getVersion());
+            assertEquals(actual, expectedRecord);
+        };
+    }
+
+    private PhoenixRegionServerEndpoint getPhoenixRegionServerEndpoint(HRegionServer regionServer) {
+        PhoenixRegionServerEndpoint coproc = regionServer
+                .getRegionServerCoprocessorHost()
+                .findCoprocessor(PhoenixRegionServerEndpoint.class);
+        return coproc;
+    }
+
+    private RegionServerEndpointProtos.GetClusterRoleRecordRequest getClusterRoleRecordRequest(String haGroupName) {
+        RegionServerEndpointProtos.GetClusterRoleRecordRequest.Builder requestBuilder
+                = RegionServerEndpointProtos.GetClusterRoleRecordRequest.newBuilder();
+        requestBuilder.setHaGroupName(ByteStringer.wrap(Bytes.toBytes(haGroupName)));
+        return requestBuilder.build();
+    }
+
+    private RegionServerEndpointProtos.InvalidateHAGroupStoreClientRequest getInvalidateHAGroupStoreClientRequest(String haGroupName) {
+        RegionServerEndpointProtos.InvalidateHAGroupStoreClientRequest.Builder requestBuilder
+                = RegionServerEndpointProtos.InvalidateHAGroupStoreClientRequest.newBuilder();
+        requestBuilder.setBroadcastUpdate(false);
+        requestBuilder.setHaGroupName(ByteStringer.wrap(Bytes.toBytes(haGroupName)));
+        return  requestBuilder.build();
+    }
+}
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryDatabaseMetaDataIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryDatabaseMetaDataIT.java
index 2549517fd0..fa6752733b 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryDatabaseMetaDataIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryDatabaseMetaDataIT.java
@@ -191,6 +191,10 @@ public class QueryDatabaseMetaDataIT extends ParallelStatsDisabledIT {
             assertEquals(PTableType.SYSTEM.toString(), rs.getString("TABLE_TYPE"));
             assertTrue(rs.next());
             assertEquals(SYSTEM_CATALOG_SCHEMA, rs.getString("TABLE_SCHEM"));
+            assertEquals(PhoenixDatabaseMetaData.SYSTEM_HA_GROUP_TABLE, rs.getString("TABLE_NAME"));
+            assertEquals(PTableType.SYSTEM.toString(), rs.getString("TABLE_TYPE"));
+            assertTrue(rs.next());
+            assertEquals(SYSTEM_CATALOG_SCHEMA, rs.getString("TABLE_SCHEM"));
             assertEquals(PhoenixDatabaseMetaData.SYSTEM_LOG_TABLE, rs.getString("TABLE_NAME"));
             assertEquals(PTableType.SYSTEM.toString(), rs.getString("TABLE_TYPE"));
             assertTrue(rs.next());
@@ -430,7 +434,7 @@ public class QueryDatabaseMetaDataIT extends ParallelStatsDisabledIT {
                 tables.add(rs.getString("TABLE_NAME"));
                 assertEquals("SYSTEM", rs.getString("TABLE_SCHEM"));
             }
-            assertEquals(11, tables.size());
+            assertEquals(12, tables.size());
             assertTrue(tables.contains("CATALOG"));
             assertTrue(tables.contains("FUNCTION"));
 
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SystemTablesCreationOnConnectionIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SystemTablesCreationOnConnectionIT.java
index 84074c349a..b997922640 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SystemTablesCreationOnConnectionIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SystemTablesCreationOnConnectionIT.java
@@ -112,12 +112,12 @@ public class SystemTablesCreationOnConnectionIT {
     private static final Set<String> PHOENIX_SYSTEM_TABLES = new HashSet<>(Arrays.asList(
             "SYSTEM.CATALOG", "SYSTEM.SEQUENCE", "SYSTEM.STATS", "SYSTEM.FUNCTION",
             "SYSTEM.MUTEX", "SYSTEM.LOG", "SYSTEM.CHILD_LINK", "SYSTEM.TASK","SYSTEM.TRANSFORM",
-            "SYSTEM.CDC_STREAM_STATUS", "SYSTEM.CDC_STREAM"));
+            "SYSTEM.CDC_STREAM_STATUS", "SYSTEM.CDC_STREAM", "SYSTEM.HA_GROUP"));
 
     private static final Set<String> PHOENIX_NAMESPACE_MAPPED_SYSTEM_TABLES = new HashSet<>(
             Arrays.asList("SYSTEM:CATALOG", "SYSTEM:SEQUENCE", "SYSTEM:STATS", "SYSTEM:FUNCTION",
                     "SYSTEM:MUTEX", "SYSTEM:LOG", "SYSTEM:CHILD_LINK", "SYSTEM:TASK", "SYSTEM:TRANSFORM",
-                    "SYSTEM:CDC_STREAM_STATUS", "SYSTEM:CDC_STREAM"));
+                    "SYSTEM:CDC_STREAM_STATUS", "SYSTEM:CDC_STREAM", "SYSTEM:HA_GROUP"));
 
     private static class PhoenixSysCatCreationServices extends ConnectionQueryServicesImpl {
 
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/TenantSpecificTablesDDLIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/TenantSpecificTablesDDLIT.java
index ab6e183103..8ae6e4cd74 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/TenantSpecificTablesDDLIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/TenantSpecificTablesDDLIT.java
@@ -561,6 +561,8 @@ public class TenantSpecificTablesDDLIT extends BaseTenantSpecificTablesIT {
             assertTrue(rs.next());
             assertTableMetaData(rs, SYSTEM_CATALOG_SCHEMA, SYSTEM_FUNCTION_TABLE, SYSTEM);
             assertTrue(rs.next());
+            assertTableMetaData(rs, PhoenixDatabaseMetaData.SYSTEM_CATALOG_SCHEMA, PhoenixDatabaseMetaData.SYSTEM_HA_GROUP_TABLE, PTableType.SYSTEM);
+            assertTrue(rs.next());
             assertTableMetaData(rs, PhoenixDatabaseMetaData.SYSTEM_CATALOG_SCHEMA, PhoenixDatabaseMetaData.SYSTEM_LOG_TABLE, PTableType.SYSTEM);
             assertTrue(rs.next());
             assertTableMetaData(rs, PhoenixDatabaseMetaData.SYSTEM_CATALOG_SCHEMA, PhoenixDatabaseMetaData.SYSTEM_MUTEX_TABLE_NAME, PTableType.SYSTEM);
@@ -695,4 +697,4 @@ public class TenantSpecificTablesDDLIT extends BaseTenantSpecificTablesIT {
         assertColumnMetaData(rs, schema, table, column);
         assertEquals(ordinalPosition, rs.getInt(ORDINAL_POSITION));
     }
-}
\ No newline at end of file
+}
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexRegionObserverMutationBlockingIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexRegionObserverMutationBlockingIT.java
new file mode 100644
index 0000000000..fdf64dd2df
--- /dev/null
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexRegionObserverMutationBlockingIT.java
@@ -0,0 +1,217 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.end2end.index;
+
+import static org.apache.phoenix.jdbc.HAGroupStoreClient.ZK_CONSISTENT_HA_NAMESPACE;
+import static org.apache.phoenix.jdbc.PhoenixHAAdmin.getLocalZkUrl;
+import static org.apache.phoenix.query.QueryServices.CLUSTER_ROLE_BASED_MUTATION_BLOCK_ENABLED;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.sql.DriverManager;
+import java.util.Map;
+
+import org.apache.hadoop.hbase.client.RetriesExhaustedWithDetailsException;
+import org.apache.phoenix.end2end.NeedsOwnMiniClusterTest;
+import org.apache.phoenix.exception.MutationBlockedIOException;
+import org.apache.phoenix.execute.CommitException;
+import org.apache.phoenix.jdbc.ClusterRoleRecord;
+import org.apache.phoenix.jdbc.HAGroupStoreRecord;
+import org.apache.phoenix.jdbc.HighAvailabilityTestingUtility;
+import org.apache.phoenix.jdbc.PhoenixConnection;
+import org.apache.phoenix.jdbc.PhoenixHAAdmin;
+import org.apache.phoenix.query.BaseTest;
+import org.apache.phoenix.thirdparty.com.google.common.collect.Maps;
+import org.apache.phoenix.util.HAGroupStoreTestUtil;
+import org.apache.phoenix.util.ReadOnlyProps;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.rules.TestName;
+
+/**
+ * Integration test for mutation blocking functionality in IndexRegionObserver.preBatchMutate()
+ * Tests that MutationBlockedIOException is properly thrown when cluster role-based mutation
+ * blocking is enabled and CRRs are in ACTIVE_TO_STANDBY state.
+ */
+@Category(NeedsOwnMiniClusterTest.class)
+public class IndexRegionObserverMutationBlockingIT extends BaseTest {
+
+    private static final Long ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS = 1000L;
+    private PhoenixHAAdmin haAdmin;
+    private static final HighAvailabilityTestingUtility.HBaseTestingUtilityPair CLUSTERS = new HighAvailabilityTestingUtility.HBaseTestingUtilityPair();
+
+    @Rule
+    public TestName testName = new TestName();
+
+    @BeforeClass
+    public static synchronized void doSetup() throws Exception {
+        Map<String, String> props = Maps.newHashMapWithExpectedSize(1);
+        // Enable cluster role-based mutation blocking
+        props.put(CLUSTER_ROLE_BASED_MUTATION_BLOCK_ENABLED, "true");
+        // No retries so that tests fail faster.
+        props.put("hbase.client.retries.number", "0");
+        setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator()));
+        CLUSTERS.start();
+    }
+
+    @Before
+    public void setUp() throws Exception {
+        haAdmin = new PhoenixHAAdmin(config, ZK_CONSISTENT_HA_NAMESPACE);
+        Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
+        String zkUrl = getLocalZkUrl(config);
+        String peerZkUrl = CLUSTERS.getZkUrl2();
+        HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(testName.getMethodName(), zkUrl, peerZkUrl,
+                ClusterRoleRecord.ClusterRole.ACTIVE, ClusterRoleRecord.ClusterRole.STANDBY, null);
+    }
+
+    @Test
+    public void testMutationBlockedOnDataTableWithIndex() throws Exception {
+        String dataTableName = generateUniqueName();
+        String indexName = generateUniqueName();
+        String haGroupName = testName.getMethodName();
+
+        try (PhoenixConnection conn = (PhoenixConnection) DriverManager.getConnection(getUrl())) {
+            conn.setHAGroupName(haGroupName);
+            // Create data table and index
+            conn.createStatement().execute("CREATE TABLE " + dataTableName +
+                    " (id VARCHAR PRIMARY KEY, name VARCHAR, age INTEGER)");
+            conn.createStatement().execute("CREATE INDEX " + indexName +
+                    " ON " + dataTableName + "(name)");
+
+            // Initially, mutations should work - verify baseline
+            conn.createStatement().execute("UPSERT INTO " + dataTableName +
+                    " VALUES ('1', 'John', 25)");
+            conn.commit();
+
+            // Set up HAGroupStoreRecord that will block mutations (ACTIVE_TO_STANDBY state)
+            HAGroupStoreRecord haGroupStoreRecord
+                    = new HAGroupStoreRecord(HAGroupStoreRecord.DEFAULT_PROTOCOL_VERSION,
+                    haGroupName, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY);
+            haAdmin.updateHAGroupStoreRecordInZooKeeper(haGroupName, haGroupStoreRecord, -1);
+
+            // Wait for the event to propagate
+            Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
+
+            // Test that UPSERT throws MutationBlockedIOException
+            try {
+                conn.createStatement().execute("UPSERT INTO " + dataTableName +
+                        " VALUES ('2', 'Jane', 30)");
+                conn.commit();
+                fail("Expected MutationBlockedIOException to be thrown");
+            } catch (CommitException e) {
+                // Verify the exception chain contains MutationBlockedIOException
+                assertTrue("Expected MutationBlockedIOException in exception chain",
+                        containsMutationBlockedException(e));
+            }
+        }
+    }
+
+
+    @Test
+    public void testMutationAllowedWhenNotBlocked() throws Exception {
+        String dataTableName = generateUniqueName();
+        String indexName = generateUniqueName();
+
+        try (PhoenixConnection conn = (PhoenixConnection) DriverManager.getConnection(getUrl())) {
+            conn.setHAGroupName(testName.getMethodName());
+            // Create data table and index
+            conn.createStatement().execute("CREATE TABLE " + dataTableName +
+                    " (id VARCHAR PRIMARY KEY, name VARCHAR, age INTEGER)");
+            conn.createStatement().execute("CREATE INDEX " + indexName +
+                    " ON " + dataTableName + "(name)");
+
+            // Mutations should work fine
+            conn.createStatement().execute("UPSERT INTO " + dataTableName +
+                    " VALUES ('1', 'Bob', 35)");
+            conn.createStatement().execute("UPSERT INTO " + dataTableName +
+                    " VALUES ('2', 'Carol', 27)");
+            conn.commit();
+
+            // Verify data was inserted successfully
+            try (java.sql.ResultSet rs = conn.createStatement().executeQuery(
+                    "SELECT COUNT(*) FROM " + dataTableName)) {
+                assertTrue(rs.next());
+                assertEquals(2, rs.getInt(1));
+            }
+        }
+    }
+
+    @Test
+    public void testMutationBlockingTransition() throws Exception {
+        String dataTableName = generateUniqueName();
+        String indexName = generateUniqueName();
+        String haGroupName = testName.getMethodName();
+
+        try (PhoenixConnection conn = (PhoenixConnection) DriverManager.getConnection(getUrl())) {
+            conn.setHAGroupName(testName.getMethodName());
+            // Create data table and index
+            conn.createStatement().execute("CREATE TABLE " + dataTableName +
+                    " (id VARCHAR PRIMARY KEY, name VARCHAR, age INTEGER)");
+            conn.createStatement().execute("CREATE INDEX " + indexName +
+                    " ON " + dataTableName + "(name)");
+
+            // Mutation should work
+            conn.createStatement().execute("UPSERT INTO " + dataTableName +
+                    " VALUES ('1', 'David', 40)");
+            conn.commit();
+            // Set up HAGroupStoreRecord that will block mutations (ACTIVE_TO_STANDBY state)
+            HAGroupStoreRecord haGroupStoreRecord
+                    = new HAGroupStoreRecord(HAGroupStoreRecord.DEFAULT_PROTOCOL_VERSION,
+                    haGroupName, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY);
+            haAdmin.updateHAGroupStoreRecordInZooKeeper(haGroupName, haGroupStoreRecord, -1);
+            Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
+
+            // Now mutations should be blocked
+            try {
+                conn.createStatement().execute("UPSERT INTO " + dataTableName +
+                        " VALUES ('2', 'Eve', 33)");
+                conn.commit();
+                fail("Expected MutationBlockedIOException after transition to ACTIVE_TO_STANDBY");
+            } catch (CommitException e) {
+                assertTrue("Expected mutation blocked exception after transition",
+                        containsMutationBlockedException(e));
+            }
+
+            // Transition back to ACTIVE (non-blocking state) and peer cluster is in ATS state
+            // Set up HAGroupStoreRecord that will block mutations (ACTIVE_TO_STANDBY state)
+            haGroupStoreRecord
+                    = new HAGroupStoreRecord(HAGroupStoreRecord.DEFAULT_PROTOCOL_VERSION,
+                    haGroupName, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+            haAdmin.updateHAGroupStoreRecordInZooKeeper(haGroupName, haGroupStoreRecord, -1);
+            Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
+
+            // Mutations should work again
+            conn.createStatement().execute("UPSERT INTO " + dataTableName +
+                    " VALUES ('3', 'Frank', 45)");
+            conn.commit();
+        }
+    }
+
+    private boolean containsMutationBlockedException(CommitException e) {
+        Throwable cause = e.getCause();
+        if (cause instanceof RetriesExhaustedWithDetailsException) {
+            RetriesExhaustedWithDetailsException re = (RetriesExhaustedWithDetailsException) cause;
+            return re.getCause(0) instanceof MutationBlockedIOException;
+        }
+        return false;
+    }
+}
\ No newline at end of file
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HAGroupStoreClientIT.java b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HAGroupStoreClientIT.java
index 5a382f77e8..5a062b3305 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HAGroupStoreClientIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HAGroupStoreClientIT.java
@@ -17,31 +17,54 @@
  */
 package org.apache.phoenix.jdbc;
 
-import org.apache.curator.framework.recipes.cache.PathChildrenCacheListener;
-import org.apache.curator.utils.ZKPaths;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.tuple.Pair;
 import org.apache.phoenix.end2end.NeedsOwnMiniClusterTest;
+import org.apache.phoenix.exception.InvalidClusterRoleTransitionException;
 import org.apache.phoenix.query.BaseTest;
 import org.apache.phoenix.thirdparty.com.google.common.collect.Maps;
+import org.apache.phoenix.util.HAGroupStoreTestUtil;
 import org.apache.phoenix.util.ReadOnlyProps;
+import org.apache.zookeeper.data.Stat;
+import org.junit.After;
 import org.junit.Before;
 import org.junit.BeforeClass;
+import org.junit.Rule;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
+import org.junit.rules.TestName;
 
 import java.io.IOException;
+import java.lang.reflect.Field;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.sql.Statement;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.concurrent.ConcurrentLinkedQueue;
 import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.Executors;
 import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
 import java.util.concurrent.TimeUnit;
 
+import org.apache.curator.framework.recipes.cache.PathChildrenCacheListener;
+
+import static org.apache.hadoop.hbase.HConstants.DEFAULT_ZK_SESSION_TIMEOUT;
+import static org.apache.phoenix.jdbc.HAGroupStoreClient.ZK_CONSISTENT_HA_NAMESPACE;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_HA_GROUP_NAME;
+import static org.apache.phoenix.jdbc.PhoenixHAAdmin.getLocalZkUrl;
 import static org.apache.phoenix.jdbc.PhoenixHAAdmin.toPath;
+import static org.apache.phoenix.util.PhoenixRuntime.JDBC_PROTOCOL_SEPARATOR;
+import static org.apache.phoenix.util.PhoenixRuntime.JDBC_PROTOCOL_ZK;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertThrows;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
 
 /**
  * Integration tests for {@link HAGroupStoreClient}
@@ -49,171 +72,299 @@ import static org.junit.Assert.assertThrows;
 @Category(NeedsOwnMiniClusterTest.class)
 public class HAGroupStoreClientIT extends BaseTest {
 
-    private final PhoenixHAAdmin haAdmin = new PhoenixHAAdmin(config);
     private static final Long ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS = 5000L;
+    private static final HighAvailabilityTestingUtility.HBaseTestingUtilityPair CLUSTERS = new HighAvailabilityTestingUtility.HBaseTestingUtilityPair();
+    private PhoenixHAAdmin haAdmin;
+    private PhoenixHAAdmin peerHaAdmin;
+    private String zkUrl;
+    private String peerZKUrl;
+
+    @Rule
+    public TestName testName = new TestName();
 
     @BeforeClass
     public static synchronized void doSetup() throws Exception {
         Map<String, String> props = Maps.newHashMapWithExpectedSize(1);
         setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator()));
+        CLUSTERS.start();
     }
 
     @Before
     public void before() throws Exception {
-        // Clean up all the existing CRRs
-        List<ClusterRoleRecord> crrs = haAdmin.listAllClusterRoleRecordsOnZookeeper();
-        for (ClusterRoleRecord crr : crrs) {
-            haAdmin.getCurator().delete().forPath(toPath(crr.getHaGroupName()));
+        haAdmin = new PhoenixHAAdmin(CLUSTERS.getHBaseCluster1().getConfiguration(), ZK_CONSISTENT_HA_NAMESPACE);
+        peerHaAdmin = new PhoenixHAAdmin(CLUSTERS.getHBaseCluster2().getConfiguration(), ZK_CONSISTENT_HA_NAMESPACE);
+        haAdmin.getCurator().delete().quietly().forPath(toPath(testName.getMethodName()));
+        peerHaAdmin.getCurator().delete().quietly().forPath(toPath(testName.getMethodName()));
+        zkUrl = getLocalZkUrl(CLUSTERS.getHBaseCluster1().getConfiguration());
+        // Clean existing records in system table
+        List<String> haGroupNames = HAGroupStoreClient.getHAGroupNames(zkUrl);
+        for (String groupName : haGroupNames) {
+            HAGroupStoreTestUtil.deleteHAGroupRecordInSystemTable(groupName, zkUrl);
+        }
+        // Insert a HAGroupStoreRecord into the system table
+        String haGroupName = testName.getMethodName();
+        this.peerZKUrl = CLUSTERS.getZkUrl2();
+        HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(haGroupName, this.zkUrl, this.peerZKUrl, ClusterRoleRecord.ClusterRole.ACTIVE, ClusterRoleRecord.ClusterRole.STANDBY, null);
+    }
+
+    @After
+    public void after() throws Exception {
+        haAdmin.getCurator().delete().quietly().forPath(toPath(testName.getMethodName()));
+        peerHaAdmin.getCurator().delete().quietly().forPath(toPath(testName.getMethodName()));
+        haAdmin.close();
+        peerHaAdmin.close();
+        HAGroupStoreTestUtil.deleteHAGroupRecordInSystemTable(testName.getMethodName(), zkUrl);
+
+    }
+
+    @Test
+    public void testHAGroupStoreClientWithBothNullZKUrl() throws Exception {
+        String haGroupName = testName.getMethodName();
+        // Clean existing record
+        HAGroupStoreTestUtil.deleteHAGroupRecordInSystemTable(haGroupName, zkUrl);
+        HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(haGroupName, null, null, ClusterRoleRecord.ClusterRole.ACTIVE, ClusterRoleRecord.ClusterRole.STANDBY, this.zkUrl);
+        HAGroupStoreClient haGroupStoreClient = HAGroupStoreClient.getInstanceForZkUrl(CLUSTERS.getHBaseCluster1().getConfiguration(), haGroupName, zkUrl);
+        assertNull(haGroupStoreClient);
+    }
+
+    @Test
+    public void testHAGroupStoreClientChangingPeerZKUrlToNullUrlToValidUrlToInvalidUrl() throws Exception {
+        String haGroupName = testName.getMethodName();
+        HAGroupStoreRecord record = new HAGroupStoreRecord("v1.0", haGroupName,
+                HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+        createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, record);
+        HAGroupStoreClient haGroupStoreClient = HAGroupStoreClient.getInstanceForZkUrl(CLUSTERS.getHBaseCluster1().getConfiguration(), haGroupName, zkUrl);
+
+        HAGroupStoreRecord currentRecord = haGroupStoreClient.getHAGroupStoreRecord();
+        assert currentRecord != null && currentRecord.getHAGroupState() == HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC;
+
+        // Base case: Check that peerPathChildrenCache is not null in HAGroupStoreClient via reflection
+        Field peerPathChildrenCache = HAGroupStoreClient.class.getDeclaredField("peerPathChildrenCache");
+        peerPathChildrenCache.setAccessible(true);
+        assertNotNull(peerPathChildrenCache.get(haGroupStoreClient));
+
+        // Clean existing record
+        HAGroupStoreTestUtil.deleteHAGroupRecordInSystemTable(haGroupName, zkUrl);
+        // Now update peerZKUrl to null and rebuild
+        HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(haGroupName, this.zkUrl, null, ClusterRoleRecord.ClusterRole.ACTIVE, ClusterRoleRecord.ClusterRole.STANDBY, null);
+        try {
+            haGroupStoreClient.rebuild();
+            fail("Should have thrown NullPointerException");
+        } catch (NullPointerException npe) {
+            // This exception is expected here.
         }
+
+        // Now update System table to contain valid peer ZK URL and also change local cluster role to STANDBY
+        HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(haGroupName,this.zkUrl, this.peerZKUrl, ClusterRoleRecord.ClusterRole.STANDBY, ClusterRoleRecord.ClusterRole.ACTIVE, null);
+        haGroupStoreClient.rebuild();
+        currentRecord = haGroupStoreClient.getHAGroupStoreRecord();
+        assertNotNull(currentRecord);
+        assertEquals(HAGroupStoreRecord.HAGroupState.DEGRADED_STANDBY, currentRecord.getHAGroupState());
+
+        // Check that peerPathChildrenCache is not null now in HAGroupStoreClient via reflection
+        assertNotNull(peerPathChildrenCache.get(haGroupStoreClient));
+
+        // Now update local HAGroupStoreRecord to STANDBY to verify that HAGroupStoreClient is working as normal
+        record = new HAGroupStoreRecord("v1.0", haGroupName,
+                HAGroupStoreRecord.HAGroupState.STANDBY);
+        createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, record);
+        Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
+
+        currentRecord = haGroupStoreClient.getHAGroupStoreRecord();
+        assertNotNull(currentRecord);
+        assertEquals(HAGroupStoreRecord.HAGroupState.STANDBY, currentRecord.getHAGroupState());
+
+
+        // Now update peerZKUrl to invalid but non-null url and rebuild
+        // This URL can also be considered unreachable url due to a connectivity issue.
+        String invalidUrl = "invalidURL";
+        HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(haGroupName, this.zkUrl, invalidUrl, ClusterRoleRecord.ClusterRole.STANDBY, ClusterRoleRecord.ClusterRole.ACTIVE, null);
+        haGroupStoreClient.rebuild();
+        currentRecord = haGroupStoreClient.getHAGroupStoreRecord();
+        assertNotNull(currentRecord);
+        assertEquals(HAGroupStoreRecord.HAGroupState.STANDBY, currentRecord.getHAGroupState());
+
+        // Check CRR, the role for peer cluster with invalid url should be UNKNOWN.
+        ClusterRoleRecord clusterRoleRecord = haGroupStoreClient.getClusterRoleRecord();
+        assertNotNull(clusterRoleRecord);
+        ClusterRoleRecord expected = new ClusterRoleRecord(haGroupName,
+                HighAvailabilityPolicy.FAILOVER,
+                this.zkUrl,
+                ClusterRoleRecord.ClusterRole.STANDBY,
+                invalidUrl,
+                ClusterRoleRecord.ClusterRole.UNKNOWN,
+                1);
+        assertEquals(expected, clusterRoleRecord);
+
+        // Check that peerPathChildrenCache is null now in HAGroupStoreClient via reflection
+        assertNull(peerPathChildrenCache.get(haGroupStoreClient));
+
+
+    }
+
+    @Test
+    public void testHAGroupStoreClientWithoutPeerZK() throws Exception {
+        String haGroupName = testName.getMethodName();
+        // Clean existing record
+        HAGroupStoreTestUtil.deleteHAGroupRecordInSystemTable(haGroupName, this.zkUrl);
+        HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(haGroupName,this.zkUrl, null, ClusterRoleRecord.ClusterRole.ACTIVE, ClusterRoleRecord.ClusterRole.STANDBY, null);
+        HAGroupStoreRecord record = new HAGroupStoreRecord("v1.0", haGroupName,
+                HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+        createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, record);
+        assertNull(HAGroupStoreClient.getInstanceForZkUrl(CLUSTERS.getHBaseCluster1().getConfiguration(), haGroupName, zkUrl));
     }
 
     @Test
-    public void testHAGroupStoreClientWithSingleCRR() throws Exception {
-        HAGroupStoreClient haGroupStoreClient = HAGroupStoreClient.getInstance(config);
-        ClusterRoleRecord crr = new ClusterRoleRecord("failover",
-                HighAvailabilityPolicy.FAILOVER, haAdmin.getZkUrl(), ClusterRoleRecord.ClusterRole.ACTIVE,
-                "random-zk-url", ClusterRoleRecord.ClusterRole.STANDBY, 1L);
-        haAdmin.createOrUpdateDataOnZookeeper(crr);
+    public void testHAGroupStoreClientWithSingleHAGroupStoreRecord() throws Exception {
+        String haGroupName = testName.getMethodName();
+        HAGroupStoreClient haGroupStoreClient = HAGroupStoreClient.getInstanceForZkUrl(CLUSTERS.getHBaseCluster1().getConfiguration(), haGroupName, zkUrl);
+
+        // Create and store HAGroupStoreRecord with ACTIVE state
+        HAGroupStoreRecord record = new HAGroupStoreRecord("v1.0", haGroupName,
+                HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+        createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, record);
         Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
-        assert haGroupStoreClient.getCRRsByClusterRole(ClusterRoleRecord.ClusterRole.ACTIVE).size() == 1;
-        assert haGroupStoreClient.getCRRsByClusterRole(ClusterRoleRecord.ClusterRole.ACTIVE_TO_STANDBY).isEmpty();
+        HAGroupStoreRecord currentRecord = haGroupStoreClient.getHAGroupStoreRecord();
+        assert currentRecord != null && currentRecord.getHAGroupState() == HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC;
 
-        // Now Update CRR so that current cluster has state ACTIVE_TO_STANDBY
-        crr = new ClusterRoleRecord("failover",
-                HighAvailabilityPolicy.FAILOVER, haAdmin.getZkUrl(), ClusterRoleRecord.ClusterRole.ACTIVE_TO_STANDBY,
-                "random-zk-url", ClusterRoleRecord.ClusterRole.STANDBY, 2L);
-        haAdmin.createOrUpdateDataOnZookeeper(crr);
+        // Now Update HAGroupStoreRecord so that current cluster has state ACTIVE_TO_STANDBY
+        record = new HAGroupStoreRecord("v1.0", haGroupName,
+                HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY);
+        createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, record);
 
         Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
         // Check that now the cluster should be in ActiveToStandby
-        assert haGroupStoreClient.getCRRsByClusterRole(ClusterRoleRecord.ClusterRole.ACTIVE).isEmpty();
-        assert haGroupStoreClient.getCRRsByClusterRole(ClusterRoleRecord.ClusterRole.ACTIVE_TO_STANDBY).size() == 1;
-
+        currentRecord = haGroupStoreClient.getHAGroupStoreRecord();
+        assert currentRecord != null && currentRecord.getHAGroupState() == HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY;
 
         // Change it back to ACTIVE so that cluster is not in ACTIVE_TO_STANDBY state
-        crr = new ClusterRoleRecord("failover",
-                HighAvailabilityPolicy.FAILOVER, haAdmin.getZkUrl(), ClusterRoleRecord.ClusterRole.ACTIVE,
-                "random-zk-url", ClusterRoleRecord.ClusterRole.STANDBY, 3L);
-        haAdmin.createOrUpdateDataOnZookeeper(crr);
+        record = new HAGroupStoreRecord("v1.0", haGroupName,
+                HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+        createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, record);
 
         Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
-        assert haGroupStoreClient.getCRRsByClusterRole(ClusterRoleRecord.ClusterRole.ACTIVE).size() == 1;
-        assert haGroupStoreClient.getCRRsByClusterRole(ClusterRoleRecord.ClusterRole.ACTIVE_TO_STANDBY).isEmpty();
-
+        currentRecord = haGroupStoreClient.getHAGroupStoreRecord();
+        assert currentRecord != null && currentRecord.getHAGroupState() == HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC;
 
         // Change it again to ACTIVE_TO_STANDBY so that we can validate watcher works repeatedly
-        crr = new ClusterRoleRecord("failover",
-                HighAvailabilityPolicy.FAILOVER, haAdmin.getZkUrl(), ClusterRoleRecord.ClusterRole.ACTIVE_TO_STANDBY,
-                "random-zk-url", ClusterRoleRecord.ClusterRole.STANDBY, 4L);
-        haAdmin.createOrUpdateDataOnZookeeper(crr);
+        record = new HAGroupStoreRecord("v1.0", haGroupName,
+                HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY);
+        createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, record);
 
         Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
-        assert haGroupStoreClient.getCRRsByClusterRole(ClusterRoleRecord.ClusterRole.ACTIVE).isEmpty();
-        assert haGroupStoreClient.getCRRsByClusterRole(ClusterRoleRecord.ClusterRole.ACTIVE_TO_STANDBY).size() == 1;
-
+        currentRecord = haGroupStoreClient.getHAGroupStoreRecord();
+        assert currentRecord != null && currentRecord.getHAGroupState() == HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY;
 
-        // Change peer cluster to ACTIVE_TO_STANDBY so that we can still process mutation on this cluster
-        crr = new ClusterRoleRecord("failover",
-                HighAvailabilityPolicy.FAILOVER, haAdmin.getZkUrl(), ClusterRoleRecord.ClusterRole.ACTIVE,
-                "random-zk-url", ClusterRoleRecord.ClusterRole.ACTIVE_TO_STANDBY, 5L);
-        haAdmin.createOrUpdateDataOnZookeeper(crr);
+        // Change it back to ACTIVE to verify transition works
+        record = new HAGroupStoreRecord("v1.0", haGroupName,
+                HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+        createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, record);
 
         Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
-        assert haGroupStoreClient.getCRRsByClusterRole(ClusterRoleRecord.ClusterRole.ACTIVE).size() == 1;
-        assert haGroupStoreClient.getCRRsByClusterRole(ClusterRoleRecord.ClusterRole.ACTIVE_TO_STANDBY).isEmpty();
+        currentRecord = haGroupStoreClient.getHAGroupStoreRecord();
+        assert currentRecord != null && currentRecord.getHAGroupState() == HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC;
     }
 
 
+    /**
+     * Helper method to create or update HAGroupStoreRecord on ZooKeeper
+     */
+    private void createOrUpdateHAGroupStoreRecordOnZookeeper(PhoenixHAAdmin haAdmin, String haGroupName, HAGroupStoreRecord record) throws Exception {
+        String path = toPath(haGroupName);
+        if (haAdmin.getCurator().checkExists().forPath(path) == null) {
+            haAdmin.createHAGroupStoreRecordInZooKeeper(record);
+        } else {
+            final Pair<HAGroupStoreRecord, Stat> currentRecord = haAdmin.getHAGroupStoreRecordInZooKeeper(haGroupName);
+            if (currentRecord.getRight() != null && currentRecord.getLeft() != null) {
+                haAdmin.updateHAGroupStoreRecordInZooKeeper(haGroupName, record, currentRecord.getRight().getVersion());
+            } else {
+                throw new IOException("Current HAGroupStoreRecord in ZK is null, cannot update HAGroupStoreRecord " + haGroupName);
+            }
+        }
+    }
+
     @Test
-    public void testHAGroupStoreClientWithMultipleCRRs() throws Exception {
-        HAGroupStoreClient haGroupStoreClient = HAGroupStoreClient.getInstance(config);
-        // Setup initial CRRs
-        ClusterRoleRecord crr1 = new ClusterRoleRecord("failover",
-                HighAvailabilityPolicy.FAILOVER, haAdmin.getZkUrl(), ClusterRoleRecord.ClusterRole.ACTIVE,
-                "random-zk-url", ClusterRoleRecord.ClusterRole.STANDBY, 1L);
-        ClusterRoleRecord crr2 = new ClusterRoleRecord("parallel",
-                HighAvailabilityPolicy.PARALLEL, haAdmin.getZkUrl(), ClusterRoleRecord.ClusterRole.ACTIVE,
-                "random-zk-url", ClusterRoleRecord.ClusterRole.STANDBY, 1L);
-        haAdmin.createOrUpdateDataOnZookeeper(crr1);
-        haAdmin.createOrUpdateDataOnZookeeper(crr2);
+    public void testHAGroupStoreClientWithMultipleHAGroupStoreRecords() throws Exception {
+        String haGroupName1 = testName.getMethodName() + "1";
+        String haGroupName2 = testName.getMethodName() + "2";
+        HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(haGroupName1,this.zkUrl, this.peerZKUrl, ClusterRoleRecord.ClusterRole.ACTIVE, ClusterRoleRecord.ClusterRole.STANDBY, null);
+        HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(haGroupName2,this.zkUrl, this.peerZKUrl, ClusterRoleRecord.ClusterRole.ACTIVE, ClusterRoleRecord.ClusterRole.STANDBY, null);
 
-        Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
-        assert haGroupStoreClient.getCRRsByClusterRole(ClusterRoleRecord.ClusterRole.ACTIVE).size() == 2;
-        assert haGroupStoreClient.getCRRsByClusterRole(ClusterRoleRecord.ClusterRole.ACTIVE_TO_STANDBY).isEmpty();
-
-        // Now Update CRR so that current cluster has state ACTIVE_TO_STANDBY for only 1 crr
-        crr1 = new ClusterRoleRecord("failover",
-                HighAvailabilityPolicy.FAILOVER, haAdmin.getZkUrl(), ClusterRoleRecord.ClusterRole.ACTIVE_TO_STANDBY,
-                "random-zk-url", ClusterRoleRecord.ClusterRole.STANDBY, 2L);
-        crr2 = new ClusterRoleRecord("parallel",
-                HighAvailabilityPolicy.PARALLEL, haAdmin.getZkUrl(), ClusterRoleRecord.ClusterRole.ACTIVE,
-                "random-zk-url", ClusterRoleRecord.ClusterRole.STANDBY, 2L);
-        haAdmin.createOrUpdateDataOnZookeeper(crr1);
-        haAdmin.createOrUpdateDataOnZookeeper(crr2);
+        // Setup initial HAGroupStoreRecords
+        HAGroupStoreRecord record1 = new HAGroupStoreRecord("v1.0", haGroupName1,
+                HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+        HAGroupStoreRecord record2 = new HAGroupStoreRecord("v1.0", haGroupName2,
+                HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+
+        createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName1, record1);
+        createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName2, record2);
+
+        HAGroupStoreClient haGroupStoreClient1 = HAGroupStoreClient.getInstanceForZkUrl(CLUSTERS.getHBaseCluster1().getConfiguration(), haGroupName1, zkUrl);
+        HAGroupStoreClient haGroupStoreClient2 = HAGroupStoreClient.getInstanceForZkUrl(CLUSTERS.getHBaseCluster1().getConfiguration(), haGroupName2, zkUrl);
 
         Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
-        // Check that now the cluster should be in ActiveToStandby
-        assert haGroupStoreClient.getCRRsByClusterRole(ClusterRoleRecord.ClusterRole.ACTIVE).size() == 1;
-        assert haGroupStoreClient.getCRRsByClusterRole(ClusterRoleRecord.ClusterRole.ACTIVE_TO_STANDBY).size() == 1;
+        HAGroupStoreRecord currentRecord1 = haGroupStoreClient1.getHAGroupStoreRecord();
+        HAGroupStoreRecord currentRecord2 = haGroupStoreClient2.getHAGroupStoreRecord();
+        assert currentRecord1 != null && currentRecord1.getHAGroupState() == HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC;
+        assert currentRecord2 != null && currentRecord2.getHAGroupState() == HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC;
 
+        // Now Update HAGroupStoreRecord so that current cluster has state ACTIVE_TO_STANDBY for only 1 record
+        record1 = new HAGroupStoreRecord("v1.0", haGroupName1,
+                HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY);
+        record2 = new HAGroupStoreRecord("v1.0", haGroupName2,
+                HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
 
-        // Change it back to ACTIVE so that cluster is not in ACTIVE_TO_STANDBY state
-        crr1 = new ClusterRoleRecord("failover",
-                HighAvailabilityPolicy.FAILOVER, haAdmin.getZkUrl(), ClusterRoleRecord.ClusterRole.ACTIVE,
-                "random-zk-url", ClusterRoleRecord.ClusterRole.STANDBY, 3L);
-        crr2 = new ClusterRoleRecord("parallel",
-                HighAvailabilityPolicy.PARALLEL, haAdmin.getZkUrl(), ClusterRoleRecord.ClusterRole.ACTIVE,
-                "random-zk-url", ClusterRoleRecord.ClusterRole.STANDBY, 3L);
-        haAdmin.createOrUpdateDataOnZookeeper(crr1);
-        haAdmin.createOrUpdateDataOnZookeeper(crr2);
+        createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName1, record1);
+        createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName2, record2);
 
         Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
-        assert haGroupStoreClient.getCRRsByClusterRole(ClusterRoleRecord.ClusterRole.ACTIVE).size() == 2;
-        assert haGroupStoreClient.getCRRsByClusterRole(ClusterRoleRecord.ClusterRole.ACTIVE_TO_STANDBY).isEmpty();
+        // Check that now the cluster should be in ActiveToStandby
+        currentRecord1 = haGroupStoreClient1.getHAGroupStoreRecord();
+        currentRecord2 = haGroupStoreClient2.getHAGroupStoreRecord();
+        assert currentRecord1 != null && currentRecord1.getHAGroupState() == HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY;
+        assert currentRecord2 != null && currentRecord2.getHAGroupState() == HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC;
 
+        // Change it back to ACTIVE so that cluster is not in ACTIVE_TO_STANDBY state
+        record1 = new HAGroupStoreRecord("v1.0", haGroupName1,
+                HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+        record2 = new HAGroupStoreRecord("v1.0", haGroupName2,
+                HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
 
-        // Change other crr to ACTIVE_TO_STANDBY and one in ACTIVE state so that we can validate watcher works repeatedly
-        crr1 = new ClusterRoleRecord("failover",
-                HighAvailabilityPolicy.FAILOVER, haAdmin.getZkUrl(), ClusterRoleRecord.ClusterRole.ACTIVE,
-                "random-zk-url", ClusterRoleRecord.ClusterRole.STANDBY, 4L);
-        crr2 = new ClusterRoleRecord("parallel",
-                HighAvailabilityPolicy.PARALLEL, haAdmin.getZkUrl(), ClusterRoleRecord.ClusterRole.ACTIVE_TO_STANDBY,
-                "random-zk-url", ClusterRoleRecord.ClusterRole.STANDBY, 4L);
-        haAdmin.createOrUpdateDataOnZookeeper(crr1);
-        haAdmin.createOrUpdateDataOnZookeeper(crr2);
+        createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName1, record1);
+        createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName2, record2);
 
         Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
-        assert haGroupStoreClient.getCRRsByClusterRole(ClusterRoleRecord.ClusterRole.ACTIVE).size() == 1;
-        assert haGroupStoreClient.getCRRsByClusterRole(ClusterRoleRecord.ClusterRole.ACTIVE_TO_STANDBY).size() == 1;
+        currentRecord1 = haGroupStoreClient1.getHAGroupStoreRecord();
+        currentRecord2 = haGroupStoreClient2.getHAGroupStoreRecord();
+        assert currentRecord1 != null && currentRecord1.getHAGroupState() == HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC;
+        assert currentRecord2 != null && currentRecord2.getHAGroupState() == HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC;
 
+        // Change other record to ACTIVE_TO_STANDBY and one in ACTIVE state so that we can validate watcher works repeatedly
+        record1 = new HAGroupStoreRecord("v1.0", haGroupName1,
+                HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+        record2 = new HAGroupStoreRecord("v1.0", haGroupName2,
+                HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY);
 
-        // Change peer cluster to ACTIVE_TO_STANDBY so that we can still process mutation on this cluster
-        crr1 = new ClusterRoleRecord("failover",
-                HighAvailabilityPolicy.FAILOVER, haAdmin.getZkUrl(), ClusterRoleRecord.ClusterRole.ACTIVE,
-                "random-zk-url", ClusterRoleRecord.ClusterRole.ACTIVE_TO_STANDBY, 5L);
-        crr2 = new ClusterRoleRecord("parallel",
-                HighAvailabilityPolicy.PARALLEL, haAdmin.getZkUrl(), ClusterRoleRecord.ClusterRole.ACTIVE,
-                "random-zk-url", ClusterRoleRecord.ClusterRole.ACTIVE_TO_STANDBY, 5L);
-        haAdmin.createOrUpdateDataOnZookeeper(crr1);
-        haAdmin.createOrUpdateDataOnZookeeper(crr2);
+        createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName1, record1);
+        createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName2, record2);
 
         Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
-        assert haGroupStoreClient.getCRRsByClusterRole(ClusterRoleRecord.ClusterRole.ACTIVE).size() == 2;
-        assert haGroupStoreClient.getCRRsByClusterRole(ClusterRoleRecord.ClusterRole.ACTIVE_TO_STANDBY).isEmpty();
+        currentRecord1 = haGroupStoreClient1.getHAGroupStoreRecord();
+        currentRecord2 = haGroupStoreClient2.getHAGroupStoreRecord();
+        assert currentRecord1 != null && currentRecord1.getHAGroupState() == HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC;
+        assert currentRecord2 != null && currentRecord2.getHAGroupState() == HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY;
     }
 
     @Test
     public void testMultiThreadedAccessToHACache() throws Exception {
-        HAGroupStoreClient haGroupStoreClient = HAGroupStoreClient.getInstance(config);
-        // Setup initial CRRs
-        ClusterRoleRecord crr1 = new ClusterRoleRecord("failover",
-                HighAvailabilityPolicy.FAILOVER, haAdmin.getZkUrl(), ClusterRoleRecord.ClusterRole.ACTIVE,
-                "random-zk-url", ClusterRoleRecord.ClusterRole.STANDBY, 1L);
-        ClusterRoleRecord crr2 = new ClusterRoleRecord("parallel",
-                HighAvailabilityPolicy.PARALLEL, haAdmin.getZkUrl(), ClusterRoleRecord.ClusterRole.ACTIVE,
-                "random-zk-url", ClusterRoleRecord.ClusterRole.STANDBY, 1L);
-        haAdmin.createOrUpdateDataOnZookeeper(crr1);
-        haAdmin.createOrUpdateDataOnZookeeper(crr2);
+        String haGroupName = testName.getMethodName();
+
+        // Setup initial HAGroupStoreRecord
+        HAGroupStoreRecord record = new HAGroupStoreRecord("v1.0", haGroupName,
+                HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+
+        createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, record);
+        HAGroupStoreClient haGroupStoreClient = HAGroupStoreClient.getInstanceForZkUrl(CLUSTERS.getHBaseCluster1().getConfiguration(), haGroupName, zkUrl);
 
         Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
 
@@ -223,8 +374,8 @@ public class HAGroupStoreClientIT extends BaseTest {
         for (int i = 0; i < threadCount; i++) {
             executor.submit(() -> {
                 try {
-                    assert haGroupStoreClient.getCRRsByClusterRole(ClusterRoleRecord.ClusterRole.ACTIVE).size() == 2;
-                    assert haGroupStoreClient.getCRRsByClusterRole(ClusterRoleRecord.ClusterRole.ACTIVE_TO_STANDBY).isEmpty();
+                    HAGroupStoreRecord currentRecord = haGroupStoreClient.getHAGroupStoreRecord();
+                    assert currentRecord != null && currentRecord.getHAGroupState() == HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC;
                     latch.countDown();
                 } catch (Exception e) {
                     throw new RuntimeException(e);
@@ -233,15 +384,11 @@ public class HAGroupStoreClientIT extends BaseTest {
         }
         assert latch.await(10, TimeUnit.SECONDS);
 
-        // Update CRRs
-        crr1 = new ClusterRoleRecord("failover",
-                HighAvailabilityPolicy.FAILOVER, haAdmin.getZkUrl(), ClusterRoleRecord.ClusterRole.ACTIVE_TO_STANDBY,
-                "random-zk-url", ClusterRoleRecord.ClusterRole.STANDBY, 2L);
-        crr2 = new ClusterRoleRecord("parallel",
-                HighAvailabilityPolicy.PARALLEL, haAdmin.getZkUrl(), ClusterRoleRecord.ClusterRole.ACTIVE,
-                "random-zk-url", ClusterRoleRecord.ClusterRole.STANDBY, 2L);
-        haAdmin.createOrUpdateDataOnZookeeper(crr1);
-        haAdmin.createOrUpdateDataOnZookeeper(crr2);
+        // Update HAGroupStoreRecord
+        record = new HAGroupStoreRecord("v1.0", haGroupName,
+                HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY);
+
+        createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, record);
 
         Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
 
@@ -250,8 +397,8 @@ public class HAGroupStoreClientIT extends BaseTest {
         for (int i = 0; i < threadCount; i++) {
             executor.submit(() -> {
                 try {
-                    assert haGroupStoreClient.getCRRsByClusterRole(ClusterRoleRecord.ClusterRole.ACTIVE).size() == 1;
-                    assert haGroupStoreClient.getCRRsByClusterRole(ClusterRoleRecord.ClusterRole.ACTIVE_TO_STANDBY).size() == 1;
+                    HAGroupStoreRecord currentRecord = haGroupStoreClient.getHAGroupStoreRecord();
+                    assert currentRecord != null && currentRecord.getHAGroupState() == HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY;
                     latch2.countDown();
                 } catch (Exception e) {
                     throw new RuntimeException(e);
@@ -263,100 +410,251 @@ public class HAGroupStoreClientIT extends BaseTest {
 
     @Test
     public void testHAGroupStoreClientWithRootPathDeletion() throws Exception {
-        HAGroupStoreClient haGroupStoreClient = HAGroupStoreClient.getInstance(config);
-        ClusterRoleRecord crr1 = new ClusterRoleRecord("failover",
-                HighAvailabilityPolicy.FAILOVER, haAdmin.getZkUrl(), ClusterRoleRecord.ClusterRole.ACTIVE_TO_STANDBY,
-                "random-zk-url", ClusterRoleRecord.ClusterRole.STANDBY, 1L);
-        ClusterRoleRecord crr2 = new ClusterRoleRecord("parallel",
-                HighAvailabilityPolicy.PARALLEL, haAdmin.getZkUrl(), ClusterRoleRecord.ClusterRole.ACTIVE,
-                "random-zk-url", ClusterRoleRecord.ClusterRole.STANDBY, 1L);
-        haAdmin.createOrUpdateDataOnZookeeper(crr1);
-        haAdmin.createOrUpdateDataOnZookeeper(crr2);
+        String haGroupName = testName.getMethodName();
+
+        HAGroupStoreRecord record1 = new HAGroupStoreRecord("v1.0", haGroupName,
+                HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+
+        createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, record1);
+        HAGroupStoreClient haGroupStoreClient = HAGroupStoreClient.getInstanceForZkUrl(CLUSTERS.getHBaseCluster1().getConfiguration(), haGroupName, zkUrl);
         Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
-        assert haGroupStoreClient.getCRRsByClusterRole(ClusterRoleRecord.ClusterRole.ACTIVE).size() == 1;
-        assert haGroupStoreClient.getCRRsByClusterRole(ClusterRoleRecord.ClusterRole.ACTIVE_TO_STANDBY).size() == 1;
+        HAGroupStoreRecord currentRecord = haGroupStoreClient.getHAGroupStoreRecord();
+        assertNotNull(currentRecord);
+        assertEquals(HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, currentRecord.getHAGroupState());
 
-        haAdmin.getCurator().delete().deletingChildrenIfNeeded().forPath(ZKPaths.PATH_SEPARATOR);
+        haAdmin.getCurator().delete().deletingChildrenIfNeeded().forPath(toPath(StringUtils.EMPTY));
         Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
-        assert haGroupStoreClient.getCRRsByClusterRole(ClusterRoleRecord.ClusterRole.ACTIVE).isEmpty();
-        assert haGroupStoreClient.getCRRsByClusterRole(ClusterRoleRecord.ClusterRole.ACTIVE_TO_STANDBY).isEmpty();
-
-
-        crr1 = new ClusterRoleRecord("failover",
-                HighAvailabilityPolicy.FAILOVER, haAdmin.getZkUrl(), ClusterRoleRecord.ClusterRole.ACTIVE_TO_STANDBY,
-                "random-zk-url", ClusterRoleRecord.ClusterRole.STANDBY, 2L);
-        crr2 = new ClusterRoleRecord("parallel",
-                HighAvailabilityPolicy.PARALLEL, haAdmin.getZkUrl(), ClusterRoleRecord.ClusterRole.ACTIVE,
-                "random-zk-url", ClusterRoleRecord.ClusterRole.STANDBY, 2L);
-        haAdmin.createOrUpdateDataOnZookeeper(crr1);
-        haAdmin.createOrUpdateDataOnZookeeper(crr2);
+        currentRecord = haGroupStoreClient.getHAGroupStoreRecord();
+
+        // The record should be automatically rebuilt from System Table as it is not in ZK
+        assertNotNull(currentRecord);
+        assertEquals(HAGroupStoreRecord.HAGroupState.ACTIVE_NOT_IN_SYNC, currentRecord.getHAGroupState());
+
+        record1 = new HAGroupStoreRecord("v1.0", haGroupName,
+                HAGroupStoreRecord.HAGroupState.ACTIVE_NOT_IN_SYNC_TO_STANDBY);
+
+        createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, record1);
         Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
-        assert haGroupStoreClient.getCRRsByClusterRole(ClusterRoleRecord.ClusterRole.ACTIVE).size() == 1;
-        assert haGroupStoreClient.getCRRsByClusterRole(ClusterRoleRecord.ClusterRole.ACTIVE_TO_STANDBY).size() == 1;
+        currentRecord = haGroupStoreClient.getHAGroupStoreRecord();
+        assertNotNull(currentRecord);
+        assertEquals(HAGroupStoreRecord.HAGroupState.ACTIVE_NOT_IN_SYNC_TO_STANDBY, currentRecord.getHAGroupState());
     }
 
     @Test
     public void testThrowsExceptionWithZKDisconnectionAndThenConnection() throws Exception {
-        HAGroupStoreClient haGroupStoreClient = HAGroupStoreClient.getInstance(config);
-        // Setup initial CRRs
-        ClusterRoleRecord crr1 = new ClusterRoleRecord("failover",
-                HighAvailabilityPolicy.FAILOVER, haAdmin.getZkUrl(), ClusterRoleRecord.ClusterRole.ACTIVE,
-                "random-zk-url", ClusterRoleRecord.ClusterRole.STANDBY, 1L);
-        ClusterRoleRecord crr2 = new ClusterRoleRecord("parallel",
-                HighAvailabilityPolicy.PARALLEL, haAdmin.getZkUrl(), ClusterRoleRecord.ClusterRole.ACTIVE,
-                "random-zk-url", ClusterRoleRecord.ClusterRole.STANDBY, 1L);
-        haAdmin.createOrUpdateDataOnZookeeper(crr1);
-        haAdmin.createOrUpdateDataOnZookeeper(crr2);
+        String haGroupName = testName.getMethodName();
+
+        // Setup initial HAGroupStoreRecord
+        HAGroupStoreRecord record = new HAGroupStoreRecord("v1.0", haGroupName,
+                HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+
+        createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, record);
+        HAGroupStoreClient haGroupStoreClient = HAGroupStoreClient.getInstanceForZkUrl(CLUSTERS.getHBaseCluster1().getConfiguration(), haGroupName, zkUrl);
 
         Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
-        assert haGroupStoreClient.getCRRsByClusterRole(ClusterRoleRecord.ClusterRole.ACTIVE).size() == 2;
+        HAGroupStoreRecord currentRecord = haGroupStoreClient.getHAGroupStoreRecord();
+        assert currentRecord != null && currentRecord.getHAGroupState() == HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC;
 
         // Shutdown the ZK Cluster to simulate CONNECTION_SUSPENDED event
-        utility.shutdownMiniZKCluster();
+        CLUSTERS.getHBaseCluster1().shutdownMiniZKCluster();
 
         Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
         //Check that HAGroupStoreClient instance is not healthy and throws IOException
-        assertThrows(IOException.class, () -> haGroupStoreClient.getCRRsByClusterRole(ClusterRoleRecord.ClusterRole.ACTIVE));
+        assertThrows(IOException.class, () -> haGroupStoreClient.getHAGroupStoreRecord());
+        // Check that the HAGroupStoreClient instance is not healthy via reflection
+        Field isHealthyField = HAGroupStoreClient.class.getDeclaredField("isHealthy");
+        isHealthyField.setAccessible(true);
+        assertFalse((boolean)isHealthyField.get(haGroupStoreClient));
 
         // Start ZK on the same port to simulate CONNECTION_RECONNECTED event
-        utility.startMiniZKCluster(1,Integer.parseInt(getZKClientPort(config)));
+        CLUSTERS.getHBaseCluster1().startMiniZKCluster(1,
+                Integer.parseInt(CLUSTERS.getHBaseCluster1().getConfiguration().get("hbase.zookeeper.property.clientPort")));
 
         Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
         //Check that HAGroupStoreClient instance is back to healthy and provides correct response
-        assert haGroupStoreClient.getCRRsByClusterRole(ClusterRoleRecord.ClusterRole.ACTIVE).size() == 2;
+        currentRecord = haGroupStoreClient.getHAGroupStoreRecord();
+        assert currentRecord != null && currentRecord.getHAGroupState() == HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC;
+        // Check that the HAGroupStoreClient instance is healthy via reflection
+        assertTrue((boolean)isHealthyField.get(haGroupStoreClient));
     }
 
 
+
+    // Tests for setHAGroupStatusIfNeeded method
+    @Test
+    public void testSetHAGroupStatusIfNeededDeleteZKAndSystemTableRecord() throws Exception {
+        String haGroupName = testName.getMethodName();
+
+        // Create HAGroupStoreClient without any existing record
+        HAGroupStoreClient haGroupStoreClient = HAGroupStoreClient.getInstanceForZkUrl(CLUSTERS.getHBaseCluster1().getConfiguration(), haGroupName, zkUrl);
+
+        // Delete the record from ZK
+        haAdmin.getCurator().delete().deletingChildrenIfNeeded().forPath(toPath(haGroupName));
+        Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
+
+        // Delete the record from System Table
+        try (PhoenixConnection conn = (PhoenixConnection) DriverManager.getConnection(JDBC_PROTOCOL_ZK + JDBC_PROTOCOL_SEPARATOR + zkUrl);
+             Statement stmt = conn.createStatement()) {
+            stmt.execute("DELETE FROM " + SYSTEM_HA_GROUP_NAME + " WHERE HA_GROUP_NAME = '" + haGroupName + "'");
+            conn.commit();
+        }
+
+        // This should fail because no record exists in either ZK or System Table
+        try {
+            haGroupStoreClient.setHAGroupStatusIfNeeded(HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+            fail("Expected IOException for missing system table record");
+        } catch (IOException e) {
+            assertTrue("Exception should mention system table", e.getMessage().contains("Current HAGroupStoreRecordStat in cache is null, "
+                    + "cannot update HAGroupStoreRecord, the record should be initialized in System Table first"));
+        }
+    }
+
     @Test
-    public void testHAGroupStoreClientWithDifferentZKURLFormats() throws Exception {
-        HAGroupStoreClient haGroupStoreClient = HAGroupStoreClient.getInstance(config);
-        final String zkClientPort = getZKClientPort(config);
-        // Setup initial CRRs
-        final String format1 = "127.0.0.1\\:"+zkClientPort+"::/hbase"; // 127.0.0.1\:53228::/hbase
-        final String format2 = "127.0.0.1:"+zkClientPort+"::/hbase"; //   127.0.0.1:53228::/hbase
-        final String format3 = "127.0.0.1\\:"+zkClientPort+":/hbase";   // 127.0.0.1\:53228:/hbase
-
-        ClusterRoleRecord crr1 = new ClusterRoleRecord("parallel1",
-                HighAvailabilityPolicy.PARALLEL, format1, ClusterRoleRecord.ClusterRole.ACTIVE,
-                "random-zk-url", ClusterRoleRecord.ClusterRole.STANDBY, 1L);
-        haAdmin.createOrUpdateDataOnZookeeper(crr1);
-
-        ClusterRoleRecord crr2 = new ClusterRoleRecord("parallel2",
-                HighAvailabilityPolicy.PARALLEL, format2, ClusterRoleRecord.ClusterRole.STANDBY,
-                "random-zk-url", ClusterRoleRecord.ClusterRole.STANDBY, 1L);
-        haAdmin.createOrUpdateDataOnZookeeper(crr2);
-
-        ClusterRoleRecord crr3 = new ClusterRoleRecord("parallel3",
-                HighAvailabilityPolicy.PARALLEL, format3, ClusterRoleRecord.ClusterRole.ACTIVE_TO_STANDBY,
-                "random-zk-url", ClusterRoleRecord.ClusterRole.STANDBY, 1L);
-        haAdmin.createOrUpdateDataOnZookeeper(crr3);
+    public void testSetHAGroupStatusIfNeededUpdateExistingRecord() throws Exception {
+        String haGroupName = testName.getMethodName();
+
+        // Create initial record
+        HAGroupStoreRecord initialRecord = new HAGroupStoreRecord("v1.0", haGroupName,
+                HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+        createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, initialRecord);
+
+        HAGroupStoreClient haGroupStoreClient = HAGroupStoreClient.getInstanceForZkUrl(CLUSTERS.getHBaseCluster1().getConfiguration(), haGroupName, zkUrl);
+        Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
 
+        // Verify initial state
+        HAGroupStoreRecord currentRecord = haGroupStoreClient.getHAGroupStoreRecord();
+        assertEquals(HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, currentRecord.getHAGroupState());
+
+        // Update to STANDBY (this should succeed as it's a valid transition)
+        haGroupStoreClient.setHAGroupStatusIfNeeded(HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY);
         Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
-        assert haGroupStoreClient.getCRRsByClusterRole(ClusterRoleRecord.ClusterRole.ACTIVE).size() == 1;
-        assert haGroupStoreClient.getCRRsByClusterRole(ClusterRoleRecord.ClusterRole.ACTIVE_TO_STANDBY).size() == 1;
-        assert haGroupStoreClient.getCRRsByClusterRole(ClusterRoleRecord.ClusterRole.STANDBY).size() == 1;
+
+        // Verify the record was updated
+        currentRecord = haGroupStoreClient.getHAGroupStoreRecord();
+        assertEquals(HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY, currentRecord.getHAGroupState());
     }
 
+    @Test
+    public void testSetHAGroupStatusIfNeededNoUpdateWhenNotNeeded() throws Exception {
+        String haGroupName = testName.getMethodName();
+
+        // Create initial record with current timestamp
+        HAGroupStoreRecord initialRecord = new HAGroupStoreRecord("v1.0", haGroupName,
+                HAGroupStoreRecord.HAGroupState.ACTIVE_NOT_IN_SYNC);
+        createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, initialRecord);
+        Stat initialRecordInZKStat = haAdmin.getHAGroupStoreRecordInZooKeeper(haGroupName).getRight();
+        int initialRecordVersion = initialRecordInZKStat.getVersion();
+
+        HAGroupStoreClient haGroupStoreClient = HAGroupStoreClient.getInstanceForZkUrl(CLUSTERS.getHBaseCluster1().getConfiguration(), haGroupName, zkUrl);
+
+        // Get the current record
+        HAGroupStoreRecord currentRecord = haGroupStoreClient.getHAGroupStoreRecord();
+
+        // Try to set to ACTIVE_IN_SYNC immediately (should not update due to timing)
+        haGroupStoreClient.setHAGroupStatusIfNeeded(HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+
+        // Add sleep if due to any bug the update might have gone through and we can assert below this.
+        Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
+        // Verify no update occurred
+        HAGroupStoreRecord afterRecord = haGroupStoreClient.getHAGroupStoreRecord();
+        Stat afterRecordInZKStat = haAdmin.getHAGroupStoreRecordInZooKeeper(haGroupName).getRight();
+        int afterRecordVersion = afterRecordInZKStat.getVersion();
+
+        assertEquals(initialRecordVersion, afterRecordVersion);
+        assertEquals("State should not change", HAGroupStoreRecord.HAGroupState.ACTIVE_NOT_IN_SYNC, afterRecord.getHAGroupState());
+    }
+
+    @Test
+    public void testSetHAGroupStatusIfNeededWithTimingLogic() throws Exception {
+        String haGroupName = testName.getMethodName();
+        // Create initial record
+        HAGroupStoreRecord initialRecord = new HAGroupStoreRecord("v1.0", haGroupName,
+                HAGroupStoreRecord.HAGroupState.ACTIVE_NOT_IN_SYNC);
+        createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, initialRecord);
+
+        HAGroupStoreClient haGroupStoreClient = HAGroupStoreClient.getInstanceForZkUrl(CLUSTERS.getHBaseCluster1().getConfiguration(), haGroupName, zkUrl);
+        Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS + DEFAULT_ZK_SESSION_TIMEOUT);
+
+        haGroupStoreClient.setHAGroupStatusIfNeeded(HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+        Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
+
+        // Verify the record was updated
+        HAGroupStoreRecord updatedRecord = haGroupStoreClient.getHAGroupStoreRecord();
+        assertEquals(HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, updatedRecord.getHAGroupState());
+    }
+
+    @Test
+    public void testSetHAGroupStatusIfNeededWithInvalidTransition() throws Exception {
+        String haGroupName = testName.getMethodName();
+
+        // Create initial record with ACTIVE state
+        HAGroupStoreRecord initialRecord = new HAGroupStoreRecord("v1.0", haGroupName,
+                HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+        createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, initialRecord);
+
+        HAGroupStoreClient haGroupStoreClient = HAGroupStoreClient.getInstanceForZkUrl(CLUSTERS.getHBaseCluster1().getConfiguration(), haGroupName, zkUrl);
+        Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
+
+        // Try to transition to STANDBY (invalid transition from ACTIVE)
+        try {
+            haGroupStoreClient.setHAGroupStatusIfNeeded(HAGroupStoreRecord.HAGroupState.STANDBY);
+            fail("Expected InvalidClusterRoleTransitionException");
+        } catch (InvalidClusterRoleTransitionException e) {
+            // Should get InvalidClusterRoleTransitionException (might be wrapped)
+            assertTrue("Exception should be about invalid transition",
+                    e.getMessage().contains("Cannot transition from ACTIVE_IN_SYNC to STANDBY") ||
+                            e.getCause() != null && e.getCause().getMessage().contains("Cannot transition from ACTIVE_IN_SYNC to STANDBY"));
+        }
+    }
+
+    @Test
+    public void testSetHAGroupStatusIfNeededWithUnhealthyClient() throws Exception {
+        String haGroupName = testName.getMethodName();
+
+        HAGroupStoreClient haGroupStoreClient = HAGroupStoreClient.getInstanceForZkUrl(CLUSTERS.getHBaseCluster1().getConfiguration(), haGroupName, zkUrl);
+
+        // Make client unhealthy by accessing private field
+        Field isHealthyField = HAGroupStoreClient.class.getDeclaredField("isHealthy");
+        isHealthyField.setAccessible(true);
+        isHealthyField.set(haGroupStoreClient, false);
+
+        // Try to set status on unhealthy client
+        try {
+            haGroupStoreClient.setHAGroupStatusIfNeeded(HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+            fail("Expected IOException for unhealthy client");
+        } catch (IOException e) {
+            assertTrue("Exception should mention unhealthy client", e.getMessage().contains("not healthy"));
+        }
+    }
+
+
+    @Test
+    public void testSetHAGroupStatusIfNeededMultipleTransitions() throws Exception {
+        String haGroupName = testName.getMethodName();
+
+        // Create initial record with old timestamp
+        HAGroupStoreRecord initialRecord = new HAGroupStoreRecord("v1.0", haGroupName,
+                HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+        createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, initialRecord);
+
+        HAGroupStoreClient haGroupStoreClient = HAGroupStoreClient.getInstanceForZkUrl(CLUSTERS.getHBaseCluster1().getConfiguration(), haGroupName, zkUrl);
+        Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
+
+        // First transition: ACTIVE -> ACTIVE_TO_STANDBY
+        haGroupStoreClient.setHAGroupStatusIfNeeded(HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY);
+        Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
+
+        HAGroupStoreRecord afterFirst = haGroupStoreClient.getHAGroupStoreRecord();
+        assertEquals(HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY, afterFirst.getHAGroupState());
+
+        // Wait and make another transition: ACTIVE_TO_STANDBY -> STANDBY
+        Thread.sleep(100); // Small delay to ensure timestamp difference
+        haGroupStoreClient.setHAGroupStatusIfNeeded(HAGroupStoreRecord.HAGroupState.STANDBY);
+        Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
+
+        HAGroupStoreRecord afterSecond = haGroupStoreClient.getHAGroupStoreRecord();
+        assertEquals(HAGroupStoreRecord.HAGroupState.STANDBY, afterSecond.getHAGroupState());
+    }
 
     /**
      * This test verifies that the updates coming via PathChildrenCacheListener are in order in which updates are sent to ZK
@@ -364,22 +662,27 @@ public class HAGroupStoreClientIT extends BaseTest {
      */
     @Test
     public void testHAGroupStoreClientWithMultiThreadedUpdates() throws Exception {
+        String haGroupName = testName.getMethodName();
+
         // Number of threads to execute
         int threadCount = 5;
 
-        // Capture versions of crr in a list(crrEventVersions)  in order they are received.
-        List<Integer> crrEventVersions = new ArrayList<>();
+        // Capture versions of records in a list(recordEventVersions) in order they are received.
+        List<Integer> recordEventVersions = new ArrayList<>();
         CountDownLatch eventsLatch = new CountDownLatch(threadCount);
         PathChildrenCacheListener pathChildrenCacheListener = (client, event) -> {
-            if(event.getData() != null && event.getData().getData() != null && ClusterRoleRecord.fromJson(event.getData().getData()).isPresent()) {
-                    ClusterRoleRecord crr = ClusterRoleRecord.fromJson(event.getData().getData()).get();
-                    crrEventVersions.add((int)crr.getVersion());
+            if(event.getData() != null && event.getData().getData() != null && HAGroupStoreRecord.fromJson(event.getData().getData()).isPresent()) {
+                HAGroupStoreRecord record = HAGroupStoreRecord.fromJson(event.getData().getData()).get();
+                if (record.getHaGroupName().equals(haGroupName)) {
+                    recordEventVersions.add(event.getData().getStat().getVersion());
                     eventsLatch.countDown();
+                }
             }
         };
 
-        // Start a new HAGroupStoreClient.
-        new HAGroupStoreClient(config, pathChildrenCacheListener);
+        // Start a new HAGroupStoreClient with custom listener.
+        new HAGroupStoreClient(CLUSTERS.getHBaseCluster1().getConfiguration(),
+                pathChildrenCacheListener, null, haGroupName, zkUrl);
 
         // Create multiple threads for update to ZK.
         final CountDownLatch updateLatch = new CountDownLatch(threadCount);
@@ -389,10 +692,10 @@ public class HAGroupStoreClientIT extends BaseTest {
         List<Integer> updateList = new ArrayList<>();
 
         // Create a queue which can be polled to send updates to ZK.
-        ConcurrentLinkedQueue<ClusterRoleRecord> updateQueue = new ConcurrentLinkedQueue<>();
+        ConcurrentLinkedQueue<HAGroupStoreRecord> updateQueue = new ConcurrentLinkedQueue<>();
         for(int i = 0; i < threadCount; i++) {
-            updateQueue.add(createCRR(i+1));
-            updateList.add(i+1);
+            updateQueue.add(createHAGroupStoreRecord(haGroupName));
+            updateList.add(i);
         }
 
         // Submit updates to ZK.
@@ -400,7 +703,8 @@ public class HAGroupStoreClientIT extends BaseTest {
             executor.submit(() -> {
                 try {
                     synchronized (HAGroupStoreClientIT.class) {
-                        haAdmin.createOrUpdateDataOnZookeeper(Objects.requireNonNull(updateQueue.poll()));
+                        HAGroupStoreRecord record = Objects.requireNonNull(updateQueue.poll());
+                        createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, record);
                     }
                     updateLatch.countDown();
                 } catch (Exception e) {
@@ -414,14 +718,204 @@ public class HAGroupStoreClientIT extends BaseTest {
         assert updateLatch.await(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS*threadCount, TimeUnit.MILLISECONDS);
 
         // Assert that the order of updates is same as order of events.
-        assert updateList.equals(crrEventVersions);
+        assert updateList.equals(recordEventVersions);
     }
 
-    private ClusterRoleRecord createCRR(Integer version) {
-        return new ClusterRoleRecord("parallel",
-                HighAvailabilityPolicy.PARALLEL, haAdmin.getZkUrl(), ClusterRoleRecord.ClusterRole.ACTIVE,
-                "random-zk-url", ClusterRoleRecord.ClusterRole.STANDBY, version);
+    @Test
+    public void testGetClusterRoleRecordNormalCase() throws Exception {
+        String haGroupName = testName.getMethodName();
+        HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(haGroupName, this.zkUrl, this.peerZKUrl,
+                ClusterRoleRecord.ClusterRole.ACTIVE, ClusterRoleRecord.ClusterRole.STANDBY, null);
+
+        // Create HAGroupStoreRecord for local cluster
+        HAGroupStoreRecord localRecord = new HAGroupStoreRecord("v1.0", haGroupName,
+                HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+        createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, localRecord);
+
+        // Create HAGroupStoreRecord for peer cluster
+        HAGroupStoreRecord peerRecord = new HAGroupStoreRecord("v1.0", haGroupName,
+                HAGroupStoreRecord.HAGroupState.STANDBY);
+        createOrUpdateHAGroupStoreRecordOnZookeeper(peerHaAdmin, haGroupName, peerRecord);
+
+        Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
+
+        HAGroupStoreClient haGroupStoreClient = HAGroupStoreClient.getInstanceForZkUrl(CLUSTERS.getHBaseCluster1().getConfiguration(), haGroupName, zkUrl);
+        assertNotNull(haGroupStoreClient);
+
+        // Test getClusterRoleRecord
+        ClusterRoleRecord clusterRoleRecord = haGroupStoreClient.getClusterRoleRecord();
+        assertNotNull(clusterRoleRecord);
+        ClusterRoleRecord expectedClusterRoleRecord = new ClusterRoleRecord(haGroupName,
+                HighAvailabilityPolicy.FAILOVER, zkUrl, ClusterRoleRecord.ClusterRole.ACTIVE, peerZKUrl,
+                ClusterRoleRecord.ClusterRole.STANDBY, 1);
+        assertEquals(expectedClusterRoleRecord, clusterRoleRecord);
+    }
+
+
+    @Test
+    public void testGetClusterRoleRecordWithValidPeerZKUrlButNoPeerRecord() throws Exception {
+        String haGroupName = testName.getMethodName();
+        HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(haGroupName, this.zkUrl, this.peerZKUrl,
+                ClusterRoleRecord.ClusterRole.ACTIVE, ClusterRoleRecord.ClusterRole.STANDBY, null);
+
+        // Create HAGroupStoreRecord for local cluster only (no peer record)
+        HAGroupStoreRecord localRecord = new HAGroupStoreRecord("v1.0", haGroupName,
+                HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+        createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, localRecord);
+
+        // Explicitly ensure no peer record exists
+        String peerPath = toPath(haGroupName);
+        if (peerHaAdmin.getCurator().checkExists().forPath(peerPath) != null) {
+            peerHaAdmin.getCurator().delete().quietly().forPath(peerPath);
+        }
+
+        Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
+
+        HAGroupStoreClient haGroupStoreClient = HAGroupStoreClient.getInstanceForZkUrl(CLUSTERS.getHBaseCluster1().getConfiguration(), haGroupName, zkUrl);
+        assertNotNull(haGroupStoreClient);
+
+        // Test getClusterRoleRecord when peer ZK URL is valid but no peer HAGroupStoreRecord exists in peer ZK
+        ClusterRoleRecord clusterRoleRecord = haGroupStoreClient.getClusterRoleRecord();
+        assertNotNull(clusterRoleRecord);
+        ClusterRoleRecord expectedClusterRoleRecord = new ClusterRoleRecord(haGroupName,
+                HighAvailabilityPolicy.FAILOVER, zkUrl, ClusterRoleRecord.ClusterRole.ACTIVE, this.peerZKUrl,
+                ClusterRoleRecord.ClusterRole.UNKNOWN, 1);
+        assertEquals(expectedClusterRoleRecord, clusterRoleRecord);
+    }
+
+    private HAGroupStoreRecord createHAGroupStoreRecord(String haGroupName) {
+        return new HAGroupStoreRecord("v1.0", haGroupName,
+                HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+    }
+
+    // Tests for getHAGroupNames static method
+    @Test
+    public void testGetHAGroupNamesWithSingleGroup() throws Exception {
+        String haGroupName = testName.getMethodName();
+        HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(haGroupName, this.zkUrl, this.peerZKUrl,
+                ClusterRoleRecord.ClusterRole.ACTIVE, ClusterRoleRecord.ClusterRole.STANDBY, null);
+
+        List<String> haGroupNames = HAGroupStoreClient.getHAGroupNames(zkUrl);
+
+        assertNotNull("HA group names list should not be null", haGroupNames);
+        assertTrue("HA group names list should contain the test group", haGroupNames.contains(haGroupName));
+
+        // Clean up
+        HAGroupStoreTestUtil.deleteHAGroupRecordInSystemTable(haGroupName, zkUrl);
+    }
+
+    @Test
+    public void testGetHAGroupNamesWithMultipleGroups() throws Exception {
+        // Delete old HA entry created by Before method
+        HAGroupStoreTestUtil.deleteHAGroupRecordInSystemTable(testName.getMethodName(), this.zkUrl);
+        // Verify that there are no groups returned
+        assertEquals(0, HAGroupStoreClient.getHAGroupNames(zkUrl).size());
+
+
+        String haGroupName1 = testName.getMethodName() + "_1";
+        String haGroupName2 = testName.getMethodName() + "_2";
+        String haGroupName3 = testName.getMethodName() + "_3";
+        String haGroupName4 = testName.getMethodName() + "_4";
+
+        // Insert multiple HA group records
+        HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(haGroupName1, this.zkUrl, this.peerZKUrl,
+                ClusterRoleRecord.ClusterRole.ACTIVE, ClusterRoleRecord.ClusterRole.STANDBY, null);
+        HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(haGroupName2, this.zkUrl, this.peerZKUrl,
+                ClusterRoleRecord.ClusterRole.STANDBY, ClusterRoleRecord.ClusterRole.ACTIVE, null);
+        HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(haGroupName3, this.zkUrl, this.peerZKUrl,
+                ClusterRoleRecord.ClusterRole.ACTIVE, ClusterRoleRecord.ClusterRole.STANDBY, null);
+        HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(haGroupName4, "bad_zk_url", this.peerZKUrl,
+                ClusterRoleRecord.ClusterRole.ACTIVE, ClusterRoleRecord.ClusterRole.STANDBY, this.zkUrl);
+
+        List<String> haGroupNames = HAGroupStoreClient.getHAGroupNames(zkUrl);
+
+        assertNotNull("HA group names list should not be null", haGroupNames);
+        assertTrue("HA group names list should contain haGroupName1", haGroupNames.contains(haGroupName1));
+        assertTrue("HA group names list should contain haGroupName2", haGroupNames.contains(haGroupName2));
+        assertTrue("HA group names list should contain haGroupName3", haGroupNames.contains(haGroupName3));
+        assertTrue("HA group names list should not contain haGroupName4", !haGroupNames.contains(haGroupName4));
+        assertEquals(3, haGroupNames.size());
+
+
+        // Clean up
+        HAGroupStoreTestUtil.deleteHAGroupRecordInSystemTable(haGroupName1, zkUrl);
+        HAGroupStoreTestUtil.deleteHAGroupRecordInSystemTable(haGroupName2, zkUrl);
+        HAGroupStoreTestUtil.deleteHAGroupRecordInSystemTable(haGroupName3, zkUrl);
+    }
+
+    @Test
+    public void testGetHAGroupNamesWithEmptyTable() throws Exception {
+        // First, delete any existing records that might interfere
+        List<String> existingGroups = HAGroupStoreClient.getHAGroupNames(zkUrl);
+        for (String groupName : existingGroups) {
+            if (groupName.startsWith("test")) { // Only clean up test groups
+                HAGroupStoreTestUtil.deleteHAGroupRecordInSystemTable(groupName, zkUrl);
+            }
+        }
+
+        // Verify the table is empty of test records
+        List<String> haGroupNames = HAGroupStoreClient.getHAGroupNames(zkUrl);
+        long testGroupCount = haGroupNames.stream()
+                .filter(name -> name.startsWith("test"))
+                .count();
+        assertEquals("Should have no test groups", 0, testGroupCount);
+    }
+
+    @Test
+    public void testGetHAGroupNamesAfterDeletingGroups() throws Exception {
+        String haGroupName1 = testName.getMethodName() + "_delete_1";
+        String haGroupName2 = testName.getMethodName() + "_delete_2";
+
+        // Insert HA group records
+        HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(haGroupName1, this.zkUrl, this.peerZKUrl,
+                ClusterRoleRecord.ClusterRole.ACTIVE, ClusterRoleRecord.ClusterRole.STANDBY, null);
+        HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(haGroupName2, this.zkUrl, this.peerZKUrl,
+                ClusterRoleRecord.ClusterRole.STANDBY, ClusterRoleRecord.ClusterRole.ACTIVE, null);
+
+        // Verify both groups exist
+        List<String> haGroupNames = HAGroupStoreClient.getHAGroupNames(zkUrl);
+        assertTrue("Should contain haGroupName1", haGroupNames.contains(haGroupName1));
+        assertTrue("Should contain haGroupName2", haGroupNames.contains(haGroupName2));
+
+        // Delete one group
+        HAGroupStoreTestUtil.deleteHAGroupRecordInSystemTable(haGroupName1, zkUrl);
+
+        // Verify only one group remains
+        haGroupNames = HAGroupStoreClient.getHAGroupNames(zkUrl);
+        assertFalse("Should not contain deleted haGroupName1", haGroupNames.contains(haGroupName1));
+        assertTrue("Should still contain haGroupName2", haGroupNames.contains(haGroupName2));
+
+        // Clean up remaining group
+        HAGroupStoreTestUtil.deleteHAGroupRecordInSystemTable(haGroupName2, zkUrl);
+
+        // Verify the group is gone
+        haGroupNames = HAGroupStoreClient.getHAGroupNames(zkUrl);
+        assertFalse("Should not contain deleted haGroupName2", haGroupNames.contains(haGroupName2));
+    }
+
+    @Test
+    public void testGetHAGroupNamesWithInvalidZkUrl() throws Exception {
+        String invalidZkUrl = "invalid:2181";
+
+        try {
+            HAGroupStoreClient.getHAGroupNames(invalidZkUrl);
+            fail("Expected SQLException for invalid ZK URL");
+        } catch (SQLException e) {
+            // Expected - the connection should fail with invalid ZK URL
+            assertNotNull("SQLException should have a message", e.getMessage());
+        }
+    }
+
+    @Test
+    public void testGetHAGroupNamesWithNullZkUrl() throws Exception {
+        try {
+            HAGroupStoreClient.getHAGroupNames(null);
+            fail("Expected SQLException for null ZK URL");
+        } catch (SQLException e) {
+            // Expected - the connection should fail with null ZK URL
+            assertNotNull("SQLException should have a message", e.getMessage());
+        }
     }
 
 
-}
+}
\ No newline at end of file
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HAGroupStoreManagerIT.java b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HAGroupStoreManagerIT.java
index 9dfbb6cb4f..34d143e5e6 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HAGroupStoreManagerIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HAGroupStoreManagerIT.java
@@ -17,72 +17,337 @@
  */
 package org.apache.phoenix.jdbc;
 
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HConstants;
 import org.apache.phoenix.end2end.NeedsOwnMiniClusterTest;
 import org.apache.phoenix.query.BaseTest;
 import org.apache.phoenix.thirdparty.com.google.common.collect.Maps;
+import org.apache.phoenix.util.HAGroupStoreTestUtil;
 import org.apache.phoenix.util.ReadOnlyProps;
 import org.junit.Before;
 import org.junit.BeforeClass;
+import org.junit.Rule;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
+import org.junit.rules.TestName;
 
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
+import java.util.stream.Collectors;
 
+import static org.apache.phoenix.jdbc.HAGroupStoreClient.ZK_CONSISTENT_HA_NAMESPACE;
+import static org.apache.phoenix.jdbc.PhoenixHAAdmin.getLocalZkUrl;
 import static org.apache.phoenix.jdbc.PhoenixHAAdmin.toPath;
 import static org.apache.phoenix.query.QueryServices.CLUSTER_ROLE_BASED_MUTATION_BLOCK_ENABLED;
+import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
 
-
+/**
+ * Integration tests for {@link HAGroupStoreManager}.
+ */
 @Category(NeedsOwnMiniClusterTest.class)
 public class HAGroupStoreManagerIT extends BaseTest {
-    private final PhoenixHAAdmin haAdmin = new PhoenixHAAdmin(config);
-    private static final Long ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS = 1000L;
+
+    @Rule
+    public TestName testName = new TestName();
+
+    private PhoenixHAAdmin haAdmin;
+    private static final Long ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS = 2000L;
+    private String zkUrl;
+    private String peerZKUrl;
+    private static final HighAvailabilityTestingUtility.HBaseTestingUtilityPair CLUSTERS = new HighAvailabilityTestingUtility.HBaseTestingUtilityPair();
 
     @BeforeClass
     public static synchronized void doSetup() throws Exception {
-        Map<String, String> props = Maps.newHashMapWithExpectedSize(1);
+        Map<String, String> props = Maps.newHashMapWithExpectedSize(2);
         props.put(CLUSTER_ROLE_BASED_MUTATION_BLOCK_ENABLED, "true");
         setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator()));
+        CLUSTERS.start();
     }
 
     @Before
     public void before() throws Exception {
-        // Clean up all the existing CRRs
-        List<ClusterRoleRecord> crrs = haAdmin.listAllClusterRoleRecordsOnZookeeper();
-        for (ClusterRoleRecord crr : crrs) {
-            haAdmin.getCurator().delete().forPath(toPath(crr.getHaGroupName()));
+        haAdmin = new PhoenixHAAdmin(config, ZK_CONSISTENT_HA_NAMESPACE);
+        zkUrl = getLocalZkUrl(config);
+        this.peerZKUrl = CLUSTERS.getZkUrl2();
+
+        // Clean up existing HAGroupStoreRecords
+        try {
+            List<String> haGroupNames = HAGroupStoreClient.getHAGroupNames(zkUrl);
+            for (String haGroupName : haGroupNames) {
+                haAdmin.getCurator().delete().quietly().forPath(toPath(haGroupName));
+            }
+        } catch (Exception e) {
+            // Ignore cleanup errors
         }
+        HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(testName.getMethodName(), zkUrl, peerZKUrl,
+                ClusterRoleRecord.ClusterRole.ACTIVE, ClusterRoleRecord.ClusterRole.STANDBY, null);
+    }
+
+    @Test
+    public void testMutationBlockingWithSingleHAGroup() throws Exception {
+        String haGroupName = testName.getMethodName();
+        HAGroupStoreManager haGroupStoreManager = HAGroupStoreManager.getInstance(config);
+
+        // Initially no mutation should be blocked
+        assertFalse(haGroupStoreManager.isMutationBlocked(haGroupName));
+
+        // Update to ACTIVE_TO_STANDBY role (should block mutations)
+        HAGroupStoreRecord transitionRecord = new HAGroupStoreRecord(
+                "1.0", haGroupName, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY);
+
+        haAdmin.updateHAGroupStoreRecordInZooKeeper(haGroupName, transitionRecord, 0);
+        Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
+
+        // Now mutations should be blocked
+        assertTrue(haGroupStoreManager.isMutationBlocked(haGroupName));
     }
 
     @Test
-    public void testHAGroupStoreManagerWithSingleCRR() throws Exception {
+    public void testMutationBlockingWithMultipleHAGroups() throws Exception {
+        String haGroupName1 = testName.getMethodName() + "_1";
+        String haGroupName2 = testName.getMethodName() + "_2";
         HAGroupStoreManager haGroupStoreManager = HAGroupStoreManager.getInstance(config);
-        // Setup initial CRRs
-        ClusterRoleRecord crr1 = new ClusterRoleRecord("failover",
-                HighAvailabilityPolicy.FAILOVER, haAdmin.getZkUrl(), ClusterRoleRecord.ClusterRole.ACTIVE,
-                "random-zk-url", ClusterRoleRecord.ClusterRole.STANDBY, 1L);
-        ClusterRoleRecord crr2 = new ClusterRoleRecord("parallel",
-                HighAvailabilityPolicy.PARALLEL, haAdmin.getZkUrl(), ClusterRoleRecord.ClusterRole.ACTIVE,
-                "random-zk-url", ClusterRoleRecord.ClusterRole.STANDBY, 1L);
-        haAdmin.createOrUpdateDataOnZookeeper(crr1);
-        haAdmin.createOrUpdateDataOnZookeeper(crr2);
 
+        // Create two HA groups with ACTIVE and ACTIVE_NOT_IN_SYNC roles
+        HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(haGroupName1, zkUrl,
+                this.peerZKUrl, ClusterRoleRecord.ClusterRole.ACTIVE, ClusterRoleRecord.ClusterRole.ACTIVE, null);
+        HAGroupStoreRecord activeRecord1 = new HAGroupStoreRecord(
+                "1.0", haGroupName1, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+        haAdmin.createHAGroupStoreRecordInZooKeeper(activeRecord1);
+
+        HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(haGroupName2, zkUrl,
+                this.peerZKUrl, ClusterRoleRecord.ClusterRole.ACTIVE, ClusterRoleRecord.ClusterRole.ACTIVE, null);
+
+        // No mutations should be blocked
+        assertFalse(haGroupStoreManager.isMutationBlocked(haGroupName1));
+        assertFalse(haGroupStoreManager.isMutationBlocked(haGroupName2));
+
+        // Update only second group to ACTIVE_NOT_IN_SYNC_TO_STANDBY
+        HAGroupStoreRecord transitionRecord2 = new HAGroupStoreRecord(
+                "1.0", haGroupName2, HAGroupStoreRecord.HAGroupState.ACTIVE_NOT_IN_SYNC_TO_STANDBY);
+
+        haAdmin.updateHAGroupStoreRecordInZooKeeper(haGroupName2, transitionRecord2, 0);
         Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
 
-        assertFalse(haGroupStoreManager.isMutationBlocked());
+        // Global mutations should be blocked due to second group
+        assertFalse(haGroupStoreManager.isMutationBlocked(haGroupName1));
+        assertTrue(haGroupStoreManager.isMutationBlocked(haGroupName2));
+    }
+
+    @Test
+    public void testGetHAGroupStoreRecord() throws Exception {
+        String haGroupName = testName.getMethodName();
+        HAGroupStoreManager haGroupStoreManager = HAGroupStoreManager.getInstance(config);
+
+        // Get record from HAGroupStoreManager
+        Optional<HAGroupStoreRecord> recordOpt = haGroupStoreManager.getHAGroupStoreRecord(haGroupName);
+        // Should be present
+        assertTrue(recordOpt.isPresent());
+
+
+        // Delete record from System Table
+        HAGroupStoreTestUtil.deleteHAGroupRecordInSystemTable(haGroupName, zkUrl);
+        // Delete record from ZK
+        haAdmin.deleteHAGroupStoreRecordInZooKeeper(haGroupName);
+        // Sleep for propagation time so that it is now reflected in cache.
+        Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
+        // Get record from HAGroupStoreManager
+        recordOpt = haGroupStoreManager.getHAGroupStoreRecord(haGroupName);
+        // Should not be present
+        assertFalse(recordOpt.isPresent());
+
+        // Create record again in System Table
+        HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(haGroupName, zkUrl, this.peerZKUrl,
+                ClusterRoleRecord.ClusterRole.ACTIVE, ClusterRoleRecord.ClusterRole.STANDBY, null);
+        // Now it should be present
+        Optional<HAGroupStoreRecord>  retrievedOpt = haGroupStoreManager.getHAGroupStoreRecord(haGroupName);
+        assertTrue(retrievedOpt.isPresent());
+
+        // Record for comparison
+        HAGroupStoreRecord record = new HAGroupStoreRecord(
+                HAGroupStoreRecord.DEFAULT_PROTOCOL_VERSION, haGroupName, HAGroupStoreRecord.HAGroupState.ACTIVE_NOT_IN_SYNC);
+
+        // Complete object comparison instead of field-by-field
+        assertEquals(record, retrievedOpt.get());
+    }
+
+    @Test
+    public void testInvalidateHAGroupStoreClient() throws Exception {
+        String haGroupName = testName.getMethodName();
+        HAGroupStoreManager haGroupStoreManager = HAGroupStoreManager.getInstance(config);
+
+        // Create a HAGroupStoreRecord first
+        HAGroupStoreRecord record = new HAGroupStoreRecord(
+                "1.0", haGroupName, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+
+        haAdmin.createHAGroupStoreRecordInZooKeeper(record);
+        Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
+
+        // Ensure we can get the record
+        Optional<HAGroupStoreRecord> recordOpt = haGroupStoreManager.getHAGroupStoreRecord(haGroupName);
+        assertTrue(recordOpt.isPresent());
+
+        // Invalidate the specific HA group client
+        haGroupStoreManager.invalidateHAGroupStoreClient(haGroupName, false);
+
+        // Should still be able to get the record after invalidation
+        recordOpt = haGroupStoreManager.getHAGroupStoreRecord(haGroupName);
+        assertTrue(recordOpt.isPresent());
+
+        // Test global invalidation
+        haGroupStoreManager.invalidateHAGroupStoreClient(false);
+
+        // Should still be able to get the record after global invalidation
+        recordOpt = haGroupStoreManager.getHAGroupStoreRecord(haGroupName);
+        assertTrue(recordOpt.isPresent());
+    }
+
+    @Test
+    public void testMutationBlockDisabled() throws Exception {
+        String haGroupName = testName.getMethodName();
+
+        // Create configuration with mutation block disabled
+        Configuration conf = new Configuration();
+        conf.set(CLUSTER_ROLE_BASED_MUTATION_BLOCK_ENABLED, "false");
+        conf.set(HConstants.ZOOKEEPER_QUORUM, getLocalZkUrl(config));
+
+        HAGroupStoreManager haGroupStoreManager = HAGroupStoreManager.getInstance(config);
 
-        crr1 = new ClusterRoleRecord("failover",
-                HighAvailabilityPolicy.FAILOVER, haAdmin.getZkUrl(), ClusterRoleRecord.ClusterRole.ACTIVE,
-                "random-zk-url", ClusterRoleRecord.ClusterRole.STANDBY, 2L);
-        crr2 = new ClusterRoleRecord("parallel",
-                HighAvailabilityPolicy.PARALLEL, haAdmin.getZkUrl(), ClusterRoleRecord.ClusterRole.ACTIVE_TO_STANDBY,
-                "random-zk-url", ClusterRoleRecord.ClusterRole.STANDBY, 2L);
-        haAdmin.createOrUpdateDataOnZookeeper(crr1);
-        haAdmin.createOrUpdateDataOnZookeeper(crr2);
+        // Create HAGroupStoreRecord with ACTIVE_TO_STANDBY role
+        HAGroupStoreRecord transitionRecord = new HAGroupStoreRecord(
+                "1.0", haGroupName, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY);
 
+        haAdmin.createHAGroupStoreRecordInZooKeeper(transitionRecord);
         Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
 
-        assert haGroupStoreManager.isMutationBlocked();
+        // Mutations should not be blocked even with ACTIVE_TO_STANDBY role
+        assertFalse(haGroupStoreManager.isMutationBlocked(haGroupName));
     }
-}
+
+    @Test
+    public void testSetHAGroupStatusToStoreAndForward() throws Exception {
+        String haGroupName = testName.getMethodName();
+        HAGroupStoreManager haGroupStoreManager = HAGroupStoreManager.getInstance(config);
+
+        // Create an initial HAGroupStoreRecord with ACTIVE status
+        HAGroupStoreRecord initialRecord = new HAGroupStoreRecord(
+                "1.0", haGroupName, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+
+        haAdmin.createHAGroupStoreRecordInZooKeeper(initialRecord);
+        Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
+
+        // Set the HA group status to store and forward (ACTIVE_NOT_IN_SYNC)
+        haGroupStoreManager.setHAGroupStatusToStoreAndForward(haGroupName);
+        Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
+
+        // Verify the status was updated to ACTIVE_NOT_IN_SYNC
+        Optional<HAGroupStoreRecord> updatedRecordOpt = haGroupStoreManager.getHAGroupStoreRecord(haGroupName);
+        assertTrue(updatedRecordOpt.isPresent());
+        HAGroupStoreRecord updatedRecord = updatedRecordOpt.get();
+        assertEquals(HAGroupStoreRecord.HAGroupState.ACTIVE_NOT_IN_SYNC, updatedRecord.getHAGroupState());
+    }
+
+    @Test
+    public void testSetHAGroupStatusRecordToSync() throws Exception {
+        String haGroupName = testName.getMethodName();
+        HAGroupStoreManager haGroupStoreManager = HAGroupStoreManager.getInstance(config);
+
+        // Create an initial HAGroupStoreRecord with ACTIVE_NOT_IN_SYNC status
+        HAGroupStoreRecord initialRecord = new HAGroupStoreRecord(
+                "1.0", haGroupName, HAGroupStoreRecord.HAGroupState.ACTIVE_NOT_IN_SYNC);
+
+        haAdmin.createHAGroupStoreRecordInZooKeeper(initialRecord);
+        Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
+
+        // Set the HA group status to sync (ACTIVE)
+        haGroupStoreManager.setHAGroupStatusRecordToSync(haGroupName);
+        Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
+
+        // Verify the status was updated to ACTIVE
+        Optional<HAGroupStoreRecord> updatedRecordOpt = haGroupStoreManager.getHAGroupStoreRecord(haGroupName);
+        assertTrue(updatedRecordOpt.isPresent());
+        HAGroupStoreRecord updatedRecord = updatedRecordOpt.get();
+        assertEquals(ClusterRoleRecord.ClusterRole.ACTIVE, updatedRecord.getClusterRole());
+    }
+
+    @Test
+    public void testGetHAGroupNamesFiltersCorrectlyByZkUrl() throws Exception {
+        HAGroupStoreManager haGroupStoreManager = HAGroupStoreManager.getInstance(config);
+
+        List<String> initialHAGroupNames = haGroupStoreManager.getHAGroupNames();
+
+        // Create HA groups with current zkUrl as ZK_URL_1
+        String haGroupWithCurrentZkUrl = testName.getMethodName() + "_current_zk";
+        HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(haGroupWithCurrentZkUrl, zkUrl, this.peerZKUrl,
+                ClusterRoleRecord.ClusterRole.ACTIVE, ClusterRoleRecord.ClusterRole.STANDBY, null);
+
+        // Create HA group with current zkUrl as ZK_URL_2 (swapped)
+        String haGroupWithCurrentZkUrlAsPeer = testName.getMethodName() + "_current_as_peer";
+        HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(haGroupWithCurrentZkUrlAsPeer, this.peerZKUrl, zkUrl,
+                ClusterRoleRecord.ClusterRole.STANDBY, ClusterRoleRecord.ClusterRole.ACTIVE, zkUrl);
+
+        // Create HA group with different zkUrl (should not appear in results)
+        String differentZkUrl = "localhost:2182:/different";
+        String haGroupWithDifferentZkUrl = testName.getMethodName() + "_different_zk";
+        HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(haGroupWithDifferentZkUrl, differentZkUrl, "localhost:2183:/other",
+                ClusterRoleRecord.ClusterRole.ACTIVE, ClusterRoleRecord.ClusterRole.STANDBY, zkUrl);
+
+        // Get HA group names - should only return groups where current zkUrl matches ZK_URL_1 or ZK_URL_2
+        List<String> filteredHAGroupNames = haGroupStoreManager.getHAGroupNames();
+
+        // Extract only new groups from filteredHAGroupNames
+        List<String> newHAGroupNames = filteredHAGroupNames.stream()
+        .filter(name -> !initialHAGroupNames.contains(name))
+        .collect(Collectors.toList());
+
+        // Check size of filteredHAGroupNames
+        assertEquals(2, newHAGroupNames.size());
+
+
+        // Should contain groups where current zkUrl is involved
+        assertTrue("Should contain HA group with current zkUrl as ZK_URL_1",
+                newHAGroupNames.contains(haGroupWithCurrentZkUrl));
+        assertTrue("Should contain HA group with current zkUrl as ZK_URL_2",
+                newHAGroupNames.contains(haGroupWithCurrentZkUrlAsPeer));
+        // Should NOT contain HA group with different zkUrl
+        assertFalse("Should NOT contain HA group with different zkUrl",
+                newHAGroupNames.contains(haGroupWithDifferentZkUrl));
+
+
+        // Clean up
+        HAGroupStoreTestUtil.deleteHAGroupRecordInSystemTable(haGroupWithCurrentZkUrl, zkUrl);
+        HAGroupStoreTestUtil.deleteHAGroupRecordInSystemTable(haGroupWithCurrentZkUrlAsPeer, zkUrl);
+        HAGroupStoreTestUtil.deleteHAGroupRecordInSystemTable(haGroupWithDifferentZkUrl, zkUrl);
+    }
+
+    @Test
+    public void testGetHAGroupNamesWhenNoMatchingZkUrl() throws Exception {
+        HAGroupStoreManager haGroupStoreManager = HAGroupStoreManager.getInstance(config);
+
+        // Clean up existing HA group created in before()
+        String testHAGroupName = testName.getMethodName();
+        HAGroupStoreTestUtil.deleteHAGroupRecordInSystemTable(testHAGroupName, zkUrl);
+
+        // Create HA groups with completely different zkUrls
+        String differentZkUrl1 = "localhost:2182:/different1";
+        String differentZkUrl2 = "localhost:2183:/different2";
+        String haGroupWithDifferentZkUrls = testName.getMethodName() + "_different";
+        HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(haGroupWithDifferentZkUrls, differentZkUrl1, differentZkUrl2,
+                ClusterRoleRecord.ClusterRole.ACTIVE, ClusterRoleRecord.ClusterRole.STANDBY, zkUrl);
+
+        // Get HA group names - should not contain the group with different zkUrls
+        List<String> filteredHAGroupNames = haGroupStoreManager.getHAGroupNames();
+
+        // Should NOT contain the HA group with different zkUrls
+        assertFalse("Should NOT contain HA group with different zkUrls",
+                filteredHAGroupNames.contains(haGroupWithDifferentZkUrls));
+
+        // Clean up
+        HAGroupStoreTestUtil.deleteHAGroupRecordInSystemTable(haGroupWithDifferentZkUrls, zkUrl);
+
+    }
+
+}
\ No newline at end of file
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HighAvailabilityGroupTestIT.java b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HighAvailabilityGroupTestIT.java
index eb7ddc5505..e23dbdb260 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HighAvailabilityGroupTestIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HighAvailabilityGroupTestIT.java
@@ -302,8 +302,9 @@ public class HighAvailabilityGroupTestIT {
         // Make ZK connectable and the cluster role record be missing
         CuratorFramework curator = mock(CuratorFramework.class);
         when(curator.blockUntilConnected(anyInt(), any(TimeUnit.class))).thenReturn(true);
-        HighAvailabilityGroup.CURATOR_CACHE.put(ZK2, curator);
-        HighAvailabilityGroup.CURATOR_CACHE.put(ZK1, curator);
+        // Use proper cache keys with default namespace
+        HighAvailabilityGroup.CURATOR_CACHE.put(HighAvailabilityGroup.generateCacheKey(ZK2, null), curator);
+        HighAvailabilityGroup.CURATOR_CACHE.put(HighAvailabilityGroup.generateCacheKey(ZK1, null), curator);
 
         ExistsBuilder eb = mock(ExistsBuilder.class);
         when(eb.forPath(anyString())).thenReturn(null);
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/ParallelPhoenixConnectionIT.java b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/ParallelPhoenixConnectionIT.java
index 32d011cd9c..9c66e1c956 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/ParallelPhoenixConnectionIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/ParallelPhoenixConnectionIT.java
@@ -36,6 +36,7 @@ import static org.apache.phoenix.monitoring.GlobalClientMetrics.GLOBAL_HA_PARALL
 import static org.apache.phoenix.monitoring.GlobalClientMetrics.GLOBAL_HA_PARALLEL_POOL2_TASK_REJECTED_COUNTER;
 import static org.apache.phoenix.query.BaseTest.extractThreadPoolExecutorFromCQSI;
 import static org.apache.phoenix.query.QueryServices.AUTO_COMMIT_ATTRIB;
+import static org.apache.phoenix.query.QueryServices.CLUSTER_ROLE_BASED_MUTATION_BLOCK_ENABLED;
 import static org.apache.phoenix.query.QueryServices.CQSI_THREAD_POOL_ALLOW_CORE_THREAD_TIMEOUT;
 import static org.apache.phoenix.query.QueryServices.CQSI_THREAD_POOL_CORE_POOL_SIZE;
 import static org.apache.phoenix.query.QueryServices.CQSI_THREAD_POOL_ENABLED;
@@ -51,14 +52,17 @@ import static org.junit.Assert.fail;
 import static org.junit.Assume.assumeTrue;
 import static org.mockito.Mockito.doAnswer;
 
-import java.lang.reflect.Field;
 import java.sql.Connection;
+import java.lang.reflect.Field;
 import java.sql.DriverManager;
 import java.sql.PreparedStatement;
 import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.sql.Statement;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ThreadPoolExecutor;
@@ -66,9 +70,11 @@ import java.util.concurrent.ThreadPoolExecutor;
 import org.apache.commons.lang3.RandomStringUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.client.RetriesExhaustedWithDetailsException;
 import org.apache.hadoop.hbase.util.VersionInfo;
 import org.apache.hadoop.test.GenericTestUtils;
 import org.apache.phoenix.end2end.NeedsOwnMiniClusterTest;
+import org.apache.phoenix.exception.MutationBlockedIOException;
 import org.apache.phoenix.jdbc.ClusterRoleRecord.ClusterRole;
 import org.apache.phoenix.jdbc.HighAvailabilityTestingUtility.HBaseTestingUtilityPair;
 import org.apache.phoenix.jdbc.PhoenixStatement.Operation;
@@ -81,6 +87,7 @@ import org.apache.phoenix.monitoring.MetricType;
 import org.apache.phoenix.query.ConnectionQueryServices;
 import org.apache.phoenix.query.ConnectionQueryServicesImpl;
 import org.apache.phoenix.query.QueryServices;
+import org.apache.phoenix.util.HAGroupStoreTestUtil;
 import org.apache.phoenix.util.PhoenixRuntime;
 import org.apache.phoenix.util.QueryUtil;
 import org.junit.AfterClass;
@@ -118,6 +125,8 @@ public class ParallelPhoenixConnectionIT {
 
     @BeforeClass
     public static void setUpBeforeClass() throws Exception {
+        CLUSTERS.getHBaseCluster1().getConfiguration().setBoolean(CLUSTER_ROLE_BASED_MUTATION_BLOCK_ENABLED, true);
+        CLUSTERS.getHBaseCluster2().getConfiguration().setBoolean(CLUSTER_ROLE_BASED_MUTATION_BLOCK_ENABLED, true);
         CLUSTERS.start();
         DriverManager.registerDriver(PhoenixDriver.INSTANCE);
         DriverManager.registerDriver(new PhoenixTestDriver());
@@ -131,6 +140,7 @@ public class ParallelPhoenixConnectionIT {
         GLOBAL_PROPERTIES.setProperty(CQSI_THREAD_POOL_MAX_QUEUE, String.valueOf(23));
         GLOBAL_PROPERTIES.setProperty(CQSI_THREAD_POOL_ALLOW_CORE_THREAD_TIMEOUT,
             String.valueOf(true));
+        GLOBAL_PROPERTIES.setProperty("hbase.client.retries.number", "0");
         GLOBAL_PROPERTIES.setProperty(CQSI_THREAD_POOL_METRICS_ENABLED, String.valueOf(true));
     }
 
@@ -356,6 +366,77 @@ public class ParallelPhoenixConnectionIT {
         }
     }
 
+    /**
+     * Test Phoenix connection creation and basic operations with HBase both cluster is ACTIVE_TO_STANDBY role.
+     */
+    @Test
+    public void testBothClusterATSRole() throws Exception {
+        String zkUrl1 = CLUSTERS.getZkUrl1();
+        String zkUrl2 = CLUSTERS.getZkUrl2();
+        String haGroupName = testName.getMethodName();
+        HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(haGroupName, zkUrl1, zkUrl2,
+                ClusterRole.ACTIVE_TO_STANDBY, ClusterRole.ACTIVE_TO_STANDBY, null);
+        HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(haGroupName, zkUrl2, zkUrl1,
+                ClusterRole.ACTIVE_TO_STANDBY, ClusterRole.ACTIVE_TO_STANDBY, null);
+        try (ParallelPhoenixConnection conn = (ParallelPhoenixConnection) getParallelConnection()) {
+            PhoenixConnection conn1 = conn.futureConnection1.get();
+            PhoenixConnection conn2 = conn.futureConnection2.get();
+            conn1.setHAGroupName(haGroupName);
+            conn2.setHAGroupName(haGroupName);
+            doTestBasicOperationsWithConnection(conn, tableName, haGroupName);
+            fail("Expected MutationBlockedIOException to be thrown");
+        } catch (SQLException e) {
+            assertTrue(containsMutationBlockedException(e));
+        } finally {
+            CLUSTERS.transitClusterRole(haGroup, ClusterRole.ACTIVE, ClusterRole.STANDBY);
+        }
+    }
+
+    /**
+     * Test Phoenix connection creation and
+     * basic operations with HBase one cluster is ACTIVE_TO_STANDBY role
+     * and other in ACTIVE role.
+     */
+    @Test
+    public void testOneClusterATSRoleWithActive() throws Exception {
+        testOneClusterATSRole(ClusterRole.ACTIVE);
+    }
+
+    /**
+     * Test Phoenix connection creation and
+     * basic operations with HBase one cluster is ACTIVE_TO_STANDBY role
+     * and other in STANDBY role.
+     */
+    @Test
+    public void testOneClusterATSRoleWithStandby() throws Exception {
+        testOneClusterATSRole(ClusterRole.STANDBY);
+    }
+
+    private void testOneClusterATSRole(ClusterRole otherRole) throws Exception {
+        CLUSTERS.transitClusterRole(haGroup, ClusterRole.ACTIVE_TO_STANDBY, otherRole);
+        try (Connection conn = getParallelConnection()) {
+            doTestBasicOperationsWithConnection(conn, tableName, haGroupName);
+        } catch (SQLException e) {
+            fail("Expected no exception to be thrown as one cluster is "
+                    + "in ACTIVE_TO_STANDBY and other in " + otherRole);
+        } finally {
+            CLUSTERS.transitClusterRole(haGroup, ClusterRole.ACTIVE, ClusterRole.STANDBY);
+        }
+    }
+
+    private boolean containsMutationBlockedException(SQLException e) {
+        Throwable cause = e.getCause();
+        // Recursively check for MutationBlockedIOException buried in exception stack.
+        while (cause != null) {
+            if (cause instanceof RetriesExhaustedWithDetailsException) {
+                RetriesExhaustedWithDetailsException re = (RetriesExhaustedWithDetailsException) cause;
+                return re.getCause(0) instanceof MutationBlockedIOException;
+            }
+            cause = cause.getCause();
+        }
+        return false;
+    }
+
     /**
      * Test Phoenix connection creation and basic operations.
      */
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/PhoenixHAAdminIT.java b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/PhoenixHAAdminIT.java
new file mode 100644
index 0000000000..5e2685d03a
--- /dev/null
+++ b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/PhoenixHAAdminIT.java
@@ -0,0 +1,481 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.jdbc;
+
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.phoenix.end2end.NeedsOwnMiniClusterTest;
+import org.apache.phoenix.exception.StaleHAGroupStoreRecordVersionException;
+import org.apache.phoenix.query.BaseTest;
+import org.apache.phoenix.thirdparty.com.google.common.collect.Maps;
+import org.apache.phoenix.util.ReadOnlyProps;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.data.Stat;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.rules.TestName;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static org.apache.phoenix.jdbc.HAGroupStoreClient.ZK_CONSISTENT_HA_NAMESPACE;
+import static org.apache.phoenix.jdbc.PhoenixHAAdmin.toPath;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+/**
+ * Integration tests for {@link PhoenixHAAdmin} HAGroupStoreRecord operations
+ */
+@Category(NeedsOwnMiniClusterTest.class)
+public class PhoenixHAAdminIT extends BaseTest {
+
+    private static final HighAvailabilityTestingUtility.HBaseTestingUtilityPair CLUSTERS = new HighAvailabilityTestingUtility.HBaseTestingUtilityPair();
+    private PhoenixHAAdmin haAdmin;
+    private PhoenixHAAdmin peerHaAdmin;
+
+    @Rule
+    public TestName testName = new TestName();
+
+    @BeforeClass
+    public static synchronized void doSetup() throws Exception {
+        Map<String, String> props = Maps.newHashMapWithExpectedSize(1);
+        setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator()));
+        CLUSTERS.start();
+    }
+
+    @Before
+    public void before() throws Exception {
+        haAdmin = new PhoenixHAAdmin(CLUSTERS.getHBaseCluster1().getConfiguration(), ZK_CONSISTENT_HA_NAMESPACE);
+        peerHaAdmin = new PhoenixHAAdmin(CLUSTERS.getHBaseCluster2().getConfiguration(), ZK_CONSISTENT_HA_NAMESPACE);
+        cleanupTestZnodes();
+    }
+
+    @After
+    public void after() throws Exception {
+        cleanupTestZnodes();
+        if (haAdmin != null) {
+            haAdmin.close();
+        }
+        if (peerHaAdmin != null) {
+            peerHaAdmin.close();
+        }
+    }
+
+    private void cleanupTestZnodes() throws Exception {
+        haAdmin.getCurator().delete().quietly().forPath(toPath(testName.getMethodName()));
+        peerHaAdmin.getCurator().delete().quietly().forPath(toPath(testName.getMethodName()));
+    }
+
+    @Test
+    public void testCreateHAGroupStoreRecordInZooKeeper() throws Exception {
+        String haGroupName = testName.getMethodName();
+        String peerZKUrl = CLUSTERS.getZkUrl2();
+
+        HAGroupStoreRecord record = new HAGroupStoreRecord(
+                "v1.0",
+                haGroupName,
+                HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC
+        );
+
+        // Create the record in ZooKeeper
+        haAdmin.createHAGroupStoreRecordInZooKeeper(record);
+
+        // Verify the record was created by reading it back
+        byte[] data = haAdmin.getCurator().getData().forPath(toPath(haGroupName));
+        HAGroupStoreRecord savedRecord = HAGroupStoreRecord.fromJson(data).get();
+
+        assertEquals(record.getHaGroupName(), savedRecord.getHaGroupName());
+        assertEquals(record.getClusterRole(), savedRecord.getClusterRole());
+    }
+
+    @Test
+    public void testCreateHAGroupStoreRecordInZooKeeperWithExistingNode() throws Exception {
+        String haGroupName = testName.getMethodName();
+
+        HAGroupStoreRecord record1 = new HAGroupStoreRecord(
+                "v1.0",
+                haGroupName,
+                HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC
+        );
+
+        // Create the first record
+        haAdmin.createHAGroupStoreRecordInZooKeeper(record1);
+
+        // Try to create again with different data
+        HAGroupStoreRecord record2 = new HAGroupStoreRecord(
+                "v1.0",
+                haGroupName,
+                HAGroupStoreRecord.HAGroupState.STANDBY
+        );
+
+        // This should throw an exception due to NodeExistsException handling
+        try {
+            haAdmin.createHAGroupStoreRecordInZooKeeper(record2);
+            fail("Expected NodeExistsException");
+        } catch (IOException e) {
+            // Expected exception
+            assertTrue(e.getCause() instanceof KeeperException.NodeExistsException);
+            assertTrue(e.getMessage().contains("Failed to create HAGroupStoreRecord for HA group"));
+        }
+
+        // Verify the original record is still there (not overwritten)
+        byte[] data = haAdmin.getCurator().getData().forPath(toPath(haGroupName));
+        HAGroupStoreRecord savedRecord = HAGroupStoreRecord.fromJson(data).get();
+
+        assertEquals(record1.getHAGroupState(), savedRecord.getHAGroupState());
+        assertEquals(record1.getProtocolVersion(), savedRecord.getProtocolVersion());
+        assertEquals(record1.getHaGroupName(), savedRecord.getHaGroupName());
+    }
+
+    @Test
+    public void testUpdateHAGroupStoreRecordInZooKeeper() throws Exception {
+        String haGroupName = testName.getMethodName();
+        String peerZKUrl = CLUSTERS.getZkUrl2();
+
+        // Create initial record
+        HAGroupStoreRecord initialRecord = new HAGroupStoreRecord(
+                "v1.0",
+                haGroupName,
+                HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC
+        );
+
+        haAdmin.createHAGroupStoreRecordInZooKeeper(initialRecord);
+
+        // Get the current stat for version checking
+        Stat stat = haAdmin.getCurator().checkExists().forPath(toPath(haGroupName));
+        assertNotNull(stat);
+        int currentVersion = stat.getVersion();
+
+        // Update the record
+        HAGroupStoreRecord updatedRecord = new HAGroupStoreRecord(
+                "v1.0",
+                haGroupName,
+                HAGroupStoreRecord.HAGroupState.STANDBY
+        );
+
+        haAdmin.updateHAGroupStoreRecordInZooKeeper(haGroupName, updatedRecord, currentVersion);
+
+        // Verify the record was updated
+        byte[] data = haAdmin.getCurator().getData().forPath(toPath(haGroupName));
+        HAGroupStoreRecord savedRecord = HAGroupStoreRecord.fromJson(data).get();
+
+        assertEquals(updatedRecord.getClusterRole(), savedRecord.getClusterRole());
+        assertEquals(updatedRecord.getHAGroupState(), savedRecord.getHAGroupState());
+        assertEquals(updatedRecord.getProtocolVersion(), savedRecord.getProtocolVersion());
+        assertEquals(updatedRecord.getHaGroupName(), savedRecord.getHaGroupName());
+    }
+
+    @Test
+    public void testUpdateHAGroupStoreRecordInZooKeeperWithStaleVersion() throws Exception {
+        String haGroupName = testName.getMethodName();
+        String peerZKUrl = CLUSTERS.getZkUrl2();
+
+        // Create initial record
+        HAGroupStoreRecord initialRecord = new HAGroupStoreRecord(
+                "v1.0",
+                haGroupName,
+                HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC
+        );
+
+        haAdmin.createHAGroupStoreRecordInZooKeeper(initialRecord);
+
+        // Get the current stat for version checking
+        Stat stat = haAdmin.getCurator().checkExists().forPath(toPath(haGroupName));
+        assertNotNull(stat);
+        int currentVersion = stat.getVersion();
+
+        // Update the record with current version (should succeed)
+        HAGroupStoreRecord updatedRecord = new HAGroupStoreRecord(
+                "v1.0",
+                haGroupName,
+                HAGroupStoreRecord.HAGroupState.STANDBY
+        );
+
+        haAdmin.updateHAGroupStoreRecordInZooKeeper(haGroupName, updatedRecord, currentVersion);
+
+        // Try to update again with the same (now stale) version - should fail
+        HAGroupStoreRecord anotherUpdate = new HAGroupStoreRecord(
+                "v1.0",
+                haGroupName,
+                HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY
+        );
+
+        try {
+            haAdmin.updateHAGroupStoreRecordInZooKeeper(haGroupName, anotherUpdate, currentVersion);
+            fail("Expected StaleHAGroupStoreRecordVersionException");
+        } catch (StaleHAGroupStoreRecordVersionException e) {
+            // Expected exception
+            assertTrue(e.getMessage().contains("with cached stat version"));
+        }
+    }
+
+    @Test
+    public void testGetHAGroupStoreRecordInZooKeeper() throws Exception {
+        String haGroupName = testName.getMethodName();
+        String peerZKUrl = CLUSTERS.getZkUrl2();
+
+        // Create initial record
+        HAGroupStoreRecord initialRecord = new HAGroupStoreRecord(
+                "v1.0",
+                haGroupName,
+                HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC
+        );
+
+        haAdmin.createHAGroupStoreRecordInZooKeeper(initialRecord);
+
+        // Get the record and stat
+        Pair<HAGroupStoreRecord, Stat> result = haAdmin.getHAGroupStoreRecordInZooKeeper(haGroupName);
+
+        assertNotNull(result);
+        assertNotNull(result.getLeft());
+        assertNotNull(result.getRight());
+
+        HAGroupStoreRecord retrievedRecord = result.getLeft();
+        Stat stat = result.getRight();
+
+        assertEquals(initialRecord.getHaGroupName(), retrievedRecord.getHaGroupName());
+        assertEquals(initialRecord.getClusterRole(), retrievedRecord.getClusterRole());
+        assertEquals(initialRecord.getHAGroupState(), retrievedRecord.getHAGroupState());
+        assertEquals(initialRecord.getProtocolVersion(), retrievedRecord.getProtocolVersion());
+
+        // Verify stat is valid
+        assertTrue(stat.getVersion() >= 0);
+        assertTrue(stat.getCtime() > 0);
+        assertTrue(stat.getMtime() > 0);
+    }
+
+    @Test
+    public void testGetHAGroupStoreRecordInZooKeeperNonExistentNode() throws Exception {
+        assertNull(haAdmin.getHAGroupStoreRecordInZooKeeper(testName.getMethodName()).getLeft());
+        assertNull(haAdmin.getHAGroupStoreRecordInZooKeeper(testName.getMethodName()).getLeft());
+    }
+
+    @Test
+    public void testCompleteWorkflowCreateUpdateGet() throws Exception {
+        String haGroupName = testName.getMethodName();
+        String peerZKUrl = CLUSTERS.getZkUrl2();
+
+        // Step 1: Create initial record
+        HAGroupStoreRecord initialRecord = new HAGroupStoreRecord(
+                "v1.0",
+                haGroupName,
+                HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC
+        );
+
+        haAdmin.createHAGroupStoreRecordInZooKeeper(initialRecord);
+
+        // Step 2: Get the record
+        Pair<HAGroupStoreRecord, Stat> result = haAdmin.getHAGroupStoreRecordInZooKeeper(haGroupName);
+        HAGroupStoreRecord retrievedRecord = result.getLeft();
+        Stat stat = result.getRight();
+
+        assertEquals(initialRecord.getHaGroupName(), retrievedRecord.getHaGroupName());
+        assertEquals(initialRecord.getClusterRole(), retrievedRecord.getClusterRole());
+        assertEquals(initialRecord.getHAGroupState(), retrievedRecord.getHAGroupState());
+        assertEquals(initialRecord.getProtocolVersion(), retrievedRecord.getProtocolVersion());
+
+        // Step 3: Update the record
+        HAGroupStoreRecord updatedRecord = new HAGroupStoreRecord(
+                "v1.0",
+                haGroupName,
+                HAGroupStoreRecord.HAGroupState.STANDBY
+        );
+
+        haAdmin.updateHAGroupStoreRecordInZooKeeper(haGroupName, updatedRecord, stat.getVersion());
+
+        // Step 4: Get the updated record
+        Pair<HAGroupStoreRecord, Stat> updatedResult = haAdmin.getHAGroupStoreRecordInZooKeeper(haGroupName);
+        HAGroupStoreRecord finalRecord = updatedResult.getLeft();
+        Stat finalStat = updatedResult.getRight();
+
+        assertEquals(updatedRecord.getHaGroupName(), finalRecord.getHaGroupName());
+        assertEquals(updatedRecord.getClusterRole(), finalRecord.getClusterRole());
+        assertEquals(updatedRecord.getHAGroupState(), finalRecord.getHAGroupState());
+        assertEquals(updatedRecord.getProtocolVersion(), finalRecord.getProtocolVersion());
+
+        // Verify stat version increased
+        assertTrue(finalStat.getVersion() > stat.getVersion());
+    }
+
+    @Test
+    public void testMultiThreadedUpdatesConcurrentVersionConflict() throws Exception {
+        String haGroupName = testName.getMethodName();
+
+        // Create initial record
+        HAGroupStoreRecord initialRecord = new HAGroupStoreRecord(
+                "v1.0",
+                haGroupName,
+                HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC
+        );
+
+        haAdmin.createHAGroupStoreRecordInZooKeeper(initialRecord);
+
+        // Get the current stat for version checking
+        Pair<HAGroupStoreRecord, Stat> result = haAdmin.getHAGroupStoreRecordInZooKeeper(haGroupName);
+        int currentVersion = result.getRight().getVersion();
+
+        // Number of threads to run concurrently
+        int threadCount = 5;
+        ExecutorService executorService = Executors.newFixedThreadPool(threadCount);
+        CountDownLatch startLatch = new CountDownLatch(1);
+        CountDownLatch finishLatch = new CountDownLatch(threadCount);
+
+        // Counters for tracking results
+        AtomicInteger successCount = new AtomicInteger(0);
+        AtomicInteger staleVersionExceptionCount = new AtomicInteger(0);
+        AtomicInteger otherExceptionCount = new AtomicInteger(0);
+
+        // Submit multiple threads that will all try to update with the same version
+        for (int i = 0; i < threadCount; i++) {
+            executorService.submit(() -> {
+                try {
+                    // Wait for all threads to be ready
+                    startLatch.await();
+
+                    // Create a unique update record for this thread
+                    HAGroupStoreRecord updatedRecord = new HAGroupStoreRecord(
+                            "v1.0",
+                            haGroupName,
+                            HAGroupStoreRecord.HAGroupState.STANDBY
+                    );
+
+                    // All threads use the same currentVersion, causing conflicts
+                    haAdmin.updateHAGroupStoreRecordInZooKeeper(haGroupName, updatedRecord, currentVersion);
+                    successCount.incrementAndGet();
+
+                } catch (StaleHAGroupStoreRecordVersionException e) {
+                    staleVersionExceptionCount.incrementAndGet();
+                } catch (Exception e) {
+                    otherExceptionCount.incrementAndGet();
+                } finally {
+                    finishLatch.countDown();
+                }
+            });
+        }
+
+        // Start all threads at the same time
+        startLatch.countDown();
+
+        // Wait for all threads to complete
+        assertTrue("Threads did not complete within timeout",
+                finishLatch.await(10, TimeUnit.SECONDS));
+
+        executorService.shutdown();
+        assertTrue("ExecutorService did not shutdown within timeout",
+                executorService.awaitTermination(5, TimeUnit.SECONDS));
+
+        // Verify results
+        assertEquals("No other exceptions should occur", 0, otherExceptionCount.get());
+        assertEquals("Exactly one thread should succeed", 1, successCount.get());
+        assertEquals("All other threads should get stale version exception",
+                threadCount - 1, staleVersionExceptionCount.get());
+
+        // Verify the final state - should contain the update from the successful thread
+        Pair<HAGroupStoreRecord, Stat> finalResult = haAdmin.getHAGroupStoreRecordInZooKeeper(haGroupName);
+        HAGroupStoreRecord finalRecord = finalResult.getLeft();
+        Stat finalStat = finalResult.getRight();
+
+        // The successful update should have changed the role to STANDBY
+        assertEquals(ClusterRoleRecord.ClusterRole.STANDBY, finalRecord.getClusterRole());
+
+        // The ZooKeeper version should have increased
+        assertTrue("ZooKeeper version should have increased", finalStat.getVersion() > currentVersion);
+    }
+
+    @Test
+    public void testMultiThreadedUpdatesWithDifferentVersions() throws Exception {
+        String haGroupName = testName.getMethodName();
+
+        // Create initial record
+        HAGroupStoreRecord initialRecord = new HAGroupStoreRecord(
+                "v1.0",
+                haGroupName,
+                HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC
+        );
+
+        haAdmin.createHAGroupStoreRecordInZooKeeper(initialRecord);
+
+        // Number of threads to run sequentially (each gets the latest version)
+        int threadCount = 10;
+        ExecutorService executorService = Executors.newFixedThreadPool(1); // Single thread pool for sequential execution
+        CountDownLatch finishLatch = new CountDownLatch(threadCount);
+
+        // Counter for tracking results
+        AtomicInteger successCount = new AtomicInteger(0);
+        AtomicInteger failureCount = new AtomicInteger(0);
+
+        // Submit multiple threads that will update sequentially
+        for (int i = 0; i < threadCount; i++) {
+            final int threadId = i;
+            executorService.submit(() -> {
+                try {
+                    // Get the current version for this thread
+                    Pair<HAGroupStoreRecord, Stat> currentResult = haAdmin.getHAGroupStoreRecordInZooKeeper(haGroupName);
+                    int currentVersion = currentResult.getRight().getVersion();
+
+                    // Create update record for this thread
+                    HAGroupStoreRecord updatedRecord = new HAGroupStoreRecord(
+                            "v1.0",
+                            haGroupName,
+                            threadId % 2 == 0 ? HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC : HAGroupStoreRecord.HAGroupState.STANDBY
+                    );
+
+                    // Update with the current version
+                    haAdmin.updateHAGroupStoreRecordInZooKeeper(haGroupName, updatedRecord, currentVersion);
+                    successCount.incrementAndGet();
+
+                } catch (Exception e) {
+                    failureCount.incrementAndGet();
+                } finally {
+                    finishLatch.countDown();
+                }
+            });
+        }
+
+        // Wait for all threads to complete
+        assertTrue("Threads did not complete within timeout",
+                finishLatch.await(15, TimeUnit.SECONDS));
+
+        executorService.shutdown();
+        assertTrue("ExecutorService did not shutdown within timeout",
+                executorService.awaitTermination(5, TimeUnit.SECONDS));
+
+        // Verify results - all should succeed since they each get the latest version
+        assertEquals("All threads should succeed when using correct versions",
+                threadCount, successCount.get());
+        assertEquals("No threads should fail", 0, failureCount.get());
+
+        // Verify the final state
+        Pair<HAGroupStoreRecord, Stat> finalResult = haAdmin.getHAGroupStoreRecordInZooKeeper(haGroupName);
+
+        // The final record should have the version from the last update
+        assertEquals(threadCount, finalResult.getRight().getVersion());
+    }
+
+}
\ No newline at end of file
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/jdbc/ClusterRoleRecordTest.java b/phoenix-core/src/test/java/org/apache/phoenix/jdbc/ClusterRoleRecordTest.java
index cb801f7d1d..24bdc2c631 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/jdbc/ClusterRoleRecordTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/jdbc/ClusterRoleRecordTest.java
@@ -275,6 +275,15 @@ public class ClusterRoleRecordTest {
         assertNotEquals(record.toString(), record.toPrettyString());
     }
 
+    @Test
+    public void testClusterRoleFromInvalidBytes() {
+        ClusterRole role = ClusterRole.from(new byte[0]);
+        assertEquals(ClusterRole.UNKNOWN, role);
+
+        role = ClusterRole.from("random".getBytes());
+        assertEquals(ClusterRole.UNKNOWN, role);
+    }
+
     //Private Helper Methods
 
     private ClusterRoleRecord getClusterRoleRecord(String name, HighAvailabilityPolicy policy,
@@ -322,4 +331,4 @@ public class ClusterRoleRecordTest {
         }
         return buffer.toByteArray();
     }
-}
+}
\ No newline at end of file
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/jdbc/HAGroupStoreRecordTest.java b/phoenix-core/src/test/java/org/apache/phoenix/jdbc/HAGroupStoreRecordTest.java
new file mode 100644
index 0000000000..8661283c0f
--- /dev/null
+++ b/phoenix-core/src/test/java/org/apache/phoenix/jdbc/HAGroupStoreRecordTest.java
@@ -0,0 +1,359 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.jdbc;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Optional;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.phoenix.util.JacksonUtil;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TestName;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Unit test for {@link HAGroupStoreRecord}.
+ */
+public class HAGroupStoreRecordTest {
+    private static final Logger LOG = LoggerFactory.getLogger(HAGroupStoreRecordTest.class);
+    private static final String PROTOCOL_VERSION = "1.0";
+
+    @Rule
+    public final TestName testName = new TestName();
+
+    /**
+     * Helper method to create a temp JSON file with the given array of HA group store records.
+     */
+    public static String createJsonFileWithRecords(HAGroupStoreRecord record)
+            throws IOException {
+        File file = File.createTempFile("phoenix.ha.group.store.records", ".test.json");
+        file.deleteOnExit();
+        JacksonUtil.getObjectWriterPretty().writeValue(file, record);
+        LOG.info("Prepared the JSON file for testing, file:{}, content:\n{}", file,
+                FileUtils.readFileToString(file, "UTF-8"));
+        return file.getPath();
+    }
+
+    @Test
+    public void testReadWriteJsonToFile() throws IOException {
+        HAGroupStoreRecord record = getHAGroupStoreRecord(testName.getMethodName(),
+                PROTOCOL_VERSION, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+        String fileName = createJsonFileWithRecords(record);
+        String fileContent = FileUtils.readFileToString(new File(fileName), "UTF-8");
+        assertTrue(fileContent.contains(record.getHaGroupName()));
+        assertTrue(fileContent.contains(record.getProtocolVersion()));
+        assertTrue(fileContent.contains(record.getHAGroupState().toString()));
+        // Create a new record from file
+        Optional<HAGroupStoreRecord> record2 = HAGroupStoreRecord.fromJson(fileContent.getBytes());
+        assertTrue(record2.isPresent());
+        // Check if same info
+        assertTrue(record.hasSameInfo(record2.get()));
+    }
+
+    @Test
+    public void testToAndFromJson() throws IOException {
+        HAGroupStoreRecord record = getHAGroupStoreRecord(testName.getMethodName(),
+                PROTOCOL_VERSION, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+        byte[] bytes = HAGroupStoreRecord.toJson(record);
+        Optional<HAGroupStoreRecord> record2 = HAGroupStoreRecord.fromJson(bytes);
+        assertTrue(record2.isPresent());
+        assertEquals(record, record2.get());
+    }
+
+    @Test
+    public void testFromJsonWithNullBytes() {
+        Optional<HAGroupStoreRecord> record = HAGroupStoreRecord.fromJson(null);
+        assertFalse(record.isPresent());
+    }
+
+    @Test
+    public void testFromJsonWithInvalidJson() {
+        byte[] invalidJson = "invalid json".getBytes();
+        Optional<HAGroupStoreRecord> record = HAGroupStoreRecord.fromJson(invalidJson);
+        assertFalse(record.isPresent());
+    }
+
+    @Test
+    public void testHasSameInfo() {
+        String haGroupName = testName.getMethodName();
+        HAGroupStoreRecord record1 = getHAGroupStoreRecord(haGroupName,
+                PROTOCOL_VERSION, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+        HAGroupStoreRecord record2 = getHAGroupStoreRecord(haGroupName,
+                PROTOCOL_VERSION, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+
+        assertTrue(record1.hasSameInfo(record2)); // Same core info despite different state
+        assertTrue(record1.hasSameInfo(record1)); // reflexive
+        assertTrue(record2.hasSameInfo(record1)); // symmetric
+    }
+
+    @Test
+    public void testHasSameInfoNegative() {
+        String haGroupName = testName.getMethodName();
+        HAGroupStoreRecord record = getHAGroupStoreRecord(haGroupName,
+                PROTOCOL_VERSION, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+
+        // Different protocol version
+        HAGroupStoreRecord recordDifferentProtocol = getHAGroupStoreRecord(haGroupName,
+                "2.0", HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+        assertFalse(record.hasSameInfo(recordDifferentProtocol));
+        assertFalse(recordDifferentProtocol.hasSameInfo(record));
+
+        // Different HA group name
+        String haGroupName2 = haGroupName + RandomStringUtils.randomAlphabetic(2);
+        HAGroupStoreRecord record2 = getHAGroupStoreRecord(haGroupName2,
+                PROTOCOL_VERSION, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+        assertFalse(record.hasSameInfo(record2));
+        assertFalse(record2.hasSameInfo(record));
+
+        // Different HA group state
+        HAGroupStoreRecord recordDifferentState = getHAGroupStoreRecord(haGroupName,
+                PROTOCOL_VERSION, HAGroupStoreRecord.HAGroupState.STANDBY);
+        assertFalse(record.hasSameInfo(recordDifferentState));
+        assertFalse(recordDifferentState.hasSameInfo(record));
+    }
+
+    @Test
+    public void testGetters() {
+        String haGroupName = testName.getMethodName();
+        String protocolVersion = "1.5";
+        HAGroupStoreRecord.HAGroupState haGroupState = HAGroupStoreRecord.HAGroupState.STANDBY;
+
+        HAGroupStoreRecord record = getHAGroupStoreRecord(haGroupName, protocolVersion, haGroupState);
+
+        assertEquals(haGroupName, record.getHaGroupName());
+        assertEquals(protocolVersion, record.getProtocolVersion());
+        assertEquals(haGroupState, record.getHAGroupState());
+        assertEquals(haGroupState.getClusterRole(), record.getClusterRole());
+    }
+
+    @Test
+    public void testEqualsAndHashCode() {
+        String haGroupName = testName.getMethodName();
+        HAGroupStoreRecord record1 = getHAGroupStoreRecord(haGroupName,
+                PROTOCOL_VERSION, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+        HAGroupStoreRecord record2 = getHAGroupStoreRecord(haGroupName,
+                PROTOCOL_VERSION, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+        HAGroupStoreRecord record3 = getHAGroupStoreRecord(haGroupName,
+                PROTOCOL_VERSION, HAGroupStoreRecord.HAGroupState.STANDBY); // Different state
+
+        // Test equals
+        assertEquals(record1, record2); // symmetric
+        assertEquals(record2, record1); // symmetric
+        assertNotEquals(record1, record3); // different state
+        assertNotEquals(null, record1); // null comparison
+        assertNotEquals("not a record", record1); // different type
+
+        // Test hashCode
+        assertEquals(record1.hashCode(), record2.hashCode()); // equal objects have same hash
+        assertNotEquals(record1.hashCode(), record3.hashCode()); // different objects likely have different hash
+    }
+
+    @Test
+    public void testToString() {
+        HAGroupStoreRecord record = getHAGroupStoreRecord(testName.getMethodName(),
+                PROTOCOL_VERSION, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+        String toString = record.toString();
+
+        // Verify all fields are present in toString
+        assertTrue(toString.contains(record.getHaGroupName()));
+        assertTrue(toString.contains(record.getProtocolVersion()));
+        assertTrue(toString.contains(record.getHAGroupState().toString()));
+    }
+
+    @Test
+    public void testToPrettyString() {
+        HAGroupStoreRecord record = getHAGroupStoreRecord(testName.getMethodName(),
+                PROTOCOL_VERSION, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+        LOG.info("toString(): {}", record.toString());
+        LOG.info("toPrettyString:\n{}", record.toPrettyString());
+        assertNotEquals(record.toString(), record.toPrettyString());
+        assertTrue(record.toPrettyString().contains(record.getHaGroupName()));
+    }
+
+    @Test(expected = NullPointerException.class)
+    public void testConstructorWithNullHaGroupName() {
+        getHAGroupStoreRecord(null, PROTOCOL_VERSION, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+    }
+
+    @Test(expected = NullPointerException.class)
+    public void testConstructorWithNullHAGroupState() {
+        getHAGroupStoreRecord(testName.getMethodName(), PROTOCOL_VERSION, null);
+    }
+
+    // Tests for HAGroupState enum
+    @Test
+    public void testHAGroupStateGetClusterRole() {
+        // Test that each HAGroupState maps to the correct ClusterRole
+        assertEquals(ClusterRoleRecord.ClusterRole.ACTIVE,
+                HAGroupStoreRecord.HAGroupState.ABORT_TO_ACTIVE_IN_SYNC.getClusterRole());
+        assertEquals(ClusterRoleRecord.ClusterRole.STANDBY,
+                HAGroupStoreRecord.HAGroupState.ABORT_TO_STANDBY.getClusterRole());
+        assertEquals(ClusterRoleRecord.ClusterRole.ACTIVE,
+                HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC.getClusterRole());
+        assertEquals(ClusterRoleRecord.ClusterRole.ACTIVE,
+                HAGroupStoreRecord.HAGroupState.ACTIVE_NOT_IN_SYNC.getClusterRole());
+        assertEquals(ClusterRoleRecord.ClusterRole.ACTIVE_TO_STANDBY,
+                HAGroupStoreRecord.HAGroupState.ACTIVE_NOT_IN_SYNC_TO_STANDBY.getClusterRole());
+        assertEquals(ClusterRoleRecord.ClusterRole.ACTIVE,
+                HAGroupStoreRecord.HAGroupState.ACTIVE_NOT_IN_SYNC_WITH_OFFLINE_PEER.getClusterRole());
+        assertEquals(ClusterRoleRecord.ClusterRole.ACTIVE_TO_STANDBY,
+                HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY.getClusterRole());
+        assertEquals(ClusterRoleRecord.ClusterRole.ACTIVE,
+                HAGroupStoreRecord.HAGroupState.ACTIVE_WITH_OFFLINE_PEER.getClusterRole());
+        assertEquals(ClusterRoleRecord.ClusterRole.STANDBY,
+                HAGroupStoreRecord.HAGroupState.DEGRADED_STANDBY.getClusterRole());
+        assertEquals(ClusterRoleRecord.ClusterRole.STANDBY,
+                HAGroupStoreRecord.HAGroupState.DEGRADED_STANDBY_FOR_READER.getClusterRole());
+        assertEquals(ClusterRoleRecord.ClusterRole.STANDBY,
+                HAGroupStoreRecord.HAGroupState.DEGRADED_STANDBY_FOR_WRITER.getClusterRole());
+        assertEquals(ClusterRoleRecord.ClusterRole.OFFLINE,
+                HAGroupStoreRecord.HAGroupState.OFFLINE.getClusterRole());
+        assertEquals(ClusterRoleRecord.ClusterRole.STANDBY,
+                HAGroupStoreRecord.HAGroupState.STANDBY.getClusterRole());
+        assertEquals(ClusterRoleRecord.ClusterRole.STANDBY_TO_ACTIVE,
+                HAGroupStoreRecord.HAGroupState.STANDBY_TO_ACTIVE.getClusterRole());
+        assertEquals(ClusterRoleRecord.ClusterRole.UNKNOWN,
+                HAGroupStoreRecord.HAGroupState.UNKNOWN.getClusterRole());
+    }
+
+    @Test
+    public void testHAGroupStateValidTransitions() {
+        // Test valid transitions for ACTIVE_NOT_IN_SYNC
+        assertTrue(HAGroupStoreRecord.HAGroupState.ACTIVE_NOT_IN_SYNC
+                .isTransitionAllowed(HAGroupStoreRecord.HAGroupState.ACTIVE_NOT_IN_SYNC));
+        assertTrue(HAGroupStoreRecord.HAGroupState.ACTIVE_NOT_IN_SYNC
+                .isTransitionAllowed(HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC));
+        assertTrue(HAGroupStoreRecord.HAGroupState.ACTIVE_NOT_IN_SYNC
+                .isTransitionAllowed(HAGroupStoreRecord.HAGroupState.ACTIVE_NOT_IN_SYNC_TO_STANDBY));
+        assertTrue(HAGroupStoreRecord.HAGroupState.ACTIVE_NOT_IN_SYNC
+                .isTransitionAllowed(HAGroupStoreRecord.HAGroupState.ACTIVE_NOT_IN_SYNC_WITH_OFFLINE_PEER));
+
+        // Test valid transitions for ACTIVE
+        assertTrue(HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC
+                .isTransitionAllowed(HAGroupStoreRecord.HAGroupState.ACTIVE_NOT_IN_SYNC));
+        assertTrue(HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC
+                .isTransitionAllowed(HAGroupStoreRecord.HAGroupState.ACTIVE_WITH_OFFLINE_PEER));
+        assertTrue(HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC
+                .isTransitionAllowed(HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY));
+
+        // Test valid transitions for STANDBY
+        assertTrue(HAGroupStoreRecord.HAGroupState.STANDBY
+                .isTransitionAllowed(HAGroupStoreRecord.HAGroupState.STANDBY_TO_ACTIVE));
+        assertTrue(HAGroupStoreRecord.HAGroupState.STANDBY
+                .isTransitionAllowed(HAGroupStoreRecord.HAGroupState.DEGRADED_STANDBY_FOR_READER));
+        assertTrue(HAGroupStoreRecord.HAGroupState.STANDBY
+                .isTransitionAllowed(HAGroupStoreRecord.HAGroupState.DEGRADED_STANDBY_FOR_WRITER));
+
+        // Test valid transitions for ACTIVE_TO_STANDBY
+        assertTrue(HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY
+                .isTransitionAllowed(HAGroupStoreRecord.HAGroupState.ABORT_TO_ACTIVE_IN_SYNC));
+        assertTrue(HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY
+                .isTransitionAllowed(HAGroupStoreRecord.HAGroupState.STANDBY));
+
+        // Test valid transitions for STANDBY_TO_ACTIVE
+        assertTrue(HAGroupStoreRecord.HAGroupState.STANDBY_TO_ACTIVE
+                .isTransitionAllowed(HAGroupStoreRecord.HAGroupState.ABORT_TO_STANDBY));
+        assertTrue(HAGroupStoreRecord.HAGroupState.STANDBY_TO_ACTIVE
+                .isTransitionAllowed(HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC));
+    }
+
+    @Test
+    public void testHAGroupStateInvalidTransitions() {
+        // Test invalid transitions - ACTIVE cannot directly go to STANDBY
+        assertFalse(HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC
+                .isTransitionAllowed(HAGroupStoreRecord.HAGroupState.STANDBY));
+
+        // Test invalid transitions - STANDBY cannot directly go to ACTIVE
+        assertFalse(HAGroupStoreRecord.HAGroupState.STANDBY
+                .isTransitionAllowed(HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC));
+
+        // Test invalid transitions - OFFLINE has no allowed transitions (manual recovery needed)
+        assertFalse(HAGroupStoreRecord.HAGroupState.OFFLINE
+                .isTransitionAllowed(HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC));
+        assertFalse(HAGroupStoreRecord.HAGroupState.OFFLINE
+                .isTransitionAllowed(HAGroupStoreRecord.HAGroupState.STANDBY));
+        assertFalse(HAGroupStoreRecord.HAGroupState.OFFLINE
+                .isTransitionAllowed(HAGroupStoreRecord.HAGroupState.UNKNOWN));
+
+        // Test invalid transitions - UNKNOWN has no allowed transitions (manual recovery needed)
+        assertFalse(HAGroupStoreRecord.HAGroupState.UNKNOWN
+                .isTransitionAllowed(HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC));
+        assertFalse(HAGroupStoreRecord.HAGroupState.UNKNOWN
+                .isTransitionAllowed(HAGroupStoreRecord.HAGroupState.STANDBY));
+        assertFalse(HAGroupStoreRecord.HAGroupState.UNKNOWN
+                .isTransitionAllowed(HAGroupStoreRecord.HAGroupState.OFFLINE));
+    }
+
+    @Test
+    public void testHAGroupStateFromBytesValidValues() {
+        // Test valid enum names (case insensitive)
+        assertEquals(HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC,
+                HAGroupStoreRecord.HAGroupState.from("ACTIVE_IN_SYNC".getBytes()));
+        assertEquals(HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC,
+                HAGroupStoreRecord.HAGroupState.from("active_in_sync".getBytes()));
+        assertEquals(HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC,
+                HAGroupStoreRecord.HAGroupState.from("Active_in_Sync".getBytes()));
+
+        assertEquals(HAGroupStoreRecord.HAGroupState.STANDBY,
+                HAGroupStoreRecord.HAGroupState.from("STANDBY".getBytes()));
+        assertEquals(HAGroupStoreRecord.HAGroupState.STANDBY,
+                HAGroupStoreRecord.HAGroupState.from("standby".getBytes()));
+
+        assertEquals(HAGroupStoreRecord.HAGroupState.OFFLINE,
+                HAGroupStoreRecord.HAGroupState.from("OFFLINE".getBytes()));
+
+        assertEquals(HAGroupStoreRecord.HAGroupState.ACTIVE_NOT_IN_SYNC,
+                HAGroupStoreRecord.HAGroupState.from("ACTIVE_NOT_IN_SYNC".getBytes()));
+
+        assertEquals(HAGroupStoreRecord.HAGroupState.DEGRADED_STANDBY_FOR_READER,
+                HAGroupStoreRecord.HAGroupState.from("DEGRADED_STANDBY_FOR_READER".getBytes()));
+    }
+
+    @Test
+    public void testHAGroupStateFromBytesInvalidValues() {
+        // Test invalid enum names return UNKNOWN
+        assertEquals(HAGroupStoreRecord.HAGroupState.UNKNOWN,
+                HAGroupStoreRecord.HAGroupState.from("INVALID_STATE".getBytes()));
+        assertEquals(HAGroupStoreRecord.HAGroupState.UNKNOWN,
+                HAGroupStoreRecord.HAGroupState.from("".getBytes()));
+        assertEquals(HAGroupStoreRecord.HAGroupState.UNKNOWN,
+                HAGroupStoreRecord.HAGroupState.from("null".getBytes()));
+        assertEquals(HAGroupStoreRecord.HAGroupState.UNKNOWN,
+                HAGroupStoreRecord.HAGroupState.from("ACTIV".getBytes())); // typo
+        assertEquals(HAGroupStoreRecord.HAGroupState.UNKNOWN,
+                HAGroupStoreRecord.HAGroupState.from("ACTIVE_EXTRA".getBytes())); // extra text
+        assertEquals(HAGroupStoreRecord.HAGroupState.UNKNOWN,
+                HAGroupStoreRecord.HAGroupState.from(null));
+
+    }
+
+    // Private Helper Methods
+    private HAGroupStoreRecord getHAGroupStoreRecord(String haGroupName, String protocolVersion,
+                                                     HAGroupStoreRecord.HAGroupState haGroupState) {
+        return new HAGroupStoreRecord(protocolVersion, haGroupName, haGroupState);
+    }
+}
\ No newline at end of file
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/jdbc/PhoenixHAAdminToolTest.java b/phoenix-core/src/test/java/org/apache/phoenix/jdbc/PhoenixHAAdminToolTest.java
index fa88b5909a..0e78da74d3 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/jdbc/PhoenixHAAdminToolTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/jdbc/PhoenixHAAdminToolTest.java
@@ -87,7 +87,7 @@ public class PhoenixHAAdminToolTest {
 
     @Before
     public void setup() throws Exception {
-        when(mockHighAvailibilityCuratorProvider.getCurator(Mockito.anyString(), any(Properties.class))).thenReturn(curator);
+        when(mockHighAvailibilityCuratorProvider.getCurator(Mockito.anyString(), any(Properties.class), anyString())).thenReturn(curator);
         haGroupName = testName.getMethodName();
         recordV1 = new ClusterRoleRecord(
                 haGroupName, HighAvailabilityPolicy.FAILOVER,
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/util/HAGroupStoreTestUtil.java b/phoenix-core/src/test/java/org/apache/phoenix/util/HAGroupStoreTestUtil.java
new file mode 100644
index 0000000000..d15e73aca2
--- /dev/null
+++ b/phoenix-core/src/test/java/org/apache/phoenix/util/HAGroupStoreTestUtil.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.util;
+
+import org.apache.phoenix.jdbc.ClusterRoleRecord;
+import org.apache.phoenix.jdbc.PhoenixConnection;
+
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.sql.Statement;
+
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_HA_GROUP_NAME;
+import static org.apache.phoenix.util.PhoenixRuntime.JDBC_PROTOCOL_SEPARATOR;
+import static org.apache.phoenix.util.PhoenixRuntime.JDBC_PROTOCOL_ZK;
+
+/**
+ * Utility class for HA Group Store testing operations.
+ */
+public class HAGroupStoreTestUtil {
+
+    /**
+     * Upserts an HA group record into the system table for testing purposes.
+     *
+     * @param haGroupName the HA group name
+     * @param zkUrl the ZooKeeper URL for the local cluster
+     * @param peerZKUrl the ZooKeeper URL for the peer cluster
+     * @param localClusterRole the role of the local cluster
+     * @param peerClusterRole the role of the peer cluster
+     * @param overrideConnZkUrl optional override for the connection ZK URL
+     * @throws SQLException if the database operation fails
+     */
+    public static void upsertHAGroupRecordInSystemTable(String haGroupName, String zkUrl, String peerZKUrl,
+                                                        ClusterRoleRecord.ClusterRole localClusterRole,
+                                                        ClusterRoleRecord.ClusterRole peerClusterRole, String overrideConnZkUrl) throws SQLException {
+        try (PhoenixConnection conn = (PhoenixConnection) DriverManager.getConnection(JDBC_PROTOCOL_ZK + JDBC_PROTOCOL_SEPARATOR + (overrideConnZkUrl != null ? overrideConnZkUrl : zkUrl));
+             Statement stmt = conn.createStatement()) {
+            // Only insert values that are not null
+            StringBuilder queryBuilder = new StringBuilder("UPSERT INTO " + SYSTEM_HA_GROUP_NAME + " (HA_GROUP_NAME, ");
+            if (zkUrl != null) {
+                queryBuilder.append("ZK_URL_1, ");
+            }
+            if (peerZKUrl != null) {
+                queryBuilder.append("ZK_URL_2, ");
+            }
+            if (localClusterRole != null) {
+                queryBuilder.append("CLUSTER_ROLE_1, ");
+            }
+            if (peerClusterRole != null) {
+                queryBuilder.append("CLUSTER_ROLE_2, ");
+            }
+            if (zkUrl != null) {
+                queryBuilder.append("CLUSTER_URL_1, ");
+            }
+            if (peerZKUrl != null) {
+                queryBuilder.append("CLUSTER_URL_2, ");
+            }
+            queryBuilder.append("POLICY, VERSION) ");
+            queryBuilder.append("VALUES ('" + haGroupName + "', ");
+            if (zkUrl != null) {
+                queryBuilder.append("'" + zkUrl + "', ");
+            }
+            if (peerZKUrl != null) {
+                queryBuilder.append("'" + peerZKUrl + "', ");
+            }
+            if (localClusterRole != null) {
+                queryBuilder.append("'" + localClusterRole + "', ");
+            }
+            if (peerClusterRole != null) {
+                queryBuilder.append("'" + peerClusterRole + "', ");
+            }
+            if (zkUrl != null) {
+                queryBuilder.append("'" + zkUrl + "', ");
+            }
+            if (peerZKUrl != null) {
+                queryBuilder.append("'" + peerZKUrl + "', ");
+            }
+            queryBuilder.append("'FAILOVER', 1)");
+            stmt.executeUpdate(queryBuilder.toString());
+            conn.commit();
+        }
+    }
+
+    /**
+     * Deletes an HA group record from the system table for testing purposes.
+     *
+     * @param haGroupName the HA group name to delete
+     * @param zkUrl the ZooKeeper URL to connect to
+     * @throws SQLException if the database operation fails
+     */
+    public static void deleteHAGroupRecordInSystemTable(String haGroupName, String zkUrl) throws SQLException {
+        // Delete the record from System Table
+        try (PhoenixConnection conn = (PhoenixConnection) DriverManager.getConnection(JDBC_PROTOCOL_ZK + JDBC_PROTOCOL_SEPARATOR + zkUrl);
+             Statement stmt = conn.createStatement()) {
+            stmt.execute("DELETE FROM " + SYSTEM_HA_GROUP_NAME + " WHERE HA_GROUP_NAME = '" + haGroupName + "'");
+            conn.commit();
+        }
+    }
+} 
\ No newline at end of file