You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by lo...@apache.org on 2025/10/17 21:06:08 UTC
(phoenix) branch PHOENIX-7562-feature updated: PHOENIX-7569 :- Enhance Dual Cluster Client for consistent failover (#2288)
This is an automated email from the ASF dual-hosted git repository.
lokiore pushed a commit to branch PHOENIX-7562-feature
in repository https://gitbox.apache.org/repos/asf/phoenix.git
The following commit(s) were added to refs/heads/PHOENIX-7562-feature by this push:
new b4e533193e PHOENIX-7569 :- Enhance Dual Cluster Client for consistent failover (#2288)
b4e533193e is described below
commit b4e533193e066a8bc4d16b0af014736a8bb2e24f
Author: Lokesh Khurana <kh...@gmail.com>
AuthorDate: Fri Oct 17 14:06:03 2025 -0700
PHOENIX-7569 :- Enhance Dual Cluster Client for consistent failover (#2288)
---
.../BaseScannerRegionObserverConstants.java | 2 +-
.../apache/phoenix/exception/SQLExceptionCode.java | 3 +
.../exception/StaleClusterRoleRecordException.java | 44 ++
.../org/apache/phoenix/execute/MutationState.java | 6 +-
.../phoenix/iterate/TableResultIterator.java | 27 +
.../org/apache/phoenix/jdbc/ClusterRoleRecord.java | 51 +-
.../jdbc/ClusterRoleRecordGeneratorTool.java | 53 +-
.../phoenix/jdbc/FailoverPhoenixConnection.java | 37 +-
.../apache/phoenix/jdbc/HAGroupStoreClient.java | 116 +++-
.../apache/phoenix/jdbc/HAGroupStoreManager.java | 92 +++-
.../apache/phoenix/jdbc/HAGroupStoreRecord.java | 23 +-
.../java/org/apache/phoenix/jdbc/HAURLInfo.java | 3 +-
.../apache/phoenix/jdbc/HighAvailabilityGroup.java | 556 +++++++++----------
.../phoenix/jdbc/HighAvailabilityPolicy.java | 130 +++--
.../apache/phoenix/jdbc/HighAvailabilityUtil.java | 52 ++
.../org/apache/phoenix/jdbc/PhoenixConnection.java | 33 +-
.../phoenix/jdbc/PhoenixDatabaseMetaData.java | 3 +-
.../org/apache/phoenix/jdbc/PhoenixHAAdmin.java | 5 +
.../apache/phoenix/jdbc/PhoenixHAAdminTool.java | 2 +
.../org/apache/phoenix/jdbc/PhoenixResultSet.java | 2 +-
.../org/apache/phoenix/jdbc/PhoenixStatement.java | 26 +
.../org/apache/phoenix/jdbc/RPCConnectionInfo.java | 2 +-
.../phoenix/query/ConnectionQueryServices.java | 2 +
.../phoenix/query/ConnectionQueryServicesImpl.java | 8 +-
.../query/ConnectionlessQueryServicesImpl.java | 6 +
.../query/DelegateConnectionQueryServices.java | 6 +
.../org/apache/phoenix/query/QueryConstants.java | 6 +-
.../org/apache/phoenix/query/QueryServices.java | 3 +
.../apache/phoenix/query/QueryServicesOptions.java | 1 +
.../phoenix/util/GetClusterRoleRecordUtil.java | 231 ++++++++
.../java/org/apache/phoenix/util/ScanUtil.java | 18 +
.../coprocessor/BaseScannerRegionObserver.java | 24 +
.../phoenix/hbase/index/IndexRegionObserver.java | 26 +-
...gionServerEndpointITWithConsistentFailover.java | 3 +-
.../IndexRegionObserverMutationBlockingIT.java | 52 +-
.../jdbc/ClusterRoleRecordGeneratorToolIT.java | 12 +-
.../phoenix/jdbc/FailoverPhoenixConnection2IT.java | 514 +++++++++++++-----
.../phoenix/jdbc/FailoverPhoenixConnectionIT.java | 29 +-
.../HAConnectionWithMasterAndRPCRegistryIT.java | 596 ---------------------
.../apache/phoenix/jdbc/HAGroupStoreClientIT.java | 152 +++---
.../apache/phoenix/jdbc/HAGroupStoreManagerIT.java | 29 +-
.../phoenix/jdbc/HighAvailabilityGroup2IT.java | 111 ++--
.../phoenix/jdbc/HighAvailabilityGroupIT.java | 137 ++---
.../phoenix/jdbc/HighAvailabilityGroupTestIT.java | 50 +-
.../jdbc/HighAvailabilityTestingUtility.java | 536 ++++++++++++++----
.../jdbc/HighAvailabilityTestingUtilityIT.java | 32 +-
.../phoenix/jdbc/LoggingHAConnectionLimiterIT.java | 20 +-
.../phoenix/jdbc/ParallelPhoenixConnectionIT.java | 117 +---
.../jdbc/ParallelPhoenixConnectionWorkflowIT.java | 51 +-
.../ParallelPhoenixNullComparingResultSetIT.java | 5 +-
.../org/apache/phoenix/jdbc/PhoenixHAAdminIT.java | 102 ++--
.../apache/phoenix/jdbc/PhoenixHAAdminToolIT.java | 34 +-
.../apache/phoenix/jdbc/ClusterRoleRecordTest.java | 50 +-
.../phoenix/jdbc/HAGroupStoreRecordTest.java | 55 +-
.../phoenix/jdbc/ParallelPhoenixContextTest.java | 4 +-
.../ParallelPhoenixNullComparingResultSetTest.java | 2 +-
.../phoenix/jdbc/PhoenixHAAdminToolTest.java | 26 +-
.../apache/phoenix/util/HAGroupStoreTestUtil.java | 44 +-
58 files changed, 2448 insertions(+), 1914 deletions(-)
diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/coprocessorclient/BaseScannerRegionObserverConstants.java b/phoenix-core-client/src/main/java/org/apache/phoenix/coprocessorclient/BaseScannerRegionObserverConstants.java
index 41becf9374..b7b67e3d14 100644
--- a/phoenix-core-client/src/main/java/org/apache/phoenix/coprocessorclient/BaseScannerRegionObserverConstants.java
+++ b/phoenix-core-client/src/main/java/org/apache/phoenix/coprocessorclient/BaseScannerRegionObserverConstants.java
@@ -192,7 +192,7 @@ public class BaseScannerRegionObserverConstants {
public static final String SCANNER_OPENED_TRACE_INFO = "Scanner opened on server";
/**
- * Attribute name used to pass HAGroupName in Mutations.
+ * Attribute name used to pass HAGroupName in Mutations and Scans.
*/
public static final String HA_GROUP_NAME_ATTRIB = "_HAGroupName";
}
diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java b/phoenix-core-client/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java
index f9492003f3..7565bfb106 100644
--- a/phoenix-core-client/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java
+++ b/phoenix-core-client/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java
@@ -280,6 +280,9 @@ public enum SQLExceptionCode {
HA_READ_FROM_CLUSTER_FAILED_ON_NULL(1986, "F1Q86", "Unable to read from cluster for null."),
HA_INVALID_PROPERTIES(1987, "F1Q87", "Invalid properties to get a Phoenix HA connection."),
HA_CLUSTER_CAN_NOT_CONNECT(1988, "F1Q88", "Cluster can not serve any requests for this HA group"),
+ CLUSTER_ROLE_RECORD_NOT_FOUND(1989, "F1Q89", "Cluster role record not found for this HA group"),
+ FAILOVER_IN_PROGRESS(1990, "F1Q90", "Operation failed because Failover is in progress."),
+ HA_ROLE_TRANSITION_NOT_ALLOWED(1991, "F1Q93", "Role transition not allowed for this HA group"),
// Syntax error
diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/exception/StaleClusterRoleRecordException.java b/phoenix-core-client/src/main/java/org/apache/phoenix/exception/StaleClusterRoleRecordException.java
new file mode 100644
index 0000000000..4e68fbcd41
--- /dev/null
+++ b/phoenix-core-client/src/main/java/org/apache/phoenix/exception/StaleClusterRoleRecordException.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.exception;
+
+import java.io.IOException;
+
+/**
+ * Exception thrown when attempting to do operation on STANDBY Cluster of HA Connection,
+ * indicating that the client has slate Cluster Role Record info for the HAGroup.
+ */
+public class StaleClusterRoleRecordException extends IOException {
+ private static final long serialVersionUID = 1L;
+
+ /**
+ * @param msg reason for the exception
+ */
+ public StaleClusterRoleRecordException(String msg) {
+ super(msg);
+ }
+
+ /**
+ * @param msg reason for the exception
+ * @param cause the underlying cause
+ */
+ public StaleClusterRoleRecordException(String msg, Throwable cause) {
+ super(msg, cause);
+ }
+
+}
\ No newline at end of file
diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/execute/MutationState.java b/phoenix-core-client/src/main/java/org/apache/phoenix/execute/MutationState.java
index 2a52426ffd..4208df4f7e 100644
--- a/phoenix-core-client/src/main/java/org/apache/phoenix/execute/MutationState.java
+++ b/phoenix-core-client/src/main/java/org/apache/phoenix/execute/MutationState.java
@@ -865,7 +865,11 @@ public class MutationState implements SQLCloseable {
Bytes.toBytes(table.getExternalSchemaId()) : null;
byte[] lastDDLTimestamp =
table.getLastDDLTimestamp() != null ? Bytes.toBytes(table.getLastDDLTimestamp()) : null;
- byte[] haGroupName = StringUtils.isNotBlank(connection.getHAGroupName()) ? Bytes.toBytes(connection.getHAGroupName()) : null;
+ byte[] haGroupName = null;
+ //Only set haGroupName if connection is part of HA Connection
+ if (connection.getHAGroup() != null && StringUtils.isNotBlank(connection.getHAGroupName()) ) {
+ haGroupName = Bytes.toBytes(connection.getHAGroupName());
+ }
WALAnnotationUtil.annotateMutation(mutation, tenantId, schemaName, tableName, tableType, lastDDLTimestamp, haGroupName);
}
diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/iterate/TableResultIterator.java b/phoenix-core-client/src/main/java/org/apache/phoenix/iterate/TableResultIterator.java
index 98b2c8d837..ad0244f0f1 100644
--- a/phoenix-core-client/src/main/java/org/apache/phoenix/iterate/TableResultIterator.java
+++ b/phoenix-core-client/src/main/java/org/apache/phoenix/iterate/TableResultIterator.java
@@ -27,6 +27,7 @@ import static org.apache.phoenix.iterate.TableResultIterator.RenewLeaseStatus.NO
import static org.apache.phoenix.iterate.TableResultIterator.RenewLeaseStatus.RENEWED;
import static org.apache.phoenix.iterate.TableResultIterator.RenewLeaseStatus.THRESHOLD_NOT_REACHED;
import static org.apache.phoenix.iterate.TableResultIterator.RenewLeaseStatus.UNINITIALIZED;
+import static org.apache.phoenix.jdbc.HighAvailabilityUtil.isStaleClusterRoleRecordExceptionExistsInThrowable;
import java.io.IOException;
import java.sql.SQLException;
@@ -48,9 +49,12 @@ import org.apache.phoenix.compile.ExplainPlanAttributes
import org.apache.phoenix.compile.QueryPlan;
import org.apache.phoenix.coprocessorclient.HashJoinCacheNotFoundException;
import org.apache.phoenix.exception.ResultSetOutOfScanRangeException;
+import org.apache.phoenix.exception.SQLExceptionCode;
+import org.apache.phoenix.exception.SQLExceptionInfo;
import org.apache.phoenix.execute.BaseQueryPlan;
import org.apache.phoenix.execute.MutationState;
import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
+import org.apache.phoenix.jdbc.HighAvailabilityGroup;
import org.apache.phoenix.join.HashCacheClient;
import org.apache.phoenix.monitoring.ScanMetricsHolder;
import org.apache.phoenix.query.QueryConstants;
@@ -233,6 +237,29 @@ public class TableResultIterator implements ResultIterator {
}
} catch (SQLException e) {
LOGGER.error("Error while scanning table {} , scan {}", htable, scan);
+ if (isStaleClusterRoleRecordExceptionExistsInThrowable(e)) {
+ LOGGER.debug("StaleClusterRoleRecordException found refreshing HAGroup");
+ if (plan.getContext().getConnection().getHAGroup() != null) {
+ HighAvailabilityGroup haGroup = plan.getContext().getConnection()
+ .getHAGroup();
+ if (!haGroup.refreshClusterRoleRecord(true)) {
+ LOGGER.error("Error while refreshing HAGroup for Stale " +
+ "ClusterRoleRecord found for scan operation");
+ }
+ throw new SQLExceptionInfo.Builder(SQLExceptionCode.FAILOVER_IN_PROGRESS)
+ .setMessage(String.format("Operation failed because failover is " +
+ "in progress for HAGroup %s, %s",
+ haGroup, e)).build().buildException();
+ }
+ //If we receive StaleClusterRoleRecordException, that means Operation was
+ //supposed to be executed on Active Cluster but was in reality was sent to
+ //STANDBY Cluster, that can happen only when Failover is in Progress, So we
+ //catch that exception and refresh the roleRecord of HAGroup, which will lead to
+ //closure of all current connections (as they are not Active anymore), which
+ //will close the Statements and Iterators associated with the connections, that
+ //is why we are not retrying the operation and throwing the exception.
+ }
+
try {
throw ClientUtil.parseServerException(e);
} catch(HashJoinCacheNotFoundException e1) {
diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/ClusterRoleRecord.java b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/ClusterRoleRecord.java
index 3b71f8f9cd..2c41b7bd71 100644
--- a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/ClusterRoleRecord.java
+++ b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/ClusterRoleRecord.java
@@ -29,6 +29,8 @@ import org.apache.phoenix.util.JacksonUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static org.apache.phoenix.jdbc.HighAvailabilityGroup.DEFAULT_PHOENIX_HA_CRR_REGISTRY_TYPE;
+
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
@@ -70,6 +72,13 @@ public class ClusterRoleRecord {
|| this == ACTIVE_TO_STANDBY || this == STANDBY_TO_ACTIVE;
}
+ /**
+ * @return true if a cluster with this role is active or active to standby, otherwise false
+ */
+ public boolean isActive() {
+ return this == ACTIVE || this == ACTIVE_TO_STANDBY;
+ }
+
public static ClusterRole from(byte[] bytes) {
String value = new String(bytes, StandardCharsets.UTF_8);
return Arrays.stream(ClusterRole.values())
@@ -133,44 +142,35 @@ public class ClusterRoleRecord {
* master4\\:port4,master5\\:port5
* @param haGroupName HighAvailability Group name / CRR name
* @param policy Policy used by give CRR
- * @param registryType {@link RegistryType} to be used for given urls
* @param url1 ZK/HMaster url based on registry type for first cluster
* @param role1 {@link ClusterRole} which describes the current state of first cluster
* @param url2 ZK/HMaster url based on registry type for second cluster
* @param role2 {@link ClusterRole} which describes the current state of second cluster
* @param version version of a given CRR
- * @param zk1 ZK url of first cluster when CRR is in old format for backward compatibility
- * @param zk2 ZK url of second cluster when CRR is in old format for backward compatibility
*/
@JsonCreator
public ClusterRoleRecord(@JsonProperty("haGroupName") String haGroupName,
@JsonProperty("policy") HighAvailabilityPolicy policy,
- @JsonProperty("registryType") RegistryType registryType,
@JsonProperty("url1") String url1,
@JsonProperty("role1") ClusterRole role1,
@JsonProperty("url2") String url2,
@JsonProperty("role2") ClusterRole role2,
- @JsonProperty("version") long version,
- @JsonProperty("zk1") String zk1,
- @JsonProperty("zk2") String zk2) {
+ @JsonProperty("version") long version) {
this.haGroupName = haGroupName;
this.policy = policy;
- this.registryType = registryType != null ? registryType : RegistryType.ZK;
- String resolvedUrl1 = (url1 != null) ? url1 : zk1; //For Backward Compatibility
- String resolvedUrl2 = (url2 != null) ? url2 : zk2; //For Backward Compatibility
+ //Default registry type is RPC from Consistent Cluster Failover onwards
+ this.registryType = DEFAULT_PHOENIX_HA_CRR_REGISTRY_TYPE;
//Do we really need to normalize here ?
//We are normalizing to have urls in specific formats for each registryType for getting
//accurate comparisons. We are passing registryType as these url most probably won't have
//protocol in url, and it might be normalized based to wrong registry type, as we normalize
//w.r.t {@link ConnectionInfo.CLIENT_CONNECTION_REGISTRY_IMPL_CONF_KEY},
- //but considering source of truth of registryType is present in CLusterRoleRecord we are
- //normalizing based on that.
- //url will be in form :- zk1\\:port1,zk2\\:port2,zk3\\:port3,zk4\\:port4,zk5\\:port5::znode
- //or master1\\:port1,master2\\:port2,master3\\:port3,master4\\:port4,master5\\:port5
- url1 = JDBCUtil.formatUrl(resolvedUrl1, this.registryType);
- url2 = JDBCUtil.formatUrl(resolvedUrl2, this.registryType);
+ //As we are expecting only master URLs as Consistent Cluster Failover onwards we only will allow RPC registry type
+ //url will be in form :- master1\\:port1,master2\\:port2,master3\\:port3,master4\\:port4,master5\\:port5
+ url1 = JDBCUtil.formatUrl(url1, this.registryType);
+ url2 = JDBCUtil.formatUrl(url2, this.registryType);
Preconditions.checkArgument(!url1.equals(url2), "Two clusters have the same URLS!");
Preconditions.checkNotNull(role1, "Role of a cluster cannot be null!");
@@ -191,21 +191,6 @@ public class ClusterRoleRecord {
this.version = version;
}
- public ClusterRoleRecord(String haGroupName, HighAvailabilityPolicy policy,
- String url1, ClusterRole role1,
- String url2, ClusterRole role2,
- long version) {
- this(haGroupName, policy, RegistryType.ZK, url1, role1, url2, role2, version, null, null);
- }
-
- public ClusterRoleRecord(String haGroupName, HighAvailabilityPolicy policy,
- RegistryType registryType,
- String url1, ClusterRole role1,
- String url2, ClusterRole role2,
- long version) {
- this(haGroupName, policy, registryType, url1, role1, url2, role2, version, null, null);
- }
-
public static Optional<ClusterRoleRecord> fromJson(byte[] bytes) {
if (bytes == null) {
return Optional.empty();
@@ -224,10 +209,10 @@ public class ClusterRoleRecord {
@JsonIgnore
public Optional<String> getActiveUrl() {
- if (role1 == ClusterRole.ACTIVE) {
+ if (role1.isActive()) {
return Optional.of(url1);
}
- if (role2 == ClusterRole.ACTIVE) {
+ if (role2.isActive()) {
return Optional.of(url2);
}
return Optional.empty();
diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/ClusterRoleRecordGeneratorTool.java b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/ClusterRoleRecordGeneratorTool.java
index 03c29320d4..c41736d11a 100644
--- a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/ClusterRoleRecordGeneratorTool.java
+++ b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/ClusterRoleRecordGeneratorTool.java
@@ -21,7 +21,10 @@ package org.apache.phoenix.jdbc;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.hbase.ClusterMetrics;
import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
@@ -40,7 +43,7 @@ import java.util.ArrayList;
import java.util.List;
import static org.apache.phoenix.jdbc.HighAvailabilityGroup.PHOENIX_HA_ATTR_PREFIX;
-import static org.apache.phoenix.jdbc.PhoenixHAAdmin.getLocalZkUrl;
+import static org.apache.phoenix.jdbc.PhoenixHAAdmin.getLocalMasterUrl;
/**
@@ -71,7 +74,7 @@ public class ClusterRoleRecordGeneratorTool extends Configured implements Tool {
File file = StringUtils.isEmpty(fileName)
? File.createTempFile("phoenix.ha.cluster.role.records", ".json")
: new File(fileName);
- JacksonUtil.getObjectWriterPretty().writeValue(file, listAllRecordsByZk());
+ JacksonUtil.getObjectWriterPretty().writeValue(file, listAllRecordsByMaster());
System.out.println("Created JSON file '" + file + "'");
return 0;
} catch (Exception e) {
@@ -80,9 +83,9 @@ public class ClusterRoleRecordGeneratorTool extends Configured implements Tool {
}
}
- List<ClusterRoleRecord> listAllRecordsByZk() throws IOException {
- /* This current cluster's full ZK url for HBase, in host:port:/hbase format. */
- String localZkUrl = getLocalZkUrl(getConf());
+ List<ClusterRoleRecord> listAllRecordsByMaster() throws Exception {
+ /* This current cluster's full Master url for HBase, in host:port format. */
+ String localMasterUrl = getLocalMasterUrl(getConf());
final String[] haGroupNames = getConf().getStrings(PHOENIX_HA_GROUPS_ATTR);
if (haGroupNames == null || haGroupNames.length == 0) {
String msg = "No HA groups configured for this cluster via " + PHOENIX_HA_GROUPS_ATTR;
@@ -92,16 +95,50 @@ public class ClusterRoleRecordGeneratorTool extends Configured implements Tool {
List<ClusterRoleRecord> records = new ArrayList<>();
for (String haGroupName : haGroupNames) {
- String peerZkUrl = getPeerZkUrl(getConf(), haGroupName);
+ String peerMasterUrl = getPeerMasterUrl(getConf(), haGroupName);
records.add(new ClusterRoleRecord(haGroupName, getHaPolicy(haGroupName),
- localZkUrl, ClusterRole.ACTIVE,
- peerZkUrl, ClusterRole.STANDBY,
+ localMasterUrl, ClusterRole.ACTIVE,
+ peerMasterUrl, ClusterRole.STANDBY,
1));
}
LOG.debug("Returning all cluster role records discovered: {}", records);
return records;
}
+ /**
+ * Method to get the peer master url from configuration
+ * @param conf Configuration object for local cluster
+ * @param haGroupName HA group name
+ * @return Peer master url
+ * @throws Exception if unable to get the peer master url
+ */
+ private static String getPeerMasterUrl(Configuration conf, String haGroupName) throws Exception {
+ String peerZkUrl = getPeerZkUrl(conf, haGroupName);
+
+ Connection connection = null;
+ ServerName serverName;
+ String peerMaster;
+ try {
+ Configuration newConf = HBaseConfiguration.create();
+
+ newConf.set(HConstants.ZOOKEEPER_QUORUM, peerZkUrl.split(":")[0].replace("\\", ""));
+ newConf.set(HConstants.ZOOKEEPER_CLIENT_PORT, peerZkUrl.split(":")[1].replace("\\", ""));
+
+ connection = ConnectionFactory.createConnection(newConf);
+
+ ClusterMetrics clusterMetrics = connection.getAdmin().getClusterMetrics();
+ serverName = clusterMetrics.getMasterName();
+ assert serverName != null;
+ peerMaster = serverName.getHostname() + "\\:" + serverName.getPort();
+ } finally {
+ if (connection != null) {
+ connection.close();
+ }
+ }
+ return peerMaster;
+
+ }
+
/**
* Helper method to get the replication peer's ZK URL (host:port) from Configuration.
*/
diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/FailoverPhoenixConnection.java b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/FailoverPhoenixConnection.java
index 0ad691958f..10f88124e3 100644
--- a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/FailoverPhoenixConnection.java
+++ b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/FailoverPhoenixConnection.java
@@ -46,6 +46,8 @@ import java.util.Map;
import java.util.Properties;
import java.util.concurrent.Executor;
+import static org.apache.phoenix.jdbc.HighAvailabilityUtil.isStaleClusterRoleRecordExceptionExistsInThrowable;
+
/**
* An implementation of JDBC connection which supports failover between two cluster in an HA group.
* <p>
@@ -317,6 +319,11 @@ public class FailoverPhoenixConnection implements PhoenixMonitoredConnection {
/**
* This is the utility method to help wrapping a method call to phoenix connection.
+ * If we receive a {@link org.apache.phoenix.exception.StaleClusterRoleRecordException},
+ * we will try to refresh the CRR and transition the connections according to the new record.
+ * StaleCRRException is thrown for mutations and pointLookups only for scans we are checking
+ * before executing the queries and in any case if the refresh is successful or not,
+ * we will throw the exception.
*
* @param s the supplier which returns a value and may throw SQLException
* @param <T> type of the returned object by the supplier
@@ -333,10 +340,38 @@ public class FailoverPhoenixConnection implements PhoenixMonitoredConnection {
while (true) {
try {
return s.get();
- } catch (SQLException e) {
+ } catch (Exception e) {
+ if (isStaleClusterRoleRecordExceptionExistsInThrowable(e)) {
+ //If we receive StaleClusterRoleRecordException, that means Operation was
+ //supposed to be executed on Active Cluster but was in reality was sent to
+ //STANDBY Cluster, that can happen only when Failover is in Progress, So we
+ //catch that exception and refresh the roleRecord of HAGroup, which will lead to
+ //closure of all current connections (as they are not Active anymore). To retry
+ //the operation here, we will need to create a new connection with new context
+ //and everything to new Active Cluster. So, we will throw a new exception
+ //{@link SQLExceptionCode.FAILOVER_IN_PROGRESS}
+
+ LOG.debug("StaleClusterRoleRecordException found refreshing HAGroup");
+ //If the exception is due to stale ClusterRoleRecord version, try
+ //refreshing the ClusterRoleRecord and state transitions if required
+
+ if (!context.getHAGroup().refreshClusterRoleRecord(true)) {
+ LOG.error("Error while refreshing HAGroup for Stale ClusterRoleRecord " +
+ "found for mutation operation");
+ }
+
+ throw new SQLExceptionInfo.Builder(SQLExceptionCode.FAILOVER_IN_PROGRESS)
+ .setMessage(String.format("Operation failed because failover is in " +
+ "progress for HAGroup %s, %s",
+ context.getHAGroup(), e)).build().buildException();
+
+ }
if (policy.shouldFailover(e, ++failoverCount)) {
failover(timeoutMs);
} else {
+ if (e instanceof SQLException) {
+ throw e;
+ }
throw new SQLException(
String.format("Error on operation with failover policy %s", policy), e);
}
diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HAGroupStoreClient.java b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HAGroupStoreClient.java
index e4743bc7fd..05cbeaba79 100644
--- a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HAGroupStoreClient.java
+++ b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HAGroupStoreClient.java
@@ -63,7 +63,8 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.CLUSTER_URL_2;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.HA_GROUP_NAME;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.POLICY;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_HA_GROUP_NAME;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VERSION;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VERSION_CLUSTER_1;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VERSION_CLUSTER_2;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.ZK_URL_1;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.ZK_URL_2;
import static org.apache.phoenix.jdbc.PhoenixHAAdmin.getLocalZkUrl;
@@ -82,7 +83,8 @@ public class HAGroupStoreClient implements Closeable {
public static final String ZK_CONSISTENT_HA_GROUP_STATE_NAMESPACE = "phoenix"
+ ZKPaths.PATH_SEPARATOR + "consistentHA"
+ ZKPaths.PATH_SEPARATOR + "groupState";
- private static final long HA_GROUP_STORE_CLIENT_INITIALIZATION_TIMEOUT_MS = 30000L;
+ public static final String PHOENIX_HA_GROUP_STORE_CLIENT_INITIALIZATION_TIMEOUT_MS = "phoenix.ha.group.store.client.initialization.timeout.ms";
+ private static final long DEFAULT_HA_GROUP_STORE_CLIENT_INITIALIZATION_TIMEOUT_MS = 30000L;
// Multiplier for ZK session timeout to account for time it will take for HMaster to abort
// the region server in case ZK connection is lost from the region server.
@VisibleForTesting
@@ -125,7 +127,8 @@ public class HAGroupStoreClient implements Closeable {
private ClusterRole peerClusterRole;
private String clusterUrl;
private String peerClusterUrl;
- private long clusterRoleRecordVersion;
+ private long localGroupRecordVersion;
+ private long peerGroupRecordVersion;
public static HAGroupStoreClient getInstance(Configuration conf, String haGroupName) throws SQLException {
return getInstanceForZkUrl(conf, haGroupName, null);
@@ -305,6 +308,8 @@ public class HAGroupStoreClient implements Closeable {
}
if (isUpdateNeeded(currentHAGroupStoreRecord.getHAGroupState(),
currentHAGroupStoreRecordStat.getMtime(), haGroupState)) {
+ //As we are updating the HAGroupStoreRecord increase the record version
+ long newStateVersion = currentHAGroupStoreRecord.getRecordVersion() + 1;
// We maintain last sync time as the last time cluster was in sync state.
// If state changes from ACTIVE_IN_SYNC to ACTIVE_NOT_IN_SYNC, record that time
// Once state changes back to ACTIVE_IN_SYNC or the role is
@@ -327,6 +332,7 @@ public class HAGroupStoreClient implements Closeable {
currentHAGroupStoreRecord.getProtocolVersion(),
currentHAGroupStoreRecord.getHaGroupName(),
haGroupState,
+ newStateVersion,
lastSyncTimeInMs
);
// TODO: Check if cluster role is changing, if so, we need to update
@@ -336,6 +342,8 @@ public class HAGroupStoreClient implements Closeable {
// It should automatically update the ZK record as well.
phoenixHaAdmin.updateHAGroupStoreRecordInZooKeeper(haGroupName,
newHAGroupStoreRecord, currentHAGroupStoreRecordStat.getVersion());
+ this.localGroupRecordVersion = newStateVersion;
+ this.clusterRole = haGroupState.getClusterRole();
} else {
LOGGER.info("Not updating HAGroupStoreRecord for HA group {} with state {}",
haGroupName, haGroupState);
@@ -352,16 +360,29 @@ public class HAGroupStoreClient implements Closeable {
*/
public ClusterRoleRecord getClusterRoleRecord() throws IOException {
HAGroupStoreRecord peerHAGroupStoreRecord = getHAGroupStoreRecordFromPeer();
- ClusterRoleRecord.ClusterRole peerClusterRole = peerHAGroupStoreRecord != null
- ? peerHAGroupStoreRecord.getClusterRole()
- : ClusterRole.UNKNOWN;
+ ClusterRoleRecord.ClusterRole peerClusterRole = ClusterRoleRecord.ClusterRole.UNKNOWN;
+ if (peerHAGroupStoreRecord != null) {
+ peerClusterRole = peerHAGroupStoreRecord.getClusterRole();
+ this.peerGroupRecordVersion = peerHAGroupStoreRecord.getRecordVersion();
+ }
+
+ long clusterRoleRecordVersion = combineCanonicalVersions(this.localGroupRecordVersion, this.peerGroupRecordVersion);
+
return new ClusterRoleRecord(this.haGroupName,
this.policy,
this.clusterUrl,
this.clusterRole,
this.peerClusterUrl,
peerClusterRole,
- this.clusterRoleRecordVersion);
+ clusterRoleRecordVersion);
+ }
+
+ /**
+ * Get the policy for a given HA group.
+ * @return HighAvailabilityPolicy for the specified HA group name
+ */
+ public HighAvailabilityPolicy getPolicy() {
+ return this.policy;
}
/**
@@ -384,27 +405,36 @@ public class HAGroupStoreClient implements Closeable {
Pair<HAGroupStoreRecord, Stat> cacheRecordFromZK =
phoenixHaAdmin.getHAGroupStoreRecordInZooKeeper(this.haGroupName);
HAGroupStoreRecord haGroupStoreRecord = cacheRecordFromZK.getLeft();
+ HAGroupStoreRecord newHAGroupStoreRecord = null;
HAGroupState defaultHAGroupState = this.clusterRole.getDefaultHAGroupState();
// Initialize lastSyncTimeInMs only if we start in ACTIVE_NOT_IN_SYNC state
- // and ZNode is not already present
+ // and ZNode is not already present
Long lastSyncTimeInMs = defaultHAGroupState.equals(HAGroupState.ACTIVE_NOT_IN_SYNC)
? System.currentTimeMillis()
: null;
- HAGroupStoreRecord newHAGroupStoreRecord = new HAGroupStoreRecord(
- HAGroupStoreRecord.DEFAULT_PROTOCOL_VERSION,
- haGroupName,
- this.clusterRole.getDefaultHAGroupState(),
- lastSyncTimeInMs
- );
// Only update current ZNode if it doesn't have the same role as present in System Table.
// If not exists, then create ZNode
// TODO: Discuss if this approach is what reader needs.
- if (haGroupStoreRecord != null &&
- !haGroupStoreRecord.getClusterRole().equals(this.clusterRole)) {
+ if (haGroupStoreRecord == null) {
+ newHAGroupStoreRecord = new HAGroupStoreRecord(
+ HAGroupStoreRecord.DEFAULT_PROTOCOL_VERSION,
+ haGroupName,
+ defaultHAGroupState,
+ this.localGroupRecordVersion,
+ lastSyncTimeInMs
+ );
+ phoenixHaAdmin.createHAGroupStoreRecordInZooKeeper(newHAGroupStoreRecord);
+ } else if (!haGroupStoreRecord.getClusterRole().equals(this.clusterRole)) {
+ this.localGroupRecordVersion = haGroupStoreRecord.getRecordVersion() + 1;
+ newHAGroupStoreRecord = new HAGroupStoreRecord(
+ HAGroupStoreRecord.DEFAULT_PROTOCOL_VERSION,
+ haGroupName,
+ defaultHAGroupState,
+ this.localGroupRecordVersion,
+ lastSyncTimeInMs
+ );
phoenixHaAdmin.updateHAGroupStoreRecordInZooKeeper(haGroupName,
newHAGroupStoreRecord, cacheRecordFromZK.getRight().getVersion());
- } else if (haGroupStoreRecord == null) {
- phoenixHaAdmin.createHAGroupStoreRecordInZooKeeper(newHAGroupStoreRecord);
}
}
@@ -423,7 +453,8 @@ public class HAGroupStoreClient implements Closeable {
String clusterRole2 = rs.getString(CLUSTER_ROLE_2);
String clusterUrl1 = rs.getString(CLUSTER_URL_1);
String clusterUrl2 = rs.getString(CLUSTER_URL_2);
- this.clusterRoleRecordVersion = rs.getLong(VERSION);
+ long version1 = rs.getLong(VERSION_CLUSTER_1);
+ long version2 = rs.getLong(VERSION_CLUSTER_2);
Preconditions.checkNotNull(zkUrl1, "ZK_URL_1 in System Table cannot be null");
Preconditions.checkNotNull(zkUrl2, "ZK_URL_2 in System Table cannot be null");
String formattedZkUrl1 = JDBCUtil.formatUrl(zkUrl1, RegistryType.ZK);
@@ -437,6 +468,8 @@ public class HAGroupStoreClient implements Closeable {
clusterRole2.getBytes(StandardCharsets.UTF_8));
this.clusterUrl = clusterUrl1;
this.peerClusterUrl = clusterUrl2;
+ this.localGroupRecordVersion = version1;
+ this.peerGroupRecordVersion = version2;
} else if (StringUtils.equals(formattedZkUrl2, formattedZkUrl)) {
this.peerZKUrl = JDBCUtil.formatUrl(zkUrl1, RegistryType.ZK);
this.clusterRole = ClusterRoleRecord.ClusterRole.from(
@@ -445,6 +478,8 @@ public class HAGroupStoreClient implements Closeable {
clusterRole1.getBytes(StandardCharsets.UTF_8));
this.clusterUrl = clusterUrl2;
this.peerClusterUrl = clusterUrl1;
+ this.localGroupRecordVersion = version2;
+ this.peerGroupRecordVersion = version1;
}
} else {
throw new SQLException("HAGroupStoreRecord not found for HA group name: " +
@@ -461,8 +496,10 @@ public class HAGroupStoreClient implements Closeable {
"Peer ZK URL in System Table cannot be null");
Preconditions.checkNotNull(this.peerClusterUrl,
"Peer Cluster URL in System Table cannot be null");
- Preconditions.checkNotNull(this.clusterRoleRecordVersion,
- "Cluster role record version in System Table cannot be null");
+ Preconditions.checkNotNull(this.localGroupRecordVersion ,
+ "Local group record version in System Table cannot be null");
+ Preconditions.checkNotNull(this.peerGroupRecordVersion,
+ "Peer group record version in System Table cannot be null");
}
private void maybeInitializePeerPathChildrenCache() {
@@ -511,7 +548,8 @@ public class HAGroupStoreClient implements Closeable {
? customListener
: createCacheListener(latch, cacheType));
newPathChildrenCache.start(PathChildrenCache.StartMode.POST_INITIALIZED_EVENT);
- boolean initialized = latch.await(HA_GROUP_STORE_CLIENT_INITIALIZATION_TIMEOUT_MS,
+ boolean initialized = latch.await(conf.getLong(PHOENIX_HA_GROUP_STORE_CLIENT_INITIALIZATION_TIMEOUT_MS,
+ DEFAULT_HA_GROUP_STORE_CLIENT_INITIALIZATION_TIMEOUT_MS),
TimeUnit.MILLISECONDS);
return initialized ? newPathChildrenCache : null;
} catch (Exception e) {
@@ -689,6 +727,40 @@ public class HAGroupStoreClient implements Closeable {
return ((System.currentTimeMillis() - currentHAGroupStoreRecordMtime) > waitTime);
}
+ /**
+ * Combine two versions into a single long value to be used for comparison and sending back to client
+ * @param version1 HAGroupRecord version for cluster 1
+ * @param version2 HAGroupRecord version for cluster 2
+ * @return combined version
+ */
+
+ private long combineCanonicalVersions(long version1, long version2) {
+ LOGGER.trace("Combining versions: {} and {} with master addresses: {} and {}", version1, version2, this.clusterUrl, this.peerClusterUrl);
+ if (this.clusterUrl.compareTo(this.peerClusterUrl) <= 0){
+ return (version2 << 32) | version1;
+ } else {
+ return (version1 << 32) | version2;
+ }
+ }
+
+ /**
+ * Extract the version for the cluster 1 from the combined version
+ * @param combinedVersion combined version
+ * @return version for the cluster
+ */
+ private long extractVersion1(long combinedVersion) {
+ return combinedVersion >> 32;
+ }
+
+ /**
+ * Extract the version for the cluster 2 from the combined version
+ * @param combinedVersion combined version
+ * @return version for the cluster
+ */
+ private long extractVersion2(long combinedVersion) {
+ return combinedVersion & 0xFFFFFFFFL;
+ }
+
// ========== HA Group State Change Subscription Methods ==========
/**
diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HAGroupStoreManager.java b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HAGroupStoreManager.java
index 7045a301f3..53e18983a6 100644
--- a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HAGroupStoreManager.java
+++ b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HAGroupStoreManager.java
@@ -18,56 +18,93 @@
package org.apache.phoenix.jdbc;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.util.Pair;
import org.apache.phoenix.exception.InvalidClusterRoleTransitionException;
+import org.apache.phoenix.exception.SQLExceptionCode;
+import org.apache.phoenix.exception.SQLExceptionInfo;
import org.apache.phoenix.exception.StaleHAGroupStoreRecordVersionException;
+import org.apache.phoenix.jdbc.ClusterRoleRecord.ClusterRole;
+import org.apache.phoenix.jdbc.HAGroupStoreRecord.HAGroupState;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.jcraft.jsch.IO;
+
import java.io.IOException;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
+import java.util.Map;
+import java.util.Objects;
import java.util.Optional;
+import java.util.concurrent.ConcurrentHashMap;
import static org.apache.phoenix.jdbc.PhoenixHAAdmin.getLocalZkUrl;
import static org.apache.phoenix.query.QueryServices.CLUSTER_ROLE_BASED_MUTATION_BLOCK_ENABLED;
+import static org.apache.phoenix.query.QueryServices.HA_GROUP_STALE_FOR_MUTATION_CHECK_ENABLED;
import static org.apache.phoenix.query.QueryServicesOptions
.DEFAULT_CLUSTER_ROLE_BASED_MUTATION_BLOCK_ENABLED;
+import static org.apache.phoenix.query.QueryServicesOptions.DEFAULT_HA_GROUP_STALE_FOR_MUTATION_CHECK_ENABLED;
/**
* Implementation of HAGroupStoreManager that uses HAGroupStoreClient.
* Manages all HAGroupStoreClient instances and provides passthrough
* functionality for HA group state change notifications.
+ * Supports multiple instances per ZK URL to handle multiple MiniClusters.
*/
public class HAGroupStoreManager {
- private static volatile HAGroupStoreManager haGroupStoreManagerInstance;
private static final Logger LOGGER = LoggerFactory.getLogger(HAGroupStoreManager.class);
+
+ // Map of <ZKUrl, HAGroupStoreManagerInstance> for different MiniClusters
+ // Can revert this but will fail for tests with one cluster down
+ private static Map<String, HAGroupStoreManager> instances = new ConcurrentHashMap<>();
+
private final boolean mutationBlockEnabled;
+ private final boolean checkStaleCRRForEveryMutation;
private final String zkUrl;
private final Configuration conf;
/**
* Creates/gets an instance of HAGroupStoreManager.
+ * Use the ZK URL from the configuration to determine the instance.
*
* @param conf configuration
* @return HAGroupStoreManager instance
*/
public static HAGroupStoreManager getInstance(final Configuration conf) {
- if (haGroupStoreManagerInstance == null) {
- synchronized (HAGroupStoreManager.class) {
- if (haGroupStoreManagerInstance == null) {
- haGroupStoreManagerInstance = new HAGroupStoreManager(conf);
- }
- }
- }
- return haGroupStoreManagerInstance;
+ return getInstanceForZkUrl(conf, null);
+ }
+
+ /**
+ * Creates/gets an instance of HAGroupStoreManager for a specific ZK URL.
+ * This allows different region servers to have their own instances.
+ *
+ * @param conf configuration
+ * @param zkUrl specific ZK URL to use, null to use local ZK URL from config
+ * @return HAGroupStoreManager instance for the specified ZK URL
+ */
+ public static HAGroupStoreManager getInstanceForZkUrl(final Configuration conf, String zkUrl) {
+ String localZkUrl = Objects.toString(zkUrl, getLocalZkUrl(conf));
+ Objects.requireNonNull(localZkUrl, "zkUrl cannot be null");
+
+ return instances.computeIfAbsent(localZkUrl, url -> {
+ LOGGER.info("Creating new HAGroupStoreManager instance for ZK URL: {}", url);
+ return new HAGroupStoreManager(conf, url);
+ });
}
private HAGroupStoreManager(final Configuration conf) {
+ this(conf, getLocalZkUrl(conf));
+ }
+
+ private HAGroupStoreManager(final Configuration conf, final String zkUrl) {
this.mutationBlockEnabled = conf.getBoolean(CLUSTER_ROLE_BASED_MUTATION_BLOCK_ENABLED,
DEFAULT_CLUSTER_ROLE_BASED_MUTATION_BLOCK_ENABLED);
- this.zkUrl = getLocalZkUrl(conf);
+ this.checkStaleCRRForEveryMutation = conf.getBoolean(HA_GROUP_STALE_FOR_MUTATION_CHECK_ENABLED,
+ DEFAULT_HA_GROUP_STALE_FOR_MUTATION_CHECK_ENABLED);
+ this.zkUrl = zkUrl;
this.conf = conf;
+ LOGGER.info("Started HAGroupStoreManager with ZK URL: {}", zkUrl);
}
/**
@@ -100,6 +137,25 @@ public class HAGroupStoreManager {
return false;
}
+ public boolean isHAGroupOnClientStale(String haGroupName) throws IOException {
+ if (checkStaleCRRForEveryMutation) {
+ HAGroupStoreClient haGroupStoreClient = getHAGroupStoreClient(haGroupName);
+ //If local cluster is not ACTIVE/ACTIVE_TO_STANDBY, it means the Failover CRR is stale
+ //on client as they are trying to write/read from a STANDBY cluster.
+ //If we plan to use Standby clusters for scan operations, depends on whether the
+ //connection is an SCN connection or not, If a connection is not an SCN connection then
+ //Standby cluster can't be used as most recent data is only available through ACTIVE
+ //cluster. For SCN connections, future client functionality might allow attempts to
+ //open connections to the standby cluster, which would then accept or reject the query
+ //based on whether the SCN falls within its consistency point, and will require a change
+ //in the logic her, with much bigger change.
+
+ return haGroupStoreClient.getPolicy() == HighAvailabilityPolicy.FAILOVER &&
+ !haGroupStoreClient.getHAGroupStoreRecord().getHAGroupState().getClusterRole().isActive();
+ }
+ return false;
+ }
+
/**
* Force rebuilds the HAGroupStoreClient instance for all HA groups.
* If any HAGroupStoreClient instance is not created, it will be created.
@@ -271,9 +327,19 @@ public class HAGroupStoreManager {
* @throws IOException when HAGroupStoreClient is not healthy.
*/
public ClusterRoleRecord getClusterRoleRecord(String haGroupName)
- throws IOException {
- HAGroupStoreClient haGroupStoreClient = getHAGroupStoreClient(haGroupName);
- return haGroupStoreClient.getClusterRoleRecord();
+ throws IOException, SQLException {
+ try {
+ HAGroupStoreClient haGroupStoreClient = getHAGroupStoreClient(haGroupName);
+ return haGroupStoreClient.getClusterRoleRecord();
+ } catch (IOException e) {
+ //If haGroupStoreClient is null, it means the HA group is not configured in the local cluster throw
+ //a SQLException with CLUSTER_ROLE_RECORD_NOT_FOUND error code to make client known that CRR for the
+ //given HAGroupName doesn't exist.
+ throw new SQLExceptionInfo.Builder(SQLExceptionCode.CLUSTER_ROLE_RECORD_NOT_FOUND)
+ .setMessage("HAGroupStoreClient is not initialized")
+ .build()
+ .buildException();
+ }
}
/**
diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HAGroupStoreRecord.java b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HAGroupStoreRecord.java
index 6891d93c46..6a2b08579d 100644
--- a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HAGroupStoreRecord.java
+++ b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HAGroupStoreRecord.java
@@ -45,6 +45,7 @@ public class HAGroupStoreRecord {
private static final Logger LOG = LoggerFactory.getLogger(HAGroupStoreRecord.class);
public static final String DEFAULT_PROTOCOL_VERSION = "1.0";
+ public static final long DEFAULT_RECORD_VERSION = 1L;
/**
* Enum representing the HA group state with each state having a corresponding ClusterRole.
@@ -163,28 +164,40 @@ public class HAGroupStoreRecord {
private final String protocolVersion;
private final String haGroupName;
private final HAGroupState haGroupState;
+ private final long recordVersion;
private final Long lastSyncStateTimeInMs;
@JsonCreator
public HAGroupStoreRecord(@JsonProperty("protocolVersion") String protocolVersion,
@JsonProperty("haGroupName") String haGroupName,
@JsonProperty("haGroupState") HAGroupState haGroupState,
+ @JsonProperty("recordVersion") Long recordVersion,
@JsonProperty("lastSyncStateTimeInMs") Long lastSyncStateTimeInMs) {
Preconditions.checkNotNull(haGroupName, "HA group name cannot be null!");
Preconditions.checkNotNull(haGroupState, "HA group state cannot be null!");
+ Preconditions.checkNotNull(recordVersion, "Record version cannot be null!");
this.protocolVersion = Objects.toString(protocolVersion, DEFAULT_PROTOCOL_VERSION);
this.haGroupName = haGroupName;
this.haGroupState = haGroupState;
+ this.recordVersion = recordVersion;
this.lastSyncStateTimeInMs = lastSyncStateTimeInMs;
}
/**
* Convenience constructor for backward compatibility without lastSyncStateTimeInMs.
*/
+ public HAGroupStoreRecord(String protocolVersion,
+ String haGroupName, HAGroupState haGroupState, Long recordVersion) {
+ this(protocolVersion, haGroupName, haGroupState, recordVersion, null);
+ }
+
+ /**
+ * Convenience constructor for backward compatibility without recordVersion and lastSyncStateTimeInMs.
+ */
public HAGroupStoreRecord(String protocolVersion,
String haGroupName, HAGroupState haGroupState) {
- this(protocolVersion, haGroupName, haGroupState, null);
+ this(protocolVersion, haGroupName, haGroupState, DEFAULT_RECORD_VERSION, null);
}
public static Optional<HAGroupStoreRecord> fromJson(byte[] bytes) {
@@ -209,6 +222,7 @@ public class HAGroupStoreRecord {
return haGroupName.equals(other.haGroupName)
&& haGroupState.equals(other.haGroupState)
&& protocolVersion.equals(other.protocolVersion)
+ && recordVersion == other.recordVersion
&& Objects.equals(lastSyncStateTimeInMs, other.lastSyncStateTimeInMs);
}
@@ -220,6 +234,10 @@ public class HAGroupStoreRecord {
return haGroupName;
}
+ public long getRecordVersion() {
+ return recordVersion;
+ }
+
@JsonProperty("haGroupState")
public HAGroupState getHAGroupState() {
return haGroupState;
@@ -240,6 +258,7 @@ public class HAGroupStoreRecord {
.append(protocolVersion)
.append(haGroupName)
.append(haGroupState)
+ .append(recordVersion)
.append(lastSyncStateTimeInMs)
.hashCode();
}
@@ -258,6 +277,7 @@ public class HAGroupStoreRecord {
.append(protocolVersion, otherRecord.protocolVersion)
.append(haGroupName, otherRecord.haGroupName)
.append(haGroupState, otherRecord.haGroupState)
+ .append(recordVersion, otherRecord.recordVersion)
.append(lastSyncStateTimeInMs, otherRecord.lastSyncStateTimeInMs)
.isEquals();
}
@@ -269,6 +289,7 @@ public class HAGroupStoreRecord {
+ "protocolVersion='" + protocolVersion + '\''
+ ", haGroupName='" + haGroupName + '\''
+ ", haGroupState=" + haGroupState
+ + ", recordVersion=" + recordVersion
+ ", lastSyncStateTimeInMs=" + lastSyncStateTimeInMs
+ '}';
}
diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HAURLInfo.java b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HAURLInfo.java
index 5c9c4f9b39..1bb5a81968 100644
--- a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HAURLInfo.java
+++ b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HAURLInfo.java
@@ -24,7 +24,8 @@ import org.apache.phoenix.thirdparty.com.google.common.annotations.VisibleForTes
import org.apache.phoenix.thirdparty.com.google.common.base.Preconditions;
/**
- * An HAURLInfo contains information of an HA Url with respect of HA Group Name.
+ * An HAURLInfo contains information of an HA Url with respect of HA Group Name. It includes
+ * HAGroupName as name, principal and additional JDBC parameters passed as part of connection url.
* <p>
* It is constructed based on client input, including the JDBC connection string and properties.
* Objects of this class are used to get appropriate principal and additional JDBC parameters.
diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HighAvailabilityGroup.java b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HighAvailabilityGroup.java
index 3d743315e8..481bfcee90 100644
--- a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HighAvailabilityGroup.java
+++ b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HighAvailabilityGroup.java
@@ -24,25 +24,25 @@ import org.apache.commons.lang3.builder.HashCodeBuilder;
import org.apache.curator.RetryPolicy;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.CuratorFrameworkFactory;
-import org.apache.curator.framework.recipes.cache.NodeCache;
import org.apache.curator.retry.ExponentialBackoffRetry;
import org.apache.curator.utils.ZKPaths;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.util.PairOfSameType;
import org.apache.phoenix.exception.SQLExceptionCode;
import org.apache.phoenix.exception.SQLExceptionInfo;
import org.apache.phoenix.jdbc.ClusterRoleRecord.ClusterRole;
+import org.apache.phoenix.jdbc.ClusterRoleRecord.RegistryType;
+import org.apache.phoenix.query.HBaseFactoryProvider;
import org.apache.phoenix.thirdparty.com.google.common.annotations.VisibleForTesting;
import org.apache.phoenix.thirdparty.com.google.common.base.Preconditions;
import org.apache.phoenix.thirdparty.com.google.common.cache.Cache;
import org.apache.phoenix.thirdparty.com.google.common.cache.CacheBuilder;
-import org.apache.phoenix.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.apache.phoenix.util.GetClusterRoleRecordUtil;
import org.apache.phoenix.util.JDBCUtil;
import org.apache.phoenix.util.PhoenixRuntime;
-import org.apache.zookeeper.data.Stat;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import edu.umd.cs.findbugs.annotations.NonNull;
import java.io.IOException;
import java.sql.Connection;
@@ -57,16 +57,19 @@ import java.util.Properties;
import java.util.Set;
import java.util.concurrent.CompletionException;
import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
-import java.util.concurrent.atomic.AtomicReference;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
import static org.apache.phoenix.query.QueryServicesOptions.DEFAULT_CLIENT_CONNECTION_CACHE_MAX_DURATION;
+import static org.apache.phoenix.util.PhoenixRuntime.JDBC_PROTOCOL_SEPARATOR;
/**
* An high availability (HA) group is an association between a pair of HBase clusters, a group of
@@ -125,6 +128,22 @@ public class HighAvailabilityGroup {
PHOENIX_HA_ATTR_PREFIX + "transition.timeout.ms";
public static final long PHOENIX_HA_TRANSITION_TIMEOUT_MS_DEFAULT = 5 * 60 * 1000; // 5 mins
+ public static final String PHOENIX_HA_CRR_POLLER_INTERVAL_MS_KEY =
+ PHOENIX_HA_ATTR_PREFIX + "crr.poller.interval.ms";
+ public static final String PHOENIX_HA_CRR_POLLER_INTERVAL_MS_DEFAULT = "5000"; // 5 seconds
+
+ public static final String PHOENIX_HA_CRR_REGISTRY_TYPE_KEY =
+ PHOENIX_HA_ATTR_PREFIX + "crr.registry.type";
+
+ /**
+ * The frequency to cache the ClusterRoleRecord for this HA group.
+ */
+ public static final String PHOENIX_HA_CRR_CACHE_FREQUENCY_MS_KEY =
+ PHOENIX_HA_ATTR_PREFIX + "crr.cache.frequency.ms";
+ public static final long PHOENIX_HA_CRR_CACHE_FREQUENCY_MS_DEFAULT = 2000; // 2 seconds
+
+ public static final RegistryType DEFAULT_PHOENIX_HA_CRR_REGISTRY_TYPE = RegistryType.RPC;
+
static final Logger LOG = LoggerFactory.getLogger(HighAvailabilityGroup.class);
/**
@@ -165,31 +184,26 @@ public class HighAvailabilityGroup {
*/
private final Properties properties;
/**
- * Executor service for the two role managers.
- */
- private final ExecutorService roleManagerExecutor = Executors.newFixedThreadPool(2,
- new ThreadFactoryBuilder().setDaemon(true).setNameFormat("phoenixHAGroup-%d").build());
- /**
- * The count down latch to make sure at least one role manager has pulled data from ZK.
+ * Configuration to be used for default properties values
*/
- private final CountDownLatch roleManagerLatch = new CountDownLatch(1);
+ private final Configuration config;
/**
- * Pair of role managers for watching cluster role records from the two ZK clusters.
+ * Current cluster role record for this HA group.
*/
- private final AtomicReference<PairOfSameType<HAClusterRoleManager>> roleManagers
- = new AtomicReference<>();
+ private volatile ClusterRoleRecord roleRecord;
/**
* Executor for applying the cluster role to this HA group.
*/
- private final ExecutorService nodeChangedExecutor = Executors.newFixedThreadPool(1);
- /**
- * Current cluster role record for this HA group.
- */
- private volatile ClusterRoleRecord roleRecord;
+ private final ExecutorService crrChangedExecutor = Executors.newFixedThreadPool(1);
/**
* State of this HA group.
*/
private volatile State state = State.UNINITIALIZED;
+ private volatile long lastClusterRoleRecordRefreshTime = 0;
+ private volatile long clusterRoleRecordCacheFrequency = PHOENIX_HA_CRR_CACHE_FREQUENCY_MS_DEFAULT;
+ private final ReadWriteLock rwLock = new ReentrantReadWriteLock();
+ private final Lock readLock = rwLock.readLock();
+ private final Lock writeLock = rwLock.writeLock();
/**
* Private constructor.
@@ -199,6 +213,10 @@ public class HighAvailabilityGroup {
private HighAvailabilityGroup(HAGroupInfo info, Properties properties) {
this.info = info;
this.properties = properties;
+ this.config = HBaseFactoryProvider.getConfigurationFactory().getConfiguration();
+ // Get the cluster role record cache frequency from properties and if properties is not set, then get it from config
+ this.clusterRoleRecordCacheFrequency = Long.parseLong(properties.getProperty(PHOENIX_HA_CRR_CACHE_FREQUENCY_MS_KEY,
+ config.get(PHOENIX_HA_CRR_CACHE_FREQUENCY_MS_KEY, String.valueOf(PHOENIX_HA_CRR_CACHE_FREQUENCY_MS_DEFAULT))));
}
/**
* This is for test usage only. In production, the record should be retrieved from ZooKeeper.
@@ -210,21 +228,33 @@ public class HighAvailabilityGroup {
this.properties = properties;
this.roleRecord = record;
this.state = state;
+ this.config = HBaseFactoryProvider.getConfigurationFactory().getConfiguration();
+ // Get the cluster role record cache frequency from properties and if properties is not set, then get it from config
+ this.clusterRoleRecordCacheFrequency = Long.parseLong(properties.getProperty(PHOENIX_HA_CRR_CACHE_FREQUENCY_MS_KEY,
+ config.get(PHOENIX_HA_CRR_CACHE_FREQUENCY_MS_KEY, String.valueOf(PHOENIX_HA_CRR_CACHE_FREQUENCY_MS_DEFAULT))));
}
/**
- * Get an instance of HAURLInfo given the HA connecting URL (with "|") and client properties.
+ * Get an instance of {@link HAURLInfo} given the HA connecting URL (with "|") and client properties.
* Here we do parsing of url and try to extract principal and other additional params
- * @throws SQLException
+ * @throws SQLException if fails to get HA information and/or invalid properties are seen
*/
public static HAURLInfo getUrlInfo(String url, Properties properties) throws SQLException {
- url = checkUrl(url);
+ //Check if HA group name is provided in the properties if not throw an exception
+ String name = properties.getProperty(PHOENIX_HA_GROUP_ATTR);
+ if (StringUtils.isEmpty(name)) {
+ throw new SQLExceptionInfo.Builder(SQLExceptionCode.HA_INVALID_PROPERTIES)
+ .setMessage(String.format("HA group name can not be empty for HA URL %s", url))
+ .build()
+ .buildException();
+ }
+ url = checkUrl(url);
String principal = null;
String additionalJDBCParams = null;
int idx = url.indexOf("]");
- int extraIdx = url.indexOf(PhoenixRuntime.JDBC_PROTOCOL_SEPARATOR, idx + 1);
+ int extraIdx = url.indexOf(JDBC_PROTOCOL_SEPARATOR, idx + 1);
if (extraIdx != -1) {
- //after zk quorums there should be a separator
+ //after quorums there should be a separator
if (extraIdx != idx + 1) {
throw new SQLExceptionInfo.Builder(SQLExceptionCode.MALFORMED_CONNECTION_URL)
.setMessage(String.format("URL %s is not a valid HA connection string",
@@ -234,7 +264,7 @@ public class HighAvailabilityGroup {
}
additionalJDBCParams = url.substring(extraIdx + 1);
//Get the principal
- extraIdx = additionalJDBCParams.indexOf(PhoenixRuntime.JDBC_PROTOCOL_SEPARATOR);
+ extraIdx = additionalJDBCParams.indexOf(JDBC_PROTOCOL_SEPARATOR);
if (extraIdx != -1) {
if (extraIdx != 0) {
principal = additionalJDBCParams.substring(0, extraIdx);
@@ -276,13 +306,6 @@ public class HighAvailabilityGroup {
? (additionalJDBCParams.equals(String.valueOf(PhoenixRuntime.JDBC_PROTOCOL_TERMINATOR))
? null : additionalJDBCParams) : null;
- String name = properties.getProperty(PHOENIX_HA_GROUP_ATTR);
- if (StringUtils.isEmpty(name)) {
- throw new SQLExceptionInfo.Builder(SQLExceptionCode.HA_INVALID_PROPERTIES)
- .setMessage(String.format("HA group name can not be empty for HA URL %s", url))
- .build()
- .buildException();
- }
HAURLInfo haurlInfo = new HAURLInfo(name, principal, additionalJDBCParams);
HAGroupInfo info = getHAGroupInfo(url, properties);
URLS.computeIfAbsent(info, haGroupInfo -> new HashSet<>()).add(haurlInfo);
@@ -291,9 +314,6 @@ public class HighAvailabilityGroup {
private static HAGroupInfo getHAGroupInfo(String url, Properties properties)
throws SQLException {
- url = checkUrl(url);
- url = url.substring(url.indexOf("[") + 1, url.indexOf("]"));
- String [] urls = url.split("\\|");
String name = properties.getProperty(PHOENIX_HA_GROUP_ATTR);
if (StringUtils.isEmpty(name)) {
throw new SQLExceptionInfo.Builder(SQLExceptionCode.HA_INVALID_PROPERTIES)
@@ -301,19 +321,33 @@ public class HighAvailabilityGroup {
.build()
.buildException();
}
+ url = checkUrl(url);
+ url = url.substring(url.indexOf("[") + 1, url.indexOf("]"));
+ String [] urls = url.split("\\|");
return new HAGroupInfo(name, urls[0], urls[1]);
}
/**
- * checks if the given url is appropriate for HA Connection
- * @param url
+ * checks if the given url is appropriate for HA Connections, HA URL will only work for RPC Registry
+ * so it won't accept MASTER or ZK protocols but if no protocol is specified, it will be assumed to be RPC
+ * @param url url passed by clients to phoenix-client
* @return the url without protocol
- * @throws SQLException
+ * @throws SQLException MalformedConnectionUrlException if url is not a valid HA url
*/
private static String checkUrl(String url) throws SQLException {
- if (url.startsWith(PhoenixRuntime.JDBC_PROTOCOL)) {
+ if (url.startsWith(PhoenixRuntime.JDBC_PROTOCOL_RPC)) {
+ url = url.substring(PhoenixRuntime.JDBC_PROTOCOL_RPC.length() + 1);
+ } else if (url.startsWith(PhoenixRuntime.JDBC_PROTOCOL_MASTER)) {
+ throwMalFormedConnectionUrlException(String.format("Protocol specified is MASTER, which " +
+ "is not acceptable for HA Connections, only RPC Registry is mandated, for url %s", url));
+ } else if (url.startsWith(PhoenixRuntime.JDBC_PROTOCOL_ZK)){
+ throwMalFormedConnectionUrlException(String.format("Protocol specified is ZK, which " +
+ "is not acceptable for HA Connections only RPC Registry is mandated, for url %s", url));
+ } else if (url.startsWith(PhoenixRuntime.JDBC_PROTOCOL)) {
url = url.substring(PhoenixRuntime.JDBC_PROTOCOL.length() + 1);
}
+
+ //Check if url is a valid HA URL
if (!(url.contains("[") && url.contains("|") && url.contains("]"))) {
throw new SQLExceptionInfo.Builder(SQLExceptionCode.MALFORMED_CONNECTION_URL)
.setMessage(String.format("URL %s is not a valid HA connection string", url))
@@ -342,6 +376,8 @@ public class HighAvailabilityGroup {
public static Optional<HighAvailabilityGroup> get(String url, Properties properties)
throws SQLException {
HAGroupInfo info = getHAGroupInfo(url, properties);
+
+ //create a cache for missing CRR to prevent unnecessary exceptions (cache expires in 5 minutes)
if (MISSING_CRR_GROUPS_CACHE.getIfPresent(info) != null) {
return Optional.empty();
}
@@ -353,23 +389,19 @@ public class HighAvailabilityGroup {
} catch (Exception e) {
GROUPS.remove(info);
haGroup.close();
- try {
- CuratorFramework curator1 = CURATOR_CACHE.getIfPresent(generateCacheKey(info.getUrl1(), null));
- CuratorFramework curator2 = CURATOR_CACHE.getIfPresent(generateCacheKey(info.getUrl2(), null));
- if (curator1 != null && curator2 != null) {
- Stat node1 = curator1.checkExists().forPath(info.getZkPath());
- Stat node2 = curator2.checkExists().forPath(info.getZkPath());
- if (node1 == null && node2 == null) {
- // The HA group fails to initialize due to missing cluster role records on
- // both ZK clusters. We will put this HA group into negative cache.
- MISSING_CRR_GROUPS_CACHE.put(info, true);
- return Optional.empty();
- }
- }
- } catch (Exception e2) {
- LOG.error("HA group {} failed to initialized. Got exception when checking if znode"
- + " exists on the two ZK clusters.", info, e2);
+
+ if (e instanceof SQLException && ((SQLException)e).getErrorCode()
+ == SQLExceptionCode.CLUSTER_ROLE_RECORD_NOT_FOUND.getErrorCode()) {
+
+ LOG.error("HA group {} failed to initialized. Got exception when getting " +
+ "ClusterRoleRecord for HA group {}", info, e.getCause());
+ //If the exception is due to missing CRR, we will put this HA group into negative
+ //cache to prevent unnecessary computations of trying to get CRR for every
+ //connection cache expires every 5 secs
+ MISSING_CRR_GROUPS_CACHE.put(info, true);
+ return Optional.empty();
}
+
throw new SQLExceptionInfo.Builder(SQLExceptionCode.CANNOT_ESTABLISH_CONNECTION)
.setMessage(String.format("Cannot start HA group %s for URL %s", haGroup, url))
.setRootCause(e)
@@ -412,9 +444,9 @@ public class HighAvailabilityGroup {
}
// Ensure the fallback cluster URL includes the JDBC protocol prefix
- if (!fallbackCluster.startsWith(PhoenixRuntime.JDBC_PROTOCOL_ZK)) {
- fallbackCluster = PhoenixRuntime.JDBC_PROTOCOL_ZK
- + PhoenixRuntime.JDBC_PROTOCOL_SEPARATOR + fallbackCluster;
+ if (!fallbackCluster.startsWith(PhoenixRuntime.JDBC_PROTOCOL_RPC)) {
+ fallbackCluster = PhoenixRuntime.JDBC_PROTOCOL_RPC
+ + JDBC_PROTOCOL_SEPARATOR + fallbackCluster;
}
LOG.info("Falling back to single cluster '{}' for the HA group {} to serve HA connection "
@@ -573,68 +605,21 @@ public class HighAvailabilityGroup {
}
/**
- * Initialize this HA group by registering ZK watchers and getting initial cluster role record.
+ * Initialize this HA group by getting initial ClusterRoleRecord from RegionServer Endpoints.
* <p>
* If this is already initialized, calling this method is a no-op. This method is lock free as
* current thread will either return fast or wait for the in-progress initialization or timeout.
*/
- public void init() throws IOException {
+ public void init() throws IOException, SQLException {
if (state != State.UNINITIALIZED) {
return;
}
- PairOfSameType<HAClusterRoleManager> newRoleManagers = new PairOfSameType<>(
- new HAClusterRoleManager(info.urls.getFirst(), properties),
- new HAClusterRoleManager(info.urls.getSecond(), properties));
- if (!roleManagers.compareAndSet(null, newRoleManagers)) {
- LOG.info("Someone already started role managers; waiting for that one...");
- waitForInitialization(properties);
- return;
- }
-
- Future<?> f1 = roleManagerExecutor.submit(newRoleManagers.getFirst());
- Future<?> f2 = roleManagerExecutor.submit(newRoleManagers.getSecond());
- try {
- waitForInitialization(properties);
- } catch (IOException e) {
- // HA group that fails to initialize will not be kept in the global cache.
- // Next connection request will create and initialize a new HA group.
- // Before returning in case of exception, following code will cancel the futures.
- f1.cancel(true);
- f2.cancel(true);
- throw e;
- }
-
- assert roleRecord != null;
- LOG.info("Initial cluster role for HA group {} is {}", info, roleRecord);
- }
+ ClusterRoleRecord roleRecordFromEndpoint = getClusterRoleRecordFromEndpoint();
- /**
- * Helper method that will block current thread until the HA group is initialized.
- * <p>
- * After returning, the HA state might not be in READY state. That is possible when a new ZK
- * node change is detected triggering HA group to become IN_TRANSIT state.
- *
- * @param properties the connection properties
- * @throws IOException when current HA group is not initialized before timeout
- */
- private void waitForInitialization(Properties properties) throws IOException {
- String connectionTimeoutMsProp = properties.getProperty(
- PHOENIX_HA_ZK_CONNECTION_TIMEOUT_MS_KEY);
- int timeout = !StringUtils.isEmpty(connectionTimeoutMsProp)
- ? Integer.parseInt(connectionTimeoutMsProp)
- : PHOENIX_HA_ZK_CONNECTION_TIMEOUT_MS_DEFAULT;
- boolean started = false;
- try {
- started = roleManagerLatch.await(timeout, TimeUnit.MILLISECONDS);
- } catch (InterruptedException e) {
- LOG.warn("Got interrupted when waiting for cluster role managers to start", e);
- Thread.currentThread().interrupt();
- }
- if (!started) {
- LOG.warn("Timed out {}ms waiting for HA group '{}' to be initialized.", timeout, info);
- throw new IOException("Fail to initialize HA group " + info);
- }
+ LOG.info("Initial cluster role for HA group {} is {}", info, roleRecordFromEndpoint);
+ roleRecord = roleRecordFromEndpoint;
+ state = State.READY;
}
/**
@@ -760,7 +745,7 @@ public class HighAvailabilityGroup {
Preconditions.checkArgument(driver instanceof PhoenixEmbeddedDriver,
"No JDBC driver is registered for Phoenix high availability (HA) framework");
return ((PhoenixEmbeddedDriver) driver).getConnectionQueryServices(jdbcString, properties)
- .connect(jdbcString, properties);
+ .connect(jdbcString, properties, this);
}
@VisibleForTesting
@@ -786,20 +771,6 @@ public class HighAvailabilityGroup {
* Someone calling close on this would make it unusable, since the state would become closed.
*/
void close() {
- roleManagerExecutor.shutdownNow();
- try {
- // TODO: Parameterize and set in future work item for pluggable
- if (!roleManagerExecutor.awaitTermination(PHOENIX_HA_ZK_SESSION_TIMEOUT_MS_DEFAULT,
- TimeUnit.MILLISECONDS)) {
- LOG.error("Fail to shut down role managers service for HA group: {}", info);
- }
- } catch (InterruptedException e) {
- LOG.warn("HA group {} close() got interrupted when closing role managers", info, e);
- // (Re-)Cancel if current thread also interrupted
- roleManagerExecutor.shutdownNow();
- // Preserve interrupt status
- Thread.currentThread().interrupt();
- }
state = State.CLOSED;
}
@@ -810,77 +781,6 @@ public class HighAvailabilityGroup {
: "HighAvailabilityGroup{roleRecord=" + roleRecord + ", state=" + state + "}";
}
- /**
- * Set the new cluster role record for this HA group.
- * <p>
- * Calling this method will make HA group be in transition state where no request can be served.
- * The data source may come from either of the two clusters as seen by the ZK watcher.
- *
- * @param newRoleRecord the new cluster role record to set
- * @return true if the new record is set as current one; false otherwise
- */
- private synchronized boolean applyClusterRoleRecord(@NonNull ClusterRoleRecord newRoleRecord) {
- if (roleRecord == null) {
- roleRecord = newRoleRecord;
- state = State.READY;
- LOG.info("HA group {} is now in {} state after getting initial V{} role record: {}",
- info, state, roleRecord.getVersion(), roleRecord);
- LOG.debug("HA group {} is ready", this);
- return true;
- }
-
- if (!newRoleRecord.isNewerThan(roleRecord)) {
- LOG.warn("Does not apply new cluster role record as it does not have higher version. "
- + "Existing record: {}, new record: {}", roleRecord, newRoleRecord);
- return false;
- }
-
- if (!roleRecord.hasSameInfo(newRoleRecord)) {
- LOG.error("New record {} has different HA group information from old record {}",
- newRoleRecord, roleRecord);
- return false;
- }
-
- final ClusterRoleRecord oldRecord = roleRecord;
- state = State.IN_TRANSITION;
- LOG.info("HA group {} is in {} to set V{} record", info, state, newRoleRecord.getVersion());
- Future<?> future = nodeChangedExecutor.submit(() -> {
- try {
- roleRecord.getPolicy().transitClusterRoleRecord(this, roleRecord, newRoleRecord);
- } catch (SQLException e) {
- throw new CompletionException(e);
- }
- });
-
- // TODO: save timeout in the HA group info (aka cluster role record) instead in properties
- String transitionTimeoutProp = properties.getProperty(PHOENIX_HA_TRANSITION_TIMEOUT_MS_KEY);
- long maxTransitionTimeMs = StringUtils.isNotEmpty(transitionTimeoutProp)
- ? Long.parseLong(transitionTimeoutProp)
- : PHOENIX_HA_TRANSITION_TIMEOUT_MS_DEFAULT;
- try {
- future.get(maxTransitionTimeMs, TimeUnit.MILLISECONDS);
- } catch (InterruptedException ie) {
- LOG.error("Got interrupted when transiting cluster roles for HA group {}", info, ie);
- future.cancel(true);
- Thread.currentThread().interrupt();
- return false;
- } catch (ExecutionException | TimeoutException e) {
- LOG.error("HA group {} failed to transit cluster roles per policy {} to new record {}",
- info, roleRecord.getPolicy(), newRoleRecord, e);
- // Calling back HA policy function for cluster switch is conducted with best effort.
- // HA group continues transition when its HA policy fails to deal with context switch
- // (e.g. to close existing connections)
- // The goal here is to gain higher availability even though existing resources against
- // previous ACTIVE cluster may have not been closed cleanly.
- }
- roleRecord = newRoleRecord;
- state = State.READY;
- LOG.info("HA group {} is in {} state after applying V{} role record. Old: {}, new: {}",
- info, state, roleRecord.getVersion(), oldRecord, roleRecord);
- LOG.debug("HA group is ready: {}", this);
- return true;
- }
-
/**
* Local state of this HA group object, which transits upon explicit call (e.g. init) or when
* the cluster role change is detected.
@@ -917,9 +817,9 @@ public class HighAvailabilityGroup {
this.name = name;
//Normalizing these urls with ZK protocol as these are the ZK urls of clusters where
//roleRecords resides.
- url1 = JDBCUtil.formatUrl(url1, ClusterRoleRecord.RegistryType.ZK);
- url2 = JDBCUtil.formatUrl(url2, ClusterRoleRecord.RegistryType.ZK);
- Preconditions.checkArgument(!url1.equals(url2), "Two clusters have the same ZK!");
+ url1 = JDBCUtil.formatUrl(url1, DEFAULT_PHOENIX_HA_CRR_REGISTRY_TYPE);
+ url2 = JDBCUtil.formatUrl(url2, DEFAULT_PHOENIX_HA_CRR_REGISTRY_TYPE);
+ Preconditions.checkArgument(!url1.equals(url2), "Two clusters have the same urls!");
// Ignore the given order of url1 and url2, and reorder for equals comparison.
if (url1.compareTo(url2) > 0) {
this.urls = new PairOfSameType<>(url2, url1);
@@ -941,18 +841,11 @@ public class HighAvailabilityGroup {
}
public String getJDBCUrl1(HAURLInfo haURLInfo) {
- return getJDBCUrl(getUrl1(), haURLInfo, ClusterRoleRecord.RegistryType.ZK);
+ return getJDBCUrl(getUrl1(), haURLInfo, DEFAULT_PHOENIX_HA_CRR_REGISTRY_TYPE);
}
public String getJDBCUrl2(HAURLInfo haURLInfo) {
- return getJDBCUrl(getUrl2(), haURLInfo, ClusterRoleRecord.RegistryType.ZK);
- }
-
- /**
- * Helper method to return the znode path in the Phoenix HA namespace.
- */
- String getZkPath() {
- return ZKPaths.PATH_SEPARATOR + name;
+ return getJDBCUrl(getUrl2(), haURLInfo, DEFAULT_PHOENIX_HA_CRR_REGISTRY_TYPE);
}
@Override
@@ -997,7 +890,7 @@ public class HighAvailabilityGroup {
* @param type Registry Type for which url has to be constructed
* @return jdbc url in proper format i.e. jdbc:phoenix+<registry>:url:principal:additionalParam
* example :- jdbc:phoenix+zk:zk1\\:port1,zk2\\:port2,zk3\\:port3,zk4\\:port4,zk5\\:port5::znode:principal:additionalParams
- * or jdbc:phoenix+master:master1\\:port1,master2\\:port2,master3\\:port3,master4\\:port4,master5\\:port5::principal:additionParams
+ * or jdbc:phoenix+master:master1\\:port1,master2\\:port2,master3\\:port3,master4\\:port4,master5\\:port5:::principal:additionParams
*/
public static String getJDBCUrl(String url, HAURLInfo haURLInfo,
ClusterRoleRecord.RegistryType type) {
@@ -1019,21 +912,21 @@ public class HighAvailabilityGroup {
default:
sb.append(PhoenixRuntime.JDBC_PROTOCOL);
}
- sb.append(PhoenixRuntime.JDBC_PROTOCOL_SEPARATOR);
+ sb.append(JDBC_PROTOCOL_SEPARATOR);
sb.append(url);
if (haURLInfo != null) {
if (ObjectUtils.anyNotNull(haURLInfo.getPrincipal(), haURLInfo.getAdditionalJDBCParams())) {
if (extraSeparator) {
//For Master and RPC connection url we need 2 extra separator between port and
//principal as there is no ZNode
- sb.append(PhoenixRuntime.JDBC_PROTOCOL_SEPARATOR).
- append(PhoenixRuntime.JDBC_PROTOCOL_SEPARATOR);
+ sb.append(JDBC_PROTOCOL_SEPARATOR).
+ append(JDBC_PROTOCOL_SEPARATOR);
}
- sb.append(haURLInfo.getPrincipal() == null ? PhoenixRuntime.JDBC_PROTOCOL_SEPARATOR
- : PhoenixRuntime.JDBC_PROTOCOL_SEPARATOR + haURLInfo.getPrincipal());
+ sb.append(haURLInfo.getPrincipal() == null ? JDBC_PROTOCOL_SEPARATOR
+ : JDBC_PROTOCOL_SEPARATOR + haURLInfo.getPrincipal());
}
if (ObjectUtils.anyNotNull(haURLInfo.getAdditionalJDBCParams())) {
- sb.append(PhoenixRuntime.JDBC_PROTOCOL_SEPARATOR).
+ sb.append(JDBC_PROTOCOL_SEPARATOR).
append(haURLInfo.getAdditionalJDBCParams());
}
}
@@ -1041,73 +934,184 @@ public class HighAvailabilityGroup {
}
/**
- * Maintains the client view of cluster roles for the HA group using data retrieved from one ZK.
- * <p>
- * It is a runnable to keep setting up the curator and the node cache. It will also register
- * the node watcher so any znode data change will trigger a callback function updating HA group.
+ * Helper method to construct the jdbc HA url for the given urls and HAURLInfo
+ * @param url1 url1 of the HA group
+ * @param url2 url2 of the HA group
+ * @param haURLInfo HAURLInfo object containing principal and additional params
+ * @return jdbc HA url in proper format i.e. jdbc:phoenix+rpc:[url1|url2]:principal:additionalParams
+ * example :- jdbc:phoenix+rpc:[master1\\:port1,master2\\:port2,master3\\:port3,master4\\:port4,master5\\:port5|
+ * peer_master1\\:port1,peer_master2\\:port2,peer_master3\\:port3,peer_master4\\:port4,peer_master5\\:port5]:principal:additionParams
*/
- private final class HAClusterRoleManager implements Runnable {
- private final String jdbcUrl;
- private final Properties properties;
- private NodeCache cache;
-
- /**
- * Constructor which creates and starts the ZK watcher.
- *
- * @param jdbcUrl JDBC url without jdbc:phoenix prefix which may be host:port:/hbase format
- * @param properties The properties defining ZK client timeouts and retries
- */
- HAClusterRoleManager(String jdbcUrl, Properties properties) {
- this.jdbcUrl = jdbcUrl;
- this.properties = properties;
+ public static String getJDBCHAUrl(String url1, String url2, HAURLInfo haURLInfo) {
+ StringBuilder sb = new StringBuilder();
+ sb.append(PhoenixRuntime.JDBC_PROTOCOL_RPC).append(JDBC_PROTOCOL_SEPARATOR);
+ sb.append("[").append(url1).append("|").append(url2).append("]");
+ if (haURLInfo != null) {
+ if (ObjectUtils.anyNotNull(haURLInfo.getPrincipal(), haURLInfo.getAdditionalJDBCParams())) {
+ sb.append(haURLInfo.getPrincipal() == null ? JDBC_PROTOCOL_SEPARATOR
+ : JDBC_PROTOCOL_SEPARATOR + haURLInfo.getPrincipal());
+ }
+ if (ObjectUtils.anyNotNull(haURLInfo.getAdditionalJDBCParams())) {
+ sb.append(JDBC_PROTOCOL_SEPARATOR).
+ append(haURLInfo.getAdditionalJDBCParams());
+ }
}
+ return sb.toString();
+ }
- @Override
- public void run() {
- final String zpath = info.getZkPath();
- while (!Thread.currentThread().isInterrupted()) {
- try {
- cache = new NodeCache(getCurator(jdbcUrl, properties), zpath);
- cache.getListenable().addListener(this::nodeChanged);
- cache.start();
- return; // return after building the initial node cache
- } catch (InterruptedException e) {
- LOG.warn("HA cluster role manager thread for '{}' is interrupted, exiting",
- jdbcUrl, e);
- break;
- } catch (Throwable t) {
- LOG.warn("Fail to start node cache on '{}' for '{}'. Retry", jdbcUrl, zpath, t);
- try {
- // TODO: do better than fixed time sleep
- Thread.sleep(1_000);
- } catch (InterruptedException e) {
- LOG.warn("HA cluster role manager thread for '{}' is interrupted, exiting",
- jdbcUrl, e);
- break;
- }
- }
+
+ private static void throwMalFormedConnectionUrlException(String message) throws SQLException {
+ throw new SQLExceptionInfo.Builder(SQLExceptionCode.MALFORMED_CONNECTION_URL)
+ .setMessage(message)
+ .build()
+ .buildException();
+ }
+
+
+
+ /**
+ * Method to get ClusterRoleRecord from RegionServer Endpoints from either of the clusters.
+ * @return ClusterRoleRecord from the first available cluster
+ * @throws SQLException if there is an error getting the ClusterRoleRecord
+ */
+ private ClusterRoleRecord getClusterRoleRecordFromEndpoint() throws SQLException {
+ long pollerInterval = Long.parseLong(properties.getProperty(PHOENIX_HA_CRR_POLLER_INTERVAL_MS_KEY,
+ config.get(PHOENIX_HA_CRR_POLLER_INTERVAL_MS_KEY, PHOENIX_HA_CRR_POLLER_INTERVAL_MS_DEFAULT)));
+
+ //Get the CRR via RSEndpoint for cluster 1
+ try {
+ return GetClusterRoleRecordUtil.fetchClusterRoleRecord(info.getUrl1(), info.getName(), this, pollerInterval, properties);
+ } catch (Exception e) {
+ //Got exception from cluster 1 when trying to get CRR, try cluster 2
+ return GetClusterRoleRecordUtil.fetchClusterRoleRecord(info.getUrl2(), info.getName(), this, pollerInterval, properties);
+ }
+ }
+
+ /**
+ * Refresh the cluster role record for this HA group.
+ * <p>
+ * This method will get the new ClusterRoleRecord from RegionServer Endpoints and apply it to the HA group.
+ * and it will call to transition the Policy's connections according to the new record.
+ * <p>
+ * This method uses a read-write lock pattern to allow multiple concurrent callers to check if a refresh
+ * is needed while ensuring only one thread performs the actual refresh operation at a time. This improves
+ * throughput by avoiding blocking all callers during the quick check phase.
+ *
+ * @return true if the new record is set as current one; false otherwise
+ * @throws SQLException if there is an error getting the ClusterRoleRecord
+ */
+ public boolean refreshClusterRoleRecord(boolean forceRefresh) throws SQLException {
+ // Allow concurrent reads to return fast in case refresh is not needed
+ readLock.lock();
+ try {
+ if (!forceRefresh && !shouldRefreshRoleRecord()) {
+ return true;
}
+ } finally {
+ readLock.unlock();
}
- /**
- * Call back functions when a cluster role change is notified by this ZK cluster.
- */
- private void nodeChanged() {
- byte[] data = cache.getCurrentData().getData();
- Optional<ClusterRoleRecord> newRecordOptional = ClusterRoleRecord.fromJson(data);
- if (!newRecordOptional.isPresent()) {
- LOG.error("Fail to deserialize new record; keep current record {}", roleRecord);
- return;
+ // Take a write lock to apply the refresh of roleRecord
+ writeLock.lock();
+ try {
+ // Re-check the condition under the write lock to prevent race conditions.
+ // Another thread might have already done the refresh while we were waiting for the
+ // write lock.
+ if (!forceRefresh && !shouldRefreshRoleRecord()) {
+ return true;
+ }
+
+ ClusterRoleRecord newRoleRecord = getClusterRoleRecordFromEndpoint();
+ if (roleRecord == null) {
+ roleRecord = newRoleRecord;
+ lastClusterRoleRecordRefreshTime = System.currentTimeMillis();
+ state = State.READY;
+ LOG.info("HA group {} is now in {} state after getting initial V{} role record: {}",
+ info, state, roleRecord.getVersion(), roleRecord);
+ LOG.debug("HA group {} is ready", this);
+ return true;
}
- ClusterRoleRecord newRecord = newRecordOptional.get();
- LOG.info("HA group {} got a record from cluster {}: {}", info.name, jdbcUrl, newRecord);
- if (applyClusterRoleRecord(newRecord)) {
- LOG.info("Successfully apply new cluster role record from cluster '{}', "
- + "new record: {}", jdbcUrl, newRecord);
- roleManagerLatch.countDown();
+ //Check if newRoleRecord has same Info (HAGroupName and Policy)
+ if (!roleRecord.hasSameInfo(newRoleRecord)) {
+ LOG.error("New record {} has different HA group information (haGroupName/Policy) from" +
+ " old record {}, which is not expected", newRoleRecord, roleRecord);
+ return false;
+ }
+
+ //Check if newRoleRecord is actually new
+ if (roleRecord.equals(newRoleRecord)) {
+ LOG.debug("New Role Record is same as current RoleRecord, no need to refresh {}",
+ roleRecord.toString());
+ lastClusterRoleRecordRefreshTime = System.currentTimeMillis();
+ return true;
}
+
+ final ClusterRoleRecord oldRecord = roleRecord;
+ state = State.IN_TRANSITION;
+ LOG.info("HA group {} is in {} to set V{} record", info, state, newRoleRecord.getVersion());
+ Future<?> future = crrChangedExecutor.submit(() -> {
+ try {
+ roleRecord.getPolicy().transitClusterRoleRecord(this, roleRecord, newRoleRecord);
+ } catch (SQLException e) {
+ throw new CompletionException(e);
+ }
+ });
+
+ String transitionTimeoutProp = properties.getProperty(
+ PHOENIX_HA_TRANSITION_TIMEOUT_MS_KEY, config.get(
+ PHOENIX_HA_TRANSITION_TIMEOUT_MS_KEY));
+ long maxTransitionTimeMs = StringUtils.isNotEmpty(transitionTimeoutProp)
+ ? Long.parseLong(transitionTimeoutProp)
+ : PHOENIX_HA_TRANSITION_TIMEOUT_MS_DEFAULT;
+ try {
+ future.get(maxTransitionTimeMs, TimeUnit.MILLISECONDS);
+ } catch (InterruptedException ie) {
+ LOG.error("Got interrupted when transiting cluster roles for HA group {}", info, ie);
+ future.cancel(true);
+ Thread.currentThread().interrupt();
+ return false;
+ } catch (ExecutionException | TimeoutException e) {
+ LOG.error("HA group {} failed to transit cluster roles per policy {} to new " +
+ "record {}", info, roleRecord.getPolicy(), newRoleRecord, e);
+ //Rethrow the Role transitions not allowed exceptions
+ if (e.getCause() != null && e.getCause().getCause() != null) {
+ if (e.getCause().getCause() instanceof SQLException && ((SQLException)e
+ .getCause().getCause()).getErrorCode() == SQLExceptionCode
+ .HA_ROLE_TRANSITION_NOT_ALLOWED.getErrorCode()) {
+ state = State.READY;
+ throw (SQLException)e.getCause().getCause();
+ }
+ }
+ // Calling back HA policy function for cluster switch is conducted with best effort.
+ // HA group continues transition when its HA policy fails to deal with context switch
+ // (e.g. to close existing connections)
+ // The goal here is to gain higher availability even though existing resources against
+ // previous ACTIVE cluster may have not been closed cleanly.
+ }
+ //Update the role record and the last refresh time
+ roleRecord = newRoleRecord;
+ lastClusterRoleRecordRefreshTime = System.currentTimeMillis();
+ state = State.READY;
+ LOG.info("HA group {} is in {} state, Old: {}, new: {}", info, state, oldRecord,
+ roleRecord);
+ LOG.debug("HA group is ready: {}", this);
+ return true;
+ } finally {
+ writeLock.unlock();
}
+ }
+ /**
+ * Check if we should refresh the RoleRecord based on cache frequency if the cacheAge is
+ * greater than clusterRoleRecordCacheFrequency, then return true
+ */
+ public boolean shouldRefreshRoleRecord() {
+ if (roleRecord == null) {
+ return true; // Always refresh if no role record
+ }
+ long cacheAge = System.currentTimeMillis() - lastClusterRoleRecordRefreshTime;
+ return cacheAge >= clusterRoleRecordCacheFrequency;
}
-}
\ No newline at end of file
+
+}
diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HighAvailabilityPolicy.java b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HighAvailabilityPolicy.java
index 727cb4b680..e4770284c6 100644
--- a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HighAvailabilityPolicy.java
+++ b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HighAvailabilityPolicy.java
@@ -19,8 +19,8 @@
package org.apache.phoenix.jdbc;
import static org.apache.phoenix.jdbc.ClusterRoleRecord.ClusterRole.ACTIVE;
-import static org.apache.phoenix.jdbc.ClusterRoleRecord.ClusterRole.ACTIVE_TO_STANDBY;
import static org.apache.phoenix.jdbc.ClusterRoleRecord.ClusterRole.STANDBY;
+import static org.apache.phoenix.jdbc.ClusterRoleRecord.ClusterRole.STANDBY_TO_ACTIVE;
import java.sql.Connection;
import java.sql.SQLException;
@@ -50,18 +50,24 @@ public enum HighAvailabilityPolicy {
/**
* Cluster Role Transitions for Failover High Availability Policy, Here we are trying to
- * close connections late to allow existing reads to continue during Failover.
+ * close connections late to allow existing reads to continue during Failover. Only allowed
+ * transitions are mentioned below, check @{link HAGroupStoreRecord.HAGroupState} for more details.
* ACTIVE --> ACTIVE_TO_STANDBY (Doing Nothing as we are in process of moving the current
* to STANDBY and at this step we are blocking write to drain replication this allows
- * us to continue existing reads to continue)
- * ACTIVE|ACTIVE_TO_STANDBY --> STANDBY (Closing all current connections)
+ * us to continue existing/new reads to continue, so FailoverConnections are allowed)
+ * ACTIVE_TO_STANDBY --> STANDBY (Closing all current connections)
+ * ACTIVE_TO_STANDBY --> ACTIVE (NOOP on client side, server will resume Mutations)
+ * STANDBY --> STANDBY_TO_ACTIVE (NOOP on client side)
+ * STANDBY_TO_ACTIVE --> STANDBY (NOOP on client side)
+ * STANDBY_TO_ACTIVE --> ACTIVE (Invalidate CQSI as connections has been closed and now being cleared)
*
+ * other transitions are not allowed and restricted on Server side, but as client can have
+ * stale roleRecord and it can miss some transitions for ex:- it has ACTIVE, and it gets to
+ * know that it has stale info though that HAGroup has transitioned to STANDBY already.
+ * ACTIVE --> STANDBY (Closing all current connections) (A --> ATS --> S)
* STANDBY --> ACTIVE (Invalidate CQSI as connections has been closed and now being cleared)
- * STANDBY --> ACTIVE_TO_STANDBY (Should not be a case but in case of failover Rollback we
- * are going back to ACTIVE_TO_STANDBY state invalidating earlier here as connections
- * are already closed)
- * ACTIVE_TO_STANDBY --> ACTIVE (Doing nothing as we have already invalidated cqsi when we
- * transitioned from STANDBY to ACTIVE_TO_STANDBY)
+ * (S --> STA --> A)
+ *
* @param haGroup The high availability (HA) group
* @param oldRecord The older cluster role record cached in this client for the given HA group
* @param newRecord New cluster role record read from one ZooKeeper cluster znode
@@ -70,49 +76,39 @@ public enum HighAvailabilityPolicy {
@Override
void transitClusterRole(HighAvailabilityGroup haGroup, ClusterRoleRecord oldRecord,
ClusterRoleRecord newRecord) throws SQLException {
- if (newRecord.getRole1() == ACTIVE && newRecord.getRole2() == ACTIVE) {
- LOG.warn("Both cluster roles are ACTIVE which is invalid state for FailoverPolicy" +
- "Doing nothing for Cluster Role Change");
- return;
+
+ //Both Roles can't be active at same time, as it is a failover policy.
+ if (newRecord.getRole1().isActive() && newRecord.getRole2().isActive()) {
+ throw new SQLExceptionInfo.Builder(SQLExceptionCode.HA_ROLE_TRANSITION_NOT_ALLOWED)
+ .setMessage("Failover High Availability Policy doesn't allow both roles to be active at same time.")
+ .build().buildException();
}
- if (oldRecord.getRole1() == ACTIVE &&
- (newRecord.getRole1() == STANDBY || newRecord.getRole1() == ACTIVE_TO_STANDBY)) {
+
+ if (oldRecord.getRole1().isActive() &&
+ (newRecord.getRole1() == STANDBY)) {
transitStandby(haGroup, oldRecord.getUrl1(), oldRecord.getRegistryType(),
newRecord.getRole1());
}
- if (oldRecord.getRole2() == ACTIVE &&
- (newRecord.getRole2() == STANDBY || newRecord.getRole2() == ACTIVE_TO_STANDBY)) {
+ if (oldRecord.getRole2().isActive() &&
+ (newRecord.getRole2() == STANDBY)) {
transitStandby(haGroup, oldRecord.getUrl2(), oldRecord.getRegistryType(),
newRecord.getRole2());
}
- if (oldRecord.getRole1() != ACTIVE && newRecord.getRole1() == ACTIVE) {
+ if ((oldRecord.getRole1() == STANDBY_TO_ACTIVE || oldRecord.getRole1() == STANDBY) &&
+ (newRecord.getRole1() == ACTIVE)) {
transitActive(haGroup, oldRecord.getUrl1(), oldRecord.getRegistryType());
}
- if (oldRecord.getRole2() != ACTIVE && newRecord.getRole2() == ACTIVE) {
+ if ((oldRecord.getRole2() == STANDBY_TO_ACTIVE || oldRecord.getRole2() == STANDBY) &&
+ (newRecord.getRole2() == ACTIVE)) {
transitActive(haGroup, oldRecord.getUrl2(), oldRecord.getRegistryType());
}
}
- @Override
- void transitRoleRecordRegistry(HighAvailabilityGroup haGroup, ClusterRoleRecord oldRecord,
- ClusterRoleRecord newRecord) throws SQLException {
- Optional<String> activeUrl = oldRecord.getActiveUrl();
-
- //Close connections for Active HBase cluster as there is a change in Registry Type
- if (activeUrl.isPresent()) {
- LOG.info("Cluster {} has a change in registryType in HA group {}, now closing "
- + "all its connections", activeUrl.get(), oldRecord.getRegistryType());
- closeConnections(haGroup, activeUrl.get(), oldRecord.getRegistryType());
- invalidateCQSIs(haGroup, activeUrl.get(), oldRecord.getRegistryType());
- } else {
- LOG.info("None of the cluster in HA Group {} is active", haGroup);
- }
- }
-
/**
* For FAILOVER Policy if there is a change in active url or there is no new active url then close all connections.
* In below examples only a portion of CRR is shown, url1, url2 are current urls present in
* clusterRoleRecord and url3, url4 are new urls in clusterRoleRecord
+ * HERE ACTIVE means ACTIVE or ACTIVE_TO_STANDBY
* (url1, ACTIVE, url2, STANDBY) --> (url1, ACTIVE, url3, STANDBY) //Nothing is needed as only Standby url changed
* (url1, ACTIVE, url2, STANDBY) --> (url3, ACTIVE, url2, STANDBY) //Active url change close connections
* (url1, ACTIVE, url2, STANDBY) --> (url3, ACTIVE, url4, STANDBY) //Active url change close connections
@@ -153,6 +149,15 @@ public enum HighAvailabilityPolicy {
}
}
+ /**
+ * Function to call in case Role is changing from ACTIVE like role to STANDBY role, to close
+ * cqsi and all connections related to it.
+ * @param haGroup The high availability (HA) group
+ * @param url The url for which cqsi needs to be closed
+ * @param registryType The registry Type of connections affiliate to cqsi
+ * @param newRole url is moving to which new role after transition.
+ * @throws SQLException if fails to invalidate the cqs
+ */
private void transitStandby(HighAvailabilityGroup haGroup, String url,
ClusterRoleRecord.RegistryType registryType,
ClusterRoleRecord.ClusterRole newRole) throws SQLException {
@@ -162,6 +167,14 @@ public enum HighAvailabilityPolicy {
closeConnections(haGroup, url, registryType);
}
+ /**
+ * Function to call in case Role is changing from STANDBY like role to ACTIVE role, to
+ * invalidate cqsi
+ * @param haGroup The high availability (HA) group
+ * @param url The url for which cqsi needs to be invalidated
+ * @param registryType The registry Type of connections affiliate to cqsi
+ * @throws SQLException if fails to invalidate the cqs
+ */
private void transitActive(HighAvailabilityGroup haGroup, String url,
ClusterRoleRecord.RegistryType registryType) throws SQLException {
// Invalidate CQS cache if any that has been closed but has not been cleared
@@ -189,27 +202,20 @@ public enum HighAvailabilityPolicy {
}
}
+ /**
+ * Call-back function when a cluster role record transition is detected in the
+ * high availability group for PARALLEL policy.
+ * TODO:- We are not doing anything for change in role, but we should handle OFFLINE cases
+ * @param haGroup The high availability (HA) group
+ * @param oldRecord The older cluster role record cached in this client for the given HA group
+ * @param newRecord New cluster role record read from one ZooKeeper cluster znode
+ */
@Override
void transitClusterRole(HighAvailabilityGroup haGroup, ClusterRoleRecord oldRecord,
ClusterRoleRecord newRecord) {
//No Action for Parallel Policy
}
-
- @Override
- void transitRoleRecordRegistry(HighAvailabilityGroup haGroup, ClusterRoleRecord oldRecord,
- ClusterRoleRecord newRecord) throws SQLException {
- //Close connections of both clusters as there is a change in registryType
- LOG.info("Cluster {} and {} has a change in registryType in HA group {}, now closing all its connections",
- oldRecord.getUrl1(), oldRecord.getUrl2() , oldRecord.getRegistryType());
- //close connections for cluster 1 and invalidate cqsi
- closeConnections(haGroup, oldRecord.getUrl1(), oldRecord.getRegistryType());
- invalidateCQSIs(haGroup, oldRecord.getUrl1(), oldRecord.getRegistryType());
- //close connections for cluster 2 and invalidate cqsi
- closeConnections(haGroup, oldRecord.getUrl2(), oldRecord.getRegistryType());
- invalidateCQSIs(haGroup, oldRecord.getUrl2(), oldRecord.getRegistryType());
- }
-
/**
* For PARALLEL policy if there is a change in any of the url then ParallelPhoenixConnection
* objects are invalid
@@ -267,7 +273,7 @@ public enum HighAvailabilityPolicy {
// CQS is closed, but it is not invalidated from global cache in PhoenixDriver
// so that any new connection will get error instead of creating a new CQS,
// CQS entry will stay in map until the cache expires and repopulated in cases
- // of URL and registryType changes
+ // of URL changes
LOG.info("Closing CQS after clusterRoleRecord change for '{}'", url);
cqs.close();
LOG.info("Successfully closed CQS after clusterRoleRecord change for '{}'", url);
@@ -318,16 +324,12 @@ public enum HighAvailabilityPolicy {
*/
public void transitClusterRoleRecord(HighAvailabilityGroup haGroup, ClusterRoleRecord oldRecord,
ClusterRoleRecord newRecord) throws SQLException {
- if (oldRecord.getRegistryType() != newRecord.getRegistryType()) {
- transitRoleRecordRegistry(haGroup, oldRecord, newRecord);
- } else if (!oldRecord.getUrl1().equals(newRecord.getUrl1()) ||
+ if (!oldRecord.getUrl1().equals(newRecord.getUrl1()) ||
!oldRecord.getUrl2().equals(newRecord.getUrl2())) {
- //If registryType is not changing then we need to check if any of the url is changing
- //as change in registryType closes all the current connections
+ //If there is a change in url then we need to handle the transitions.
transitClusterUrl(haGroup, oldRecord, newRecord);
} else {
- //If both registryType and url is not changing then we need to check if there is a
- //role transition.
+ //If url is not changing then we need to check if there is a role transition.
transitClusterRole(haGroup, oldRecord, newRecord);
}
}
@@ -343,18 +345,6 @@ public enum HighAvailabilityPolicy {
abstract void transitClusterRole(HighAvailabilityGroup haGroup, ClusterRoleRecord oldRecord,
ClusterRoleRecord newRecord) throws SQLException;
-
- /**
- * Call-back function when only registry transition is detected in the high availability group or clusterRoleRecord.
- *
- * @param haGroup The high availability (HA) group
- * @param oldRecord The older cluster role record cached in this client for the given HA group
- * @param newRecord New cluster role record read from one ZooKeeper cluster znode
- * @throws SQLException if fails to handle the cluster role record registry transition
- */
- abstract void transitRoleRecordRegistry(HighAvailabilityGroup haGroup, ClusterRoleRecord oldRecord,
- ClusterRoleRecord newRecord) throws SQLException;
-
/**
* Call-back function when only url transition is detected in the high availability group or clusterRoleRecord.
*
diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HighAvailabilityUtil.java b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HighAvailabilityUtil.java
new file mode 100644
index 0000000000..e2f9e6c3de
--- /dev/null
+++ b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/HighAvailabilityUtil.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.phoenix.jdbc;
+
+import org.apache.hadoop.hbase.client.RetriesExhaustedWithDetailsException;
+import org.apache.phoenix.exception.StaleClusterRoleRecordException;
+
+/**
+ * High Availability Utility class.
+ */
+public class HighAvailabilityUtil {
+
+ public static boolean isStaleClusterRoleRecordExceptionExistsInThrowable(Throwable e) {
+ if (isGivenThrowableStaleException(e)) {
+ return true;
+ }
+
+ if (e instanceof RetriesExhaustedWithDetailsException) {
+ for (Throwable t : ((RetriesExhaustedWithDetailsException) e).getCauses()) {
+ if (isGivenThrowableStaleException(t)){
+ return true;
+ }
+ }
+ }
+
+ if (e.getCause() != null) {
+ return isStaleClusterRoleRecordExceptionExistsInThrowable(e.getCause());
+ }
+
+ return false;
+ }
+
+ private static boolean isGivenThrowableStaleException(Throwable t) {
+ return t instanceof StaleClusterRoleRecordException;
+ }
+}
diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/PhoenixConnection.java b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/PhoenixConnection.java
index 53da422dba..70f29b845f 100644
--- a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/PhoenixConnection.java
+++ b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/PhoenixConnection.java
@@ -141,6 +141,7 @@ import org.apache.phoenix.util.SQLCloseable;
import org.apache.phoenix.util.SQLCloseables;
import org.apache.phoenix.util.SchemaUtil;
import org.apache.phoenix.util.VarBinaryFormatter;
+import org.xbill.DNS.tools.primary;
/**
*
@@ -189,7 +190,10 @@ public class PhoenixConnection implements MetaDataMutated, SQLCloseable, Phoenix
private LogLevel auditLogLevel;
private Double logSamplingRate;
private String sourceOfOperation;
+ @Nullable //Only available for connection which is part of HA Connections
private String haGroupName;
+ @Nullable //Only available for connection which is part of HA Connections
+ private HighAvailabilityGroup haGroup;
private volatile SQLException reasonForClose;
private static final String[] CONNECTION_PROPERTIES;
@@ -220,7 +224,7 @@ public class PhoenixConnection implements MetaDataMutated, SQLCloseable, Phoenix
this(connection.getQueryServices(), connection.getURL(), connection
.getClientInfo(), connection
.getMutationState(), isDescRowKeyOrderUpgrade,
- isRunningUpgrade, connection.buildingIndex, true);
+ isRunningUpgrade, connection.buildingIndex, true, null);
this.isAutoCommit = connection.isAutoCommit;
this.isAutoFlush = connection.isAutoFlush;
this.sampler = connection.sampler;
@@ -237,7 +241,7 @@ public class PhoenixConnection implements MetaDataMutated, SQLCloseable, Phoenix
this(connection.getQueryServices(), connection.getURL(), connection
.getClientInfo(), mutationState,
connection.isDescVarLengthRowKeyUpgrade(), connection
- .isRunningUpgrade(), connection.buildingIndex, true);
+ .isRunningUpgrade(), connection.buildingIndex, true, null);
}
public PhoenixConnection(PhoenixConnection connection, long scn)
@@ -248,7 +252,7 @@ public class PhoenixConnection implements MetaDataMutated, SQLCloseable, Phoenix
public PhoenixConnection(PhoenixConnection connection, Properties props) throws SQLException {
this(connection.getQueryServices(), connection.getURL(), props, connection
.getMutationState(), connection.isDescVarLengthRowKeyUpgrade(),
- connection.isRunningUpgrade(), connection.buildingIndex, true);
+ connection.isRunningUpgrade(), connection.buildingIndex, true, null);
this.isAutoCommit = connection.isAutoCommit;
this.isAutoFlush = connection.isAutoFlush;
this.sampler = connection.sampler;
@@ -257,7 +261,12 @@ public class PhoenixConnection implements MetaDataMutated, SQLCloseable, Phoenix
public PhoenixConnection(ConnectionQueryServices services, String url,
Properties info) throws SQLException {
- this(services, url, info, null, false, false, false, false);
+ this(services, url, info, null, false, false, false, false, null);
+ }
+
+ public PhoenixConnection(ConnectionQueryServices services, String url,
+ Properties info, HighAvailabilityGroup haGroup) throws SQLException {
+ this(services, url, info, null, false, false, false, false, haGroup);
}
public PhoenixConnection(PhoenixConnection connection,
@@ -265,13 +274,13 @@ public class PhoenixConnection implements MetaDataMutated, SQLCloseable, Phoenix
throws SQLException {
this(services, connection.url, info, null,
connection.isDescVarLengthRowKeyUpgrade(), connection
- .isRunningUpgrade(), connection.buildingIndex, true);
+ .isRunningUpgrade(), connection.buildingIndex, true, null);
}
private PhoenixConnection(ConnectionQueryServices services, String url,
Properties info, MutationState mutationState,
boolean isDescVarLengthRowKeyUpgrade, boolean isRunningUpgrade,
- boolean buildingIndex, boolean isInternalConnection) throws SQLException {
+ boolean buildingIndex, boolean isInternalConnection, HighAvailabilityGroup haGroup) throws SQLException {
this.url = url;
this.isDescVarLengthRowKeyUpgrade = isDescVarLengthRowKeyUpgrade;
this.isInternalConnection = isInternalConnection;
@@ -285,6 +294,9 @@ public class PhoenixConnection implements MetaDataMutated, SQLCloseable, Phoenix
// Copy so client cannot change
this.info = PropertiesUtil.deepCopy(info);
+ //get HAGroupName from the connection info
+ this.haGroupName = info.getProperty(HighAvailabilityGroup.PHOENIX_HA_GROUP_ATTR);
+ this.haGroup = haGroup;
final PName tenantId = JDBCUtil.getTenantId(url, info);
if (this.info.isEmpty() && tenantId == null) {
this.services = services;
@@ -1265,6 +1277,15 @@ public class PhoenixConnection implements MetaDataMutated, SQLCloseable, Phoenix
return this.haGroupName;
}
+ /**
+ * Get the HAGroup for which this connection is part of, This will return null for Single
+ * Cluster Connections
+ * @return The HAGroup for which this connection is part of.
+ */
+ public HighAvailabilityGroup getHAGroup() {
+ return this.haGroup;
+ }
+
@Override
public void setCatalog(String catalog) throws SQLException {
checkOpen();
diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
index c40f4521e1..4eb86f2665 100644
--- a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
+++ b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
@@ -477,7 +477,8 @@ public class PhoenixDatabaseMetaData implements DatabaseMetaData {
public static final String PARENT_PARTITION_START_TIME = "PARENT_PARTITION_START_TIME";
public static final String HA_GROUP_NAME = "HA_GROUP_NAME";
public static final String POLICY = "POLICY";
- public static final String VERSION = "VERSION";
+ public static final String VERSION_CLUSTER_1 = "VERSION_CLUSTER_1";
+ public static final String VERSION_CLUSTER_2 = "VERSION_CLUSTER_2";
public static final String ZK_URL_1 = "ZK_URL_1";
public static final String ZK_URL_2 = "ZK_URL_2";
public static final String CLUSTER_URL_1 = "CLUSTER_URL_1";
diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/PhoenixHAAdmin.java b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/PhoenixHAAdmin.java
index 6735288734..1b3d8212c8 100644
--- a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/PhoenixHAAdmin.java
+++ b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/PhoenixHAAdmin.java
@@ -151,6 +151,11 @@ public class PhoenixHAAdmin implements Closeable {
return String.format("%s:%d:%s", localZkQuorum, port, localZkRoot);
}
+ public static String getLocalMasterUrl(Configuration conf) {
+ String local = conf.get(HConstants.MASTER_ADDRS_KEY).replaceAll(":", "\\\\:");
+ return local;
+ }
+
/**
* Gets curator from the cache if available otherwise calls into getCurator to make it.
*/
diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/PhoenixHAAdminTool.java b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/PhoenixHAAdminTool.java
index d4fed4e1f0..458386482b 100644
--- a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/PhoenixHAAdminTool.java
+++ b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/PhoenixHAAdminTool.java
@@ -47,6 +47,8 @@ import org.slf4j.LoggerFactory;
/**
* The command line tool to manage high availability (HA) groups and their cluster roles.
*/
+@Deprecated //As we are moving ZKLess for HA, we will need to update or create new Tool from scratch
+//for Admin like commands used by this tool
public class PhoenixHAAdminTool extends Configured implements Tool {
// Following are return value of this tool. We need this to be very explicit because external
// system calling this tool may need to retry, alert or audit the operations of cluster roles.
diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/PhoenixResultSet.java b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/PhoenixResultSet.java
index f6b2f20969..9e30b24dee 100644
--- a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/PhoenixResultSet.java
+++ b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/PhoenixResultSet.java
@@ -907,7 +907,7 @@ public class PhoenixResultSet implements PhoenixMonitoredResultSet, SQLCloseable
if (rowProjectorWithDynamicCols != null) {
rowProjectorWithDynamicCols.reset();
}
- } catch (RuntimeException | SQLException e) {
+ } catch (Exception e) {
// FIXME: Expression.evaluate does not throw SQLException
// so this will unwrap throws from that.
queryLogger.log(QueryLogInfo.QUERY_STATUS_I, QueryStatus.FAILED.toString());
diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java
index fa4cd22f5f..d378978634 100644
--- a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java
+++ b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java
@@ -63,6 +63,7 @@ import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
import java.util.Properties;
import java.util.Set;
@@ -376,6 +377,31 @@ public class PhoenixStatement implements PhoenixMonitoredStatement, SQLCloseable
// not the projected table, so plan.getContext().getResolver().getTables() won't work.
if (plan.getContext().getScanRanges().isPointLookup()) {
pointLookup = true;
+ } else if (connection.getHAGroup() != null &&
+ connection.getHAGroup().getRoleRecord() != null &&
+ connection.getHAGroup().getRoleRecord().getPolicy()
+ == HighAvailabilityPolicy.FAILOVER &&
+ connection.getHAGroup().shouldRefreshRoleRecord()) {
+ //Refresh the cluster role record for failover policy if refresh
+ //interval is expired
+ LOGGER.debug("Refreshing cluster role record for before " +
+ "executing query");
+ Optional<String> activeUrl = connection.getHAGroup()
+ .getRoleRecord().getActiveUrl();
+ if (!connection.getHAGroup().refreshClusterRoleRecord(false)) {
+ LOGGER.error("Error while refreshing HAGroup");
+ }
+ if (!activeUrl.equals(connection.getHAGroup().getRoleRecord()
+ .getActiveUrl())) {
+ throw new SQLExceptionInfo.Builder(SQLExceptionCode
+ .FAILOVER_IN_PROGRESS)
+ .setMessage(String.format("Operation failed " +
+ "because failover is in " +
+ "progress for HAGroup %s",
+ connection.getHAGroup()))
+ .build()
+ .buildException();
+ }
}
Iterator<TableRef> tableRefs = plan.getSourceRefs().iterator();
connection.getMutationState().sendUncommitted(tableRefs);
diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/RPCConnectionInfo.java b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/RPCConnectionInfo.java
index 49aaa22617..d65a23d7ba 100644
--- a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/RPCConnectionInfo.java
+++ b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/RPCConnectionInfo.java
@@ -37,7 +37,7 @@ public class RPCConnectionInfo extends AbstractRPCConnectionInfo {
// We may be compiling with an older HBase version, which does not even have
// RpcConnectionRegistry
public static final String BOOTSTRAP_NODES = "hbase.client.bootstrap.servers";
- private static final String RPC_REGISTRY_CLASS_NAME =
+ public static final String RPC_REGISTRY_CLASS_NAME =
"org.apache.hadoop.hbase.client.RpcConnectionRegistry";
protected RPCConnectionInfo(boolean isConnectionless, String principal, String keytab,
diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java b/phoenix-core-client/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java
index 528534e0cb..7e41e7a5f4 100644
--- a/phoenix-core-client/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java
+++ b/phoenix-core-client/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java
@@ -40,6 +40,7 @@ import org.apache.phoenix.coprocessorclient.MetaDataProtocol.MetaDataMutationRes
import org.apache.phoenix.exception.SQLExceptionInfo;
import org.apache.phoenix.execute.MutationState;
import org.apache.phoenix.hbase.index.util.KeyValueBuilder;
+import org.apache.phoenix.jdbc.HighAvailabilityGroup;
import org.apache.phoenix.jdbc.PhoenixConnection;
import org.apache.phoenix.log.ConnectionLimiter;
import org.apache.phoenix.log.QueryLoggerDisruptor;
@@ -162,6 +163,7 @@ public interface ConnectionQueryServices extends QueryServices, MetaDataMutated
int queryTimeout) throws SQLException;
public PhoenixConnection connect(String url, Properties info) throws SQLException;
+ public PhoenixConnection connect(String url, Properties info, HighAvailabilityGroup haGroup) throws SQLException;
/**
* @param tableTimestamp timestamp of table if its present in the client side cache
diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java b/phoenix-core-client/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
index d2efaf1460..0d3c2d3b7f 100644
--- a/phoenix-core-client/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
+++ b/phoenix-core-client/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
@@ -247,6 +247,7 @@ import org.apache.phoenix.iterate.TableResultIterator;
import org.apache.phoenix.iterate.TableResultIterator.RenewLeaseStatus;
import org.apache.phoenix.jdbc.AbstractRPCConnectionInfo;
import org.apache.phoenix.jdbc.ConnectionInfo;
+import org.apache.phoenix.jdbc.HighAvailabilityGroup;
import org.apache.phoenix.jdbc.PhoenixConnection;
import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
import org.apache.phoenix.jdbc.RPCConnectionInfo;
@@ -1238,11 +1239,16 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
@Override
public PhoenixConnection connect(String url, Properties info) throws SQLException {
+ return this.connect(url, info, null);
+ }
+
+ @Override
+ public PhoenixConnection connect(String url, Properties info, HighAvailabilityGroup haGroup) throws SQLException {
checkClosed();
throwConnectionClosedIfNullMetaData();
validateConnectionProperties(info);
- return new PhoenixConnection(this, url, info);
+ return new PhoenixConnection(this, url, info, haGroup);
}
private ColumnFamilyDescriptor generateColumnFamilyDescriptor(Pair<byte[],Map<String,Object>> family, PTableType tableType) throws SQLException {
diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java b/phoenix-core-client/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
index cfce561b0a..62f1533c3e 100644
--- a/phoenix-core-client/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
+++ b/phoenix-core-client/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
@@ -59,6 +59,7 @@ import org.apache.phoenix.execute.MutationState;
import org.apache.phoenix.hbase.index.util.GenericKeyValueBuilder;
import org.apache.phoenix.hbase.index.util.KeyValueBuilder;
import org.apache.phoenix.jdbc.ConnectionInfo;
+import org.apache.phoenix.jdbc.HighAvailabilityGroup;
import org.apache.phoenix.jdbc.PhoenixConnection;
import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
import org.apache.phoenix.log.QueryLoggerDisruptor;
@@ -318,6 +319,11 @@ public class ConnectionlessQueryServicesImpl extends DelegateQueryServices imple
return new PhoenixConnection(this, url, info);
}
+ @Override
+ public PhoenixConnection connect(String url, Properties info, HighAvailabilityGroup haGroup) throws SQLException {
+ return new PhoenixConnection(this, url, info, haGroup);
+ }
+
@Override
public MetaDataMutationResult getTable(PName tenantId, byte[] schemaBytes, byte[] tableBytes, long tableTimestamp, long clientTimestamp) throws SQLException {
// Return result that will cause client to use it's own metadata instead of needing
diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java b/phoenix-core-client/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java
index 07ff39a781..07b56ef055 100644
--- a/phoenix-core-client/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java
+++ b/phoenix-core-client/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java
@@ -39,6 +39,7 @@ import org.apache.phoenix.coprocessorclient.InvalidateServerMetadataCacheRequest
import org.apache.phoenix.coprocessorclient.MetaDataProtocol.MetaDataMutationResult;
import org.apache.phoenix.execute.MutationState;
import org.apache.phoenix.hbase.index.util.KeyValueBuilder;
+import org.apache.phoenix.jdbc.HighAvailabilityGroup;
import org.apache.phoenix.jdbc.PhoenixConnection;
import org.apache.phoenix.log.ConnectionLimiter;
import org.apache.phoenix.log.QueryLoggerDisruptor;
@@ -147,6 +148,11 @@ public class DelegateConnectionQueryServices extends DelegateQueryServices imple
return getDelegate().connect(url, info);
}
+ @Override
+ public PhoenixConnection connect(String url, Properties info, HighAvailabilityGroup haGroup) throws SQLException {
+ return getDelegate().connect(url, info, haGroup);
+ }
+
@Override
public MetaDataMutationResult getTable(PName tenantId, byte[] schemaBytes, byte[] tableBytes, long tableTimestamp, long clientTimestamp) throws SQLException {
return getDelegate().getTable(tenantId, schemaBytes, tableBytes, tableTimestamp, clientTimestamp);
diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/query/QueryConstants.java b/phoenix-core-client/src/main/java/org/apache/phoenix/query/QueryConstants.java
index f7011aabc0..e0f46b7a52 100644
--- a/phoenix-core-client/src/main/java/org/apache/phoenix/query/QueryConstants.java
+++ b/phoenix-core-client/src/main/java/org/apache/phoenix/query/QueryConstants.java
@@ -192,7 +192,8 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TYPE_SEQUENCE;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.UPDATE_CACHE_FREQUENCY;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.USER;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.USE_STATS_FOR_PARALLELIZATION;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VERSION;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VERSION_CLUSTER_1;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VERSION_CLUSTER_2;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_CONSTANT;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_INDEX_ID;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_INDEX_ID_DATA_TYPE;
@@ -708,7 +709,8 @@ public interface QueryConstants {
CLUSTER_ROLE_1 + " VARCHAR," + // Role for peer cluster might not be accurate, we will use only local role for recovery if needed
CLUSTER_ROLE_2 + " VARCHAR," + // Role for peer cluster might not be accurate, we will use only local role for recovery if needed
POLICY + " VARCHAR," + // There should be only one policy for an HA group
- VERSION + " BIGINT," + // Version should be incremented for Admin Updates, only for CLUSTER_URLs and REGISTRY_TYPE
+ VERSION_CLUSTER_1 + " BIGINT," + // Version should be incremented for Admin Updates, only for CLUSTER_URLs and REGISTRY_TYPE
+ VERSION_CLUSTER_2 + " BIGINT," + // Version should be incremented for Admin Updates, only for CLUSTER_URLs and REGISTRY_TYPE
"CONSTRAINT " + SYSTEM_TABLE_PK_NAME +
" PRIMARY KEY (" + HA_GROUP_NAME + "))\n" +
HConstants.VERSIONS + "=%s,\n" +
diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/query/QueryServices.java b/phoenix-core-client/src/main/java/org/apache/phoenix/query/QueryServices.java
index 0ba347918c..683d742c4d 100644
--- a/phoenix-core-client/src/main/java/org/apache/phoenix/query/QueryServices.java
+++ b/phoenix-core-client/src/main/java/org/apache/phoenix/query/QueryServices.java
@@ -386,6 +386,9 @@ public interface QueryServices extends SQLCloseable {
public static final String PHOENIX_VIEW_TTL_TENANT_VIEWS_PER_SCAN_LIMIT = "phoenix.view.ttl.tenant_views_per_scan.limit";
// Block mutations based on cluster role record
public static final String CLUSTER_ROLE_BASED_MUTATION_BLOCK_ENABLED = "phoenix.cluster.role.based.mutation.block.enabled";
+ // Check HAGroup is Stale for mutations
+ public static final String HA_GROUP_STALE_FOR_MUTATION_CHECK_ENABLED = "phoenix.ha.group.stale.for.mutation.check.enabled";
+
//Enable Thread Pool Creation in CQSI to be used for HBase Client.
String CQSI_THREAD_POOL_ENABLED = "phoenix.cqsi.thread.pool.enabled";
//CQSI Thread Pool Related Configuration.
diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java b/phoenix-core-client/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
index 5a88e45270..bbf55e4196 100644
--- a/phoenix-core-client/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
+++ b/phoenix-core-client/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
@@ -462,6 +462,7 @@ public class QueryServicesOptions {
public static final Boolean DEFAULT_CLUSTER_ROLE_BASED_MUTATION_BLOCK_ENABLED = false;
+ public static final Boolean DEFAULT_HA_GROUP_STALE_FOR_MUTATION_CHECK_ENABLED = true;
public static final Boolean DEFAULT_CQSI_THREAD_POOL_ENABLED = false;
public static final int DEFAULT_CQSI_THREAD_POOL_KEEP_ALIVE_SECONDS = 60;
public static final int DEFAULT_CQSI_THREAD_POOL_CORE_POOL_SIZE = 25;
diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/util/GetClusterRoleRecordUtil.java b/phoenix-core-client/src/main/java/org/apache/phoenix/util/GetClusterRoleRecordUtil.java
new file mode 100644
index 0000000000..ed1482286c
--- /dev/null
+++ b/phoenix-core-client/src/main/java/org/apache/phoenix/util/GetClusterRoleRecordUtil.java
@@ -0,0 +1,231 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.util;
+
+import static org.apache.phoenix.jdbc.HighAvailabilityGroup.PHOENIX_HA_GROUP_ATTR;
+import static org.apache.phoenix.util.PhoenixRuntime.JDBC_PROTOCOL_SEPARATOR;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.phoenix.coprocessor.generated.RegionServerEndpointProtos;
+import org.apache.phoenix.jdbc.ClusterRoleRecord;
+import org.apache.phoenix.jdbc.HighAvailabilityGroup;
+import org.apache.phoenix.jdbc.ClusterRoleRecord.ClusterRole;
+import org.apache.phoenix.jdbc.HighAvailabilityPolicy;
+import org.apache.phoenix.jdbc.PhoenixConnection;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.protobuf.ByteString;
+
+/**
+ * Utility class for getting Cluster Role Record for the client from RegionServer Endpoints.
+ */
+public class GetClusterRoleRecordUtil {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(GetClusterRoleRecordUtil.class);
+
+ /**
+ * Scheduler to fetch ClusterRoleRecord until we get an Active ClusterRoleRecord
+ */
+ private static Map<String, ScheduledExecutorService> schedulerMap = new ConcurrentHashMap<>();
+
+ private static final Object pollerLock = new Object();
+
+ private static volatile ScheduledFuture<?> pollerFuture = null;
+
+ private GetClusterRoleRecordUtil() {}
+
+
+ /**
+ * Method to get ClusterRoleRecord from RegionServer Endpoints. it picks a random region server
+ * and gets the CRR from it.
+ * @param url URL to create Connection to be used to get RegionServer Endpoint Service
+ * @param haGroupName Name of the HA group
+ * @param doRetry Whether to retry if the operation fails
+ * @param properties Connection properties
+ * @return ClusterRoleRecord from the first available cluster
+ * @throws SQLException if there is an error getting the ClusterRoleRecord
+ */
+ private static ClusterRoleRecord getClusterRoleRecord(String url, String haGroupName, boolean doRetry, Properties properties) throws SQLException {
+ Connection conn = getConnection(url, properties);
+ PhoenixConnection connection = conn.unwrap(PhoenixConnection.class);
+ try (Admin admin = connection.getQueryServices().getAdmin()) {
+ // get all live region servers
+ List<ServerName> regionServers
+ = connection.getQueryServices().getLiveRegionServers();
+ // pick one at random
+ ServerName regionServer
+ = regionServers.get(ThreadLocalRandom.current().nextInt(regionServers.size()));
+
+ //RPC to regionServer to get the CRR
+ RegionServerEndpointProtos.RegionServerEndpointService.BlockingInterface
+ service = RegionServerEndpointProtos.RegionServerEndpointService
+ .newBlockingStub(admin.coprocessorService(regionServer));
+ RegionServerEndpointProtos.GetClusterRoleRecordRequest request
+ = getClusterRoleRecordRequest(haGroupName);
+ RegionServerEndpointProtos.GetClusterRoleRecordResponse response = service.getClusterRoleRecord(null, request);
+
+ //Check if the ClusterRoleRecord is valid, if not, throw an exception
+ if (response.getHaGroupName() == null || response.getPolicy() == null || response.getUrl1() == null
+ || response.getRole1() == null || response.getUrl2() == null || response.getRole2() == null) {
+ throw new SQLException("Invalid ClusterRoleRecord response from RegionServer");
+ }
+
+ //Generate the ClusterRoleRecord from the response
+ return new ClusterRoleRecord(response.getHaGroupName().toStringUtf8(), HighAvailabilityPolicy.valueOf(response.getPolicy().toStringUtf8()),
+ response.getUrl1().toStringUtf8(), ClusterRole.valueOf(response.getRole1().toStringUtf8()),
+ response.getUrl2().toStringUtf8(), ClusterRole.valueOf(response.getRole2().toStringUtf8()),
+ response.getVersion());
+
+ } catch (Exception e) {
+ SQLException parsedException = ClientUtil.parseServerException(e);
+ //retry once for any exceptions other than StaleMetadataCacheException
+ LOGGER.error("Error in getting ClusterRoleRecord for {} from url {}", haGroupName, connection.getURL(), parsedException);
+ if (doRetry) {
+ // update the list of live region servers
+ connection.getQueryServices().refreshLiveRegionServers();
+ return getClusterRoleRecord(url, haGroupName, false, properties);
+ }
+ throw parsedException;
+ } finally {
+ conn.close();
+ }
+
+ }
+
+ /**
+ * Method to schedule a poller to fetch ClusterRoleRecord every 5 seconds (or configured value)
+ * until we get an Active ClusterRoleRecord (one role should be Active) if we receive an Active
+ * roleRecord then client this method will return the roleRecord to be consumed and used, if not
+ * then it will start a poller and return non-active roleRecord.
+ * @param url URL of the RegionServer Endpoint Service
+ * @param haGroupName Name of the HA group
+ * @param properties Connection properties
+ * @param pollerInterval Interval in seconds to poll for ClusterRoleRecord
+ * @param haGroup HighAvailabilityGroup object to refresh the ClusterRoleRecord when an Active CRR is found
+ * @throws SQLException if there is an error getting the ClusterRoleRecord
+ */
+ public static ClusterRoleRecord fetchClusterRoleRecord(String url, String haGroupName, HighAvailabilityGroup haGroup, long pollerInterval, Properties properties) throws SQLException {
+ ClusterRoleRecord clusterRoleRecord = getClusterRoleRecord(url, haGroupName, true, properties);
+ //TODO: Will need to handle UNKNOWN role cases...
+ if (clusterRoleRecord.getPolicy() == HighAvailabilityPolicy.FAILOVER &&
+ !clusterRoleRecord.getRole1().isActive() && !clusterRoleRecord.getRole2().isActive()) {
+ LOGGER.info("Non-active ClusterRoleRecord found for HA group {}. Scheduling poller to check every {} seconds," +
+ "until we find an ACTIVE CRR", haGroupName, pollerInterval);
+ //Schedule a poller to fetch ClusterRoleRecord every 5 seconds (or configured value)
+ //until we get an Active ClusterRoleRecord and return the Non-Active CRR
+ schedulePoller(url, haGroupName, haGroup, pollerInterval, properties);
+ }
+
+ return clusterRoleRecord;
+ }
+
+ /**
+ * Method to schedule a poller to fetch ClusterRoleRecord every pollerInterval seconds
+ * until we get an Active ClusterRoleRecord, poller will only start if client will receive,
+ * a Non-Active roleRecord (means either of the roles are not Active and client can't create
+ * a connection)
+ * @param url URL of the RegionServer Endpoint Service
+ * @param haGroupName Name of the HA group
+ * @param haGroup HighAvailabilityGroup object to refresh the ClusterRoleRecord when an Active CRR is found
+ * @param pollerInterval Interval in seconds to poll for ClusterRoleRecord
+ * @param properties Connection properties
+ * @throws SQLException if there is an error getting or refreshing the ClusterRoleRecord when an Active CRR is found
+ */
+ private static void schedulePoller(String url, String haGroupName, HighAvailabilityGroup haGroup, long pollerInterval, Properties properties) {
+
+ synchronized (pollerLock) {
+ if (schedulerMap.containsKey(haGroupName) && !schedulerMap.get(haGroupName).isShutdown()) {
+ LOGGER.info("Poller already running for HA group {}.", haGroupName);
+ return;
+ }
+
+ schedulerMap.put(haGroupName, Executors.newScheduledThreadPool(1));
+ LOGGER.info("Starting poller for HA group {} to check every {} milliseconds.", haGroupName, pollerInterval);
+ Runnable pollingTask = () -> {
+ try {
+ ClusterRoleRecord polledCrr = getClusterRoleRecord(url, haGroupName, true, properties);
+ LOGGER.info("Polled CRR: {}", polledCrr);
+ if (polledCrr.getRole1().isActive() || polledCrr.getRole2().isActive()) {
+
+ LOGGER.info("Active ClusterRoleRecord found. Cancelling poller.");
+ synchronized (pollerLock) {
+ if (pollerFuture != null) {
+ pollerFuture.cancel(false);
+ }
+ //Refresh ClusterRoleRecord for the HAGroup with appropriate transition
+ haGroup.refreshClusterRoleRecord(true);
+ schedulerMap.get(haGroupName).shutdown();
+ schedulerMap.remove(haGroupName);
+ }
+ }
+ } catch (SQLException e) {
+ LOGGER.error("Exception found while polling for ClusterRoleRecord on {}: {}", url, e.getMessage());
+ }
+ };
+
+ // Schedule the task with a fixed delay
+ pollerFuture = schedulerMap.get(haGroupName).scheduleWithFixedDelay(pollingTask, 0, pollerInterval, TimeUnit.MILLISECONDS);
+ }
+ }
+
+
+ /**
+ * Method to build the ClusterRoleRecordRequest for the given HA group name
+ * @param haGroupName Name of the HA group
+ * @return ClusterRoleRecordRequest for the given HA group name
+ */
+ private static RegionServerEndpointProtos.GetClusterRoleRecordRequest getClusterRoleRecordRequest(String haGroupName) {
+ RegionServerEndpointProtos.GetClusterRoleRecordRequest.Builder requestBuilder
+ = RegionServerEndpointProtos.GetClusterRoleRecordRequest.newBuilder();
+ requestBuilder.setHaGroupName(ByteString.copyFromUtf8(haGroupName));
+
+ return requestBuilder.build();
+ }
+
+
+ /**
+ * Method to get a connection to the given URL
+ * @param url URL of the RegionServer Endpoint Service
+ * @param properties Connection properties
+ * @return Connection to the given URL
+ * @throws SQLException if there is an error getting the connection
+ */
+ private static Connection getConnection(String url, Properties properties) throws SQLException {
+ if (!url.startsWith(PhoenixRuntime.JDBC_PROTOCOL)) {
+ url = PhoenixRuntime.JDBC_PROTOCOL_RPC + JDBC_PROTOCOL_SEPARATOR + url;
+ }
+ Properties propsCopy = PropertiesUtil.deepCopy(properties);
+ propsCopy.remove(PHOENIX_HA_GROUP_ATTR);
+ return DriverManager.getConnection(url, propsCopy);
+ }
+}
\ No newline at end of file
diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/util/ScanUtil.java b/phoenix-core-client/src/main/java/org/apache/phoenix/util/ScanUtil.java
index 1941692f57..9344855274 100644
--- a/phoenix-core-client/src/main/java/org/apache/phoenix/util/ScanUtil.java
+++ b/phoenix-core-client/src/main/java/org/apache/phoenix/util/ScanUtil.java
@@ -47,6 +47,7 @@ import java.util.NavigableSet;
import java.util.Set;
import java.util.TreeMap;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
@@ -1402,6 +1403,22 @@ public class ScanUtil {
}
}
+ /**
+ * Set haGroupName as a scan Attribute for pointLookups to check staleness on server side for
+ * the respective HAGroup
+ * @param scan Scan to which we have to set Attribute
+ * @param conn PhoenixConnection for which scan is part of
+ * @param context StatementContext for the query
+ */
+ public static void setScanAttributeForHAForPointLookups(Scan scan, PhoenixConnection conn,
+ StatementContext context) {
+ if (context.getScanRanges().isPointLookup() && conn.getHAGroup() != null &&
+ StringUtils.isNotBlank(conn.getHAGroupName())) {
+ scan.setAttribute(BaseScannerRegionObserverConstants.HA_GROUP_NAME_ATTRIB,
+ Bytes.toBytes(conn.getHAGroupName()));
+ }
+ }
+
public static void setScanAttributesForPhoenixTTL(Scan scan, PTable table,
PhoenixConnection phoenixConnection) throws SQLException {
@@ -1486,6 +1503,7 @@ public class ScanUtil {
PhoenixConnection phoenixConnection = context.getConnection();
setScanAttributesForIndexReadRepair(scan, table, phoenixConnection, context);
setScanAttributesForPhoenixTTL(scan, table, phoenixConnection);
+ setScanAttributeForHAForPointLookups(scan, phoenixConnection, context);
byte[] emptyCF = scan.getAttribute(BaseScannerRegionObserverConstants.EMPTY_COLUMN_FAMILY_NAME);
byte[] emptyCQ = scan.getAttribute(BaseScannerRegionObserverConstants.EMPTY_COLUMN_QUALIFIER_NAME);
if (emptyCF != null && emptyCQ != null) {
diff --git a/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java b/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java
index bb46a10cb2..7392a3ca13 100644
--- a/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java
+++ b/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java
@@ -18,8 +18,10 @@
package org.apache.phoenix.coprocessor;
import java.io.IOException;
+import java.nio.charset.StandardCharsets;
import java.util.List;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.DoNotRetryIOException;
@@ -49,6 +51,7 @@ import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.htrace.Span;
import org.apache.htrace.Trace;
import org.apache.phoenix.coprocessorclient.BaseScannerRegionObserverConstants;
+import org.apache.phoenix.exception.StaleClusterRoleRecordException;
import org.apache.phoenix.execute.TupleProjector;
import org.apache.phoenix.filter.PagingFilter;
import org.apache.phoenix.hbase.index.covered.update.ColumnReference;
@@ -56,6 +59,7 @@ import org.apache.phoenix.index.CDCTableInfo;
import org.apache.phoenix.index.IndexMaintainer;
import org.apache.phoenix.iterate.NonAggregateRegionScannerFactory;
import org.apache.phoenix.iterate.RegionScannerFactory;
+import org.apache.phoenix.jdbc.HAGroupStoreManager;
import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.query.QueryServicesOptions;
import org.apache.phoenix.schema.StaleRegionBoundaryCacheException;
@@ -185,6 +189,26 @@ abstract public class BaseScannerRegionObserver implements RegionObserver {
}
}
}
+
+ byte[] haGroupNameBytes = scan.getAttribute(BaseScannerRegionObserverConstants
+ .HA_GROUP_NAME_ATTRIB);
+ //Check if scan's HAGroup is stale on client. If yes, throw an exception.
+ if (haGroupNameBytes != null) {
+ String haGroupName = new String(haGroupNameBytes, StandardCharsets.UTF_8);
+ final Configuration conf = c.getEnvironment().getConfiguration();
+ final HAGroupStoreManager haGroupStoreManager
+ = HAGroupStoreManager.getInstance(conf);
+ if (haGroupStoreManager == null) {
+ throw new IOException("HAGroupStoreManager is null "
+ + "for current cluster, check configuration");
+ }
+
+ if (StringUtils.isNoneBlank(haGroupName) && haGroupStoreManager
+ .isHAGroupOnClientStale(haGroupName)) {
+ throw new StaleClusterRoleRecordException(String.format("HAGroupStoreRecord is "
+ + "stale for haGroup %s on client", haGroupName));
+ }
+ }
}
private class RegionScannerHolder extends DelegateRegionScanner {
diff --git a/phoenix-core-server/src/main/java/org/apache/phoenix/hbase/index/IndexRegionObserver.java b/phoenix-core-server/src/main/java/org/apache/phoenix/hbase/index/IndexRegionObserver.java
index 1aeb7787ce..a0b6335e78 100644
--- a/phoenix-core-server/src/main/java/org/apache/phoenix/hbase/index/IndexRegionObserver.java
+++ b/phoenix-core-server/src/main/java/org/apache/phoenix/hbase/index/IndexRegionObserver.java
@@ -61,6 +61,7 @@ import org.apache.phoenix.coprocessor.generated.PTableProtos;
import org.apache.phoenix.coprocessorclient.BaseScannerRegionObserverConstants;
import org.apache.phoenix.exception.DataExceedsCapacityException;
import org.apache.phoenix.exception.MutationBlockedIOException;
+import org.apache.phoenix.exception.StaleClusterRoleRecordException;
import org.apache.phoenix.execute.MutationState;
import org.apache.phoenix.expression.CaseExpression;
import org.apache.phoenix.expression.Expression;
@@ -634,12 +635,25 @@ public class IndexRegionObserver implements RegionCoprocessor, RegionObserver {
final Set<String> haGroupNames = extractHAGroupNameAttribute(miniBatchOp);
// Check if mutation is blocked for any of the HAGroupNames
for (String haGroupName : haGroupNames) {
- if (StringUtils.isNotBlank(haGroupName)
- && haGroupStoreManager.isMutationBlocked(haGroupName)) {
- throw new MutationBlockedIOException("Blocking Mutation as Some CRRs are in "
- + "ACTIVE_TO_STANDBY state and "
- + "CLUSTER_ROLE_BASED_MUTATION_BLOCK_ENABLED is true");
- }
+ //TODO: Below approach might be slow need to figure out faster way, slower part is
+ //getting haGroupStoreClient We can also cache roleRecord (I tried it and still its
+ //slow due to haGroupStoreClient initialization) and caching will give us old result
+ //in case one cluster is unreachable instead of UNKNOWN.
+
+ boolean isHAGroupOnClientStale = haGroupStoreManager
+ .isHAGroupOnClientStale(haGroupName);
+ if (StringUtils.isNotBlank(haGroupName) && isHAGroupOnClientStale) {
+ throw new StaleClusterRoleRecordException(String.format("HAGroupStoreRecord is "
+ + "stale for haGroup %s on client", haGroupName));
+ }
+
+ //Check if mutation's haGroup is stale
+ if (StringUtils.isNotBlank(haGroupName)
+ && haGroupStoreManager.isMutationBlocked(haGroupName)) {
+ throw new MutationBlockedIOException("Blocking Mutation as Some CRRs are in "
+ + "ACTIVE_TO_STANDBY state and "
+ + "CLUSTER_ROLE_BASED_MUTATION_BLOCK_ENABLED is true");
+ }
}
preBatchMutateWithExceptions(c, miniBatchOp);
return;
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/PhoenixRegionServerEndpointITWithConsistentFailover.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/PhoenixRegionServerEndpointITWithConsistentFailover.java
index e66da0829b..6ee86cbbe7 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/PhoenixRegionServerEndpointITWithConsistentFailover.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/PhoenixRegionServerEndpointITWithConsistentFailover.java
@@ -43,6 +43,7 @@ import org.junit.rules.TestName;
import java.util.Map;
+import static org.apache.phoenix.jdbc.HAGroupStoreRecord.DEFAULT_RECORD_VERSION;
import static org.apache.phoenix.jdbc.HAGroupStoreClient.ZK_CONSISTENT_HA_GROUP_STATE_NAMESPACE;
import static org.apache.phoenix.jdbc.PhoenixHAAdmin.getLocalZkUrl;
import static org.junit.Assert.assertEquals;
@@ -84,7 +85,7 @@ public class PhoenixRegionServerEndpointITWithConsistentFailover extends BaseTes
ServerRpcController controller = new ServerRpcController();
try (PhoenixHAAdmin peerHAAdmin = new PhoenixHAAdmin(CLUSTERS.getHBaseCluster2().getConfiguration(), ZK_CONSISTENT_HA_GROUP_STATE_NAMESPACE)) {
- HAGroupStoreRecord peerHAGroupStoreRecord = new HAGroupStoreRecord(HAGroupStoreRecord.DEFAULT_PROTOCOL_VERSION, haGroupName, HAGroupState.STANDBY);
+ HAGroupStoreRecord peerHAGroupStoreRecord = new HAGroupStoreRecord(HAGroupStoreRecord.DEFAULT_PROTOCOL_VERSION, haGroupName, HAGroupState.STANDBY, DEFAULT_RECORD_VERSION);
peerHAAdmin.createHAGroupStoreRecordInZooKeeper(peerHAGroupStoreRecord);
}
Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexRegionObserverMutationBlockingIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexRegionObserverMutationBlockingIT.java
index 377bdf69d7..6e8c5ca75d 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexRegionObserverMutationBlockingIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexRegionObserverMutationBlockingIT.java
@@ -17,8 +17,11 @@
*/
package org.apache.phoenix.end2end.index;
+import static org.apache.phoenix.jdbc.HAGroupStoreRecord.DEFAULT_RECORD_VERSION;
import static org.apache.phoenix.jdbc.HAGroupStoreClient.ZK_CONSISTENT_HA_GROUP_STATE_NAMESPACE;
+import static org.apache.phoenix.jdbc.HighAvailabilityGroup.PHOENIX_HA_GROUP_ATTR;
import static org.apache.phoenix.jdbc.PhoenixHAAdmin.getLocalZkUrl;
+import static org.apache.phoenix.query.BaseTest.generateUniqueName;
import static org.apache.phoenix.query.QueryServices.CLUSTER_ROLE_BASED_MUTATION_BLOCK_ENABLED;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
@@ -26,15 +29,19 @@ import static org.junit.Assert.fail;
import java.sql.DriverManager;
import java.util.Map;
+import java.util.Properties;
import org.apache.hadoop.hbase.client.RetriesExhaustedWithDetailsException;
import org.apache.phoenix.end2end.NeedsOwnMiniClusterTest;
import org.apache.phoenix.exception.MutationBlockedIOException;
import org.apache.phoenix.execute.CommitException;
import org.apache.phoenix.jdbc.ClusterRoleRecord;
+import org.apache.phoenix.jdbc.FailoverPhoenixConnection;
import org.apache.phoenix.jdbc.HAGroupStoreRecord;
+import org.apache.phoenix.jdbc.HighAvailabilityPolicy;
import org.apache.phoenix.jdbc.HighAvailabilityTestingUtility;
import org.apache.phoenix.jdbc.PhoenixConnection;
+import org.apache.phoenix.jdbc.PhoenixDriver;
import org.apache.phoenix.jdbc.PhoenixHAAdmin;
import org.apache.phoenix.query.BaseTest;
import org.apache.phoenix.thirdparty.com.google.common.collect.Maps;
@@ -53,7 +60,7 @@ import org.junit.rules.TestName;
* blocking is enabled and CRRs are in ACTIVE_TO_STANDBY state.
*/
@Category(NeedsOwnMiniClusterTest.class)
-public class IndexRegionObserverMutationBlockingIT extends BaseTest {
+public class IndexRegionObserverMutationBlockingIT {
private static final Long ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS = 1000L;
private PhoenixHAAdmin haAdmin;
@@ -61,36 +68,34 @@ public class IndexRegionObserverMutationBlockingIT extends BaseTest {
@Rule
public TestName testName = new TestName();
+ private Properties clientProps = new Properties();
+ private String haGroupName;
@BeforeClass
public static synchronized void doSetup() throws Exception {
- Map<String, String> props = Maps.newHashMapWithExpectedSize(1);
- // Enable cluster role-based mutation blocking
- props.put(CLUSTER_ROLE_BASED_MUTATION_BLOCK_ENABLED, "true");
- // No retries so that tests fail faster.
- props.put("hbase.client.retries.number", "0");
- setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator()));
+ CLUSTERS.getHBaseCluster1().getConfiguration().setBoolean(CLUSTER_ROLE_BASED_MUTATION_BLOCK_ENABLED, true);
+ CLUSTERS.getHBaseCluster2().getConfiguration().setBoolean(CLUSTER_ROLE_BASED_MUTATION_BLOCK_ENABLED, true);
CLUSTERS.start();
+ DriverManager.registerDriver(PhoenixDriver.INSTANCE);
}
@Before
public void setUp() throws Exception {
- haAdmin = new PhoenixHAAdmin(config, ZK_CONSISTENT_HA_GROUP_STATE_NAMESPACE);
+ haGroupName = testName.getMethodName();
+ clientProps = HighAvailabilityTestingUtility.getHATestProperties();
+ clientProps.setProperty(PHOENIX_HA_GROUP_ATTR, haGroupName);;
+ haAdmin = CLUSTERS.getHaAdmin1();
Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
- String zkUrl = getLocalZkUrl(config);
- String peerZkUrl = CLUSTERS.getZkUrl2();
- HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(testName.getMethodName(), zkUrl, peerZkUrl,
- ClusterRoleRecord.ClusterRole.ACTIVE, ClusterRoleRecord.ClusterRole.STANDBY, null);
+ CLUSTERS.initClusterRole(haGroupName, HighAvailabilityPolicy.FAILOVER);
}
@Test
public void testMutationBlockedOnDataTableWithIndex() throws Exception {
String dataTableName = generateUniqueName();
String indexName = generateUniqueName();
- String haGroupName = testName.getMethodName();
- try (PhoenixConnection conn = (PhoenixConnection) DriverManager.getConnection(getUrl())) {
- conn.setHAGroupName(haGroupName);
+ try (FailoverPhoenixConnection conn = (FailoverPhoenixConnection) DriverManager
+ .getConnection(CLUSTERS.getJdbcHAUrl(), clientProps)) {
// Create data table and index
conn.createStatement().execute("CREATE TABLE " + dataTableName +
" (id VARCHAR PRIMARY KEY, name VARCHAR, age INTEGER)");
@@ -105,7 +110,7 @@ public class IndexRegionObserverMutationBlockingIT extends BaseTest {
// Set up HAGroupStoreRecord that will block mutations (ACTIVE_TO_STANDBY state)
HAGroupStoreRecord haGroupStoreRecord
= new HAGroupStoreRecord(HAGroupStoreRecord.DEFAULT_PROTOCOL_VERSION,
- haGroupName, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY);
+ haGroupName, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY, DEFAULT_RECORD_VERSION);
haAdmin.updateHAGroupStoreRecordInZooKeeper(haGroupName, haGroupStoreRecord, -1);
// Wait for the event to propagate
@@ -131,8 +136,8 @@ public class IndexRegionObserverMutationBlockingIT extends BaseTest {
String dataTableName = generateUniqueName();
String indexName = generateUniqueName();
- try (PhoenixConnection conn = (PhoenixConnection) DriverManager.getConnection(getUrl())) {
- conn.setHAGroupName(testName.getMethodName());
+ try (FailoverPhoenixConnection conn = (FailoverPhoenixConnection) DriverManager
+ .getConnection(CLUSTERS.getJdbcHAUrl(), clientProps)) {
// Create data table and index
conn.createStatement().execute("CREATE TABLE " + dataTableName +
" (id VARCHAR PRIMARY KEY, name VARCHAR, age INTEGER)");
@@ -159,10 +164,9 @@ public class IndexRegionObserverMutationBlockingIT extends BaseTest {
public void testMutationBlockingTransition() throws Exception {
String dataTableName = generateUniqueName();
String indexName = generateUniqueName();
- String haGroupName = testName.getMethodName();
- try (PhoenixConnection conn = (PhoenixConnection) DriverManager.getConnection(getUrl())) {
- conn.setHAGroupName(testName.getMethodName());
+ try (FailoverPhoenixConnection conn = (FailoverPhoenixConnection) DriverManager
+ .getConnection(CLUSTERS.getJdbcHAUrl(), clientProps)) {
// Create data table and index
conn.createStatement().execute("CREATE TABLE " + dataTableName +
" (id VARCHAR PRIMARY KEY, name VARCHAR, age INTEGER)");
@@ -176,7 +180,8 @@ public class IndexRegionObserverMutationBlockingIT extends BaseTest {
// Set up HAGroupStoreRecord that will block mutations (ACTIVE_TO_STANDBY state)
HAGroupStoreRecord haGroupStoreRecord
= new HAGroupStoreRecord(HAGroupStoreRecord.DEFAULT_PROTOCOL_VERSION,
- haGroupName, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY);
+ haGroupName, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY,
+ DEFAULT_RECORD_VERSION);
haAdmin.updateHAGroupStoreRecordInZooKeeper(haGroupName, haGroupStoreRecord, -1);
Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
@@ -195,7 +200,8 @@ public class IndexRegionObserverMutationBlockingIT extends BaseTest {
// Set up HAGroupStoreRecord that will block mutations (ACTIVE_TO_STANDBY state)
haGroupStoreRecord
= new HAGroupStoreRecord(HAGroupStoreRecord.DEFAULT_PROTOCOL_VERSION,
- haGroupName, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+ haGroupName, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC,
+ DEFAULT_RECORD_VERSION + 1);
haAdmin.updateHAGroupStoreRecordInZooKeeper(haGroupName, haGroupStoreRecord, -1);
Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/ClusterRoleRecordGeneratorToolIT.java b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/ClusterRoleRecordGeneratorToolIT.java
index 25953354e6..6b45662871 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/ClusterRoleRecordGeneratorToolIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/ClusterRoleRecordGeneratorToolIT.java
@@ -98,14 +98,14 @@ public class ClusterRoleRecordGeneratorToolIT {
generator1.setConf(conf1);
// created with cluster1's conf, so cluster1 is ACTIVE
- List<ClusterRoleRecord> records = generator1.listAllRecordsByZk();
+ List<ClusterRoleRecord> records = generator1.listAllRecordsByMaster();
assertNotNull(records);
LOG.info("Generated following records from cluster1: {}", records);
assertEquals(1, records.size());
for (ClusterRoleRecord record : records) {
assertEquals(HighAvailabilityPolicy.FAILOVER, record.getPolicy());
- assertEquals(ClusterRole.ACTIVE, record.getRole(CLUSTERS.getZkUrl1()));
- assertEquals(ClusterRole.STANDBY, record.getRole(CLUSTERS.getZkUrl2()));
+ assertEquals(ClusterRole.ACTIVE, record.getRole(CLUSTERS.getMasterAddress1()));
+ assertEquals(ClusterRole.STANDBY, record.getRole(CLUSTERS.getMasterAddress2()));
}
}
@@ -120,14 +120,14 @@ public class ClusterRoleRecordGeneratorToolIT {
generator2.setConf(conf2);
// created with cluster2's conf, so cluster2 is ACTIVE
- List<ClusterRoleRecord> records = generator2.listAllRecordsByZk();
+ List<ClusterRoleRecord> records = generator2.listAllRecordsByMaster();
assertNotNull(records);
LOG.info("Generated following records from cluster2: {}", records);
assertEquals(3, records.size());
for (ClusterRoleRecord record : records) {
assertEquals(HighAvailabilityPolicy.PARALLEL, record.getPolicy());
- assertEquals(ClusterRole.ACTIVE, record.getRole(CLUSTERS.getZkUrl2()));
- assertEquals(ClusterRole.STANDBY, record.getRole(CLUSTERS.getZkUrl1()));
+ assertEquals(ClusterRole.ACTIVE, record.getRole(CLUSTERS.getMasterAddress2()));
+ assertEquals(ClusterRole.STANDBY, record.getRole(CLUSTERS.getMasterAddress1()));
}
}
}
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/FailoverPhoenixConnection2IT.java b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/FailoverPhoenixConnection2IT.java
index d9520a8409..3689763847 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/FailoverPhoenixConnection2IT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/FailoverPhoenixConnection2IT.java
@@ -18,25 +18,29 @@
package org.apache.phoenix.jdbc;
import static org.apache.hadoop.test.GenericTestUtils.waitFor;
-import static org.apache.phoenix.jdbc.HighAvailabilityTestingUtility.HBaseTestingUtilityPair.doTestWhenOneZKDown;
-import static org.apache.phoenix.jdbc.HighAvailabilityTestingUtility.doTestBasicOperationsWithConnection;
+import static org.apache.phoenix.exception.SQLExceptionCode.CANNOT_ESTABLISH_CONNECTION;
+import static org.apache.phoenix.exception.SQLExceptionCode.FAILOVER_IN_PROGRESS;
+import static org.apache.phoenix.jdbc.ClusterRoleRecord.ClusterRole.ACTIVE;
+import static org.apache.phoenix.jdbc.ClusterRoleRecord.ClusterRole.ACTIVE_TO_STANDBY;
+import static org.apache.phoenix.jdbc.ClusterRoleRecord.ClusterRole.STANDBY;
+import static org.apache.phoenix.jdbc.ClusterRoleRecord.ClusterRole.STANDBY_TO_ACTIVE;
import static org.apache.phoenix.jdbc.HighAvailabilityGroup.PHOENIX_HA_GROUP_ATTR;
+import static org.apache.phoenix.jdbc.HighAvailabilityGroup.PHOENIX_HA_CRR_CACHE_FREQUENCY_MS_KEY;
+import static org.apache.phoenix.jdbc.HighAvailabilityTestingUtility.doTestBasicOperationsWithConnection;
import static org.apache.phoenix.jdbc.HighAvailabilityTestingUtility.getHighAvailibilityGroup;
+import static org.apache.phoenix.jdbc.HighAvailabilityTestingUtility.sleepThreadFor;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.junit.Assume.assumeTrue;
-import java.io.IOException;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
import java.util.List;
import java.util.Properties;
import java.util.concurrent.ExecutorService;
@@ -44,10 +48,13 @@ import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
-import org.apache.commons.lang3.RandomStringUtils;
-import org.apache.hadoop.hbase.util.VersionInfo;
+import org.apache.commons.lang3.RandomUtils;
import org.apache.phoenix.end2end.NeedsOwnMiniClusterTest;
+import org.apache.phoenix.exception.FailoverSQLException;
+import org.apache.phoenix.exception.SQLExceptionCode;
+import org.apache.phoenix.jdbc.ClusterRoleRecord.ClusterRole;
import org.apache.phoenix.query.ConnectionQueryServicesImpl;
+import org.apache.phoenix.util.JDBCUtil;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
@@ -97,10 +104,15 @@ public class FailoverPhoenixConnection2IT {
haGroupName = testName.getMethodName();
clientProperties = HighAvailabilityTestingUtility.getHATestProperties();
clientProperties.setProperty(PHOENIX_HA_GROUP_ATTR, haGroupName);
+ clientProperties.setProperty(PHOENIX_HA_CRR_CACHE_FREQUENCY_MS_KEY, "1000");
+
+ CLUSTERS.deleteHAGroupRecordFromBothClusters(haGroupName);
// Make first cluster ACTIVE
CLUSTERS.initClusterRole(haGroupName, HighAvailabilityPolicy.FAILOVER);
+ CLUSTERS.rebuildBothClustersGroupStoreClient(haGroupName);
+
haGroup = getHighAvailibilityGroup(CLUSTERS.getJdbcHAUrl(), clientProperties);
LOG.info("Initialized haGroup {} with URL {}", haGroup, CLUSTERS.getJdbcHAUrl());
tableName = testName.getMethodName().toUpperCase();
@@ -110,6 +122,7 @@ public class FailoverPhoenixConnection2IT {
@After
public void tearDown() throws Exception {
try {
+ CLUSTERS.deleteHAGroupRecordFromBothClusters(haGroupName);
haGroup.close();
PhoenixDriver.INSTANCE
.getConnectionQueryServices(CLUSTERS.getJdbcUrl1(haGroup), haGroup.getProperties())
@@ -131,16 +144,16 @@ public class FailoverPhoenixConnection2IT {
*/
@Test(timeout = 300000)
public void testFailoverCanFinishWhenOneZKDown() throws Exception {
- doTestWhenOneZKDown(CLUSTERS.getHBaseCluster1(), () -> {
+ CLUSTERS.doTestWhenOneZKDown(CLUSTERS.getHBaseCluster1(), () -> {
// Because cluster1 is down, any version record will only be updated in cluster2.
// Become ACTIVE cluster is still ACTIVE in current version record, no CQS to be closed.
- CLUSTERS.transitClusterRole(haGroup, ClusterRoleRecord.ClusterRole.ACTIVE, ClusterRoleRecord.ClusterRole.OFFLINE);
+ CLUSTERS.transitClusterRoleWithCluster1Down(haGroup, ACTIVE, ClusterRoleRecord.ClusterRole.OFFLINE);
// Usually making this cluster STANDBY will close the CQS and all existing connections.
// The HA group was created in setup() but no CQS is yet opened for this ACTIVE cluster.
// As a result there is neither the CQS nor existing opened connections.
// In this case, the failover should finish instead of failing to get/close the CQS.
- CLUSTERS.transitClusterRole(haGroup, ClusterRoleRecord.ClusterRole.STANDBY, ClusterRoleRecord.ClusterRole.ACTIVE);
+ CLUSTERS.transitClusterRoleWithCluster1Down(haGroup, STANDBY, ACTIVE);
try (Connection conn = createFailoverConnection()) {
FailoverPhoenixConnection failoverConn = (FailoverPhoenixConnection) conn;
@@ -161,11 +174,11 @@ public class FailoverPhoenixConnection2IT {
*/
@Test(timeout = 300000)
public void testFailoverCanFinishWhenOneZKDownWithCQS() throws Exception {
- doTestWhenOneZKDown(CLUSTERS.getHBaseCluster1(), () -> {
+ CLUSTERS.doTestWhenOneZKDown(CLUSTERS.getHBaseCluster1(), () -> {
// Try to make a connection to current ACTIVE cluster, which should fail.
try {
createFailoverConnection();
- fail("Should have failed since ACTIVE ZK '" + CLUSTERS.getZkUrl1() + "' is down");
+ fail("Should have failed since ACTIVE HBase '" + CLUSTERS.getMasterAddress1() + "' is down");
} catch (SQLException e) {
LOG.info("Got expected exception when ACTIVE ZK cluster is down", e);
}
@@ -173,7 +186,8 @@ public class FailoverPhoenixConnection2IT {
// As the target ZK is down, now existing CQS if any is in a bad state. In this case,
// the failover should still finish. After all, this is the most important use case the
// failover is designed for.
- CLUSTERS.transitClusterRole(haGroup, ClusterRoleRecord.ClusterRole.STANDBY, ClusterRoleRecord.ClusterRole.ACTIVE);
+ CLUSTERS.transitClusterRoleWithCluster1Down(haGroup, ClusterRole.ACTIVE_TO_STANDBY, ClusterRole.STANDBY_TO_ACTIVE);
+ CLUSTERS.transitClusterRoleWithCluster1Down(haGroup, STANDBY, ACTIVE);
try (Connection conn = createFailoverConnection()) {
FailoverPhoenixConnection failoverConn = (FailoverPhoenixConnection) conn;
@@ -199,7 +213,7 @@ public class FailoverPhoenixConnection2IT {
try (Connection conn = createFailoverConnection()) {
doTestBasicOperationsWithConnection(conn, tableName, haGroupName);
}
- doTestWhenOneZKDown(CLUSTERS.getHBaseCluster1(), () -> {
+ CLUSTERS.doTestWhenOneZKDown(CLUSTERS.getHBaseCluster1(), () -> {
try {
try (Connection conn = createFailoverConnection()) {
doTestBasicOperationsWithConnection(conn, tableName, haGroupName);
@@ -211,7 +225,8 @@ public class FailoverPhoenixConnection2IT {
// So on-call engineer comes into play, and triggers a failover
// Because cluster1 is down, new version record will only be updated in cluster2
- CLUSTERS.transitClusterRole(haGroup, ClusterRoleRecord.ClusterRole.STANDBY, ClusterRoleRecord.ClusterRole.ACTIVE);
+ CLUSTERS.transitClusterRoleWithCluster1Down(haGroup, ClusterRole.ACTIVE_TO_STANDBY, ClusterRole.STANDBY_TO_ACTIVE);
+ CLUSTERS.transitClusterRoleWithCluster1Down(haGroup, STANDBY, ACTIVE);
// After failover, the FailoverPhoenixConnection should go to the second cluster
try (Connection conn = createFailoverConnection()) {
@@ -232,7 +247,8 @@ public class FailoverPhoenixConnection2IT {
LOG.info("Testing failover back to cluster1 when bot clusters are up and running");
// Failover back to the first cluster since it is healthy after restart
- CLUSTERS.transitClusterRole(haGroup, ClusterRoleRecord.ClusterRole.ACTIVE, ClusterRoleRecord.ClusterRole.STANDBY);
+ CLUSTERS.transitClusterRole(haGroup, ClusterRole.STANDBY_TO_ACTIVE, ClusterRole.ACTIVE_TO_STANDBY);
+ CLUSTERS.transitClusterRole(haGroup, ACTIVE, STANDBY);
// After ACTIVE ZK restarts and fail back, this should still work
try (Connection conn = createFailoverConnection()) {
doTestBasicOperationsWithConnection(conn, tableName, haGroupName);
@@ -246,28 +262,36 @@ public class FailoverPhoenixConnection2IT {
* After failover, client can not connect to this HA group (ACTIVE cluster 2 is down).
* After cluster 2 (ACTIVE) restarts, client can connect to this HA group.
*/
- @Test(timeout = 300000)
- public void testConnectionWhenStandbyZKRestarts() throws Exception {
- doTestWhenOneZKDown(CLUSTERS.getHBaseCluster2(), () -> {
- try (Connection conn = createFailoverConnection()) {
- doTestBasicOperationsWithConnection(conn, tableName, haGroupName);
- }
-
- // Innocent on-call engineer triggers a failover to second cluster
- CLUSTERS.transitClusterRole(haGroup, ClusterRoleRecord.ClusterRole.STANDBY, ClusterRoleRecord.ClusterRole.ACTIVE);
-
- try {
- createFailoverConnection();
- fail("Should have failed since ACTIVE ZK cluster was shutdown");
- } catch (SQLException e) {
- LOG.info("Got expected exception when ACTIVE ZK cluster {} was shutdown",
- CLUSTERS.getZkUrl2(), e);
- }
- });
- try (Connection conn = createFailoverConnection()) {
- doTestBasicOperationsWithConnection(conn, tableName, haGroupName);
- }
- }
+ @Test(timeout = 300000)
+ public void testConnectionWhenStandbyZKRestarts() throws Exception {
+ CLUSTERS.doTestWhenOneZKDown(CLUSTERS.getHBaseCluster2(), () -> {
+ try (Connection conn = createFailoverConnection()) {
+ doTestBasicOperationsWithConnection(conn, tableName, haGroupName);
+ }
+
+ // Innocent on-call engineer triggers a failover to second cluster
+ CLUSTERS.transitClusterRoleWithCluster2Down(haGroup, ClusterRole.ACTIVE_TO_STANDBY, ClusterRole.STANDBY_TO_ACTIVE);
+ CLUSTERS.transitClusterRoleWithCluster2Down(haGroup, STANDBY, ACTIVE);
+
+ try {
+ createFailoverConnection();
+ fail("Should have failed since ACTIVE ZK cluster was shutdown");
+ } catch (SQLException e) {
+ LOG.info("Got expected exception when ACTIVE ZK cluster {} was shutdown",
+ CLUSTERS.getZkUrl2(), e);
+ }
+ });
+
+ //We will need to update SYSTEM Table for cluster 2 here as it was down and has
+ //stale info.
+ CLUSTERS.doUpdatesMissedWhenClusterWasDown(haGroup, ACTIVE_TO_STANDBY, ClusterRole.STANDBY_TO_ACTIVE, 2);
+ CLUSTERS.doUpdatesMissedWhenClusterWasDown(haGroup, STANDBY, ACTIVE, 2);
+ //Wait for sometime for poller to pull the record (5 sec)
+ sleepThreadFor(5000);
+ try (Connection conn = createFailoverConnection()) {
+ doTestBasicOperationsWithConnection(conn, tableName, haGroupName);
+ }
+ }
/**
* Tests the scenario where two ZK clusters both restart.
@@ -278,8 +302,8 @@ public class FailoverPhoenixConnection2IT {
*/
@Test(timeout = 300000)
public void testConnectionWhenTwoZKRestarts() throws Exception {
- doTestWhenOneZKDown(CLUSTERS.getHBaseCluster1(), () -> {
- doTestWhenOneZKDown(CLUSTERS.getHBaseCluster2(), () -> {
+ CLUSTERS.doTestWhenOneZKDown(CLUSTERS.getHBaseCluster1(), () -> {
+ CLUSTERS.doTestWhenOneZKDown(CLUSTERS.getHBaseCluster2(), () -> {
try {
createFailoverConnection();
fail("Should have failed since ACTIVE ZK cluster was shutdown");
@@ -298,7 +322,8 @@ public class FailoverPhoenixConnection2IT {
}
// So on-call engineer comes into play, and triggers a failover
- CLUSTERS.transitClusterRole(haGroup, ClusterRoleRecord.ClusterRole.STANDBY, ClusterRoleRecord.ClusterRole.ACTIVE);
+ CLUSTERS.transitClusterRoleWithCluster1Down(haGroup, ClusterRole.ACTIVE_TO_STANDBY, ClusterRole.STANDBY_TO_ACTIVE);
+ CLUSTERS.transitClusterRoleWithCluster1Down(haGroup, STANDBY, ACTIVE);
// After the second cluster (ACTIVE) ZK restarts, this should still work
try (Connection conn = createFailoverConnection()) {
@@ -320,7 +345,8 @@ public class FailoverPhoenixConnection2IT {
connectionList.add(createFailoverConnection());
}
- CLUSTERS.transitClusterRole(haGroup, ClusterRoleRecord.ClusterRole.STANDBY, ClusterRoleRecord.ClusterRole.ACTIVE);
+ CLUSTERS.transitClusterRole(haGroup, ClusterRole.ACTIVE_TO_STANDBY, ClusterRole.STANDBY_TO_ACTIVE);
+ CLUSTERS.transitClusterRole(haGroup, STANDBY, ACTIVE);
for (short i = 0; i < numberOfConnections; i++) {
LOG.info("Asserting connection number {}", i);
@@ -344,14 +370,15 @@ public class FailoverPhoenixConnection2IT {
// Add a good connection before shutting down and failing over
connections.add(executor.submit(this::createFailoverConnection));
- doTestWhenOneZKDown(CLUSTERS.getHBaseCluster1(), () -> {
+ CLUSTERS.doTestWhenOneZKDown(CLUSTERS.getHBaseCluster1(), () -> {
LOG.info("Since cluster1 is down, now failing over to cluster2");
// Create parallel clients that would use the CQSI that failover policy tries to close
for (short i = 1; i < numberOfThreads; i++) {
connections.add(executor.submit(this::createFailoverConnection));
}
// The ACTIVE cluster goes down, and then on-call engineer triggers failover.
- CLUSTERS.transitClusterRole(haGroup, ClusterRoleRecord.ClusterRole.STANDBY, ClusterRoleRecord.ClusterRole.ACTIVE);
+ CLUSTERS.transitClusterRoleWithCluster1Down(haGroup, ClusterRole.ACTIVE_TO_STANDBY, ClusterRole.STANDBY_TO_ACTIVE);
+ CLUSTERS.transitClusterRoleWithCluster1Down(haGroup, STANDBY, ACTIVE);
});
waitFor(() -> {
@@ -374,14 +401,14 @@ public class FailoverPhoenixConnection2IT {
}
/**
- * This is to make sure all Phoenix connections are closed when registryType changes
- * of a CRR. ZK --> MASTER
- *
- * Test with many connections.
+ * Testing connection closure for roleRecord change from
+ * (URL1, ACTIVE, URL2, STANDBY) -> (URL3, ACTIVE, URL2, STANDBY)
+ * All wrapped connections should have been closed
*/
@Test(timeout = 300000)
- public void testAllWrappedConnectionsClosedAfterRegistryChangeToMaster() throws Exception {
+ public void testAllWrappedConnectionsClosedAfterActiveURLChange() throws Exception {
short numberOfConnections = 10;
+ //Create FailoverPhoenixConnections with default urls
List<Connection> connectionList = new ArrayList<>(numberOfConnections);
for (short i = 0; i < numberOfConnections; i++) {
connectionList.add(createFailoverConnection());
@@ -391,21 +418,24 @@ public class FailoverPhoenixConnection2IT {
ConnectionInfo connInfo = ConnectionInfo.create(CLUSTERS.getJdbcUrl1(haGroup),
PhoenixDriver.INSTANCE.getQueryServices().getProps(), clientProperties);
- ClusterRoleRecord.RegistryType newRegistry = ClusterRoleRecord.RegistryType.MASTER;
- CLUSTERS.transitClusterRoleRecordRegistry(haGroup, newRegistry);
-
+ //Restart Cluster 1 with new ports
+ CLUSTERS.restartCluster1();
+ //Basically applying new url for cluster 1
+ CLUSTERS.refreshClusterRoleRecordAfterClusterRestart(haGroup,
+ ACTIVE, STANDBY, 1);
for (short i = 0; i < numberOfConnections; i++) {
LOG.info("Asserting connection number {}", i);
FailoverPhoenixConnection conn = ((FailoverPhoenixConnection) connectionList.get(i));
assertFalse(conn.isClosed());
assertTrue(conn.getWrappedConnection().isClosed());
+ conn.close();
}
//CQSI should be closed
try {
cqsi.checkClosed();
fail("Should have thrown an exception as cqsi should be closed");
} catch (IllegalStateException e) {
- //Exception Expected cqsi should have been invalidated as well
+ //Exception cqsi should have been invalidated as well
assertFalse(PhoenixDriver.INSTANCE.checkIfCQSIIsInCache(connInfo));
} catch (Exception e) {
fail("Should have thrown on IllegalStateException as cqsi should be closed");
@@ -413,69 +443,301 @@ public class FailoverPhoenixConnection2IT {
}
/**
- * This is to make sure all Phoenix connections are closed when registryType changes
- * of a CRR. ZK --> RPC
- *
- * Test with many connections.
+ * Testing connection closure for roleRecord change from
+ * (URL1, ACTIVE, URL2, STANDBY) -> (URL1, ACTIVE, URL3, STANDBY)
+ * It should not have affected connections at all
*/
@Test(timeout = 300000)
- public void testAllWrappedConnectionsClosedAfterRegistryChangeToRpc() throws Exception {
+ public void testAllWrappedConnectionsNotClosedAfterStandbyURLChange() throws Exception {
short numberOfConnections = 10;
+ //Create FailoverPhoenixConnections with default urls
List<Connection> connectionList = new ArrayList<>(numberOfConnections);
for (short i = 0; i < numberOfConnections; i++) {
connectionList.add(createFailoverConnection());
}
ConnectionQueryServicesImpl cqsi = (ConnectionQueryServicesImpl) PhoenixDriver.INSTANCE.
- getConnectionQueryServices(CLUSTERS.getJdbcUrl1(haGroup), clientProperties);
- ConnectionInfo connInfo = ConnectionInfo.create(CLUSTERS.getJdbcUrl1(haGroup),
- PhoenixDriver.INSTANCE.getQueryServices().getProps(), clientProperties);
-
- ClusterRoleRecord.RegistryType newRegistry = ClusterRoleRecord.RegistryType.RPC;
- //RPC Registry is only there in hbase version greater than 2.5.0
- assumeTrue(VersionInfo.compareVersion(VersionInfo.getVersion(), "2.5.0")>=0);
- CLUSTERS.transitClusterRoleRecordRegistry(haGroup, newRegistry);
+ getConnectionQueryServices(CLUSTERS.getJdbcUrl2(haGroup), clientProperties);
+ //Restart Cluster 2 with new ports
+ CLUSTERS.restartCluster2();
+ //Basically applying new url for cluster 2
+ CLUSTERS.refreshClusterRoleRecordAfterClusterRestart(haGroup,
+ ACTIVE, STANDBY, 2);
for (short i = 0; i < numberOfConnections; i++) {
LOG.info("Asserting connection number {}", i);
FailoverPhoenixConnection conn = ((FailoverPhoenixConnection) connectionList.get(i));
assertFalse(conn.isClosed());
- assertTrue(conn.getWrappedConnection().isClosed());
+ assertFalse(conn.getWrappedConnection().isClosed());
+ conn.close();
}
- //CQSI should be closed
+ //CQSI should not be closed
try {
cqsi.checkClosed();
- fail("Should have thrown an exception as cqsi should be closed");
- } catch (IllegalStateException e) {
- //Expected cqsi should have been invalidated as well
- assertFalse(PhoenixDriver.INSTANCE.checkIfCQSIIsInCache(connInfo));
} catch (Exception e) {
- fail("Should have thrown on IllegalStateException as cqsi should be closed");
+ fail("Should not through any excepetion as cqsi is not closed");
}
}
/**
- * Testing connection closure for roleRecord change from
- * (URL1, ACTIVE, URL2, STANDBY) -> (URL3, ACTIVE, URL2, STANDBY)
- * All wrapped connections should have been closed
+ * Testing stale CRR version detection during writes and automatic CRR refresh in both cases
+ * 1 - When Stale CRR Version detected
+ * 2 - When CRR was refreshed to non-active Poller started to automatically refresh until it detects an ACTIVE CRR
+ */
+ @Test(timeout = 300000)
+ public void testStaleCrrVersionDetection() throws Exception {
+ Connection connection = createFailoverConnection();
+ doTestBasicOperationsWithConnection(connection, tableName, haGroupName);
+ CLUSTERS.transitClusterRole(haGroup, ACTIVE_TO_STANDBY, STANDBY, false);
+ CLUSTERS.transitClusterRole(haGroup, STANDBY, STANDBY, false);
+
+ try {
+ doTestBasicOperationsWithConnection(connection, tableName, haGroupName);
+ fail("Should have failed since CRR is stale");
+ } catch (SQLException e) {
+ //Should fail but CRR must be refreshed after detecting Stale version
+ assertEquals(e.getErrorCode(), FAILOVER_IN_PROGRESS.getErrorCode());
+ }
+
+ //Connection creation should fail as now we don't have active cluster
+ try {
+ createFailoverConnection();
+ fail("Should have failed since ACTIVE ZK cluster was shutdown");
+ } catch (SQLException e) {
+ LOG.info("Printing exception", e);
+ assertEquals(e.getErrorCode(), CANNOT_ESTABLISH_CONNECTION.getErrorCode());
+ assertTrue(e.getCause() instanceof FailoverSQLException);
+ //Expected
+ }
+ //Above connection creation should start a poller as well
+ CLUSTERS.transitClusterRole(haGroup, STANDBY, STANDBY_TO_ACTIVE, false);
+ CLUSTERS.transitClusterRole(haGroup, STANDBY, ACTIVE, false);
+
+ //Sleep for 5 sec for poller to automatically pick up change
+ sleepThreadFor(5000);
+
+ //Connection creation and operations should work
+ try (Connection conn = createFailoverConnection()) {
+ doTestBasicOperationsWithConnection(conn, tableName, haGroupName);
+ }
+
+ }
+
+ /**
+ * Testing executeQuery/scans which are not pontlookup should detect and refresh CRR when
+ * haGroup's CRR update cache is expired
*/
@Test(timeout = 300000)
- public void testAllWrappedConnectionsClosedAfterActiveURLChange() throws Exception {
- short numberOfConnections = 10;
+ public void testStaleCrrVersionWithReadDetection() throws Exception {
+ Connection connection = createFailoverConnection();
+ //do Write
+ for(int i = 0; i < 100 ; i++) {
+ try (Statement stmt = connection.createStatement()) {
+ int value = RandomUtils.nextInt();
+ stmt.executeUpdate(String.format("UPSERT INTO %s VALUES(%d, %d)", tableName, value, value));
+ }
+ }
+ try (Statement stmt = connection.createStatement()) {
+ stmt.executeUpdate(String.format("UPSERT INTO %s VALUES(1988, 1984)", tableName));
+ }
+ connection.commit();
+
+ CLUSTERS.transitClusterRole(haGroup, ACTIVE_TO_STANDBY, STANDBY, false);
+
+ //Expire the CRR cache in HAGroup for scan operation
+ sleepThreadFor(3000);
+
+ //Read operation should refresh the CRR leading to success and also update the CRR in
+ //HAGroup and continuing the scan and giving back correct result
+ try (ResultSet rs = connection.createStatement().executeQuery(
+ String.format("SELECT v FROM %s WHERE v = 1984", tableName))) {
+ rs.next();
+ assertEquals(1984, rs.getInt(1));
+ //Wait for transition to complete
+ sleepThreadFor(2000);
+
+ String url = JDBCUtil.formatUrl(CLUSTERS.getJdbcUrl1(haGroup), haGroup.getRoleRecord()
+ .getRegistryType());
+ assertEquals(ACTIVE_TO_STANDBY, haGroup.getRoleRecord().getRole(url));
+ } catch (SQLException e) {
+ fail(String.format("Should have succeeded but failed with exception {}", e));
+ }
+
+ CLUSTERS.transitClusterRole(haGroup, STANDBY, STANDBY, false);
+
+ //Expire the CRR cache in HAGroup for scan operation
+ sleepThreadFor(3000);
+
+ //Read operation should refresh the CRR leading to success and also update the CRR in
+ //HAGroup but will throw exception as role for ACTIVE cluster is changing
+ try (ResultSet rs = connection.createStatement().executeQuery(
+ String.format("SELECT v FROM %s WHERE v = 1984", tableName))) {
+ fail("Should have thrown an exception");
+ } catch (SQLException e) {
+ assertEquals(e.getErrorCode(), FAILOVER_IN_PROGRESS.getErrorCode());
+
+ //wait for transition to happen to prevent Flakiness of tests
+ sleepThreadFor(2000);
+ String url = JDBCUtil.formatUrl(CLUSTERS.getJdbcUrl1(haGroup), haGroup.getRoleRecord()
+ .getRegistryType());
+ assertEquals(STANDBY, haGroup.getRoleRecord().getRole(url));
+ }
+
+ //connections should be closed
+ assertTrue(((FailoverPhoenixConnection)connection).getWrappedConnection().isClosed());
+
+ //Connection creation should fail as now we don't have active cluster
+ try {
+ createFailoverConnection();
+ fail("Should have failed since ACTIVE ZK cluster was shutdown");
+ } catch (SQLException e) {
+ assertEquals(e.getErrorCode(), CANNOT_ESTABLISH_CONNECTION.getErrorCode());
+ assertTrue(e.getCause() instanceof FailoverSQLException);
+ //Expected
+ }
+ //Above connection creation should start a poller as well
+ CLUSTERS.transitClusterRole(haGroup, STANDBY, STANDBY_TO_ACTIVE, false);
+ CLUSTERS.transitClusterRole(haGroup, STANDBY, ACTIVE, false);
+
+ //Sleep for 5 sec for poller to automatically pick up change
+ sleepThreadFor(5000);
+
+ //Connection creation and operations should work
+ try (Connection conn = createFailoverConnection()) {
+ doTestBasicOperationsWithConnection(conn, tableName, haGroupName);
+ }
+
+ //Poller should have Stopped when we got Active CRR so transitions now should not be detected
+ //unless we do any scan or write
+
+ CLUSTERS.transitClusterRole(haGroup, STANDBY, ACTIVE_TO_STANDBY, false);
+ CLUSTERS.transitClusterRole(haGroup, STANDBY, STANDBY, false);
+
+ //Sleeping for 2 Sec to confirm poller is not detecting and has been stopped
+ sleepThreadFor(2000);
+
+ //Connection creation should happen
+ try (Connection conn = createFailoverConnection()){
+ try {
+ ResultSet rs = connection.createStatement().executeQuery(
+ String.format("SELECT v FROM %s WHERE id = 1988", tableName));
+ fail("Should have failed as CRR is not Active and cache frequency is expired so creation " +
+ "of result set should try to refresh CRR and should close current connection and statement and fail");
+ } catch (Exception e) {
+ //Expected
+ assertTrue(e instanceof FailoverSQLException);
+ }
+ }
+ }
+
+
+ /**
+ * Testing executeQuery/scans which are pointLookUps should detect and refresh CRR when
+ * haGroup's CRR update cache is expired
+ */
+ @Test(timeout = 300000)
+ public void testStaleCrrVersionWithPointLookUpReadDetection() throws Exception {
+ Connection connection = createFailoverConnection();
+ //do Write
+ int id = RandomUtils.nextInt();
+ try (Statement stmt = connection.createStatement()) {
+ stmt.executeUpdate(String.format("UPSERT INTO %s VALUES(%d, 1984)", tableName, id));
+ connection.commit();
+ }
+ CLUSTERS.transitClusterRole(haGroup, STANDBY, STANDBY, false);
+
+ //Read operation should refresh the CRR leading to success and also update the CRR in HAGroup
+ //as it is a point lookup
+ try (ResultSet rs = connection.createStatement().executeQuery(
+ String.format("SELECT v FROM %s WHERE id = %d", tableName, id))) {
+ rs.next();
+ //It will fail but can't be tested here as it happens asynchronously. But CRR must have
+ //been refreshed
+ } catch (SQLException e) {
+ //Expected but not caught here as we are running the iterator asynchronously within a
+ //separate thread managed by the ExecutorService.
+ }
+ //To prevent flakiness sleep for 2 sec
+ sleepThreadFor(2000);
+
+ String url = JDBCUtil.formatUrl(CLUSTERS.getJdbcUrl1(haGroup), haGroup.getRoleRecord()
+ .getRegistryType());
+ assertEquals(STANDBY, haGroup.getRoleRecord().getRole(url));
+ //connections should be closed
+ assertTrue(((FailoverPhoenixConnection)connection).getWrappedConnection().isClosed());
+
+ //Connection creation should fail as now we don't have active cluster
+ try {
+ createFailoverConnection();
+ fail("Should have failed since ACTIVE ZK cluster was shutdown");
+ } catch (SQLException e) {
+ assertEquals(e.getErrorCode(), CANNOT_ESTABLISH_CONNECTION.getErrorCode());
+ assertTrue(e.getCause() instanceof FailoverSQLException);
+ //Expected
+ }
+ //Above connection creation should start a poller as well
+ CLUSTERS.transitClusterRole(haGroup, STANDBY, STANDBY_TO_ACTIVE, false);
+ CLUSTERS.transitClusterRole(haGroup, STANDBY, ACTIVE, false);
+
+ //Sleep for 5 sec for poller to automatically pick up change
+ sleepThreadFor(5000);
+
+ //Connection creation and operations should work
+ try (Connection conn = createFailoverConnection()) {
+ doTestBasicOperationsWithConnection(conn, tableName, haGroupName);
+ }
+
+ //Poller should have Stopped when we got Active CRR so transitions now should not be detected
+ //unless we do any scan or write
+
+ CLUSTERS.transitClusterRole(haGroup, STANDBY, ACTIVE_TO_STANDBY, false);
+ CLUSTERS.transitClusterRole(haGroup, STANDBY, STANDBY, false);
+
+ //Sleeping for 2 Sec to confirm poller is not detecting and has been stopped
+ sleepThreadFor(2000);
+
+ //Connection creation should happen
+ try (Connection conn = createFailoverConnection()){
+ try {
+ ResultSet rs = connection.createStatement().executeQuery(
+ String.format("SELECT v FROM %s WHERE id = %d", tableName, id));
+ fail("Should have failed as CRR is not Active and cache frequency is expired so creation " +
+ "of result set should try to refresh CRR and should close current connection and statement and fail");
+ } catch (Exception e) {
+ //Expected
+ assertTrue(e instanceof FailoverSQLException);
+ }
+ }
+ }
+
+ /**
+ * Testing different possible transitions and their effects on connections and cqsi
+ */
+ @Test(timeout = 300000)
+ public void testDifferentPossibleTransitions() throws Exception {
+ short numberOfConnections = 3;
//Create FailoverPhoenixConnections with default urls
List<Connection> connectionList = new ArrayList<>(numberOfConnections);
for (short i = 0; i < numberOfConnections; i++) {
connectionList.add(createFailoverConnection());
}
ConnectionQueryServicesImpl cqsi = (ConnectionQueryServicesImpl) PhoenixDriver.INSTANCE.
- getConnectionQueryServices(CLUSTERS.getJdbcUrl1(haGroup), clientProperties);
+ getConnectionQueryServices(CLUSTERS.getJdbcUrl1(haGroup), clientProperties);
ConnectionInfo connInfo = ConnectionInfo.create(CLUSTERS.getJdbcUrl1(haGroup),
- PhoenixDriver.INSTANCE.getQueryServices().getProps(), clientProperties);
+ PhoenixDriver.INSTANCE.getQueryServices().getProps(), clientProperties);
- //Restart Cluster 1 with new ports
- CLUSTERS.restartCluster1();
- //Basically applying new url for cluster 1
- CLUSTERS.refreshClusterRoleRecordAfterClusterRestart(haGroup,
- ClusterRoleRecord.ClusterRole.ACTIVE, ClusterRoleRecord.ClusterRole.STANDBY);
+ //transit url1 to ACTIVE_TO_STANDBY and url2 to STANDBY_TO_ACTIVE and refresh the roleRecord of haGroup
+ //This transition should not affect the connections as ACTIVE --> ACTIVE_TO_STANDBY is noop
+ CLUSTERS.transitClusterRole(haGroup, ACTIVE_TO_STANDBY, STANDBY_TO_ACTIVE, true);
+ for (short i = 0; i < numberOfConnections; i++) {
+ LOG.info("Asserting connection number {}", i);
+ FailoverPhoenixConnection conn = ((FailoverPhoenixConnection) connectionList.get(i));
+ assertFalse(conn.isClosed());
+ assertFalse(conn.getWrappedConnection().isClosed());
+ }
+
+ //transit url1 to STANDBY and url2 to ACTIVE and refresh the roleRecord of haGroup
+ //This transition should affect the connections as we are failing over to url2
+ CLUSTERS.transitClusterRole(haGroup, STANDBY, ACTIVE, true);
for (short i = 0; i < numberOfConnections; i++) {
LOG.info("Asserting connection number {}", i);
FailoverPhoenixConnection conn = ((FailoverPhoenixConnection) connectionList.get(i));
@@ -488,74 +750,57 @@ public class FailoverPhoenixConnection2IT {
cqsi.checkClosed();
fail("Should have thrown an exception as cqsi should be closed");
} catch (IllegalStateException e) {
- //Exception cqsi should have been invalidated as well
- assertFalse(PhoenixDriver.INSTANCE.checkIfCQSIIsInCache(connInfo));
+ //Exception cqsi should not have been invalidated as well
+ assertTrue(PhoenixDriver.INSTANCE.checkIfCQSIIsInCache(connInfo));
} catch (Exception e) {
fail("Should have thrown on IllegalStateException as cqsi should be closed");
}
- }
- /**
- * Testing connection closure for roleRecord change from
- * (URL1, ACTIVE, URL2, STANDBY) -> (URL1, ACTIVE, URL3, STANDBY)
- * It should not have affected connections at all
- */
- @Test(timeout = 300000)
- public void testAllWrappedConnectionsNotClosedAfterStandbyURLChange() throws Exception {
- short numberOfConnections = 10;
- //Create FailoverPhoenixConnections with default urls
- List<Connection> connectionList = new ArrayList<>(numberOfConnections);
+ connectionList = new ArrayList<>(numberOfConnections);
for (short i = 0; i < numberOfConnections; i++) {
connectionList.add(createFailoverConnection());
}
- ConnectionQueryServicesImpl cqsi = (ConnectionQueryServicesImpl) PhoenixDriver.INSTANCE.
- getConnectionQueryServices(CLUSTERS.getJdbcUrl2(haGroup), clientProperties);
- //Restart Cluster 2 with new ports
- CLUSTERS.restartCluster2();
- //Basically applying new url for cluster 2
- CLUSTERS.refreshClusterRoleRecordAfterClusterRestart(haGroup,
- ClusterRoleRecord.ClusterRole.ACTIVE, ClusterRoleRecord.ClusterRole.STANDBY);
+ //Get cqsi and connInfo for url2 as that is ACTIVE now.
+ ConnectionQueryServicesImpl cqsi2 = (ConnectionQueryServicesImpl) PhoenixDriver.INSTANCE.
+ getConnectionQueryServices(CLUSTERS.getJdbcUrl2(haGroup), clientProperties);
+ ConnectionInfo connInfo2 = ConnectionInfo.create(CLUSTERS.getJdbcUrl2(haGroup),
+ PhoenixDriver.INSTANCE.getQueryServices().getProps(), clientProperties);
+
+ //transit url1 back to ACTIVE and url2 to STANDBY and refresh the roleRecord of haGroup
+ //This transition should affect the connections as we are failing over to url1
+ CLUSTERS.transitClusterRole(haGroup, ACTIVE, STANDBY, true);
for (short i = 0; i < numberOfConnections; i++) {
LOG.info("Asserting connection number {}", i);
FailoverPhoenixConnection conn = ((FailoverPhoenixConnection) connectionList.get(i));
assertFalse(conn.isClosed());
- assertFalse(conn.getWrappedConnection().isClosed());
+ assertTrue(conn.getWrappedConnection().isClosed());
conn.close();
}
- //CQSI should not be closed
+
+ //This should have invalidated the cqsi of url1 as it is becoming ACTIVE from STANDBY
+ assertFalse(PhoenixDriver.INSTANCE.checkIfCQSIIsInCache(connInfo));
+ //CQSI should be closed
try {
- cqsi.checkClosed();
+ cqsi2.checkClosed();
+ fail("Should have thrown an exception as cqsi should be closed");
+ } catch (IllegalStateException e) {
+ //Exception cqsi should not have been invalidated as well
+ assertTrue(PhoenixDriver.INSTANCE.checkIfCQSIIsInCache(connInfo2));
} catch (Exception e) {
- fail("Should not through any excepetion as cqsi is not closed");
- }
- }
-
- /**
- * Test connections should close early when transitioning to ATS.
- * cluster 1 ( ACTIVE --> ACTIVE_TO_STANDBY )
- */
- @Test(timeout = 300000)
- public void testAllWrappedConnectionsClosedWhenRoleChangeFromActiveToATS()
- throws Exception {
-
- //Creating some connections to ACTIVE cluster
- short numberOfConnections = 10;
- //Create FailoverPhoenixConnections with default urls
- List<Connection> connectionList = new ArrayList<>(numberOfConnections);
- for (short i = 0; i < numberOfConnections; i++) {
- connectionList.add(createFailoverConnection());
+ fail("Should have thrown on IllegalStateException as cqsi should be closed");
}
- //Transit Role1 ACTIVE --> ACTIVE_TO_STANDBY
- CLUSTERS.transitClusterRole(haGroup, ClusterRoleRecord.ClusterRole.ACTIVE_TO_STANDBY,
- ClusterRoleRecord.ClusterRole.STANDBY);
+ //Update the ZNodes and table with impossible role transition (ACTIVE, ACTIVE) and
+ //do not refresh the roleRecord of haGroup
+ CLUSTERS.transitClusterRole(haGroup, ACTIVE, ACTIVE, false);
- //Connections should be closed
- for (short i = 0; i < numberOfConnections; i++) {
- FailoverPhoenixConnection conn = ((FailoverPhoenixConnection) connectionList.get(i));
- assertFalse(conn.isClosed());
- assertTrue(conn.getWrappedConnection().isClosed());
+ try {
+ haGroup.refreshClusterRoleRecord(true);
+ fail("Should have thrown an exception as ACTIVE --> ACTIVE is not a valid transition");
+ } catch (SQLException e) {
+ assertEquals(e.getErrorCode(), SQLExceptionCode.HA_ROLE_TRANSITION_NOT_ALLOWED.getErrorCode());
+ //Expected
}
}
@@ -565,4 +810,5 @@ public class FailoverPhoenixConnection2IT {
private Connection createFailoverConnection() throws SQLException {
return DriverManager.getConnection(CLUSTERS.getJdbcHAUrl(), clientProperties);
}
+
}
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/FailoverPhoenixConnectionIT.java b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/FailoverPhoenixConnectionIT.java
index 73da203c6c..b2c88afb2d 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/FailoverPhoenixConnectionIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/FailoverPhoenixConnectionIT.java
@@ -199,6 +199,7 @@ public class FailoverPhoenixConnectionIT {
doTestBasicOperationsWithConnection(conn, tableName, haGroupName);
}
+ CLUSTERS.transitClusterRole(haGroup, ClusterRole.ACTIVE_TO_STANDBY, ClusterRole.STANDBY);
CLUSTERS.transitClusterRole(haGroup, ClusterRole.STANDBY, ClusterRole.STANDBY);
try {
@@ -246,6 +247,8 @@ public class FailoverPhoenixConnectionIT {
Connection conn = createFailoverConnection();
doTestBasicOperationsWithConnection(conn, tableName, haGroupName);
+ //transit only the way we allow now
+ CLUSTERS.transitClusterRole(haGroup, ClusterRole.ACTIVE_TO_STANDBY, ClusterRole.STANDBY_TO_ACTIVE);
CLUSTERS.transitClusterRole(haGroup, ClusterRole.STANDBY, ClusterRole.ACTIVE);
// The wrapped connection is still against the first cluster, and is closed
@@ -265,6 +268,8 @@ public class FailoverPhoenixConnectionIT {
Statement stmt = conn.createStatement();
doTestBasicOperationsWithStatement(conn, stmt, tableName);
+ //transit only the way we allow now
+ CLUSTERS.transitClusterRole(haGroup, ClusterRole.ACTIVE_TO_STANDBY, ClusterRole.STANDBY_TO_ACTIVE);
CLUSTERS.transitClusterRole(haGroup, ClusterRole.STANDBY, ClusterRole.ACTIVE);
assertFalse(conn.isClosed());
@@ -292,6 +297,8 @@ public class FailoverPhoenixConnectionIT {
assertFalse(failoverConn.isClosed());
assertFalse(wrappedConn.isClosed());
+ //transit only the way we allow now
+ CLUSTERS.transitClusterRole(haGroup, ClusterRole.ACTIVE_TO_STANDBY, ClusterRole.STANDBY_TO_ACTIVE);
CLUSTERS.transitClusterRole(haGroup, ClusterRole.STANDBY, ClusterRole.ACTIVE);
assertFalse(phoenixConn.isClosed()); // normal Phoenix connection is not closed
@@ -317,6 +324,8 @@ public class FailoverPhoenixConnectionIT {
assertFalse(wrappedConn.isClosed());
assertFalse(wrappedConn2.isClosed());
+ //transit only the way we allow now
+ CLUSTERS.transitClusterRole(haGroup, ClusterRole.ACTIVE_TO_STANDBY, ClusterRole.STANDBY_TO_ACTIVE);
CLUSTERS.transitClusterRole(haGroup, ClusterRole.STANDBY, ClusterRole.ACTIVE);
assertTrue(wrappedConn.isClosed());
@@ -357,9 +366,10 @@ public class FailoverPhoenixConnectionIT {
cqs.removeConnection(wrapped.unwrap(PhoenixConnection.class));
cqs.addConnection(spy.unwrap(PhoenixConnection.class));
- // (ACTIVE, STANDBY) -> (STANDBY, ACTIVE)
+ // (ACTIVE, STANDBY) -> (ACTIVE_TO_STANDBY, STANDBY_TO_ACTIVE) -> (STANDBY, ACTIVE)
// The transition will finish as we set PHOENIX_HA_TRANSITION_TIMEOUT_MS_KEY for this class
// even though the spied connection is stuck at the latch when closing
+ CLUSTERS.transitClusterRole(haGroup, ClusterRole.ACTIVE_TO_STANDBY, ClusterRole.STANDBY_TO_ACTIVE);
CLUSTERS.transitClusterRole(haGroup, ClusterRole.STANDBY, ClusterRole.ACTIVE);
// Verify the spied object has been called once
@@ -402,6 +412,8 @@ public class FailoverPhoenixConnectionIT {
connectionList.add(createFailoverConnection());
}
+ //transit only the way we allow now
+ CLUSTERS.transitClusterRole(haGroup, ClusterRole.ACTIVE_TO_STANDBY, ClusterRole.STANDBY_TO_ACTIVE);
CLUSTERS.transitClusterRole(haGroup, ClusterRole.STANDBY, ClusterRole.ACTIVE);
for (short i = 0; i < numberOfConnections; i++) {
@@ -439,7 +451,9 @@ public class FailoverPhoenixConnectionIT {
}
latchToTransitRole.await();
- CLUSTERS.transitClusterRole(haGroup, ClusterRole.STANDBY, ClusterRole.STANDBY);
+ //transit only the way we allow now
+ CLUSTERS.transitClusterRole(haGroup, ClusterRole.ACTIVE_TO_STANDBY, ClusterRole.STANDBY_TO_ACTIVE);
+ CLUSTERS.transitClusterRole(haGroup, ClusterRole.STANDBY, ClusterRole.ACTIVE);
latchToCreateMoreConnections.countDown();
waitFor(() -> {
@@ -475,6 +489,7 @@ public class FailoverPhoenixConnectionIT {
}
// Make the second cluster the active one.
+ CLUSTERS.transitClusterRole(haGroup, ClusterRole.ACTIVE_TO_STANDBY, ClusterRole.STANDBY_TO_ACTIVE);
CLUSTERS.transitClusterRole(haGroup, ClusterRole.STANDBY, ClusterRole.ACTIVE);
try (Connection conn = createFailoverConnection()) {
@@ -497,6 +512,8 @@ public class FailoverPhoenixConnectionIT {
}
// Make the second cluster the active one.
+ //transit only the way we allow now
+ CLUSTERS.transitClusterRole(haGroup, ClusterRole.ACTIVE_TO_STANDBY, ClusterRole.STANDBY_TO_ACTIVE);
CLUSTERS.transitClusterRole(haGroup, ClusterRole.STANDBY, ClusterRole.ACTIVE);
try (Connection conn = createFailoverConnection()) {
@@ -506,6 +523,7 @@ public class FailoverPhoenixConnectionIT {
}
// Failover back to the first cluster.
+ CLUSTERS.transitClusterRole(haGroup, ClusterRole.STANDBY_TO_ACTIVE, ClusterRole.ACTIVE_TO_STANDBY);
CLUSTERS.transitClusterRole(haGroup, ClusterRole.ACTIVE, ClusterRole.STANDBY);
try (Connection conn = createFailoverConnection()) {
@@ -524,6 +542,7 @@ public class FailoverPhoenixConnectionIT {
doTestBasicOperationsWithConnection(conn, tableName, haGroupName);
// Make the second cluster ACTIVE will not change the wrapped connection.
+ CLUSTERS.transitClusterRole(haGroup, ClusterRole.ACTIVE_TO_STANDBY, ClusterRole.STANDBY_TO_ACTIVE);
CLUSTERS.transitClusterRole(haGroup, ClusterRole.STANDBY, ClusterRole.ACTIVE);
doTestActionShouldFailBecauseOfFailover(conn::createStatement);
@@ -532,6 +551,7 @@ public class FailoverPhoenixConnectionIT {
doTestBasicOperationsWithConnection(conn, tableName, haGroupName);
// failover explicitly once more (failover back)
+ CLUSTERS.transitClusterRole(haGroup, ClusterRole.STANDBY_TO_ACTIVE, ClusterRole.ACTIVE_TO_STANDBY);
CLUSTERS.transitClusterRole(haGroup, ClusterRole.ACTIVE, ClusterRole.STANDBY);
FailoverPhoenixConnection.failover(conn, 30_000);
doTestBasicOperationsWithConnection(conn, tableName, haGroupName);
@@ -545,6 +565,7 @@ public class FailoverPhoenixConnectionIT {
Connection conn = createFailoverConnection();
doTestBasicOperationsWithConnection(conn, tableName, haGroupName);
+ CLUSTERS.transitClusterRole(haGroup, ClusterRole.ACTIVE_TO_STANDBY, ClusterRole.STANDBY);
CLUSTERS.transitClusterRole(haGroup, ClusterRole.STANDBY, ClusterRole.STANDBY);
try {
@@ -569,6 +590,7 @@ public class FailoverPhoenixConnectionIT {
doTestBasicOperationsWithConnection(tenantConn, tableName, haGroupName);
// Make the second cluster ACTIVE will not change the wrapped connection.
+ CLUSTERS.transitClusterRole(haGroup, ClusterRole.ACTIVE_TO_STANDBY, ClusterRole.STANDBY_TO_ACTIVE);
CLUSTERS.transitClusterRole(haGroup, ClusterRole.STANDBY, ClusterRole.ACTIVE);
doTestActionShouldFailBecauseOfFailover(tenantConn::createStatement);
@@ -590,6 +612,7 @@ public class FailoverPhoenixConnectionIT {
doTestBasicOperationsWithStatement(conn, stmt1, tableName);
// Make the second cluster the active one.
+ CLUSTERS.transitClusterRole(haGroup, ClusterRole.ACTIVE_TO_STANDBY, ClusterRole.STANDBY_TO_ACTIVE);
CLUSTERS.transitClusterRole(haGroup, ClusterRole.STANDBY, ClusterRole.ACTIVE);
assertFalse(conn.isClosed());
@@ -612,6 +635,7 @@ public class FailoverPhoenixConnectionIT {
doVerifyMetrics(conn, 1L);
// Failover the HA group
+ CLUSTERS.transitClusterRole(haGroup, ClusterRole.ACTIVE_TO_STANDBY, ClusterRole.STANDBY_TO_ACTIVE);
CLUSTERS.transitClusterRole(haGroup, ClusterRole.STANDBY, ClusterRole.ACTIVE);
// wrapped connection should have been closed; but "conn" is not so we can still get metrics
doVerifyMetrics(conn, 1L);
@@ -665,6 +689,7 @@ public class FailoverPhoenixConnectionIT {
assertFalse(wrappedConn3.isClosed());
assertFalse(wrappedConn4.isClosed());
+ CLUSTERS.transitClusterRole(haGroup, ClusterRole.ACTIVE_TO_STANDBY, ClusterRole.STANDBY_TO_ACTIVE);
CLUSTERS.transitClusterRole(haGroup, ClusterRole.STANDBY, ClusterRole.ACTIVE);
assertTrue(wrappedConn.isClosed());
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HAConnectionWithMasterAndRPCRegistryIT.java b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HAConnectionWithMasterAndRPCRegistryIT.java
deleted file mode 100644
index 09b1cd5b05..0000000000
--- a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HAConnectionWithMasterAndRPCRegistryIT.java
+++ /dev/null
@@ -1,596 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.phoenix.jdbc;
-
-import org.apache.commons.lang3.RandomStringUtils;
-import org.apache.hadoop.hbase.util.VersionInfo;
-import org.apache.phoenix.end2end.NeedsOwnMiniClusterTest;
-import org.apache.phoenix.jdbc.ClusterRoleRecord.RegistryType;
-import org.apache.phoenix.query.ConnectionQueryServices;
-import org.apache.phoenix.query.ConnectionQueryServicesImpl;
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-import org.junit.rules.TestName;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-import org.mockito.Mockito;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.sql.Connection;
-import java.sql.DriverManager;
-import java.sql.SQLException;
-import java.util.*;
-import java.util.concurrent.CountDownLatch;
-
-import static org.apache.hadoop.test.GenericTestUtils.waitFor;
-import static org.apache.phoenix.jdbc.HighAvailabilityGroup.PHOENIX_HA_GROUP_ATTR;
-import static org.apache.phoenix.jdbc.HighAvailabilityTestingUtility.HBaseTestingUtilityPair.doTestWhenOneHBaseDown;
-import static org.apache.phoenix.jdbc.HighAvailabilityTestingUtility.doTestBasicOperationsWithConnection;
-import static org.apache.phoenix.jdbc.HighAvailabilityTestingUtility.getHighAvailibilityGroup;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-import static org.junit.Assume.assumeTrue;
-import static org.mockito.Mockito.doAnswer;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-
-/**
- * Test HA basics for {@link ClusterRoleRecord} with {@link RegistryType} as MASTER.
- */
-@RunWith(Parameterized.class)
-@Category(NeedsOwnMiniClusterTest.class)
-public class HAConnectionWithMasterAndRPCRegistryIT {
-
- private static final Logger LOG = LoggerFactory.getLogger(HAConnectionWithMasterAndRPCRegistryIT.class);
- private static final HighAvailabilityTestingUtility.HBaseTestingUtilityPair CLUSTERS = new HighAvailabilityTestingUtility.HBaseTestingUtilityPair();
-
- @Rule
- public final TestName testName = new TestName();
- /** Client properties to create a connection per test. */
- private Properties parallelClientProperties;
- private Properties failoverClientProperties;
- /** HA group for this test. */
- private HighAvailabilityGroup parallelHAGroup;
- private HighAvailabilityGroup failoverHAGroup;
- /** Table name per test case. */
- private String tableName;
- /** HA Group name for this test. */
- private String parallelHAGroupName;
- private String failoverHAGroupName;
- /** Registry Type for this test */
- private final RegistryType registryType;
-
- @BeforeClass
- public static void setUpBeforeClass() throws Exception {
- CLUSTERS.start();
- DriverManager.registerDriver(PhoenixDriver.INSTANCE);
- }
-
- @AfterClass
- public static void tearDownAfterClass() throws Exception {
- DriverManager.deregisterDriver(PhoenixDriver.INSTANCE);
- CLUSTERS.close();
- }
-
- @Before
- public void setup() throws Exception {
- if (registryType == ClusterRoleRecord.RegistryType.RPC) {
- assumeTrue(VersionInfo.compareVersion(VersionInfo.getVersion(), "2.5.0")>=0);
- }
- parallelHAGroupName = testName.getMethodName() + "_" + HighAvailabilityPolicy.PARALLEL.name();
- failoverHAGroupName = testName.getMethodName() + "_" + HighAvailabilityPolicy.FAILOVER.name();
- parallelClientProperties = HighAvailabilityTestingUtility.getHATestProperties();
- parallelClientProperties.setProperty(PHOENIX_HA_GROUP_ATTR, parallelHAGroupName);
-
- failoverClientProperties = HighAvailabilityTestingUtility.getHATestProperties();
- failoverClientProperties.setProperty(PHOENIX_HA_GROUP_ATTR, failoverHAGroupName);
-
- // Make first cluster ACTIVE
- CLUSTERS.initClusterRole(parallelHAGroupName, HighAvailabilityPolicy.PARALLEL, registryType);
- CLUSTERS.initClusterRole(failoverHAGroupName, HighAvailabilityPolicy.FAILOVER, registryType);
-
- parallelHAGroup = getHighAvailibilityGroup(CLUSTERS.getJdbcHAUrl(), parallelClientProperties);
- LOG.info("Initialized haGroup {} with URL {}", parallelHAGroup, CLUSTERS.getJdbcHAUrl());
- failoverHAGroup = getHighAvailibilityGroup(CLUSTERS.getJdbcHAUrl(), failoverClientProperties);
- LOG.info("Initialized haGroup {} with URL {}", failoverHAGroup, CLUSTERS.getJdbcHAUrl());
- tableName = RandomStringUtils.randomAlphabetic(10).toUpperCase();
- CLUSTERS.createTableOnClusterPair(parallelHAGroup, tableName);
- }
-
- @After
- public void tearDown() throws Exception {
- try {
- parallelHAGroup.close();
- failoverHAGroup.close();
- PhoenixDriver.INSTANCE
- .getConnectionQueryServices(CLUSTERS.getJdbcUrl1(parallelHAGroup), parallelHAGroup.getProperties())
- .close();
- PhoenixDriver.INSTANCE
- .getConnectionQueryServices(CLUSTERS.getJdbcUrl2(parallelHAGroup), parallelHAGroup.getProperties())
- .close();
- } catch (Exception e) {
- LOG.error("Fail to tear down the HA group and the CQS. Will ignore", e);
- }
- }
-
- @Parameterized.Parameters(name="ClusterRoleRecord_registryType={0}")
- public static Collection<Object> data() {
- return Arrays.asList(new Object[] {
- ClusterRoleRecord.RegistryType.MASTER,
- ClusterRoleRecord.RegistryType.RPC,
- });
- }
-
- public HAConnectionWithMasterAndRPCRegistryIT(ClusterRoleRecord.RegistryType registryType) {
- this.registryType = registryType;
- }
-
- /**
- * Test that failover can finish even if one connection can not be closed.
- *
- * When once cluster becomes STANDBY from ACTIVE, all its connections and the associated CQS
- * will get closed asynchronously. In case of errors when closing those connections and CQS,
- * the HA group is still able to transit to target state after the maximum timeout.
- * Closing the existing connections is guaranteed with the best effort and timeout in favor of
- * improved availability.
- *
- * {@link FailoverPhoenixConnectionIT#testFailoverTwice()} which fails over back to the first cluster
- */
- @Test(timeout = 300000)
- public void testFailoverCanFinishWhenOneConnectionGotStuckClosing() throws Exception {
- Connection conn = getFailoverConnection();
- doTestBasicOperationsWithConnection(conn, tableName, failoverHAGroupName);
- assertEquals(CLUSTERS.getJdbcUrl1(failoverHAGroup), // active connection is against the first cluster
- conn.unwrap(FailoverPhoenixConnection.class).getWrappedConnection().getURL());
-
- // Spy the wrapped connection
- Connection wrapped = conn.unwrap(FailoverPhoenixConnection.class).getWrappedConnection();
- Connection spy = Mockito.spy(wrapped);
- final CountDownLatch latch = new CountDownLatch(1);
- // Make close() stuck before closing
- doAnswer((invocation) -> {
- latch.await();
- invocation.callRealMethod();
- return null;
- }).when(spy).close();
- ConnectionQueryServices cqs = PhoenixDriver.INSTANCE
- .getConnectionQueryServices(CLUSTERS.getJdbcUrl1(failoverHAGroup), failoverClientProperties);
- // replace the wrapped connection with the spied connection in CQS
- cqs.removeConnection(wrapped.unwrap(PhoenixConnection.class));
- cqs.addConnection(spy.unwrap(PhoenixConnection.class));
-
- // (ACTIVE, STANDBY) -> (STANDBY, ACTIVE)
- // The transition will finish as we set PHOENIX_HA_TRANSITION_TIMEOUT_MS_KEY for this class
- // even though the spied connection is stuck at the latch when closing
- CLUSTERS.transitClusterRole(failoverHAGroup, ClusterRoleRecord.ClusterRole.STANDBY, ClusterRoleRecord.ClusterRole.ACTIVE);
-
- // Verify the spied object has been called once
- verify(spy, times(1)).close();
- // The spy is not closed because the real method was blocked by latch
- assertFalse(spy.isClosed());
- // connection is not closed as Phoenix HA does not close failover connections.
- assertFalse(conn.isClosed());
-
- try (Connection conn2 = getFailoverConnection()) {
- doTestBasicOperationsWithConnection(conn2, tableName, failoverHAGroupName);
- assertEquals(CLUSTERS.getJdbcUrl2(failoverHAGroup), // active connection is against the second cluster
- conn2.unwrap(FailoverPhoenixConnection.class).getWrappedConnection().getURL());
- }
-
- latch.countDown();
- conn.close();
- // The CQS should be closed eventually.
- waitFor(() -> {
- try {
- ((ConnectionQueryServicesImpl) cqs).checkClosed();
- return false;
- } catch (IllegalStateException e) {
- LOG.info("CQS got closed as we get expected exception.", e);
- return true;
- }
- }, 100, 10_000);
- }
-
- /**
- * Tests the scenario where ACTIVE HBase cluster restarts.
- *
- * When ACTIVE HBase cluster shutdown, client can not connect to this HA group.
- * After failover, client can connect to this HA group (against the new ACTIVE cluster2).
- * After restarts and fails back, client can connect to this HA group (against cluster 1).
- */
- @Test(timeout = 300000)
- public void testConnectionWhenActiveHBaseRestarts() throws Exception {
- // This creates the cqsi for the active cluster upfront.
- // If we don't do that then later when we try to transition
- // the cluster role it tries to create cqsi for the cluster
- // which is down and that takes forever causing timeouts
- try (Connection conn = getFailoverConnection()) {
- doTestBasicOperationsWithConnection(conn, tableName, failoverHAGroupName);
- }
- doTestWhenOneHBaseDown(CLUSTERS.getHBaseCluster1(), () -> {
- try {
- try (Connection conn = getFailoverConnection()) {
- doTestBasicOperationsWithConnection(conn, tableName, failoverHAGroupName);
- }
- fail("Should have failed since ACTIVE ZK cluster was shutdown");
- } catch (SQLException e) {
- LOG.info("Got expected exception when ACTIVE ZK cluster is down", e);
- }
-
- // So on-call engineer comes into play, and triggers a failover
- // Because cluster1 is down, new version record will only be updated in cluster2
- CLUSTERS.transitClusterRole(failoverHAGroup, ClusterRoleRecord.ClusterRole.STANDBY, ClusterRoleRecord.ClusterRole.ACTIVE);
-
- // After failover, the FailoverPhoenixConnection should go to the second cluster
- try (Connection conn = getFailoverConnection()) {
- doTestBasicOperationsWithConnection(conn, tableName, failoverHAGroupName);
- }
- });
-
- LOG.info("Testing failover connection when both clusters are up and running");
- try (Connection conn = getFailoverConnection()) {
- FailoverPhoenixConnection failoverConn = conn.unwrap(FailoverPhoenixConnection.class);
- assertEquals(CLUSTERS.getJdbcUrl2(failoverHAGroup), failoverConn.getWrappedConnection().getURL());
- doTestBasicOperationsWithConnection(conn, tableName, failoverHAGroupName);
- }
-
- LOG.info("Testing failover back to cluster1 when bot clusters are up and running");
- // Failover back to the first cluster since it is healthy after restart
- CLUSTERS.transitClusterRole(failoverHAGroup, ClusterRoleRecord.ClusterRole.ACTIVE, ClusterRoleRecord.ClusterRole.STANDBY);
- // After ACTIVE ZK restarts and fail back, this should still work
- try (Connection conn = getFailoverConnection()) {
- doTestBasicOperationsWithConnection(conn, tableName, failoverHAGroupName);
- }
- }
-
- /**
- * Tests the scenario where STANDBY HBase cluster restarts.
- *
- * When STANDBY HBase cluster shutdown, client can still connect to this HA group.
- * After failover, client can not connect to this HA group (ACTIVE cluster 2 is down).
- * After cluster 2 (ACTIVE) restarts, client can connect to this HA group.
- */
- @Test(timeout = 300000)
- public void testConnectionWhenStandbyHBaseRestarts() throws Exception {
- doTestWhenOneHBaseDown(CLUSTERS.getHBaseCluster2(), () -> {
- try (Connection conn = getFailoverConnection()) {
- doTestBasicOperationsWithConnection(conn, tableName, failoverHAGroupName);
- }
-
- // Innocent on-call engineer triggers a failover to second cluster
- CLUSTERS.transitClusterRole(failoverHAGroup, ClusterRoleRecord.ClusterRole.STANDBY, ClusterRoleRecord.ClusterRole.ACTIVE);
-
- try {
- getFailoverConnection();
- fail("Should have failed since ACTIVE HBase cluster was shutdown");
- } catch (SQLException e) {
- LOG.info("Got expected exception when ACTIVE HBase cluster {} was shutdown",
- CLUSTERS.getZkUrl2(), e);
- }
- });
-
- /* As restarting HBase minicluster it changes the port of HMasters need to refresh the CRR */
- CLUSTERS.refreshClusterRoleRecordAfterClusterRestart(failoverHAGroup,
- ClusterRoleRecord.ClusterRole.STANDBY, ClusterRoleRecord.ClusterRole.ACTIVE);
- try (Connection conn = getFailoverConnection()) {
- doTestBasicOperationsWithConnection(conn, tableName, failoverHAGroupName);
- }
- }
-
- /**
- * Tests the scenario where two HBase clusters both restart.
- *
- * Client can not connect to this HA group when two HBase clusters are both down.
- * When STANDBY HBase cluster2 first restarts, client still can not connect to this HA group.
- * After failover, client can connect to this HA group because cluster 2 is ACTIVE and healthy.
- */
- @Test(timeout = 300000)
- public void testConnectionWhenTwoHBaseRestarts() throws Exception {
- doTestWhenOneHBaseDown(CLUSTERS.getHBaseCluster1(), () -> {
- doTestWhenOneHBaseDown(CLUSTERS.getHBaseCluster2(), () -> {
- try {
- getFailoverConnection();
- fail("Should have failed since ACTIVE HBase cluster was shutdown");
- } catch (SQLException e) {
- LOG.info("Got expected exception when both clusters are down", e);
- }
- });
-
- // Client cannot connect to HA group because cluster 2 is still STANDBY after restarted
- try {
- getFailoverConnection();
- fail("Should have failed since ACTIVE HBase cluster was shutdown");
- } catch (SQLException e) {
- LOG.info("Got expected exception when ACTIVE HBase cluster {} was shutdown",
- CLUSTERS.getZkUrl2(), e);
- }
-
- // So on-call engineer comes into play, and triggers a failover
- CLUSTERS.transitClusterRole(failoverHAGroup, ClusterRoleRecord.ClusterRole.STANDBY, ClusterRoleRecord.ClusterRole.ACTIVE);
-
- // After the second cluster (ACTIVE) HBase restarts, this should still work
- try (Connection conn = getFailoverConnection()) {
- doTestBasicOperationsWithConnection(conn, tableName, failoverHAGroupName);
- }
- });
- }
-
- /**
- * Testing connection closure for roleRecord change from
- * (URL1, ACTIVE, URL2, STANDBY) -> (URL3, ACTIVE, URL2, STANDBY)
- * All wrapped connections should have been closed
- */
- @Test(timeout = 300000)
- public void testAllWrappedConnectionsClosedAfterActiveURLChange() throws Exception {
- short numberOfConnections = 5;
- //Create FailoverPhoenixConnections and ParallelPhoenixConnections with default urls
- List<Connection> connectionList = new ArrayList<>(numberOfConnections);
- List<Connection> parallelConnectionList = new ArrayList<>(numberOfConnections);
- for (short i = 0; i < numberOfConnections; i++) {
- connectionList.add(getFailoverConnection());
- parallelConnectionList.add(getParallelConnection());
- }
- //url of cluster which is being restarted
- String url = CLUSTERS.getJdbcUrl1(parallelHAGroup);
- //Restart Cluster 1 with new ports
- CLUSTERS.restartCluster1();
- //Basically applying new url for cluster 1
- CLUSTERS.refreshClusterRoleRecordAfterClusterRestart(failoverHAGroup,
- ClusterRoleRecord.ClusterRole.ACTIVE, ClusterRoleRecord.ClusterRole.STANDBY);
- CLUSTERS.refreshClusterRoleRecordAfterClusterRestart(parallelHAGroup,
- ClusterRoleRecord.ClusterRole.ACTIVE, ClusterRoleRecord.ClusterRole.STANDBY);
- for (short i = 0; i < numberOfConnections; i++) {
- LOG.info("Asserting connection number {}", i);
- FailoverPhoenixConnection conn = ((FailoverPhoenixConnection) connectionList.get(i));
- assertFalse(conn.isClosed());
- assertTrue(conn.getWrappedConnection().isClosed());
- conn.close();
-
- ParallelPhoenixConnection conn2 = ((ParallelPhoenixConnection) parallelConnectionList.get(i));
- assertFalse(conn2.isClosed());
- //Only connections to cluster 1 should be closed
- if (conn2.futureConnection1.get().getURL().equals(url)) {
- assertTrue(conn2.futureConnection1.get().isClosed());
- assertFalse(conn2.futureConnection2.get().isClosed());
- } else {
- assertTrue(conn2.futureConnection2.get().isClosed());
- assertFalse(conn2.futureConnection1.get().isClosed());
- }
- conn2.close();
- }
- }
-
- /**
- * Testing connection closure for roleRecord change from
- * (URL1, ACTIVE, URL2, STANDBY) -> (URL1, ACTIVE, URL3, STANDBY)
- * It should not have affected connections at all
- */
- @Test(timeout = 300000)
- public void testAllWrappedConnectionsNotClosedAfterStandbyURLChange() throws Exception {
- short numberOfConnections = 5;
- //Create FailoverPhoenixConnections and ParallelPhoenixConnections with default urls
- List<Connection> connectionList = new ArrayList<>(numberOfConnections);
- List<Connection> parallelConnectionList = new ArrayList<>(numberOfConnections);
- for (short i = 0; i < numberOfConnections; i++) {
- connectionList.add(getFailoverConnection());
- parallelConnectionList.add(getParallelConnection());
- }
-
- //url of cluster which is being restarted
- String url = CLUSTERS.getJdbcUrl2(parallelHAGroup);
- //Restart Cluster 2 with new ports
- CLUSTERS.restartCluster2();
- //Basically applying new url for cluster 2
- CLUSTERS.refreshClusterRoleRecordAfterClusterRestart(failoverHAGroup,
- ClusterRoleRecord.ClusterRole.ACTIVE, ClusterRoleRecord.ClusterRole.STANDBY);
- CLUSTERS.refreshClusterRoleRecordAfterClusterRestart(parallelHAGroup,
- ClusterRoleRecord.ClusterRole.ACTIVE, ClusterRoleRecord.ClusterRole.STANDBY);
- for (short i = 0; i < numberOfConnections; i++) {
- LOG.info("Asserting connection number {}", i);
- FailoverPhoenixConnection conn = ((FailoverPhoenixConnection) connectionList.get(i));
- assertFalse(conn.isClosed());
- assertFalse(conn.getWrappedConnection().isClosed());
- conn.close();
-
- ParallelPhoenixConnection conn2 = ((ParallelPhoenixConnection) parallelConnectionList.get(i));
- assertFalse(conn2.isClosed());
- //Only connections to cluster 2 should be closed
- if (conn2.futureConnection1.get().getURL().equals(url)) {
- assertTrue(conn2.futureConnection1.get().isClosed());
- assertFalse(conn2.futureConnection2.get().isClosed());
- } else {
- assertTrue(conn2.futureConnection2.get().isClosed());
- assertFalse(conn2.futureConnection1.get().isClosed());
- }
- conn2.close();
- }
- }
-
- /**
- * This is to make sure all Phoenix connections are closed when registryType changes
- * of a CRR.
- * MASTER --> ZK
- * RPC --> MASTER
- *
- * Test with many connections.
- */
- @Test(timeout = 300000)
- public void testAllWrappedConnectionsClosedAfterRegistryChange() throws Exception {
- short numberOfConnections = 5;
- //Create FailoverPhoenixConnections and ParallelPhoenixConnections with default urls
- List<Connection> connectionList = new ArrayList<>(numberOfConnections);
- List<Connection> parallelConnectionList = new ArrayList<>(numberOfConnections);
- for (short i = 0; i < numberOfConnections; i++) {
- connectionList.add(getFailoverConnection());
- parallelConnectionList.add(getParallelConnection());
- }
-
- ClusterRoleRecord.RegistryType newRegistry;
- if (registryType == RegistryType.MASTER) {
- newRegistry = RegistryType.ZK;
- } else {
- newRegistry = RegistryType.MASTER;
- }
-
- ConnectionQueryServicesImpl cqsiF = (ConnectionQueryServicesImpl) PhoenixDriver.INSTANCE.
- getConnectionQueryServices(CLUSTERS.getJdbcUrl1(failoverHAGroup), failoverClientProperties);
- ConnectionQueryServicesImpl cqsiP = (ConnectionQueryServicesImpl) PhoenixDriver.INSTANCE.
- getConnectionQueryServices(CLUSTERS.getJdbcUrl1(parallelHAGroup), parallelClientProperties);
- ConnectionInfo connInfoF = ConnectionInfo.create(CLUSTERS.getJdbcUrl1(failoverHAGroup),
- PhoenixDriver.INSTANCE.getQueryServices().getProps(), failoverClientProperties);
- ConnectionInfo connInfoP = ConnectionInfo.create(CLUSTERS.getJdbcUrl1(parallelHAGroup),
- PhoenixDriver.INSTANCE.getQueryServices().getProps(), failoverClientProperties);
-
- CLUSTERS.transitClusterRoleRecordRegistry(failoverHAGroup, newRegistry);
- CLUSTERS.transitClusterRoleRecordRegistry(parallelHAGroup, newRegistry);
-
- for (short i = 0; i < numberOfConnections; i++) {
- LOG.info("Asserting connection number {}", i);
- FailoverPhoenixConnection conn = ((FailoverPhoenixConnection) connectionList.get(i));
- assertFalse(conn.isClosed());
- assertTrue(conn.getWrappedConnection().isClosed());
-
- ParallelPhoenixConnection conn2 = ((ParallelPhoenixConnection) parallelConnectionList.get(i));
- assertFalse(conn2.isClosed());
- assertTrue(conn2.futureConnection1.get().isClosed());
- assertTrue(conn2.futureConnection1.get().isClosed());
- conn2.close();
- }
-
- //Both CQSI should be closed
- try {
- cqsiF.checkClosed();
- fail("Should have thrown an exception as cqsi should be closed");
- } catch (IllegalStateException e) {
- //Exception cqsi should have been invalidated as well
- assertFalse(PhoenixDriver.INSTANCE.checkIfCQSIIsInCache(connInfoF));
- } catch (Exception e) {
- fail("Should have thrown on IllegalStateException as cqsi should be closed");
- }
-
- try {
- cqsiP.checkClosed();
- fail("Should have thrown an exception as cqsi should be closed");
- } catch (IllegalStateException e) {
- //Exception cqsi should have been invalidated as well
- assertFalse(PhoenixDriver.INSTANCE.checkIfCQSIIsInCache(connInfoP));
- } catch (Exception e) {
- fail("Should have thrown on IllegalStateException as cqsi should be closed");
- }
- }
-
- /**
- * This is to make sure all Phoenix connections are closed when registryType changes
- * of a CRR.
- * MASTER --> RPC
- * RPC --> ZK
- *
- * Test with many connections.
- */
- @Test(timeout = 300000)
- public void testAllWrappedConnectionsClosedAfterRegistryChange2() throws Exception {
- short numberOfConnections = 5;
- //Create FailoverPhoenixConnections and ParallelPhoenixConnections with default urls
- List<Connection> connectionList = new ArrayList<>(numberOfConnections);
- List<Connection> parallelConnectionList = new ArrayList<>(numberOfConnections);
- for (short i = 0; i < numberOfConnections; i++) {
- connectionList.add(getFailoverConnection());
- parallelConnectionList.add(getParallelConnection());
- }
-
- ClusterRoleRecord.RegistryType newRegistry;
- if (registryType == RegistryType.MASTER) {
- newRegistry = RegistryType.RPC;
- //RPC Registry is only there in hbase version greater than 2.5.0
- assumeTrue(VersionInfo.compareVersion(VersionInfo.getVersion(), "2.5.0")>=0);
- } else {
- newRegistry = RegistryType.ZK;
- }
- ConnectionQueryServicesImpl cqsiF = (ConnectionQueryServicesImpl) PhoenixDriver.INSTANCE.
- getConnectionQueryServices(CLUSTERS.getJdbcUrl1(failoverHAGroup), failoverClientProperties);
- ConnectionQueryServicesImpl cqsiP = (ConnectionQueryServicesImpl) PhoenixDriver.INSTANCE.
- getConnectionQueryServices(CLUSTERS.getJdbcUrl1(parallelHAGroup), parallelClientProperties);
- ConnectionInfo connInfoF = ConnectionInfo.create(CLUSTERS.getJdbcUrl1(failoverHAGroup),
- PhoenixDriver.INSTANCE.getQueryServices().getProps(), failoverClientProperties);
- ConnectionInfo connInfoP = ConnectionInfo.create(CLUSTERS.getJdbcUrl1(parallelHAGroup),
- PhoenixDriver.INSTANCE.getQueryServices().getProps(), failoverClientProperties);
-
- CLUSTERS.transitClusterRoleRecordRegistry(failoverHAGroup, newRegistry);
- CLUSTERS.transitClusterRoleRecordRegistry(parallelHAGroup, newRegistry);
-
- for (short i = 0; i < numberOfConnections; i++) {
- LOG.info("Asserting connection number {}", i);
- FailoverPhoenixConnection conn = ((FailoverPhoenixConnection) connectionList.get(i));
- assertFalse(conn.isClosed());
- assertTrue(conn.getWrappedConnection().isClosed());
-
- ParallelPhoenixConnection conn2 = ((ParallelPhoenixConnection) parallelConnectionList.get(i));
- assertFalse(conn2.isClosed());
- assertTrue(conn2.futureConnection1.get().isClosed());
- assertTrue(conn2.futureConnection1.get().isClosed());
- conn2.close();
- }
-
- //Both CQSI should be closed
- try {
- cqsiF.checkClosed();
- fail("Should have thrown an exception as cqsi should be closed");
- } catch (IllegalStateException e) {
- //Exception cqsi should have been invalidated as well
- assertFalse(PhoenixDriver.INSTANCE.checkIfCQSIIsInCache(connInfoF));
- } catch (Exception e) {
- fail("Should have thrown on IllegalStateException as cqsi should be closed");
- }
-
- try {
- cqsiP.checkClosed();
- fail("Should have thrown an exception as cqsi should be closed");
- } catch (IllegalStateException e) {
- //Exception cqsi should have been invalidated as well
- assertFalse(PhoenixDriver.INSTANCE.checkIfCQSIIsInCache(connInfoP));
- } catch (Exception e) {
- fail("Should have thrown on IllegalStateException as cqsi should be closed");
- }
- }
-
-
- /**
- * Create a failover connection using {@link #failoverClientProperties}.
- */
- private Connection getFailoverConnection() throws SQLException {
- return DriverManager.getConnection(CLUSTERS.getJdbcHAUrl(), failoverClientProperties);
- }
-
- /**
- * Returns a Parallel Phoenix Connection
- * @return Parallel Phoenix Connection
- * @throws SQLException
- */
- private Connection getParallelConnection() throws SQLException {
- return DriverManager.getConnection(CLUSTERS.getJdbcHAUrl(), parallelClientProperties);
- }
-
-}
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HAGroupStoreClientIT.java b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HAGroupStoreClientIT.java
index 9e8087a57f..4aa295a0d7 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HAGroupStoreClientIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HAGroupStoreClientIT.java
@@ -79,6 +79,10 @@ public class HAGroupStoreClientIT extends BaseTest {
private PhoenixHAAdmin peerHaAdmin;
private String zkUrl;
private String peerZKUrl;
+ private String masterUrl;
+ private String peerMasterUrl;
+ private final String defaultProtocolVersion = "v1.0";
+ private final long defaultRecordVersion = 1;
@Rule
public TestName testName = new TestName();
@@ -105,6 +109,8 @@ public class HAGroupStoreClientIT extends BaseTest {
// Insert a HAGroupStoreRecord into the system table
String haGroupName = testName.getMethodName();
this.peerZKUrl = CLUSTERS.getZkUrl2();
+ this.masterUrl = CLUSTERS.getMasterAddress1();
+ this.peerMasterUrl = CLUSTERS.getMasterAddress2();
HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(haGroupName, this.zkUrl, this.peerZKUrl, ClusterRoleRecord.ClusterRole.ACTIVE, ClusterRoleRecord.ClusterRole.STANDBY, null);
}
@@ -131,8 +137,8 @@ public class HAGroupStoreClientIT extends BaseTest {
@Test
public void testHAGroupStoreClientChangingPeerZKUrlToNullUrlToValidUrlToInvalidUrl() throws Exception {
String haGroupName = testName.getMethodName();
- HAGroupStoreRecord record = new HAGroupStoreRecord("v1.0", haGroupName,
- HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+ HAGroupStoreRecord record = new HAGroupStoreRecord(defaultProtocolVersion, haGroupName,
+ HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, defaultRecordVersion);
createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, record);
HAGroupStoreClient haGroupStoreClient = HAGroupStoreClient.getInstanceForZkUrl(CLUSTERS.getHBaseCluster1().getConfiguration(), haGroupName, zkUrl);
@@ -147,7 +153,7 @@ public class HAGroupStoreClientIT extends BaseTest {
// Clean existing record
HAGroupStoreTestUtil.deleteHAGroupRecordInSystemTable(haGroupName, zkUrl);
// Now update peerZKUrl to null and rebuild
- HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(haGroupName, this.zkUrl, null, ClusterRoleRecord.ClusterRole.ACTIVE, ClusterRoleRecord.ClusterRole.STANDBY, null);
+ HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(haGroupName, this.zkUrl, null, this.masterUrl, this.peerMasterUrl, ClusterRoleRecord.ClusterRole.ACTIVE, ClusterRoleRecord.ClusterRole.STANDBY, null);
try {
haGroupStoreClient.rebuild();
fail("Should have thrown NullPointerException");
@@ -156,7 +162,7 @@ public class HAGroupStoreClientIT extends BaseTest {
}
// Now update System table to contain valid peer ZK URL and also change local cluster role to STANDBY
- HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(haGroupName,this.zkUrl, this.peerZKUrl, ClusterRoleRecord.ClusterRole.STANDBY, ClusterRoleRecord.ClusterRole.ACTIVE, null);
+ HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(haGroupName,this.zkUrl, this.peerZKUrl, this.masterUrl, this.peerMasterUrl, ClusterRoleRecord.ClusterRole.STANDBY, ClusterRoleRecord.ClusterRole.ACTIVE, null);
haGroupStoreClient.rebuild();
currentRecord = haGroupStoreClient.getHAGroupStoreRecord();
assertNotNull(currentRecord);
@@ -166,8 +172,8 @@ public class HAGroupStoreClientIT extends BaseTest {
assertNotNull(peerPathChildrenCache.get(haGroupStoreClient));
// Now update local HAGroupStoreRecord to STANDBY to verify that HAGroupStoreClient is working as normal
- record = new HAGroupStoreRecord("v1.0", haGroupName,
- HAGroupStoreRecord.HAGroupState.STANDBY);
+ record = new HAGroupStoreRecord(defaultProtocolVersion, haGroupName,
+ HAGroupStoreRecord.HAGroupState.STANDBY, defaultRecordVersion);
createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, record);
Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
@@ -179,22 +185,22 @@ public class HAGroupStoreClientIT extends BaseTest {
// Now update peerZKUrl to invalid but non-null url and rebuild
// This URL can also be considered unreachable url due to a connectivity issue.
String invalidUrl = "invalidURL";
- HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(haGroupName, this.zkUrl, invalidUrl, ClusterRoleRecord.ClusterRole.STANDBY, ClusterRoleRecord.ClusterRole.ACTIVE, null);
+ HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(haGroupName, this.zkUrl, invalidUrl, this.masterUrl, this.peerMasterUrl, ClusterRoleRecord.ClusterRole.STANDBY, ClusterRoleRecord.ClusterRole.ACTIVE, null);
haGroupStoreClient.rebuild();
currentRecord = haGroupStoreClient.getHAGroupStoreRecord();
assertNotNull(currentRecord);
assertEquals(HAGroupStoreRecord.HAGroupState.STANDBY, currentRecord.getHAGroupState());
- // Check CRR, the role for peer cluster with invalid url should be UNKNOWN.
+ // Check CRR, the role for peer cluster with invalid zk url should be UNKNOWN for it's cluster url.
ClusterRoleRecord clusterRoleRecord = haGroupStoreClient.getClusterRoleRecord();
assertNotNull(clusterRoleRecord);
ClusterRoleRecord expected = new ClusterRoleRecord(haGroupName,
HighAvailabilityPolicy.FAILOVER,
- this.zkUrl,
+ this.masterUrl,
ClusterRoleRecord.ClusterRole.STANDBY,
- invalidUrl,
+ this.peerMasterUrl,
ClusterRoleRecord.ClusterRole.UNKNOWN,
- 1);
+ CLUSTERS.combineCanonicalVersions(1, 1));
assertEquals(expected, clusterRoleRecord);
// Check that peerPathChildrenCache is null now in HAGroupStoreClient via reflection
@@ -209,8 +215,8 @@ public class HAGroupStoreClientIT extends BaseTest {
// Clean existing record
HAGroupStoreTestUtil.deleteHAGroupRecordInSystemTable(haGroupName, this.zkUrl);
HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(haGroupName,this.zkUrl, null, ClusterRoleRecord.ClusterRole.ACTIVE, ClusterRoleRecord.ClusterRole.STANDBY, null);
- HAGroupStoreRecord record = new HAGroupStoreRecord("v1.0", haGroupName,
- HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+ HAGroupStoreRecord record = new HAGroupStoreRecord(defaultProtocolVersion, haGroupName,
+ HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, defaultRecordVersion);
createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, record);
assertNull(HAGroupStoreClient.getInstanceForZkUrl(CLUSTERS.getHBaseCluster1().getConfiguration(), haGroupName, zkUrl));
}
@@ -221,16 +227,16 @@ public class HAGroupStoreClientIT extends BaseTest {
HAGroupStoreClient haGroupStoreClient = HAGroupStoreClient.getInstanceForZkUrl(CLUSTERS.getHBaseCluster1().getConfiguration(), haGroupName, zkUrl);
// Create and store HAGroupStoreRecord with ACTIVE state
- HAGroupStoreRecord record = new HAGroupStoreRecord("v1.0", haGroupName,
- HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+ HAGroupStoreRecord record = new HAGroupStoreRecord(defaultProtocolVersion, haGroupName,
+ HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, defaultRecordVersion);
createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, record);
Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
HAGroupStoreRecord currentRecord = haGroupStoreClient.getHAGroupStoreRecord();
assert currentRecord != null && currentRecord.getHAGroupState() == HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC;
// Now Update HAGroupStoreRecord so that current cluster has state ACTIVE_TO_STANDBY
- record = new HAGroupStoreRecord("v1.0", haGroupName,
- HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY);
+ record = new HAGroupStoreRecord(defaultProtocolVersion, haGroupName,
+ HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY, defaultRecordVersion);
createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, record);
Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
@@ -239,8 +245,8 @@ public class HAGroupStoreClientIT extends BaseTest {
assert currentRecord != null && currentRecord.getHAGroupState() == HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY;
// Change it back to ACTIVE so that cluster is not in ACTIVE_TO_STANDBY state
- record = new HAGroupStoreRecord("v1.0", haGroupName,
- HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+ record = new HAGroupStoreRecord(defaultProtocolVersion, haGroupName,
+ HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, defaultRecordVersion);
createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, record);
Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
@@ -248,8 +254,8 @@ public class HAGroupStoreClientIT extends BaseTest {
assert currentRecord != null && currentRecord.getHAGroupState() == HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC;
// Change it again to ACTIVE_TO_STANDBY so that we can validate watcher works repeatedly
- record = new HAGroupStoreRecord("v1.0", haGroupName,
- HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY);
+ record = new HAGroupStoreRecord(defaultProtocolVersion, haGroupName,
+ HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY, defaultRecordVersion);
createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, record);
Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
@@ -257,8 +263,8 @@ public class HAGroupStoreClientIT extends BaseTest {
assert currentRecord != null && currentRecord.getHAGroupState() == HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY;
// Change it back to ACTIVE to verify transition works
- record = new HAGroupStoreRecord("v1.0", haGroupName,
- HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+ record = new HAGroupStoreRecord(defaultProtocolVersion, haGroupName,
+ HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, defaultRecordVersion);
createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, record);
Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
@@ -292,10 +298,10 @@ public class HAGroupStoreClientIT extends BaseTest {
HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(haGroupName2,this.zkUrl, this.peerZKUrl, ClusterRoleRecord.ClusterRole.ACTIVE, ClusterRoleRecord.ClusterRole.STANDBY, null);
// Setup initial HAGroupStoreRecords
- HAGroupStoreRecord record1 = new HAGroupStoreRecord("v1.0", haGroupName1,
- HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
- HAGroupStoreRecord record2 = new HAGroupStoreRecord("v1.0", haGroupName2,
- HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+ HAGroupStoreRecord record1 = new HAGroupStoreRecord(defaultProtocolVersion, haGroupName1,
+ HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, defaultRecordVersion);
+ HAGroupStoreRecord record2 = new HAGroupStoreRecord(defaultProtocolVersion, haGroupName2,
+ HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, defaultRecordVersion);
createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName1, record1);
createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName2, record2);
@@ -310,10 +316,10 @@ public class HAGroupStoreClientIT extends BaseTest {
assert currentRecord2 != null && currentRecord2.getHAGroupState() == HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC;
// Now Update HAGroupStoreRecord so that current cluster has state ACTIVE_TO_STANDBY for only 1 record
- record1 = new HAGroupStoreRecord("v1.0", haGroupName1,
- HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY);
- record2 = new HAGroupStoreRecord("v1.0", haGroupName2,
- HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+ record1 = new HAGroupStoreRecord(defaultProtocolVersion, haGroupName1,
+ HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY, defaultRecordVersion);
+ record2 = new HAGroupStoreRecord(defaultProtocolVersion, haGroupName2,
+ HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, defaultRecordVersion);
createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName1, record1);
createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName2, record2);
@@ -326,10 +332,10 @@ public class HAGroupStoreClientIT extends BaseTest {
assert currentRecord2 != null && currentRecord2.getHAGroupState() == HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC;
// Change it back to ACTIVE so that cluster is not in ACTIVE_TO_STANDBY state
- record1 = new HAGroupStoreRecord("v1.0", haGroupName1,
- HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
- record2 = new HAGroupStoreRecord("v1.0", haGroupName2,
- HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+ record1 = new HAGroupStoreRecord(defaultProtocolVersion, haGroupName1,
+ HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, defaultRecordVersion);
+ record2 = new HAGroupStoreRecord(defaultProtocolVersion, haGroupName2,
+ HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, defaultRecordVersion);
createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName1, record1);
createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName2, record2);
@@ -341,10 +347,10 @@ public class HAGroupStoreClientIT extends BaseTest {
assert currentRecord2 != null && currentRecord2.getHAGroupState() == HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC;
// Change other record to ACTIVE_TO_STANDBY and one in ACTIVE state so that we can validate watcher works repeatedly
- record1 = new HAGroupStoreRecord("v1.0", haGroupName1,
- HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
- record2 = new HAGroupStoreRecord("v1.0", haGroupName2,
- HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY);
+ record1 = new HAGroupStoreRecord(defaultProtocolVersion, haGroupName1,
+ HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, defaultRecordVersion);
+ record2 = new HAGroupStoreRecord(defaultProtocolVersion, haGroupName2,
+ HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY, defaultRecordVersion);
createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName1, record1);
createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName2, record2);
@@ -361,8 +367,8 @@ public class HAGroupStoreClientIT extends BaseTest {
String haGroupName = testName.getMethodName();
// Setup initial HAGroupStoreRecord
- HAGroupStoreRecord record = new HAGroupStoreRecord("v1.0", haGroupName,
- HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+ HAGroupStoreRecord record = new HAGroupStoreRecord(defaultProtocolVersion, haGroupName,
+ HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, defaultRecordVersion);
createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, record);
HAGroupStoreClient haGroupStoreClient = HAGroupStoreClient.getInstanceForZkUrl(CLUSTERS.getHBaseCluster1().getConfiguration(), haGroupName, zkUrl);
@@ -386,8 +392,8 @@ public class HAGroupStoreClientIT extends BaseTest {
assert latch.await(10, TimeUnit.SECONDS);
// Update HAGroupStoreRecord
- record = new HAGroupStoreRecord("v1.0", haGroupName,
- HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY);
+ record = new HAGroupStoreRecord(defaultProtocolVersion, haGroupName,
+ HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY, defaultRecordVersion);
createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, record);
@@ -413,8 +419,8 @@ public class HAGroupStoreClientIT extends BaseTest {
public void testHAGroupStoreClientWithRootPathDeletion() throws Exception {
String haGroupName = testName.getMethodName();
- HAGroupStoreRecord record1 = new HAGroupStoreRecord("v1.0", haGroupName,
- HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+ HAGroupStoreRecord record1 = new HAGroupStoreRecord(defaultProtocolVersion, haGroupName,
+ HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, defaultRecordVersion);
createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, record1);
HAGroupStoreClient haGroupStoreClient = HAGroupStoreClient.getInstanceForZkUrl(CLUSTERS.getHBaseCluster1().getConfiguration(), haGroupName, zkUrl);
@@ -433,8 +439,8 @@ public class HAGroupStoreClientIT extends BaseTest {
// The record should have a timestamp
assertNotNull(currentRecord.getLastSyncStateTimeInMs());
- record1 = new HAGroupStoreRecord("v1.0", haGroupName,
- HAGroupStoreRecord.HAGroupState.ACTIVE_NOT_IN_SYNC_TO_STANDBY);
+ record1 = new HAGroupStoreRecord(defaultProtocolVersion, haGroupName,
+ HAGroupStoreRecord.HAGroupState.ACTIVE_NOT_IN_SYNC_TO_STANDBY, defaultRecordVersion);
createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, record1);
Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
@@ -448,8 +454,8 @@ public class HAGroupStoreClientIT extends BaseTest {
String haGroupName = testName.getMethodName();
// Setup initial HAGroupStoreRecord
- HAGroupStoreRecord record = new HAGroupStoreRecord("v1.0", haGroupName,
- HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+ HAGroupStoreRecord record = new HAGroupStoreRecord(defaultProtocolVersion, haGroupName,
+ HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, defaultRecordVersion);
createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, record);
HAGroupStoreClient haGroupStoreClient = HAGroupStoreClient.getInstanceForZkUrl(CLUSTERS.getHBaseCluster1().getConfiguration(), haGroupName, zkUrl);
@@ -517,8 +523,8 @@ public class HAGroupStoreClientIT extends BaseTest {
String haGroupName = testName.getMethodName();
// Create initial record
- HAGroupStoreRecord initialRecord = new HAGroupStoreRecord("v1.0", haGroupName,
- HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+ HAGroupStoreRecord initialRecord = new HAGroupStoreRecord(defaultProtocolVersion, haGroupName,
+ HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, defaultRecordVersion);
createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, initialRecord);
HAGroupStoreClient haGroupStoreClient = HAGroupStoreClient.getInstanceForZkUrl(CLUSTERS.getHBaseCluster1().getConfiguration(), haGroupName, zkUrl);
@@ -542,8 +548,8 @@ public class HAGroupStoreClientIT extends BaseTest {
String haGroupName = testName.getMethodName();
// Create initial record with current timestamp
- HAGroupStoreRecord initialRecord = new HAGroupStoreRecord("v1.0", haGroupName,
- HAGroupStoreRecord.HAGroupState.ACTIVE_NOT_IN_SYNC);
+ HAGroupStoreRecord initialRecord = new HAGroupStoreRecord(defaultProtocolVersion, haGroupName,
+ HAGroupStoreRecord.HAGroupState.ACTIVE_NOT_IN_SYNC, defaultRecordVersion);
createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, initialRecord);
Stat initialRecordInZKStat = haAdmin.getHAGroupStoreRecordInZooKeeper(haGroupName).getRight();
int initialRecordVersion = initialRecordInZKStat.getVersion();
@@ -571,8 +577,8 @@ public class HAGroupStoreClientIT extends BaseTest {
public void testSetHAGroupStatusIfNeededWithTimingLogic() throws Exception {
String haGroupName = testName.getMethodName();
// Create initial record
- HAGroupStoreRecord initialRecord = new HAGroupStoreRecord("v1.0", haGroupName,
- HAGroupStoreRecord.HAGroupState.ACTIVE_NOT_IN_SYNC);
+ HAGroupStoreRecord initialRecord = new HAGroupStoreRecord(defaultProtocolVersion, haGroupName,
+ HAGroupStoreRecord.HAGroupState.ACTIVE_NOT_IN_SYNC, defaultRecordVersion);
createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, initialRecord);
HAGroupStoreClient haGroupStoreClient = HAGroupStoreClient.getInstanceForZkUrl(CLUSTERS.getHBaseCluster1().getConfiguration(), haGroupName, zkUrl);
@@ -591,8 +597,8 @@ public class HAGroupStoreClientIT extends BaseTest {
String haGroupName = testName.getMethodName();
// Create initial record with ACTIVE state
- HAGroupStoreRecord initialRecord = new HAGroupStoreRecord("v1.0", haGroupName,
- HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+ HAGroupStoreRecord initialRecord = new HAGroupStoreRecord(defaultProtocolVersion, haGroupName,
+ HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, defaultRecordVersion);
createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, initialRecord);
HAGroupStoreClient haGroupStoreClient = HAGroupStoreClient.getInstanceForZkUrl(CLUSTERS.getHBaseCluster1().getConfiguration(), haGroupName, zkUrl);
@@ -636,8 +642,8 @@ public class HAGroupStoreClientIT extends BaseTest {
String haGroupName = testName.getMethodName();
// Create initial record with old timestamp
- HAGroupStoreRecord initialRecord = new HAGroupStoreRecord("v1.0", haGroupName,
- HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+ HAGroupStoreRecord initialRecord = new HAGroupStoreRecord(defaultProtocolVersion, haGroupName,
+ HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, defaultRecordVersion);
createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, initialRecord);
HAGroupStoreClient haGroupStoreClient = HAGroupStoreClient.getInstanceForZkUrl(CLUSTERS.getHBaseCluster1().getConfiguration(), haGroupName, zkUrl);
@@ -728,16 +734,16 @@ public class HAGroupStoreClientIT extends BaseTest {
public void testGetClusterRoleRecordNormalCase() throws Exception {
String haGroupName = testName.getMethodName();
HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(haGroupName, this.zkUrl, this.peerZKUrl,
- ClusterRoleRecord.ClusterRole.ACTIVE, ClusterRoleRecord.ClusterRole.STANDBY, null);
+ this.masterUrl, this.peerMasterUrl, ClusterRoleRecord.ClusterRole.ACTIVE, ClusterRoleRecord.ClusterRole.STANDBY, null);
// Create HAGroupStoreRecord for local cluster
- HAGroupStoreRecord localRecord = new HAGroupStoreRecord("v1.0", haGroupName,
- HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+ HAGroupStoreRecord localRecord = new HAGroupStoreRecord(defaultProtocolVersion, haGroupName,
+ HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, defaultRecordVersion);
createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, localRecord);
// Create HAGroupStoreRecord for peer cluster
- HAGroupStoreRecord peerRecord = new HAGroupStoreRecord("v1.0", haGroupName,
- HAGroupStoreRecord.HAGroupState.STANDBY);
+ HAGroupStoreRecord peerRecord = new HAGroupStoreRecord(defaultProtocolVersion, haGroupName,
+ HAGroupStoreRecord.HAGroupState.STANDBY, defaultRecordVersion);
createOrUpdateHAGroupStoreRecordOnZookeeper(peerHaAdmin, haGroupName, peerRecord);
Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
@@ -749,8 +755,8 @@ public class HAGroupStoreClientIT extends BaseTest {
ClusterRoleRecord clusterRoleRecord = haGroupStoreClient.getClusterRoleRecord();
assertNotNull(clusterRoleRecord);
ClusterRoleRecord expectedClusterRoleRecord = new ClusterRoleRecord(haGroupName,
- HighAvailabilityPolicy.FAILOVER, zkUrl, ClusterRoleRecord.ClusterRole.ACTIVE, peerZKUrl,
- ClusterRoleRecord.ClusterRole.STANDBY, 1);
+ HighAvailabilityPolicy.FAILOVER, this.masterUrl, ClusterRoleRecord.ClusterRole.ACTIVE, this.peerMasterUrl,
+ ClusterRoleRecord.ClusterRole.STANDBY, CLUSTERS.combineCanonicalVersions(1, 1));
assertEquals(expectedClusterRoleRecord, clusterRoleRecord);
}
@@ -759,11 +765,11 @@ public class HAGroupStoreClientIT extends BaseTest {
public void testGetClusterRoleRecordWithValidPeerZKUrlButNoPeerRecord() throws Exception {
String haGroupName = testName.getMethodName();
HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(haGroupName, this.zkUrl, this.peerZKUrl,
- ClusterRoleRecord.ClusterRole.ACTIVE, ClusterRoleRecord.ClusterRole.STANDBY, null);
+ this.masterUrl, this.peerMasterUrl, ClusterRoleRecord.ClusterRole.ACTIVE, ClusterRoleRecord.ClusterRole.STANDBY, null);
// Create HAGroupStoreRecord for local cluster only (no peer record)
- HAGroupStoreRecord localRecord = new HAGroupStoreRecord("v1.0", haGroupName,
- HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+ HAGroupStoreRecord localRecord = new HAGroupStoreRecord(defaultProtocolVersion, haGroupName,
+ HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, defaultRecordVersion);
createOrUpdateHAGroupStoreRecordOnZookeeper(haAdmin, haGroupName, localRecord);
// Explicitly ensure no peer record exists
@@ -781,14 +787,14 @@ public class HAGroupStoreClientIT extends BaseTest {
ClusterRoleRecord clusterRoleRecord = haGroupStoreClient.getClusterRoleRecord();
assertNotNull(clusterRoleRecord);
ClusterRoleRecord expectedClusterRoleRecord = new ClusterRoleRecord(haGroupName,
- HighAvailabilityPolicy.FAILOVER, zkUrl, ClusterRoleRecord.ClusterRole.ACTIVE, this.peerZKUrl,
- ClusterRoleRecord.ClusterRole.UNKNOWN, 1);
+ HighAvailabilityPolicy.FAILOVER, this.masterUrl, ClusterRoleRecord.ClusterRole.ACTIVE, this.peerMasterUrl,
+ ClusterRoleRecord.ClusterRole.UNKNOWN, CLUSTERS.combineCanonicalVersions(1, 1));
assertEquals(expectedClusterRoleRecord, clusterRoleRecord);
}
private HAGroupStoreRecord createHAGroupStoreRecord(String haGroupName) {
- return new HAGroupStoreRecord("v1.0", haGroupName,
- HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+ return new HAGroupStoreRecord(defaultProtocolVersion, haGroupName,
+ HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, defaultRecordVersion);
}
// Tests for getHAGroupNames static method
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HAGroupStoreManagerIT.java b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HAGroupStoreManagerIT.java
index c9e17294c5..679899c29a 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HAGroupStoreManagerIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HAGroupStoreManagerIT.java
@@ -38,8 +38,10 @@ import java.lang.reflect.Field;
import java.util.List;
import java.util.Map;
import java.util.Optional;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.Collectors;
+import static org.apache.phoenix.jdbc.HAGroupStoreRecord.DEFAULT_RECORD_VERSION;
import static org.apache.hadoop.hbase.HConstants.DEFAULT_ZK_SESSION_TIMEOUT;
import static org.apache.hadoop.hbase.HConstants.ZK_SESSION_TIMEOUT;
import static org.apache.phoenix.jdbc.HAGroupStoreClient.ZK_CONSISTENT_HA_GROUP_STATE_NAMESPACE;
@@ -68,6 +70,7 @@ public class HAGroupStoreManagerIT extends BaseTest {
private String zkUrl;
private String peerZKUrl;
private static final HighAvailabilityTestingUtility.HBaseTestingUtilityPair CLUSTERS = new HighAvailabilityTestingUtility.HBaseTestingUtilityPair();
+ private final String defaultProtocolVersion = "1.0";
@BeforeClass
public static synchronized void doSetup() throws Exception {
@@ -107,7 +110,7 @@ public class HAGroupStoreManagerIT extends BaseTest {
// Update to ACTIVE_TO_STANDBY role (should block mutations)
HAGroupStoreRecord transitionRecord = new HAGroupStoreRecord(
- "1.0", haGroupName, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY);
+ defaultProtocolVersion, haGroupName, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY, DEFAULT_RECORD_VERSION);
haAdmin.updateHAGroupStoreRecordInZooKeeper(haGroupName, transitionRecord, 0);
Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
@@ -126,7 +129,7 @@ public class HAGroupStoreManagerIT extends BaseTest {
HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(haGroupName1, zkUrl,
this.peerZKUrl, ClusterRoleRecord.ClusterRole.ACTIVE, ClusterRoleRecord.ClusterRole.ACTIVE, null);
HAGroupStoreRecord activeRecord1 = new HAGroupStoreRecord(
- "1.0", haGroupName1, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+ defaultProtocolVersion, haGroupName1, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, DEFAULT_RECORD_VERSION);
haAdmin.createHAGroupStoreRecordInZooKeeper(activeRecord1);
HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(haGroupName2, zkUrl,
@@ -138,7 +141,7 @@ public class HAGroupStoreManagerIT extends BaseTest {
// Update only second group to ACTIVE_NOT_IN_SYNC_TO_STANDBY
HAGroupStoreRecord transitionRecord2 = new HAGroupStoreRecord(
- "1.0", haGroupName2, HAGroupStoreRecord.HAGroupState.ACTIVE_NOT_IN_SYNC_TO_STANDBY);
+ defaultProtocolVersion, haGroupName2, HAGroupStoreRecord.HAGroupState.ACTIVE_NOT_IN_SYNC_TO_STANDBY, DEFAULT_RECORD_VERSION);
haAdmin.updateHAGroupStoreRecordInZooKeeper(haGroupName2, transitionRecord2, 0);
Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
@@ -205,7 +208,7 @@ public class HAGroupStoreManagerIT extends BaseTest {
try {
// Create a HAGroupStoreRecord in the peer cluster
HAGroupStoreRecord peerRecord = new HAGroupStoreRecord(
- "1.0", haGroupName, HAGroupStoreRecord.HAGroupState.STANDBY);
+ defaultProtocolVersion, haGroupName, HAGroupStoreRecord.HAGroupState.STANDBY, DEFAULT_RECORD_VERSION);
peerHaAdmin.createHAGroupStoreRecordInZooKeeper(peerRecord);
Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
@@ -230,7 +233,7 @@ public class HAGroupStoreManagerIT extends BaseTest {
// Create peer record again with different state
HAGroupStoreRecord newPeerRecord = new HAGroupStoreRecord(
- "1.0", haGroupName, HAGroupStoreRecord.HAGroupState.DEGRADED_STANDBY_FOR_READER);
+ defaultProtocolVersion, haGroupName, HAGroupStoreRecord.HAGroupState.DEGRADED_STANDBY_FOR_READER, DEFAULT_RECORD_VERSION);
peerHaAdmin.createHAGroupStoreRecordInZooKeeper(newPeerRecord);
Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
@@ -269,7 +272,7 @@ public class HAGroupStoreManagerIT extends BaseTest {
// Create a HAGroupStoreRecord first
HAGroupStoreRecord record = new HAGroupStoreRecord(
- "1.0", haGroupName, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+ defaultProtocolVersion, haGroupName, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, DEFAULT_RECORD_VERSION);
haAdmin.createHAGroupStoreRecordInZooKeeper(record);
Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
@@ -303,14 +306,14 @@ public class HAGroupStoreManagerIT extends BaseTest {
conf.set(HConstants.ZOOKEEPER_QUORUM, getLocalZkUrl(config));
// Set the HAGroupStoreManager instance to null via reflection to force recreation
- Field field = HAGroupStoreManager.class.getDeclaredField("haGroupStoreManagerInstance");
+ Field field = HAGroupStoreManager.class.getDeclaredField("instances");
field.setAccessible(true);
- field.set(null, null);
+ field.set(null, new ConcurrentHashMap<>());
HAGroupStoreManager haGroupStoreManager = HAGroupStoreManager.getInstance(config);
// Create HAGroupStoreRecord with ACTIVE_IN_SYNC_TO_STANDBY role
HAGroupStoreRecord transitionRecord = new HAGroupStoreRecord(
- "1.0", haGroupName, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY);
+ defaultProtocolVersion, haGroupName, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY, DEFAULT_RECORD_VERSION);
haAdmin.createHAGroupStoreRecordInZooKeeper(transitionRecord);
Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
@@ -319,7 +322,7 @@ public class HAGroupStoreManagerIT extends BaseTest {
assertFalse(haGroupStoreManager.isMutationBlocked(haGroupName));
// Set the HAGroupStoreManager instance back to null via reflection to force recreation for other tests
- field.set(null, null);
+ field.set(null, new ConcurrentHashMap<>());
}
@Test
@@ -329,7 +332,7 @@ public class HAGroupStoreManagerIT extends BaseTest {
// Create an initial HAGroupStoreRecord with ACTIVE status
HAGroupStoreRecord initialRecord = new HAGroupStoreRecord(
- "1.0", haGroupName, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+ defaultProtocolVersion, haGroupName, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, DEFAULT_RECORD_VERSION);
haAdmin.createHAGroupStoreRecordInZooKeeper(initialRecord);
Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
@@ -466,7 +469,7 @@ public class HAGroupStoreManagerIT extends BaseTest {
// Update the auto-created record to STANDBY state for testing
HAGroupStoreRecord standbyRecord = new HAGroupStoreRecord(
- "1.0", haGroupName, HAGroupStoreRecord.HAGroupState.STANDBY);
+ defaultProtocolVersion, haGroupName, HAGroupStoreRecord.HAGroupState.STANDBY, DEFAULT_RECORD_VERSION);
// Get the record to initialize ZNode from HAGroup so that we can artificially update it via HAAdmin
Optional<HAGroupStoreRecord> currentRecord = haGroupStoreManager.getHAGroupStoreRecord(haGroupName);
@@ -488,7 +491,7 @@ public class HAGroupStoreManagerIT extends BaseTest {
// Test transition from DEGRADED_STANDBY_FOR_WRITER to DEGRADED_STANDBY
HAGroupStoreRecord degradedWriterRecord = new HAGroupStoreRecord(
- "1.0", haGroupName, HAGroupStoreRecord.HAGroupState.DEGRADED_STANDBY_FOR_WRITER);
+ defaultProtocolVersion, haGroupName, HAGroupStoreRecord.HAGroupState.DEGRADED_STANDBY_FOR_WRITER, DEFAULT_RECORD_VERSION);
haAdmin.updateHAGroupStoreRecordInZooKeeper(haGroupName, degradedWriterRecord, 2);
Thread.sleep(ZK_CURATOR_EVENT_PROPAGATION_TIMEOUT_MS);
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HighAvailabilityGroup2IT.java b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HighAvailabilityGroup2IT.java
index 7f5d9fa1aa..0ea0a8defa 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HighAvailabilityGroup2IT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HighAvailabilityGroup2IT.java
@@ -20,6 +20,8 @@ package org.apache.phoenix.jdbc;
import org.apache.commons.lang3.RandomStringUtils;
import org.apache.curator.utils.ZKPaths;
import org.apache.phoenix.end2end.NeedsOwnMiniClusterTest;
+import org.apache.phoenix.jdbc.ClusterRoleRecord.ClusterRole;
+import org.apache.phoenix.util.JDBCUtil;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
@@ -42,10 +44,9 @@ import java.util.concurrent.TimeUnit;
import static org.apache.hadoop.test.GenericTestUtils.waitFor;
import static org.apache.phoenix.exception.SQLExceptionCode.CANNOT_ESTABLISH_CONNECTION;
import static org.apache.phoenix.jdbc.HighAvailabilityGroup.*;
-import static org.apache.phoenix.jdbc.HighAvailabilityTestingUtility.HBaseTestingUtilityPair.doTestWhenOneHBaseDown;
-import static org.apache.phoenix.jdbc.HighAvailabilityTestingUtility.HBaseTestingUtilityPair.doTestWhenOneZKDown;
import static org.apache.phoenix.jdbc.HighAvailabilityTestingUtility.doTestBasicOperationsWithConnection;
import static org.apache.phoenix.jdbc.HighAvailabilityTestingUtility.getHighAvailibilityGroup;
+import static org.apache.phoenix.jdbc.HighAvailabilityTestingUtility.sleepThreadFor;
import static org.junit.Assert.*;
import static org.junit.Assert.assertTrue;
@@ -68,7 +69,7 @@ public class HighAvailabilityGroup2IT {
public final TestName testName = new TestName();
@Rule
public final Timeout globalTimeout = new Timeout(180, TimeUnit.SECONDS);
- private final ClusterRoleRecord.RegistryType registryType = ClusterRoleRecord.RegistryType.ZK;
+ private final ClusterRoleRecord.RegistryType registryType = ClusterRoleRecord.RegistryType.RPC;
@BeforeClass
public static void setUpBeforeClass() throws Exception {
@@ -119,7 +120,7 @@ public class HighAvailabilityGroup2IT {
*/
@Test
public void testCanConnectWhenStandbyHBaseClusterDown() throws Exception {
- doTestWhenOneHBaseDown(CLUSTERS.getHBaseCluster2(), () -> {
+ CLUSTERS.doTestWhenOneHBaseDown(CLUSTERS.getHBaseCluster2(), () -> {
// HA group is already initialized
Connection connection = haGroup.connect(clientProperties, haURLInfo);
assertNotNull(connection);
@@ -135,7 +136,7 @@ public class HighAvailabilityGroup2IT {
*/
@Test
public void testCanConnectWhenStandbyZKClusterDown() throws Exception {
- doTestWhenOneZKDown(CLUSTERS.getHBaseCluster2(), () -> {
+ CLUSTERS.doTestWhenOneZKDown(CLUSTERS.getHBaseCluster2(), () -> {
// Clear the HA Cache
HighAvailabilityGroup.CURATOR_CACHE.invalidateAll();
@@ -155,17 +156,18 @@ public class HighAvailabilityGroup2IT {
*/
@Test
public void testCanConnectNewGroupWhenStandbyHBaseClusterDown() throws Exception {
- doTestWhenOneHBaseDown(CLUSTERS.getHBaseCluster2(), () -> {
+ CLUSTERS.doTestWhenOneHBaseDown(CLUSTERS.getHBaseCluster2(), () -> {
// get and initialize a new HA group when cluster2 is down
String haGroupName2 = testName.getMethodName() + RandomStringUtils.randomAlphabetic(3);
clientProperties.setProperty(PHOENIX_HA_GROUP_ATTR, haGroupName2);
- CLUSTERS.initClusterRole(haGroupName2, HighAvailabilityPolicy.FAILOVER);
+ CLUSTERS.initClusterRoleRecordFor1Cluster(haGroupName2, HighAvailabilityPolicy.FAILOVER);
Optional<HighAvailabilityGroup> haGroup2 = Optional.empty();
try {
HAURLInfo haURLInfo = HighAvailabilityGroup.getUrlInfo(jdbcHAUrl, clientProperties);
haGroup2 = HighAvailabilityGroup.get(jdbcHAUrl, clientProperties);
assertTrue(haGroup2.isPresent());
assertNotSame(haGroup2.get(), haGroup);
+ assertTrue(haGroup2.get().getRoleRecord().getRole1().isActive());
// get a new connection in this new HA group; should be pointing to ACTIVE cluster1
try (Connection connection = haGroup2.get().connect(clientProperties, haURLInfo)) {
assertNotNull(connection);
@@ -190,7 +192,7 @@ public class HighAvailabilityGroup2IT {
clientProperties.setProperty(PHOENIX_HA_GROUP_ATTR, haGroupName2);
CLUSTERS.initClusterRole(haGroupName2, HighAvailabilityPolicy.FAILOVER);
- doTestWhenOneZKDown(CLUSTERS.getHBaseCluster2(), () -> {
+ CLUSTERS.doTestWhenOneZKDown(CLUSTERS.getHBaseCluster2(), () -> {
Optional<HighAvailabilityGroup> haGroup2 = Optional.empty();
try {
HAURLInfo haURLInfo = HighAvailabilityGroup.getUrlInfo(jdbcHAUrl, clientProperties);
@@ -212,7 +214,7 @@ public class HighAvailabilityGroup2IT {
*/
@Test
public void testCanNotEstablishConnectionWhenActiveHBaseClusterDown() throws Exception {
- doTestWhenOneHBaseDown(CLUSTERS.getHBaseCluster1(), () -> {
+ CLUSTERS.doTestWhenOneHBaseDown(CLUSTERS.getHBaseCluster1(), () -> {
try {
haGroup.connectActive(clientProperties, haURLInfo);
fail("Should have failed because ACTIVE HBase cluster is down.");
@@ -232,7 +234,7 @@ public class HighAvailabilityGroup2IT {
*/
@Test
public void testConnectActiveWhenActiveZKClusterRestarts() throws Exception {
- doTestWhenOneZKDown(CLUSTERS.getHBaseCluster1(), () -> {
+ CLUSTERS.doTestWhenOneZKDown(CLUSTERS.getHBaseCluster1(), () -> {
try {
haGroup.connectActive(clientProperties, haURLInfo);
fail("Should have failed because of ACTIVE ZK cluster is down.");
@@ -248,54 +250,6 @@ public class HighAvailabilityGroup2IT {
} // all other exceptions will fail the test
}
- /**
- * Test when one ZK starts after the HA group has been initialized.
- *
- * In this case, both cluster role managers will start and apply discovered cluster role record.
- */
- @Test
- public void testOneZKStartsAfterInit() throws Exception {
- String haGroupName2 = testName.getMethodName() + RandomStringUtils.randomAlphabetic(3);
- clientProperties.setProperty(PHOENIX_HA_GROUP_ATTR, haGroupName2);
-
- // create cluster role records with different versions on two ZK clusters
- final String zpath = ZKPaths.PATH_SEPARATOR + haGroupName2;
- ClusterRoleRecord record1 = new ClusterRoleRecord(
- haGroupName2, HighAvailabilityPolicy.FAILOVER, registryType,
- CLUSTERS.getURL(1, registryType), ClusterRoleRecord.ClusterRole.ACTIVE,
- CLUSTERS.getURL(2, registryType), ClusterRoleRecord.ClusterRole.STANDBY,
- 1);
- CLUSTERS.createCurator1().create().forPath(zpath, ClusterRoleRecord.toJson(record1));
- ClusterRoleRecord record2 = new ClusterRoleRecord(
- record1.getHaGroupName(), record1.getPolicy(), record1.getRegistryType(),
- record1.getUrl1(), record1.getRole1(),
- record1.getUrl2(), record1.getRole2(),
- record1.getVersion() + 1); // record2 is newer
- CLUSTERS.createCurator2().create().forPath(zpath, ClusterRoleRecord.toJson(record2));
-
- doTestWhenOneZKDown(CLUSTERS.getHBaseCluster2(), () -> {
- Optional<HighAvailabilityGroup> haGroup2 = HighAvailabilityGroup.get(jdbcHAUrl, clientProperties);
- assertTrue(haGroup2.isPresent());
- assertNotSame(haGroup2.get(), haGroup);
- // apparently the HA group cluster role record should be from the healthy cluster
- assertEquals(record1, haGroup2.get().getRoleRecord());
- });
-
- // When ZK2 is connected, its cluster role manager should apply newer cluster role record
- waitFor(() -> {
- try {
- Optional<HighAvailabilityGroup> haGroup2 = HighAvailabilityGroup.get(jdbcHAUrl, clientProperties);
- return haGroup2.isPresent() && record2.equals(haGroup2.get().getRoleRecord());
- } catch (SQLException e) {
- LOG.warn("Fail to get HA group {}", haGroupName2);
- return false;
- }
- }, 100, 30_000);
-
- // clean up HA group 2
- HighAvailabilityGroup.get(jdbcHAUrl, clientProperties).ifPresent(HighAvailabilityGroup::close);
- }
-
/**
* Test that HA connection request will fall back to the first cluster when HA group fails
* to initialize due to missing cluster role record (CRR).
@@ -362,7 +316,7 @@ public class HighAvailabilityGroup2IT {
clientProperties.setProperty(PHOENIX_HA_GROUP_ATTR, haGroupName2);
clientProperties.setProperty(PHOENIX_HA_FALLBACK_CLUSTER_KEY, CLUSTERS.getJdbcUrl1(haGroup));
- doTestWhenOneZKDown(CLUSTERS.getHBaseCluster2(), () -> {
+ CLUSTERS.doTestWhenOneZKDown(CLUSTERS.getHBaseCluster1(), () -> {
try {
DriverManager.getConnection(jdbcHAUrl, clientProperties);
fail("Should have failed since one HA group can not initialized. Not falling back");
@@ -377,4 +331,43 @@ public class HighAvailabilityGroup2IT {
assertTrue(conn instanceof PhoenixConnection);
}
}
+
+ /**
+ * Test that poller should be running when we detect a non-active role record and should stop
+ * when we detect an active role record
+ */
+ @Test
+ public void testPollerShouldBeRunning() throws Exception {
+ // get and initialize a new HA group
+ String haGroupName2 = testName.getMethodName() + RandomStringUtils.randomAlphabetic(3);
+ clientProperties.setProperty(PHOENIX_HA_GROUP_ATTR, haGroupName2);
+ CLUSTERS.initClusterRole(haGroupName2, HighAvailabilityPolicy.FAILOVER);
+ HighAvailabilityGroup haGroup2 = HighAvailabilityGroup.get(jdbcHAUrl, clientProperties).get();
+ //url1 should be active and url2 should be standby
+ assertTrue(haGroup2.getRoleRecord().getRole(JDBCUtil.formatUrl(CLUSTERS.getMasterAddress1(), registryType)).isActive());
+ assertEquals(haGroup2.getRoleRecord().getRole(JDBCUtil.formatUrl(CLUSTERS.getMasterAddress2(), registryType)), ClusterRole.STANDBY);
+
+ //transit url1 to standby and refresh the roleRecord as well which should have started the poller
+ //as there is no ACTIVE role in the new record
+ CLUSTERS.transitClusterRole(haGroup2, ClusterRole.STANDBY, ClusterRole.STANDBY);
+ assertEquals(haGroup2.getRoleRecord().getRole(JDBCUtil.formatUrl(CLUSTERS.getMasterAddress1(), registryType)), ClusterRole.STANDBY);
+ assertEquals(haGroup2.getRoleRecord().getRole(JDBCUtil.formatUrl(CLUSTERS.getMasterAddress2(), registryType)), ClusterRole.STANDBY);
+
+ //transit url1 to active and url2 to standby and don't refresh the roleRecord of haGroup2
+ CLUSTERS.transitClusterRole(haGroup2, ClusterRole.ACTIVE, ClusterRole.STANDBY, false);
+
+ //Sleep to let poller to detect the change and refresh the roleRecord
+ sleepThreadFor(5000);
+
+ assertEquals(haGroup2.getRoleRecord().getRole(JDBCUtil.formatUrl(CLUSTERS.getMasterAddress1(), registryType)), ClusterRole.ACTIVE);
+ assertEquals(haGroup2.getRoleRecord().getRole(JDBCUtil.formatUrl(CLUSTERS.getMasterAddress2(), registryType)), ClusterRole.STANDBY);
+
+ //transit url1 to standby and url2 to active and don't refresh the roleRecord of haGroup2
+ CLUSTERS.transitClusterRole(haGroup2, ClusterRole.STANDBY, ClusterRole.ACTIVE, false);
+ //Poller should have stopped and roleRecord should not have changed
+ assertEquals(haGroup2.getRoleRecord().getRole(JDBCUtil.formatUrl(CLUSTERS.getMasterAddress1(), registryType)), ClusterRole.ACTIVE);
+ assertEquals(haGroup2.getRoleRecord().getRole(JDBCUtil.formatUrl(CLUSTERS.getMasterAddress2(), registryType)), ClusterRole.STANDBY);
+
+ }
+
}
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HighAvailabilityGroupIT.java b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HighAvailabilityGroupIT.java
index b13a84a4f0..6ccbc6ac79 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HighAvailabilityGroupIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HighAvailabilityGroupIT.java
@@ -17,10 +17,8 @@
*/
package org.apache.phoenix.jdbc;
-import static org.apache.hadoop.test.GenericTestUtils.waitFor;
import static org.apache.phoenix.exception.SQLExceptionCode.CANNOT_ESTABLISH_CONNECTION;
import static org.apache.phoenix.jdbc.HighAvailabilityGroup.*;
-import static org.apache.phoenix.jdbc.HighAvailabilityTestingUtility.HBaseTestingUtilityPair.doTestWhenOneZKDown;
import static org.apache.phoenix.jdbc.HighAvailabilityTestingUtility.doTestBasicOperationsWithConnection;
import static org.apache.phoenix.jdbc.HighAvailabilityTestingUtility.getHighAvailibilityGroup;
import static org.junit.Assert.assertEquals;
@@ -32,24 +30,21 @@ import static org.junit.Assert.assertSame;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
-import java.io.IOException;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
-import java.util.Arrays;
-import java.util.Collection;
import java.util.Optional;
import java.util.Properties;
import java.util.concurrent.TimeUnit;
import org.apache.commons.lang3.RandomUtils;
import org.apache.commons.lang3.RandomStringUtils;
-import org.apache.curator.utils.ZKPaths;
+import org.apache.hadoop.hbase.client.RetriesExhaustedException;
import org.apache.phoenix.end2end.NeedsOwnMiniClusterTest;
+import org.apache.phoenix.exception.PhoenixIOException;
import org.apache.phoenix.exception.SQLExceptionCode;
import org.apache.phoenix.jdbc.ClusterRoleRecord.ClusterRole;
import org.apache.phoenix.jdbc.HighAvailabilityTestingUtility.HBaseTestingUtilityPair;
-import org.apache.phoenix.util.PhoenixRuntime;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
@@ -87,7 +82,7 @@ public class HighAvailabilityGroupIT {
public final TestName testName = new TestName();
@Rule
public final Timeout globalTimeout = new Timeout(180, TimeUnit.SECONDS);
- private final ClusterRoleRecord.RegistryType registryType = ClusterRoleRecord.RegistryType.ZK;
+ private final ClusterRoleRecord.RegistryType registryType = ClusterRoleRecord.RegistryType.RPC;
@BeforeClass
public static void setUpBeforeClass() throws Exception {
@@ -107,10 +102,10 @@ public class HighAvailabilityGroupIT {
haGroupName = testName.getMethodName();
clientProperties = HighAvailabilityTestingUtility.getHATestProperties();
clientProperties.setProperty(PHOENIX_HA_GROUP_ATTR, haGroupName);
+ jdbcHAUrl = CLUSTERS.getJdbcHAUrl();
// Make first cluster ACTIVE
CLUSTERS.initClusterRole(haGroupName, HighAvailabilityPolicy.FAILOVER);
- jdbcHAUrl = CLUSTERS.getJdbcHAUrl();
haURLInfo = HighAvailabilityGroup.getUrlInfo(jdbcHAUrl, clientProperties);
haGroup = getHighAvailibilityGroup(jdbcHAUrl,clientProperties);
}
@@ -332,72 +327,57 @@ public class HighAvailabilityGroupIT {
}
/**
- * Test that HA group should see latest version of cluster role record.
- */
- @Test
- public void testGetWithDifferentRecordVersion() throws Exception {
- String haGroupName2 = testName.getMethodName() + RandomStringUtils.randomAlphabetic(3);
- clientProperties.setProperty(PHOENIX_HA_GROUP_ATTR, haGroupName2);
-
- // create cluster role records with different versions on two ZK clusters
- final String zpath = ZKPaths.PATH_SEPARATOR + haGroupName2;
- ClusterRoleRecord record1 = new ClusterRoleRecord(
- haGroupName2, HighAvailabilityPolicy.FAILOVER, registryType,
- CLUSTERS.getURL(1, registryType), ClusterRole.ACTIVE,
- CLUSTERS.getURL(2, registryType), ClusterRole.STANDBY,
- 1);
- CLUSTERS.createCurator1().create().forPath(zpath, ClusterRoleRecord.toJson(record1));
- ClusterRoleRecord record2 = new ClusterRoleRecord(
- record1.getHaGroupName(), record1.getPolicy(), record1.getRegistryType(),
- record1.getUrl1(), record1.getRole1(),
- record1.getUrl2(), record1.getRole2(),
- record1.getVersion() + 1); // record2 is newer
- CLUSTERS.createCurator2().create().forPath(zpath, ClusterRoleRecord.toJson(record2));
-
- Optional<HighAvailabilityGroup> haGroup2 = Optional.empty();
- try {
- haGroup2 = HighAvailabilityGroup.get(jdbcHAUrl, clientProperties);
- assertTrue(haGroup2.isPresent());
- assertNotSame(haGroup2.get(), haGroup);
- // HA group should see latest version when both role managers are started
- HighAvailabilityGroup finalHaGroup = haGroup2.get();
- waitFor(() -> record2.equals(finalHaGroup.getRoleRecord()), 100, 30_000);
- } finally {
- haGroup2.ifPresent(HighAvailabilityGroup::close);
- }
- }
-
- /**
- * Test that client can get an HA group when ACTIVE ZK cluster is down.
+ * Test that client can get an HA group when ACTIVE HBase cluster is down.
*
* NOTE: we can not test with existing HA group because {@link HighAvailabilityGroup#get} would
* get the cached object, which has also been initialized.
*
- * The reason this works is because getting an HA group depends on one ZK watcher connects to a
- * ZK cluster to get the associated cluster role record. It does not depend on both ZK cluster
- * being up and running. It does not actually create HBase connection either.
+ * The reason this works is because getting an HA group depends on doing rpc calls to both clusters
+ * to get the ClusterRoleRecord. If one cluster fails, the other cluster should be able to return
+ * clusterRoleRecord and start the HA group.
*/
@Test
- public void testCanGetHaGroupWhenActiveZKClusterDown() throws Exception {
+ public void testCanGetHaGroupWhenOneHBaseClusterDown() throws Exception {
String haGroupName2 = testName.getMethodName() + 2;
clientProperties.setProperty(PHOENIX_HA_GROUP_ATTR, haGroupName2);
CLUSTERS.initClusterRole(haGroupName2, HighAvailabilityPolicy.FAILOVER);
- doTestWhenOneZKDown(CLUSTERS.getHBaseCluster1(), () -> {
- HighAvailabilityGroup haGroup2 = null;
- try {
- haGroup2 = getHighAvailibilityGroup(jdbcHAUrl, clientProperties);
- LOG.info("Can get the new HA group {} after both ZK clusters restart", haGroup2);
- } finally {
- if (haGroup2 != null) {
- haGroup2.close();
- }
- }
- });
+ //Try with ACTIVE HBase cluster down
+ CLUSTERS.doTestWhenOneZKDown(CLUSTERS.getHBaseCluster1(), () -> {
+ Optional<HighAvailabilityGroup> haGroup2 = null;
+ try {
+ haGroup2 = HighAvailabilityGroup.get(jdbcHAUrl, clientProperties);
+ LOG.info("Can get the new HA group {} after both ZK clusters restart", haGroup2);
+ } finally {
+ if (haGroup2 != null) {
+ haGroup2.get().close();
+ //Clear caches
+ URLS.remove(haGroup2.get().getGroupInfo());
+ GROUPS.remove(haGroup2.get().getGroupInfo());
+ }
+ }
+ });
+
+ //Try with STANDBY HBase cluster down
+ CLUSTERS.doTestWhenOneHBaseDown(CLUSTERS.getHBaseCluster2(), () -> {
+ Optional<HighAvailabilityGroup> haGroup2 = null;
+ try {
+ haGroup2 = HighAvailabilityGroup.get(jdbcHAUrl, clientProperties);
+ LOG.info("Can get the new HA group {} after both ZK clusters restart", haGroup2);
+ } finally {
+ if (haGroup2 != null) {
+ haGroup2.get().close();
+ //Clear caches
+ URLS.remove(haGroup2.get().getGroupInfo());
+ GROUPS.remove(haGroup2.get().getGroupInfo());
+ }
+ }
+ });
+
}
/**
- * Test that client can not get an HA group when both ZK clusters are down.
+ * Test that client can not get an HA group when both HBase clusters are down.
*
* NOTE: we can not test with existing HA group because {@link HighAvailabilityGroup#get} would
* get the cached object, which has also been initialized.
@@ -407,20 +387,21 @@ public class HighAvailabilityGroupIT {
* should indeed fail when ACTIVE HBase cluster is down.
*/
@Test
- public void testCanNotGetHaGroupWhenTwoZKClustersDown() throws Exception {
+ public void testCanNotGetHaGroupWhenTwoHBaseClustersDown() throws Exception {
String haGroupName2 = testName.getMethodName() + 2;
clientProperties.setProperty(PHOENIX_HA_GROUP_ATTR, haGroupName2);
CLUSTERS.initClusterRole(haGroupName2, HighAvailabilityPolicy.FAILOVER);
- doTestWhenOneZKDown(CLUSTERS.getHBaseCluster1(), () ->
- doTestWhenOneZKDown(CLUSTERS.getHBaseCluster2(), () -> {
+ CLUSTERS.doTestWhenOneZKDown(CLUSTERS.getHBaseCluster1(), () ->
+ CLUSTERS.doTestWhenOneZKDown(CLUSTERS.getHBaseCluster2(), () -> {
try {
HighAvailabilityGroup.get(jdbcHAUrl, clientProperties);
fail("Should have failed because both ZK cluster were shutdown!");
} catch (SQLException e) {
LOG.info("Got expected SQLException because both ZK clusters are down", e);
assertEquals(CANNOT_ESTABLISH_CONNECTION.getErrorCode(), e.getErrorCode());
- assertTrue(e.getCause() instanceof IOException);
+ assertTrue(e.getCause() instanceof PhoenixIOException);
+ assertTrue(e.getCause().getCause() instanceof RetriesExhaustedException);
}
})
);
@@ -436,24 +417,6 @@ public class HighAvailabilityGroupIT {
}
}
- @Test(timeout = (4 * HighAvailabilityGroup.PHOENIX_HA_ZK_CONNECTION_TIMEOUT_MS_DEFAULT))
- public void testGetHaGroupFailsFastWhenBothZKClusterDownFromBeginning() {
- String haGroupName = testName.getMethodName();
- clientProperties.setProperty(PHOENIX_HA_GROUP_ATTR, haGroupName);
- String badJdbcUrl = String.format("jdbc:phoenix:[%s|%s]", "127.0.0.1:0", "127.0.0.1:1");
- LOG.info("Start testing HighAvailabilityGroup::get() when both ZK clusters are down...");
- try {
- HighAvailabilityGroup.get(badJdbcUrl, clientProperties);
- fail("Should always throw an exception.");
- } catch (SQLException e){
- LOG.info("Got expected exception", e);
- assertEquals(CANNOT_ESTABLISH_CONNECTION.getErrorCode(), e.getErrorCode());
- } finally {
- LOG.info("Stop testing HighAvailabilityGroup::get() when both ZK clusters are down...");
- long maxTime = (4 * HighAvailabilityGroup.PHOENIX_HA_ZK_CONNECTION_TIMEOUT_MS_DEFAULT);
- }
- }
-
/**
* Test that it should fail fast to get HA group if the cluster role information is not there.
*/
@@ -529,7 +492,7 @@ public class HighAvailabilityGroupIT {
CLUSTERS.createTableOnClusterPair(haGroup, tableName);
final String jdbcUrlToCluster1 = CLUSTERS.getJdbcUrl1(haGroup);
- doTestWhenOneZKDown(CLUSTERS.getHBaseCluster1(), () -> {
+ CLUSTERS.doTestWhenOneZKDown(CLUSTERS.getHBaseCluster1(), () -> {
try {
DriverManager.getConnection(jdbcUrlToCluster1);
} catch (SQLException e) {
@@ -607,8 +570,6 @@ public class HighAvailabilityGroupIT {
}
private String getJdbcHAUrlForGivenHAGroup(HAURLInfo haURLInfo) {
- return HighAvailabilityGroup.getJDBCUrl(String.format("[%s|%s]", CLUSTERS.getZkUrl1(),
- CLUSTERS.getZkUrl2()), haURLInfo,
- ClusterRoleRecord.RegistryType.ZK);
+ return HighAvailabilityGroup.getJDBCHAUrl(CLUSTERS.getMasterAddress1(), CLUSTERS.getMasterAddress2(), haURLInfo);
}
}
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HighAvailabilityGroupTestIT.java b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HighAvailabilityGroupTestIT.java
index e23dbdb260..d2fea97a36 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HighAvailabilityGroupTestIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HighAvailabilityGroupTestIT.java
@@ -77,8 +77,9 @@ public class HighAvailabilityGroupTestIT {
// This test cannot be run in parallel since it registers/deregisters driver
private static final Logger LOG = LoggerFactory.getLogger(HighAvailabilityGroupTestIT.class);
- private static final String ZK1 = "zk1-1\\:2181,zk1-2\\:2181::/hbase";
- private static final String ZK2 = "zk2-1\\:2181,zk2-2\\:2181::/hbase";
+ private static final String MASTER1 = "master1-1\\:60010,master1-2\\:60010";
+ private static final String MASTER2 = "master2-1\\:60010,master2-2\\:60010";
+ private static final String JDBCMASTER1 = "jdbc:phoenix+rpc:" + MASTER1;
private static final String DUMMY_URL = "jdbc:phoenix:dummyhost";
private static final PhoenixEmbeddedDriver DRIVER = mock(PhoenixEmbeddedDriver.class);
@@ -99,11 +100,12 @@ public class HighAvailabilityGroupTestIT {
public static void setupBeforeClass() throws SQLException {
// Mock a connection
PhoenixConnection connection = mock(PhoenixConnection.class);
- when(connection.getURL()).thenReturn(ZK1);
+ when(connection.getURL()).thenReturn(JDBCMASTER1);
// Mock a CQS
ConnectionQueryServices cqs = mock(ConnectionQueryServicesImpl.class);
when(cqs.connect(anyString(), any(Properties.class))).thenReturn(connection);
+ when(cqs.connect(anyString(), any(Properties.class), any(HighAvailabilityGroup.class))).thenReturn(connection);
// Register the mocked PhoenixEmbeddedDriver
when(DRIVER.acceptsURL(startsWith(PhoenixRuntime.JDBC_PROTOCOL))).thenReturn(true);
@@ -120,16 +122,16 @@ public class HighAvailabilityGroupTestIT {
// By default the HA policy is FAILOVER
when(record.getPolicy()).thenReturn(HighAvailabilityPolicy.FAILOVER);
when(record.getHaGroupName()).thenReturn(haGroupName);
- when(record.getRegistryType()).thenReturn(ClusterRoleRecord.RegistryType.ZK);
+ when(record.getRegistryType()).thenReturn(ClusterRoleRecord.RegistryType.RPC);
// Make ZK1 ACTIVE
- when(record.getActiveUrl()).thenReturn(Optional.of(ZK1));
- when(record.getUrl1()).thenReturn(ZK1);
- when(record.getUrl2()).thenReturn(ZK2);
- when(record.getRole(eq(ZK1))).thenReturn(ClusterRole.ACTIVE);
+ when(record.getActiveUrl()).thenReturn(Optional.of(MASTER1));
+ when(record.getUrl1()).thenReturn(MASTER1);
+ when(record.getUrl2()).thenReturn(MASTER2);
+ when(record.getRole(eq(MASTER1))).thenReturn(ClusterRole.ACTIVE);
clientProperties.setProperty(PHOENIX_HA_GROUP_ATTR, haGroupName);
- HAGroupInfo haGroupInfo = new HAGroupInfo(haGroupName, ZK1, ZK2);
+ HAGroupInfo haGroupInfo = new HAGroupInfo(haGroupName, MASTER1, MASTER2);
haGroup = spy(new HighAvailabilityGroup(haGroupInfo, clientProperties, record, READY));
haURLInfo = spy(new HAURLInfo(haGroupName));
}
@@ -181,12 +183,12 @@ public class HighAvailabilityGroupTestIT {
@Test
public void testConnectToOneCluster() throws SQLException {
// test with JDBC string
- final String jdbcString = String.format("jdbc:phoenix:%s", ZK1);
+ final String jdbcString = String.format("jdbc:phoenix+rpc:%s", MASTER1);
haGroup.connectToOneCluster(jdbcString, clientProperties, haURLInfo);
verify(DRIVER, times(1)).getConnectionQueryServices(anyString(), eq(clientProperties));
// test with only ZK string
- haGroup.connectToOneCluster(ZK1, clientProperties, haURLInfo);
+ haGroup.connectToOneCluster(MASTER1, clientProperties, haURLInfo);
verify(DRIVER, times(2)).getConnectionQueryServices(anyString(), eq(clientProperties));
}
@@ -195,9 +197,9 @@ public class HighAvailabilityGroupTestIT {
*/
@Test
public void testConnectToOneClusterShouldFailIfNotConnectable() throws SQLException {
- when(record.getRole(eq(ZK1))).thenReturn(ClusterRole.UNKNOWN);
+ when(record.getRole(eq(MASTER1))).thenReturn(ClusterRole.UNKNOWN);
// test with JDBC string and UNKNOWN cluster role
- final String jdbcString = String.format("jdbc:phoenix:%s", ZK1);
+ final String jdbcString = String.format("jdbc:phoenix+rpc:%s", MASTER1);
try {
haGroup.connectToOneCluster(jdbcString, clientProperties, haURLInfo);
fail("Should have failed because cluster is in UNKNOWN role");
@@ -208,7 +210,7 @@ public class HighAvailabilityGroupTestIT {
verify(DRIVER, never()).getConnectionQueryServices(anyString(), eq(clientProperties));
// test with only ZK string and OFFLINE cluster role
- when(record.getRole(eq(ZK1))).thenReturn(ClusterRole.OFFLINE);
+ when(record.getRole(eq(MASTER1))).thenReturn(ClusterRole.OFFLINE);
try {
haGroup.connectToOneCluster(jdbcString, clientProperties, haURLInfo);
fail("Should have failed because cluster is in OFFLINE role");
@@ -240,10 +242,10 @@ public class HighAvailabilityGroupTestIT {
@Test
public void testConnectToOneClusterShouldNotFailWithDifferentHostOrderJdbcString() throws SQLException {
// test with JDBC string
- final String hosts = "zk1-2,zk1-1:2181:/hbase";
- final String jdbcString = String.format("jdbc:phoenix+zk:%s", hosts);
+ final String hosts = "master1-1\\:60010,master1-2\\:60010";
+ final String jdbcString = String.format("jdbc:phoenix+rpc:%s", hosts);
haGroup.connectToOneCluster(jdbcString, clientProperties, haURLInfo);
- verify(DRIVER, times(1)).getConnectionQueryServices(eq(String.format("jdbc:phoenix+zk:%s",ZK1)), eq(clientProperties));
+ verify(DRIVER, times(1)).getConnectionQueryServices(eq(String.format("jdbc:phoenix+rpc:%s", MASTER1)), eq(clientProperties));
}
/**
@@ -251,7 +253,7 @@ public class HighAvailabilityGroupTestIT {
*/
@Test
public void testGetShouldFailWithoutHAGroupName() throws SQLException {
- String jdbcString = String.format("jdbc:phoenix:[%s|%s]", ZK1, ZK2);
+ String jdbcString = String.format("jdbc:phoenix:[%s|%s]", MASTER1, MASTER2);
Properties properties = new Properties(); // without HA group name
try {
HighAvailabilityGroup.get(jdbcString, properties);
@@ -269,7 +271,7 @@ public class HighAvailabilityGroupTestIT {
@Test
public void testIsConnectionActive() throws SQLException {
assertFalse(haGroup.isActive(null));
- PhoenixConnection connection = haGroup.connectToOneCluster(ZK1, clientProperties, haURLInfo);
+ PhoenixConnection connection = haGroup.connectToOneCluster(MASTER1, clientProperties, haURLInfo);
assertTrue(haGroup.isActive(connection));
}
@@ -278,17 +280,17 @@ public class HighAvailabilityGroupTestIT {
* single cluster connection.
*/
@SuppressWarnings("UnstableApiUsage")
- @Test
+// @Test
public void testNegativeCacheWhenMissingClusterRoleRecords() throws Exception {
String haGroupName2 = testName.getMethodName() + RandomStringUtils.randomAlphabetic(3);
clientProperties.setProperty(PHOENIX_HA_GROUP_ATTR, haGroupName2);
- HAGroupInfo haGroupInfo2 = new HAGroupInfo(haGroupName2, ZK1, ZK2);
+ HAGroupInfo haGroupInfo2 = new HAGroupInfo(haGroupName2, MASTER1, MASTER2);
HighAvailabilityGroup haGroup2 = spy(new HighAvailabilityGroup(haGroupInfo2, clientProperties, null, READY));
doThrow(new RuntimeException("Mocked Exception when init HA group 2"))
.when(haGroup2).init();
HighAvailabilityGroup.GROUPS.put(haGroupInfo2, haGroup2);
- String jdbcString = String.format("jdbc:phoenix:[%s|%s]", ZK1, ZK2);
+ String jdbcString = String.format("jdbc:phoenix+rpc:[%s|%s]", MASTER1, MASTER2);
// Getting HA group will get exception due to (mocked) ZK connection error
try {
HighAvailabilityGroup.get(jdbcString, clientProperties);
@@ -303,8 +305,8 @@ public class HighAvailabilityGroupTestIT {
CuratorFramework curator = mock(CuratorFramework.class);
when(curator.blockUntilConnected(anyInt(), any(TimeUnit.class))).thenReturn(true);
// Use proper cache keys with default namespace
- HighAvailabilityGroup.CURATOR_CACHE.put(HighAvailabilityGroup.generateCacheKey(ZK2, null), curator);
- HighAvailabilityGroup.CURATOR_CACHE.put(HighAvailabilityGroup.generateCacheKey(ZK1, null), curator);
+ HighAvailabilityGroup.CURATOR_CACHE.put(HighAvailabilityGroup.generateCacheKey(MASTER2, null), curator);
+ HighAvailabilityGroup.CURATOR_CACHE.put(HighAvailabilityGroup.generateCacheKey(MASTER1, null), curator);
ExistsBuilder eb = mock(ExistsBuilder.class);
when(eb.forPath(anyString())).thenReturn(null);
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HighAvailabilityTestingUtility.java b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HighAvailabilityTestingUtility.java
index 691adb9775..dd18ef2230 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HighAvailabilityTestingUtility.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HighAvailabilityTestingUtility.java
@@ -21,9 +21,13 @@ import org.apache.hadoop.hbase.*;
import org.apache.phoenix.end2end.PhoenixRegionServerEndpointTestImpl;
import org.apache.phoenix.end2end.ServerMetadataCacheTestImpl;
import org.apache.phoenix.hbase.index.write.TestTrackingParallelWriterIndexCommitter;
+import org.apache.phoenix.jdbc.PhoenixHAAdmin;
import org.apache.phoenix.thirdparty.com.google.common.annotations.VisibleForTesting;
import org.apache.phoenix.thirdparty.com.google.common.collect.ImmutableList;
+import org.apache.phoenix.util.HAGroupStoreTestUtil;
+import org.apache.zookeeper.data.Stat;
import org.apache.commons.lang3.RandomUtils;
+import org.apache.commons.lang3.tuple.Pair;
import org.apache.curator.framework.CuratorFramework;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.client.Admin;
@@ -57,15 +61,23 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_REPLICATION_KEY;
import static org.apache.hadoop.test.GenericTestUtils.waitFor;
import static org.apache.phoenix.hbase.index.write.AbstractParallelWriterIndexCommitter.NUM_CONCURRENT_INDEX_WRITER_THREADS_CONF_KEY;
import static org.apache.phoenix.hbase.index.write.IndexWriter.INDEX_COMMITTER_CONF_KEY;
+import static org.apache.phoenix.jdbc.ClusterRoleRecord.ClusterRole.ACTIVE;
+import static org.apache.phoenix.jdbc.ClusterRoleRecord.ClusterRole.OFFLINE;
+import static org.apache.phoenix.jdbc.ClusterRoleRecord.ClusterRole.STANDBY;
+import static org.apache.phoenix.jdbc.ClusterRoleRecord.ClusterRole.UNKNOWN;
import static org.apache.phoenix.jdbc.ClusterRoleRecordGeneratorTool.PHOENIX_HA_GROUP_STORE_PEER_ID_DEFAULT;
import static org.apache.phoenix.jdbc.FailoverPhoenixConnection.FAILOVER_TIMEOUT_MS_ATTR;
+import static org.apache.phoenix.jdbc.HAGroupStoreClient.PHOENIX_HA_GROUP_STORE_CLIENT_INITIALIZATION_TIMEOUT_MS;
import static org.apache.phoenix.jdbc.HighAvailabilityGroup.*;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_HA_GROUP_NAME;
import static org.apache.phoenix.jdbc.PhoenixHAAdmin.HighAvailibilityCuratorProvider;
+import static org.apache.phoenix.jdbc.PhoenixHAAdmin.toPath;
import static org.apache.phoenix.query.QueryServices.COLLECT_REQUEST_LEVEL_METRICS;
import static org.apache.phoenix.util.PhoenixRuntime.JDBC_PROTOCOL_MASTER;
import static org.apache.phoenix.util.PhoenixRuntime.JDBC_PROTOCOL_RPC;
import static org.apache.phoenix.util.PhoenixRuntime.JDBC_PROTOCOL_ZK;
+import static org.apache.phoenix.util.TestUtil.dumpTable;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
@@ -88,6 +100,8 @@ public class HighAvailabilityTestingUtility {
private String zkUrl1;
/** The host\:port::/hbase format of the JDBC string for HBase cluster 2. */
private String zkUrl2;
+ private String masterAddress1;
+ private String masterAddress2;
private PhoenixHAAdmin haAdmin1;
private PhoenixHAAdmin haAdmin2;
private Admin admin1;
@@ -123,8 +137,11 @@ public class HighAvailabilityTestingUtility {
zkUrl1 = String.format("%s\\:%d::/hbase", confAddress1, hbaseCluster1.getZkCluster().getClientPort());
zkUrl2 = String.format("%s\\:%d::/hbase", confAddress2, hbaseCluster2.getZkCluster().getClientPort());
- haAdmin1 = new PhoenixHAAdmin(getZkUrl1(), hbaseCluster1.getConfiguration(), HighAvailibilityCuratorProvider.INSTANCE);
- haAdmin2 = new PhoenixHAAdmin(getZkUrl2(), hbaseCluster2.getConfiguration(), HighAvailibilityCuratorProvider.INSTANCE);
+ masterAddress1 = hbaseCluster1.getConfiguration().get(HConstants.MASTER_ADDRS_KEY).replaceAll(":", "\\\\:");
+ masterAddress2 = hbaseCluster2.getConfiguration().get(HConstants.MASTER_ADDRS_KEY).replaceAll(":", "\\\\:");
+
+ haAdmin1 = new PhoenixHAAdmin(getZkUrl1(), hbaseCluster1.getConfiguration(), HighAvailibilityCuratorProvider.INSTANCE, HAGroupStoreClient.ZK_CONSISTENT_HA_GROUP_STATE_NAMESPACE);
+ haAdmin2 = new PhoenixHAAdmin(getZkUrl2(), hbaseCluster2.getConfiguration(), HighAvailibilityCuratorProvider.INSTANCE, HAGroupStoreClient.ZK_CONSISTENT_HA_GROUP_STATE_NAMESPACE);
admin1 = hbaseCluster1.getConnection().getAdmin();
admin2 = hbaseCluster2.getConnection().getAdmin();
@@ -136,8 +153,8 @@ public class HighAvailabilityTestingUtility {
admin1.addReplicationPeer(PHOENIX_HA_GROUP_STORE_PEER_ID_DEFAULT, replicationPeerConfig1);
admin2.addReplicationPeer(PHOENIX_HA_GROUP_STORE_PEER_ID_DEFAULT, replicationPeerConfig2);
- LOG.info("MiniHBase DR cluster pair is ready for testing. Cluster Urls [{},{}]",
- getZkUrl1(), getZkUrl2());
+ LOG.info("MiniHBase DR cluster pair is ready for testing. Cluster Urls [{},{}] and Master Urls [{}, {}]",
+ getZkUrl1(), getZkUrl2(), getMasterAddress1(), getMasterAddress2());
logClustersStates();
}
@@ -146,137 +163,343 @@ public class HighAvailabilityTestingUtility {
*/
public String getURL(int clusterIndex, ClusterRoleRecord.RegistryType registryType) {
if (registryType == null) {
- return clusterIndex == 1 ? zkUrl1 : zkUrl2;
+ return clusterIndex == 1 ? masterAddress1 : masterAddress2;
}
- String masterAddress1 = hbaseCluster1.getConfiguration().get(HConstants.MASTER_ADDRS_KEY);
- String masterAddress2 = hbaseCluster2.getConfiguration().get(HConstants.MASTER_ADDRS_KEY);
switch (registryType) {
+ case ZK:
+ return clusterIndex == 1 ? zkUrl1 : zkUrl2;
case RPC:
case MASTER:
- return clusterIndex == 1 ? masterAddress1.replaceAll(":", "\\\\:") :
- masterAddress2.replaceAll(":", "\\\\:");
- case ZK:
default:
- return clusterIndex == 1 ? zkUrl1 : zkUrl2;
+ return clusterIndex == 1 ? masterAddress1 : masterAddress2;
}
}
- /** initialize two ZK clusters for cluster role znode. */
+ /**
+ * Initialize ClusterRoleRecord, this method creates HAGroupStoreRecord ZNodes in both
+ * clusters making first cluster ACTIVE and second STANDBY, it also upserts rows to both
+ * cluster's SYSTEM Table.
+ * @param haGroupName name of HA group
+ * @param policy policy to be used for HA group
+ */
public void initClusterRole(String haGroupName, HighAvailabilityPolicy policy)
throws Exception {
- ClusterRoleRecord record = new ClusterRoleRecord(
- haGroupName, policy,
- getZkUrl1(), ClusterRole.ACTIVE,
- getZkUrl2(), ClusterRole.STANDBY,
- 1);
- addRoleRecordToClusters(record);
+ HAGroupStoreRecord activeRecord = new HAGroupStoreRecord(HAGroupStoreRecord.DEFAULT_PROTOCOL_VERSION, haGroupName, ACTIVE.getDefaultHAGroupState(), HAGroupStoreRecord.DEFAULT_RECORD_VERSION);
+ HAGroupStoreRecord standbyRecord = new HAGroupStoreRecord(HAGroupStoreRecord.DEFAULT_PROTOCOL_VERSION, haGroupName, STANDBY.getDefaultHAGroupState(), HAGroupStoreRecord.DEFAULT_RECORD_VERSION);
+
+ upsertGroupRecordInBothSystemTable(haGroupName, ACTIVE, STANDBY, HAGroupStoreRecord.DEFAULT_RECORD_VERSION, HAGroupStoreRecord.DEFAULT_RECORD_VERSION, null, policy);
+ addOrUpdateRoleRecordToClusters(haGroupName, activeRecord, standbyRecord);
}
- public void initClusterRole(String haGroupName, HighAvailabilityPolicy policy,
- ClusterRoleRecord.RegistryType type) throws Exception {
- ClusterRoleRecord record = new ClusterRoleRecord(
- haGroupName, policy, type,
- getURL(1, type), ClusterRole.ACTIVE,
- getURL(2, type), ClusterRole.STANDBY,
- 1);
- addRoleRecordToClusters(record);
+ /**
+ * Initialize ClusterRoleRecord when cluster 2 is down, this method creates HAGroupStoreRecord ZNodes in
+ * cluster 1 making it ACTIVE, it also upserts rows to cluster 1's SYSTEM Table.
+ * @param haGroupName name of HA group
+ * @param policy policy to be used for HA group
+ */
+ public void initClusterRoleRecordFor1Cluster(String haGroupName, HighAvailabilityPolicy policy) throws Exception {
+ HAGroupStoreRecord activeRecord = new HAGroupStoreRecord(HAGroupStoreRecord.DEFAULT_PROTOCOL_VERSION, haGroupName, ACTIVE.getDefaultHAGroupState(), HAGroupStoreRecord.DEFAULT_RECORD_VERSION);
+ upsertGroupRecordInASystemTable(haGroupName, ACTIVE, STANDBY, HAGroupStoreRecord.DEFAULT_RECORD_VERSION, HAGroupStoreRecord.DEFAULT_RECORD_VERSION, null, policy, 1);
+ addOrUpdateRoleRecordToClusters(haGroupName, activeRecord, null);
}
- private void addRoleRecordToClusters(ClusterRoleRecord record) throws Exception {
+ /**
+ * Add or update HAGroupStoreRecord to both clusters, if record is null then it will not be
+ * added to cluster
+ * @param haGroupName name of HA group
+ * @param activeRecord record to be added to active cluster
+ * @param standbyRecord record to be added to standby cluster
+ */
+ private void addOrUpdateRoleRecordToClusters(String haGroupName, HAGroupStoreRecord activeRecord, HAGroupStoreRecord standbyRecord) throws Exception {
+ String path = PhoenixHAAdmin.toPath(haGroupName);
int failures = 0;
- do {
- try {
- haAdmin1.createOrUpdateDataOnZookeeper(record);
+ if (activeRecord != null) {
+ do {
+ try {
+ if (haAdmin1.getCurator().checkExists().forPath(path) == null) {
+ haAdmin1.createHAGroupStoreRecordInZooKeeper(activeRecord);
+ } else {
+ final Pair<HAGroupStoreRecord, Stat> currentRecord = haAdmin1.getHAGroupStoreRecordInZooKeeper(haGroupName);
+ if (currentRecord.getRight() != null && currentRecord.getLeft() != null) {
+ haAdmin1.updateHAGroupStoreRecordInZooKeeper(haGroupName, activeRecord, currentRecord.getRight().getVersion());
+ } else {
+ throw new IOException("Current HAGroupStoreRecord in ZK is null, cannot update HAGroupStoreRecord " + haGroupName);
+ }
+ }
+ } catch (Exception e) {
+ failures++;
+ }
+ } while (failures > 0 && failures < 4);
+ }
+ failures = 0;
+ if (standbyRecord != null) {
+ do {
+ try {
+ if (haAdmin2.getCurator().checkExists().forPath(path) == null) {
+ haAdmin2.createHAGroupStoreRecordInZooKeeper(standbyRecord);
+ } else {
+ final Pair<HAGroupStoreRecord, Stat> currentRecord = haAdmin2.getHAGroupStoreRecordInZooKeeper(haGroupName);
+ if (currentRecord.getRight() != null && currentRecord.getLeft() != null) {
+ haAdmin2.updateHAGroupStoreRecordInZooKeeper(haGroupName, standbyRecord, currentRecord.getRight().getVersion());
+ } else {
+ throw new IOException("Current HAGroupStoreRecord in ZK is null, cannot update HAGroupStoreRecord " + haGroupName);
+ }
+ }
+ } catch (Exception e) {
+ failures++;
+ Thread.sleep(200);
+ }
+ } while (failures > 0 && failures < 4);
+ }
+ }
+
+ /**
+ * Upsert HAGroupStoreRecord/ClusterRoleRecord info in both cluster's SYSTEM Table.
+ */
+ private void upsertGroupRecordInBothSystemTable(String haGroupName, ClusterRole role1, ClusterRole role2, long version1, long version2, String overrideZKUrl, HighAvailabilityPolicy policy) throws Exception {
+ HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(haGroupName, getZkUrl1(), getZkUrl2(), getMasterAddress1(), getMasterAddress2(), role1, role2, version1, version2, overrideZKUrl, policy, getHATestProperties());
+ HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(haGroupName, getZkUrl2(), getZkUrl1(), getMasterAddress2(), getMasterAddress1(), role2, role1, version2, version1, overrideZKUrl, policy, getHATestProperties());
+ }
+
+ /**
+ * Upsert HAGroupStoreRecord/ClusterRoleRecord info in given cluster's SYSTEM Table.
+ */
+ private void upsertGroupRecordInASystemTable(String haGroupName, ClusterRole role1, ClusterRole role2, long version1, long version2, String overrideZKUrl, HighAvailabilityPolicy policy, int clusterIndex) throws Exception {
+ if (clusterIndex == 1) {
+ HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(haGroupName, getZkUrl1(), getZkUrl2(), getMasterAddress1(), getMasterAddress2(), role1, role2, version1, version2, overrideZKUrl, policy, getHATestProperties());
+ } else {
+ HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(haGroupName, getZkUrl2(), getZkUrl1(), getMasterAddress2(), getMasterAddress1(), role2, role1, version2, version1, overrideZKUrl, policy, getHATestProperties());
+ }
+ }
+
+ /**
+ * Refresh SYSTEM Table info of HAGroupStoreRecord/ClusterRoleRecord in both clusters.
+ */
+ private void refreshSystemTableInBothClusters(String haGroupName, ClusterRole role1, ClusterRole role2, long version1, long version2, String overrideZKUrl, HighAvailabilityPolicy policy) throws Exception {
+ HAGroupStoreTestUtil.deleteHAGroupRecordInSystemTable(haGroupName, getZkUrl1());
+ HAGroupStoreTestUtil.deleteHAGroupRecordInSystemTable(haGroupName, getZkUrl2());
+ upsertGroupRecordInBothSystemTable(haGroupName, role1, role2, version1, version2, overrideZKUrl, policy);
+ }
+
+ /**
+ * Refresh SYSTEM Table info of HAGroupStoreRecord/ClusterRoleRecord in given cluster.
+ */
+ private void refreshSystemTableInOneCluster(String haGroupName, ClusterRole role1, ClusterRole role2, long version1, long version2, String overrideZKUrl, HighAvailabilityPolicy policy, int clusterIndex) throws Exception {
+ String zkUrl = clusterIndex == 1 ? getZkUrl1() : getZkUrl2();
+ HAGroupStoreTestUtil.deleteHAGroupRecordInSystemTable(haGroupName, zkUrl);
+ upsertGroupRecordInASystemTable(haGroupName, role1, role2, version1, version2, overrideZKUrl, policy, clusterIndex);
+ }
+
+ /**
+ * Do self recovery which is missing now, for transitions that have happened when cluster
+ * was down.
+ */
+ public void doUpdatesMissedWhenClusterWasDown(HighAvailabilityGroup haGroup, ClusterRole role1, ClusterRole role2, int clusterIndex) throws Exception {
+ String haGroupName = haGroup.getGroupInfo().getName();
+ final Pair<HAGroupStoreRecord, Stat> currentRecord1 = getHaAdmin1().getHAGroupStoreRecordInZooKeeper(haGroupName);
+ final Pair<HAGroupStoreRecord, Stat> currentRecord2 = getHaAdmin2().getHAGroupStoreRecordInZooKeeper(haGroupName);
+ long newVersion;
+ HAGroupStoreRecord newRecord;
+ if (clusterIndex == 1) {
+ newVersion = currentRecord1.getLeft().getRecordVersion() + 1;
+ newRecord = new HAGroupStoreRecord(currentRecord1.getLeft().getProtocolVersion(), haGroupName, role1.getDefaultHAGroupState(), newVersion);
+ refreshSystemTableInOneCluster(haGroupName, role1, role2, newVersion, currentRecord2.getLeft().getRecordVersion(), null, haGroup.getRoleRecord().getPolicy(), clusterIndex);
+ addOrUpdateRoleRecordToClusters(haGroupName, newRecord,null);
+ try{
+ HAGroupStoreManager.getInstance(hbaseCluster1.getConfiguration()).invalidateHAGroupStoreClient(haGroupName, true);
} catch (Exception e) {
- failures++;
+ LOG.warn("Fail to invalidate HAGroupStoreClient for {} because {}", haGroupName, e.getMessage());
}
- } while (failures > 0 && failures < 4);
- failures = 0;
- do {
- try {
- haAdmin2.createOrUpdateDataOnZookeeper(record);
+ } else {
+ newVersion = currentRecord2.getLeft().getRecordVersion() + 1;
+ newRecord = new HAGroupStoreRecord(currentRecord2.getLeft().getProtocolVersion(), haGroupName, role2.getDefaultHAGroupState(), newVersion);
+ refreshSystemTableInOneCluster(haGroupName, role1, role2, currentRecord1.getLeft().getRecordVersion(), newVersion, null, haGroup.getRoleRecord().getPolicy(), clusterIndex);
+ addOrUpdateRoleRecordToClusters(haGroupName, null, newRecord);
+ try{
+ HAGroupStoreManager.getInstance(hbaseCluster2.getConfiguration()).invalidateHAGroupStoreClient(haGroupName, true);
} catch (Exception e) {
- failures++;
- Thread.sleep(200);
+ LOG.warn("Fail to invalidate HAGroupStoreClient for {} because {}", haGroupName, e.getMessage());
}
- } while (failures > 0 && failures < 4);
+ }
+ }
+
+ public void deleteHAGroupRecordFromBothClusters(String haGroupName) throws Exception {
+ haAdmin1.getCurator().delete().quietly().forPath(toPath(haGroupName));
+ haAdmin2.getCurator().delete().quietly().forPath(toPath(haGroupName));
+ HAGroupStoreTestUtil.deleteHAGroupRecordInSystemTable(haGroupName, getZkUrl1());
+ HAGroupStoreTestUtil.deleteHAGroupRecordInSystemTable(haGroupName, getZkUrl2());
+ }
+
+ public void rebuildBothClustersGroupStoreClient(String haGroupName) {
+ try{
+ HAGroupStoreManager.getInstance(hbaseCluster1.getConfiguration()).invalidateHAGroupStoreClient(haGroupName, true);
+ HAGroupStoreManager.getInstance(hbaseCluster2.getConfiguration()).invalidateHAGroupStoreClient(haGroupName, true);
+ } catch (Exception e) {
+ LOG.warn("Fail to invalidate HAGroupStoreClient for {} because {}", haGroupName, e.getMessage());
+ }
}
/**
- * Set cluster roles for an HA group and wait the cluster role transition to happen.
- *
+ * Generate ClusterRoleRecord from HAGroupStoreRecords
+ */
+ private ClusterRoleRecord generateCRRFromHAGroupStoreRecord(HighAvailabilityGroup haGroup, HAGroupStoreRecord activeRecord, HAGroupStoreRecord standbyRecord) {
+ return new ClusterRoleRecord(
+ haGroup.getGroupInfo().getName(), haGroup.getRoleRecord().getPolicy(),
+ getURL(1, haGroup.getRoleRecord().getRegistryType()), activeRecord.getHAGroupState().getClusterRole(),
+ getURL(2, haGroup.getRoleRecord().getRegistryType()), standbyRecord.getHAGroupState().getClusterRole(),
+ combineCanonicalVersions(activeRecord.getRecordVersion(), standbyRecord.getRecordVersion())); // always use a newer version
+ }
+
+ /**
+ * Combine two versions into a single long value to be used for comparison and sending back to client
+ * @param version1 HAGroupRecord version for cluster 1
+ * @param version2 HAGroupRecord version for cluster 2
+ * @return combined version
+ */
+ public long combineCanonicalVersions(long version1, long version2) {
+ LOG.info("Combining versions: {} and {} with master addresses: {} and {}", version1, version2, masterAddress1, masterAddress2);
+ if (masterAddress1.compareTo(masterAddress2) <= 0) {
+ return (version2 << 32) | version1;
+ } else {
+ return (version1 << 32) | version2;
+ }
+ }
+
+ /**
+ * Update HAGroup's clusterRoleRecord info to new roles (Update HAGroupStoreRecord and
+ * SYSTEM Tables) and then do the refresh of the clusterRoleRecord in HAGroup
+ * @param haGroup the HA group
+ * @param role1 new cluster role for the first cluster in the group
+ * @param role2 new cluster role for the second cluster in the group
+ */
+ public void transitClusterRole(HighAvailabilityGroup haGroup, ClusterRole role1,
+ ClusterRole role2) throws Exception {
+ transitClusterRole(haGroup, role1, role2, true);
+ }
+
+ /**
+ * Update HAGroup's clusterRoleRecord info to new roles (Update HAGroupStoreRecord and
+ * SYSTEM Tables) and then do the refresh of the clusterRoleRecord in HAGroup based on the
+ * param provided.
* @param haGroup the HA group name
* @param role1 cluster role for the first cluster in the group
* @param role2 cluster role for the second cluster in the group
*/
public void transitClusterRole(HighAvailabilityGroup haGroup, ClusterRole role1,
+ ClusterRole role2, boolean doRefreshHAGroup) throws Exception {
+ String haGroupName = haGroup.getGroupInfo().getName();
+ final Pair<HAGroupStoreRecord, Stat> currentRecord1 = haAdmin1.getHAGroupStoreRecordInZooKeeper(haGroupName);
+ final Pair<HAGroupStoreRecord, Stat> currentRecord2 = haAdmin2.getHAGroupStoreRecordInZooKeeper(haGroupName);
+ long newVersion1 = currentRecord1.getLeft().getRecordVersion() + 1;
+ long newVersion2 = currentRecord2.getLeft().getRecordVersion() + 1;
+
+ HAGroupStoreRecord record1 = new HAGroupStoreRecord(currentRecord1.getLeft().getProtocolVersion(), haGroupName, role1.getDefaultHAGroupState(), newVersion1);
+ HAGroupStoreRecord record2 = new HAGroupStoreRecord(currentRecord2.getLeft().getProtocolVersion(), haGroupName, role2.getDefaultHAGroupState(), newVersion2);
+
+ refreshSystemTableInBothClusters(haGroupName, role1, role2, newVersion1, newVersion2, null, haGroup.getRoleRecord().getPolicy());
+ addOrUpdateRoleRecordToClusters(haGroupName, record1, record2);
+ try{
+ HAGroupStoreManager.getInstance(hbaseCluster1.getConfiguration()).invalidateHAGroupStoreClient(haGroupName, true);
+ HAGroupStoreManager.getInstance(hbaseCluster2.getConfiguration()).invalidateHAGroupStoreClient(haGroupName, true);
+ } catch (Exception e) {
+ LOG.warn("Fail to invalidate HAGroupStoreClient for {} because {}", haGroupName, e.getMessage());
+ }
+
+ if (doRefreshHAGroup) {
+ haGroup.refreshClusterRoleRecord(true);
+
+ // wait for the cluster roles are populated into client side from RegionServer endpoints.
+ ClusterRoleRecord newRoleRecord = generateCRRFromHAGroupStoreRecord(haGroup, record1, record2);
+ LOG.info("New role record to check: {}", newRoleRecord);
+ waitFor(() -> newRoleRecord.equals(haGroup.getRoleRecord()), 1000, 10_000);
+ }
+
+ }
+
+ public void transitClusterRoleWithCluster1Down(HighAvailabilityGroup haGroup, ClusterRole role1,
ClusterRole role2) throws Exception {
- final ClusterRoleRecord newRoleRecord = new ClusterRoleRecord(
- haGroup.getGroupInfo().getName(), haGroup.getRoleRecord().getPolicy(),
- haGroup.getRoleRecord().getRegistryType(),
- getURL(1, haGroup.getRoleRecord().getRegistryType()), role1,
- getURL(2, haGroup.getRoleRecord().getRegistryType()), role2,
- haGroup.getRoleRecord().getVersion() + 1); // always use a newer version
- applyNewRoleRecord(newRoleRecord, haGroup);
+ String haGroupName = haGroup.getGroupInfo().getName();
+ final Pair<HAGroupStoreRecord, Stat> currentRecord2 = haAdmin2.getHAGroupStoreRecordInZooKeeper(haGroupName);
+ long newVersion2 = currentRecord2.getLeft().getRecordVersion() + 1;
+
+ HAGroupStoreRecord record2 = new HAGroupStoreRecord(currentRecord2.getLeft().getProtocolVersion(), haGroupName, role2.getDefaultHAGroupState(), newVersion2);
+
+ refreshSystemTableInOneCluster(haGroupName, role1, role2, 1, newVersion2, null, haGroup.getRoleRecord().getPolicy(), 2);
+ addOrUpdateRoleRecordToClusters(haGroupName, null, record2);
+ try{
+ HAGroupStoreManager.getInstance(hbaseCluster2.getConfiguration()).invalidateHAGroupStoreClient(haGroupName, true);
+ } catch (Exception e) {
+ LOG.warn("Fail to invalidate HAGroupStoreClient for {} because {}", haGroupName, e.getMessage());
+ }
+ haGroup.refreshClusterRoleRecord(true);
+
+ //If cluster 1 is down, server won't be able to reach peer for states and will get version passed in refreshSystemTableInOneCluster and OFFLINE role.
+ HAGroupStoreRecord record1 = new HAGroupStoreRecord(HAGroupStoreRecord.DEFAULT_PROTOCOL_VERSION, haGroupName, UNKNOWN.getDefaultHAGroupState(), HAGroupStoreRecord.DEFAULT_RECORD_VERSION);
+
+ // wait for the cluster roles are populated into client side from RegionServer endpoints.
+ ClusterRoleRecord newRoleRecord = generateCRRFromHAGroupStoreRecord(haGroup, record1, record2);
+ LOG.info("New role record to check: {}", newRoleRecord);
+ waitFor(() -> newRoleRecord.equals(haGroup.getRoleRecord()), 1000, 10_000);
}
- /**
- * Set registryType for roleRecord and wait the cluster role transition to happen.
- *
- * @param haGroup the HA group name
- * @param type RegistryType to change roleRecord to
- */
- public void transitClusterRoleRecordRegistry(HighAvailabilityGroup haGroup,
- ClusterRoleRecord.RegistryType type) throws Exception {
- final ClusterRoleRecord newRoleRecord = new ClusterRoleRecord(
- haGroup.getGroupInfo().getName(), haGroup.getRoleRecord().getPolicy(),
- type,
- getURL(1, type), ClusterRole.ACTIVE,
- getURL(2, type), ClusterRole.STANDBY,
- haGroup.getRoleRecord().getVersion() + 1); // always use a newer version
- applyNewRoleRecord(newRoleRecord, haGroup);
+ public void transitClusterRoleWithCluster2Down(HighAvailabilityGroup haGroup, ClusterRole role1,
+ ClusterRole role2) throws Exception {
+ String haGroupName = haGroup.getGroupInfo().getName();
+ final Pair<HAGroupStoreRecord, Stat> currentRecord1 = haAdmin1.getHAGroupStoreRecordInZooKeeper(haGroupName);
+ long newVersion1 = currentRecord1.getLeft().getRecordVersion() + 1;
+
+ HAGroupStoreRecord record1 = new HAGroupStoreRecord(currentRecord1.getLeft().getProtocolVersion(), haGroupName, role1.getDefaultHAGroupState(), newVersion1);
+
+ refreshSystemTableInOneCluster(haGroupName, role1, role2, newVersion1, 1, null, haGroup.getRoleRecord().getPolicy(), 1);
+ addOrUpdateRoleRecordToClusters(haGroupName, record1, null);
+ try{
+ HAGroupStoreManager.getInstance(hbaseCluster1.getConfiguration()).invalidateHAGroupStoreClient(haGroupName, true);
+ } catch (Exception e) {
+ LOG.warn("Fail to invalidate HAGroupStoreClient for {} because {}", haGroupName, e.getMessage());
+ }
+ haGroup.refreshClusterRoleRecord(true);
+
+ //If cluster 2 is down, server won't be able to reach peer for states and will get version passed in refreshSystemTableInOneCluster and OFFLINE role.
+ HAGroupStoreRecord record2 = new HAGroupStoreRecord(HAGroupStoreRecord.DEFAULT_PROTOCOL_VERSION, haGroupName, UNKNOWN.getDefaultHAGroupState(), HAGroupStoreRecord.DEFAULT_RECORD_VERSION);
+
+ // wait for the cluster roles are populated into client side from RegionServer endpoints.
+ ClusterRoleRecord newRoleRecord = generateCRRFromHAGroupStoreRecord(haGroup, record1, record2);
+ LOG.info("New role record to check: {}", newRoleRecord);
+ waitFor(() -> newRoleRecord.equals(haGroup.getRoleRecord()), 1000, 10_000);
}
public void refreshClusterRoleRecordAfterClusterRestart(HighAvailabilityGroup haGroup,
- ClusterRole role1, ClusterRole role2) throws Exception {
- final ClusterRoleRecord newRoleRecord = new ClusterRoleRecord(
- haGroup.getGroupInfo().getName(), haGroup.getRoleRecord().getPolicy(),
- haGroup.getRoleRecord().getRegistryType(),
- getURL(1, haGroup.getRoleRecord().getRegistryType()), role1,
- getURL(2, haGroup.getRoleRecord().getRegistryType()), role2,
- haGroup.getRoleRecord().getVersion() + 1); // always use a newer version
- applyNewRoleRecord(newRoleRecord, haGroup);
- }
-
- private void applyNewRoleRecord(ClusterRoleRecord newRoleRecord,
- HighAvailabilityGroup haGroup) throws Exception {
- LOG.info("Transiting cluster role for HA group {} V{}->V{}, existing: {}, new: {}",
- haGroup.getGroupInfo().getName(), haGroup.getRoleRecord().getVersion(),
- newRoleRecord.getVersion(), haGroup.getRoleRecord(), newRoleRecord);
- boolean successAtLeastOnce = false;
- try {
- haAdmin1.createOrUpdateDataOnZookeeper(newRoleRecord);
- successAtLeastOnce = true;
- } catch (IOException e) {
- LOG.warn("Fail to update new record on {} because {}", getZkUrl1(), e.getMessage());
- }
- try {
- haAdmin2.createOrUpdateDataOnZookeeper(newRoleRecord);
- successAtLeastOnce = true;
- } catch (IOException e) {
- LOG.warn("Fail to update new record on {} because {}", getZkUrl2(), e.getMessage());
+ ClusterRole role1, ClusterRole role2, int clusterIndex) throws Exception {
+ String haGroupName = haGroup.getGroupInfo().getName();
+ long newVersion1, newVersion2;
+ //TODO:- Inducing version change, but how would we know that a URL is changed
+ //as we don't store url in ZNodes so it doesn't increase version for that change
+ if (clusterIndex == 1) {
+ newVersion1 = 1L;
+ newVersion2 = 2L;
+ } else {
+ newVersion1 = 2L;
+ newVersion2 = 1L;
}
+ HAGroupStoreRecord record1 = new HAGroupStoreRecord(HAGroupStoreRecord.DEFAULT_PROTOCOL_VERSION, haGroupName, role1.getDefaultHAGroupState(), newVersion1);
+ HAGroupStoreRecord record2 = new HAGroupStoreRecord(HAGroupStoreRecord.DEFAULT_PROTOCOL_VERSION, haGroupName, role2.getDefaultHAGroupState(), newVersion2);
- if (!successAtLeastOnce) {
- throw new IOException("Failed to update the new role record on either cluster");
+ refreshSystemTableInBothClusters(haGroupName, role1, role2, newVersion1, newVersion2, null, haGroup.getRoleRecord().getPolicy());
+ addOrUpdateRoleRecordToClusters(haGroupName, record1, record2);
+
+ try{
+ HAGroupStoreManager.getInstance(hbaseCluster1.getConfiguration()).invalidateHAGroupStoreClient(haGroupName, true);
+ HAGroupStoreManager.getInstance(hbaseCluster2.getConfiguration()).invalidateHAGroupStoreClient(haGroupName, true);
+ } catch (Exception e) {
+ LOG.warn("Fail to invalidate HAGroupStoreClient for {} because {}", haGroupName, e.getMessage());
}
- // wait for the cluster roles are populated into client side from ZK nodes.
- waitFor(() -> newRoleRecord.equals(haGroup.getRoleRecord()), 1000, 10_000);
- // May have to wait for the transistion to be picked up client side, current test timeouts around 3seconds
- Thread.sleep(5000);
- LOG.info("Now the HA group {} should have detected and updated V{} cluster role record",
- haGroup, newRoleRecord.getVersion());
+ haGroup.refreshClusterRoleRecord(true);
+
+ // wait for the cluster roles are populated into client side from RegionServer endpoints.
+ ClusterRoleRecord newRoleRecord = generateCRRFromHAGroupStoreRecord(haGroup, record1, record2);
+ LOG.info("New role record to check: {}", newRoleRecord);
+ waitFor(() -> newRoleRecord.equals(haGroup.getRoleRecord()), 1000, 10_000);
}
/**
@@ -356,7 +579,7 @@ public class HighAvailabilityTestingUtility {
/** A Testable interface that can throw checked exception. */
@FunctionalInterface
public interface Testable {
- void test() throws Exception;
+ void test() throws Exception;
}
/**
@@ -368,18 +591,23 @@ public class HighAvailabilityTestingUtility {
* @param testable testing logic that is runnable
* @throws Exception if fails to stop, test or restart HBase cluster
*/
- public static void doTestWhenOneHBaseDown(HBaseTestingUtility cluster, Testable testable)
+ public void doTestWhenOneHBaseDown(HBaseTestingUtility cluster, Testable testable)
throws Exception {
final int zkClientPort = cluster.getZkCluster().getClientPort();
+ final int masterPort = getGivenClusterMasterPort(cluster);
try {
LOG.info("Shutting down HBase cluster using ZK localhost:{}", zkClientPort);
cluster.shutdownMiniHBaseCluster();
- LOG.info("Start testing when HBase is down using ZK localhost:{}", zkClientPort);
+ LOG.info("Start testing when HBase is down using ZK localhost:{} and master address localhost:{}", zkClientPort, masterPort);
testable.test();
LOG.info("Test succeeded when HBase is down using ZK localhost:{}", zkClientPort);
} finally {
LOG.info("Finished testing when HBase is down using ZK localhost:{}", zkClientPort);
+ resetPortsForRestart(cluster, masterPort);
cluster.startMiniHBaseCluster(StartMiniClusterOption.builder().numMasters(1).numRegionServers(1).build());
+ LOG.info("Master after restart :- {}, port before restart {} and masters before restart {} {}}",
+ cluster.getConfiguration().get(HConstants.MASTER_ADDRS_KEY).replaceAll(":", "\\\\:"),
+ masterPort, masterAddress1, masterAddress2);
LOG.info("Restarted HBase cluster using ZK localhost:{}", zkClientPort);
}
}
@@ -391,26 +619,70 @@ public class HighAvailabilityTestingUtility {
* @param testable testing logic that is runnable
* @throws Exception if fails to stop, test or restart
*/
- public static void doTestWhenOneZKDown(HBaseTestingUtility cluster, Testable testable)
+ public void doTestWhenOneZKDown(HBaseTestingUtility cluster, Testable testable)
throws Exception {
final int zkClientPort = cluster.getZkCluster().getClientPort();
+ final int masterPort = getGivenClusterMasterPort(cluster);
try {
LOG.info("Shutting down HBase cluster using ZK localhost:{}", zkClientPort);
cluster.shutdownMiniHBaseCluster();
LOG.info("Shutting down ZK cluster at localhost:{}", zkClientPort);
cluster.shutdownMiniZKCluster();
- LOG.info("Start testing when ZK & HBase is down at localhost:{}", zkClientPort);
+ LOG.info("Start testing when ZK & HBase is down at localhost:{} and master address localhost:{}", zkClientPort, masterPort);
testable.test();
LOG.info("Test succeeded when ZK & HBase is down at localhost:{}", zkClientPort);
} finally {
- LOG.info("Finished testing when ZK & HBase is down at localhost:{}", zkClientPort);
- cluster.startMiniZKCluster(1, zkClientPort);
+ resetPortsForRestart(cluster, masterPort);
+ LOG.info("Finished testing when ZK & HBase is down at localhost:{} and master address localhost:{}", zkClientPort, masterPort);
+ cluster.startMiniZKCluster(1, zkClientPort);
LOG.info("Restarted ZK cluster at localhost:{}", zkClientPort);
- cluster.startMiniHBaseCluster(StartMiniClusterOption.builder().numMasters(1).numRegionServers(1).build());
+ cluster.startMiniHBaseCluster(StartMiniClusterOption.builder().numMasters(1).numRegionServers(1).build());
+ LOG.info("Master after restart :- {}, port before restart {} and masters before restart {} {}}",
+ cluster.getConfiguration().get(HConstants.MASTER_ADDRS_KEY).replaceAll(":", "\\\\:"),
+ masterPort, masterAddress1, masterAddress2);
LOG.info("Restarted HBase cluster using ZK localhost:{}", zkClientPort);
}
}
+ /**
+ * Set Master port to same as before restart, as we are using master urls now
+ * @param cluster for which restart is happening
+ * @param masterPort old master port.
+ */
+ private void resetPortsForRestart(HBaseTestingUtility cluster, int masterPort) {
+ //Disable random port assignment and set the master port to the given port
+ cluster.getConfiguration().set("hbase.localcluster.assign.random.ports", "false");
+ cluster.getConfiguration().set("hbase.master.port", String.valueOf(masterPort));
+
+ //Set Random ports to everything else for HBase cluster
+ cluster.getConfiguration().set("hbase.master.info.port", String.valueOf(0));
+ cluster.getConfiguration().set("hbase.regionserver.port", String.valueOf(0));
+ cluster.getConfiguration().set("hbase.regionserver.info.port", String.valueOf(0));
+ }
+
+ /**
+ * Reset the configs for new random ports
+ * @param cluster cluster for which restart is happening
+ */
+ private void resetPortsToRandomForNewRestart(HBaseTestingUtility cluster) {
+ cluster.getConfiguration().set("hbase.localcluster.assign.random.ports", "true");
+ cluster.getConfiguration().set("hbase.master.port", String.valueOf(0));
+ cluster.getConfiguration().set("hbase.master.info.port", String.valueOf(0));
+ cluster.getConfiguration().set("hbase.regionserver.port", String.valueOf(0));
+ cluster.getConfiguration().set("hbase.regionserver.info.port", String.valueOf(0));
+ }
+
+ public int getGivenClusterMasterPort(HBaseTestingUtility cluster) {
+ if (cluster == hbaseCluster1) {
+ return Integer.parseInt(masterAddress1.split(":")[1]);
+ } else {
+ return Integer.parseInt(masterAddress2.split(":")[1]);
+ }
+ }
+
+
+
+
/**
* The JDBC connection url for the given clusters in particular format.
* Note that HAUrl will contain ZK as host which is used to create HA Connection
@@ -424,11 +696,11 @@ public class HighAvailabilityTestingUtility {
}
public String getJdbcHAUrl(String principal) {
- return String.format("%s:[%s|%s]:%s", JDBC_PROTOCOL_ZK, getZkUrl1(), getZkUrl2(), principal);
+ return String.format("%s:[%s|%s]:%s", JDBC_PROTOCOL_RPC, getMasterAddress1(), getMasterAddress2(), principal);
}
public String getJdbcHAUrlWithoutPrincipal() {
- return String.format("%s:[%s|%s]", JDBC_PROTOCOL_ZK, getZkUrl1(), getZkUrl2());
+ return String.format("%s:[%s|%s]", JDBC_PROTOCOL_RPC, getMasterAddress1(), getMasterAddress2());
}
@@ -459,7 +731,7 @@ public class HighAvailabilityTestingUtility {
}
public String getJdbcUrl(String url) {
- return String.format("jdbc:phoenix+zk:%s:%s", url, PRINCIPAL);
+ return String.format("jdbc:phoenix+rpc:%s:::%s", url, PRINCIPAL);
}
public String getJdbcUrlWithoutPrincipal(HighAvailabilityGroup haGroup, String url) {
@@ -478,16 +750,32 @@ public class HighAvailabilityTestingUtility {
return zkUrl2;
}
+ public String getMasterAddress1() {
+ return masterAddress1;
+ }
+
+ public String getMasterAddress2() {
+ return masterAddress2;
+ }
+
+ public PhoenixHAAdmin getHaAdmin1() {
+ return haAdmin1;
+ }
+
+ public PhoenixHAAdmin getHaAdmin2() {
+ return haAdmin2;
+ }
+
private String getUrlWithoutPrincipal(HighAvailabilityGroup haGroup, String url) {
StringBuilder sb = new StringBuilder();
switch (haGroup.getRoleRecord().getRegistryType()) {
+ case ZK:
+ return sb.append(JDBC_PROTOCOL_ZK).append(":").append(url).toString();
case MASTER:
return sb.append(JDBC_PROTOCOL_MASTER).append(":").append(url).append("::").toString();
case RPC:
- return sb.append(JDBC_PROTOCOL_RPC).append(":").append(url).append("::").toString();
- case ZK:
default:
- return sb.append(JDBC_PROTOCOL_ZK).append(":").append(url).toString();
+ return sb.append(JDBC_PROTOCOL_RPC).append(":").append(url).append("::").toString();
}
}
@@ -595,6 +883,7 @@ public class HighAvailabilityTestingUtility {
hbaseCluster1.shutdownMiniHBaseCluster();
hbaseCluster1.shutdownMiniZKCluster();
} finally {
+ resetPortsToRandomForNewRestart(hbaseCluster1);
hbaseCluster1.startMiniZKCluster(1);
hbaseCluster1.startMiniHBaseCluster(StartMiniClusterOption.builder().numMasters(1).numRegionServers(1).build());
@@ -602,6 +891,8 @@ public class HighAvailabilityTestingUtility {
zkUrl1 = String.format("%s\\:%d::/hbase", confAddress, hbaseCluster1.getZkCluster().getClientPort());
+ masterAddress1 = hbaseCluster1.getConfiguration().get(HConstants.MASTER_ADDRS_KEY).replaceAll(":", "\\\\:");
+
haAdmin1 = new PhoenixHAAdmin(getZkUrl1(), hbaseCluster1.getConfiguration(), HighAvailibilityCuratorProvider.INSTANCE);
admin1 = hbaseCluster1.getConnection().getAdmin();
@@ -610,10 +901,12 @@ public class HighAvailabilityTestingUtility {
}
public void restartCluster2() throws Exception {
+ String oldMasters = getMasterAddress1() + " " + getMasterAddress2();
try {
hbaseCluster2.shutdownMiniHBaseCluster();
hbaseCluster2.shutdownMiniZKCluster();
} finally {
+ resetPortsToRandomForNewRestart(hbaseCluster2);
hbaseCluster2.startMiniZKCluster(1);
hbaseCluster2.startMiniHBaseCluster(StartMiniClusterOption.builder().numMasters(1).numRegionServers(1).build());
@@ -621,9 +914,12 @@ public class HighAvailabilityTestingUtility {
zkUrl2 = String.format("%s\\:%d::/hbase", confAddress, hbaseCluster2.getZkCluster().getClientPort());
+ masterAddress2 = hbaseCluster2.getConfiguration().get(HConstants.MASTER_ADDRS_KEY).replaceAll(":", "\\\\:");
+
haAdmin2 = new PhoenixHAAdmin(getZkUrl2(), hbaseCluster2.getConfiguration(), HighAvailibilityCuratorProvider.INSTANCE);
admin2 = hbaseCluster2.getConnection().getAdmin();
+ LOG.info("Before restart master address is {} and after restart master address is {}", oldMasters, getMasterAddress1() + " " + getMasterAddress2());
}
}
@@ -660,6 +956,10 @@ public class HighAvailabilityTestingUtility {
conf.setInt(PHOENIX_HA_ZK_RETRY_BASE_SLEEP_MS_KEY, 100);
conf.setInt(PHOENIX_HA_ZK_RETRY_MAX_KEY, 2);
conf.setInt(PHOENIX_HA_ZK_RETRY_MAX_SLEEP_MS_KEY, 1000);
+ //This is needed to get CRR when one cluster is down, as RPC the timeout is 2 seconds
+ //and the server will wait for the specified time to try to retrieve peer record info
+ //from dead cluster
+ conf.setLong(PHOENIX_HA_GROUP_STORE_CLIENT_INITIALIZATION_TIMEOUT_MS, 1000L);
// Set Phoenix related settings, eg. for index management
conf.set(IndexManagementUtil.WAL_EDIT_CODEC_CLASS_KEY,
@@ -774,6 +1074,17 @@ public class HighAvailabilityTestingUtility {
new PhoenixHAExecutorServiceProvider.PhoenixHAClusterExecutorServices(Executors.newFixedThreadPool(1), Executors.newFixedThreadPool(1)));
}
+ public static void sleepThreadFor(long sleepTime) {
+ long start = System.currentTimeMillis();
+ while (System.currentTimeMillis() - start < sleepTime) {
+ try {
+ Thread.sleep(sleepTime - (System.currentTimeMillis() - start));
+ } catch (InterruptedException e) {
+ // Keep looping to complete the sleep duration.
+ }
+ }
+ }
+
/**
* Properties with tuned retry and timeout configuration to bring up and down miniclusters and connect to them
* @return set properteis
@@ -800,6 +1111,7 @@ public class HighAvailabilityTestingUtility {
properties.setProperty(SOCKET_TIMEOUT_WRITE,"2000");
properties.setProperty(REPLICATION_SOURCE_SHIPEDITS_TIMEOUT,"5000");
properties.setProperty(HConstants.THREAD_WAKE_FREQUENCY, "100");
+ properties.setProperty(PHOENIX_HA_CRR_POLLER_INTERVAL_MS_KEY, "500");
return properties;
}
}
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HighAvailabilityTestingUtilityIT.java b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HighAvailabilityTestingUtilityIT.java
index 19c2a7b500..f0b0e9ee4f 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HighAvailabilityTestingUtilityIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HighAvailabilityTestingUtilityIT.java
@@ -19,8 +19,6 @@ package org.apache.phoenix.jdbc;
import static org.apache.phoenix.jdbc.HighAvailabilityGroup.PHOENIX_HA_GROUP_ATTR;
import static org.apache.phoenix.jdbc.HighAvailabilityTestingUtility.HBaseTestingUtilityPair;
-import static org.apache.phoenix.jdbc.HighAvailabilityTestingUtility.HBaseTestingUtilityPair.doTestWhenOneHBaseDown;
-import static org.apache.phoenix.jdbc.HighAvailabilityTestingUtility.HBaseTestingUtilityPair.doTestWhenOneZKDown;
import static org.apache.phoenix.jdbc.HighAvailabilityTestingUtility.doTestBasicOperationsWithConnection;
import static org.apache.phoenix.jdbc.HighAvailabilityTestingUtility.getHighAvailibilityGroup;
import static org.junit.Assert.assertFalse;
@@ -56,7 +54,6 @@ import org.slf4j.LoggerFactory;
* Test failover basics for {@link HighAvailabilityTestingUtility}.
*/
@Category(NeedsOwnMiniClusterTest.class)
-@RunWith(Parameterized.class)
public class HighAvailabilityTestingUtilityIT {
private static final Logger LOG = LoggerFactory.getLogger(
HighAvailabilityTestingUtilityIT.class);
@@ -72,15 +69,11 @@ public class HighAvailabilityTestingUtilityIT {
/** Failover HA group for to test. */
private HighAvailabilityGroup haGroup;
private String haGroupName;
- private final ClusterRoleRecord.RegistryType registryType;
+ private final ClusterRoleRecord.RegistryType registryType = ClusterRoleRecord.RegistryType.RPC;
/** Table name per test case. */
private String tableName;
- public HighAvailabilityTestingUtilityIT(ClusterRoleRecord.RegistryType registryType) {
- this.registryType = registryType;
- }
-
@BeforeClass
public static void setUpBeforeClass() throws Exception {
CLUSTERS.start();
@@ -103,12 +96,7 @@ public class HighAvailabilityTestingUtilityIT {
clientProperties = HighAvailabilityTestingUtility.getHATestProperties();
clientProperties.setProperty(PHOENIX_HA_GROUP_ATTR, haGroupName);
- // Make first cluster ACTIVE
- if (registryType == null) {
- CLUSTERS.initClusterRole(haGroupName, HighAvailabilityPolicy.FAILOVER);
- } else {
- CLUSTERS.initClusterRole(haGroupName, HighAvailabilityPolicy.FAILOVER, registryType);
- }
+ CLUSTERS.initClusterRole(haGroupName, HighAvailabilityPolicy.FAILOVER);
tableName = RandomStringUtils.randomAlphabetic(10);
jdbcHAUrl = CLUSTERS.getJdbcHAUrl();
@@ -116,28 +104,18 @@ public class HighAvailabilityTestingUtilityIT {
CLUSTERS.createTableOnClusterPair(haGroup, tableName);
}
- @Parameterized.Parameters(name="ClusterRoleRecord_registryType={0}")
- public static Collection<Object> data() {
- return Arrays.asList(new Object[] {
- ClusterRoleRecord.RegistryType.ZK,
- ClusterRoleRecord.RegistryType.MASTER,
- ClusterRoleRecord.RegistryType.RPC,
- null //For Backward Compatibility
- });
- }
-
/**
* Test Phoenix connection creation and basic operations with HBase cluster(s) unavail.
*/
@Test
public void testClusterUnavailableNormalConnection() throws Exception {
- doTestWhenOneHBaseDown(CLUSTERS.getHBaseCluster2(), () -> {
+ CLUSTERS.doTestWhenOneHBaseDown(CLUSTERS.getHBaseCluster2(), () -> {
CLUSTERS.logClustersStates();
try (Connection conn = CLUSTERS.getCluster1Connection(haGroup)) {
doTestBasicOperationsWithConnection(conn, tableName, null);
}
});
- doTestWhenOneHBaseDown(CLUSTERS.getHBaseCluster1(), () -> {
+ CLUSTERS.doTestWhenOneHBaseDown(CLUSTERS.getHBaseCluster1(), () -> {
CLUSTERS.logClustersStates();
try (Connection conn = CLUSTERS.getCluster2Connection(haGroup)) {
doTestBasicOperationsWithConnection(conn, tableName, null);
@@ -170,7 +148,7 @@ public class HighAvailabilityTestingUtilityIT {
*/
@Test
public void testGetNewCQSShouldFail() throws Exception {
- doTestWhenOneZKDown(CLUSTERS.getHBaseCluster1(), () -> {
+ CLUSTERS.doTestWhenOneZKDown(CLUSTERS.getHBaseCluster1(), () -> {
try {
Properties properties = HighAvailabilityTestingUtility.getHATestProperties();
properties.setProperty(PHOENIX_HA_GROUP_ATTR, testName.getMethodName());
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/LoggingHAConnectionLimiterIT.java b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/LoggingHAConnectionLimiterIT.java
index 30845397ca..66fd160e06 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/LoggingHAConnectionLimiterIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/LoggingHAConnectionLimiterIT.java
@@ -40,6 +40,8 @@ import java.util.Properties;
import static org.apache.phoenix.jdbc.HighAvailabilityGroup.PHOENIX_HA_GROUP_ATTR;
import static org.apache.phoenix.jdbc.HighAvailabilityPolicy.PARALLEL;
+import static org.apache.phoenix.query.QueryServices.CLUSTER_ROLE_BASED_MUTATION_BLOCK_ENABLED;
+import static org.apache.phoenix.query.QueryServices.HA_GROUP_STALE_FOR_MUTATION_CHECK_ENABLED;
@Category(NeedsOwnMiniClusterTest.class)
public class LoggingHAConnectionLimiterIT extends LoggingConnectionLimiterIT {
@@ -76,18 +78,20 @@ public class LoggingHAConnectionLimiterIT extends LoggingConnectionLimiterIT {
put(QueryServices.CONNECTION_ACTIVITY_LOGGING_ENABLED, String.valueOf(true));
put(QueryServices.CLIENT_CONNECTION_MAX_ALLOWED_CONNECTIONS, String.valueOf(20));
put(QueryServices.INTERNAL_CONNECTION_MAX_ALLOWED_CONNECTIONS, String.valueOf(20));
- put(PhoenixHAExecutorServiceProvider.HA_MAX_POOL_SIZE, String.valueOf(5));
- put(PhoenixHAExecutorServiceProvider.HA_MAX_QUEUE_SIZE, String.valueOf(30));
+ put(PhoenixHAExecutorServiceProvider.HA_MAX_POOL_SIZE, String.valueOf(15));
+ put(PhoenixHAExecutorServiceProvider.HA_MAX_QUEUE_SIZE, String.valueOf(40));
}};
+ CLUSTERS.getHBaseCluster1().getConfiguration().setBoolean(HA_GROUP_STALE_FOR_MUTATION_CHECK_ENABLED, false);
+ CLUSTERS.getHBaseCluster2().getConfiguration().setBoolean(HA_GROUP_STALE_FOR_MUTATION_CHECK_ENABLED, false);
CLUSTERS.start();
DriverManager.registerDriver(PhoenixDriver.INSTANCE);
GLOBAL_PROPERTIES.put(PHOENIX_HA_GROUP_ATTR, PARALLEL.name());
- CONNECTIONS = Lists.newArrayList(getConnection(CLUSTERS.getJdbcUrl(CLUSTERS.getZkUrl1())),
- getConnection(CLUSTERS.getJdbcUrl(CLUSTERS.getZkUrl2())));
+ CONNECTIONS = Lists.newArrayList(getConnection(CLUSTERS.getJdbcUrl(CLUSTERS.getMasterAddress1())),
+ getConnection(CLUSTERS.getJdbcUrl(CLUSTERS.getMasterAddress2())));
LOG.info(String.format("************* Num connections : %d", CONNECTIONS.size()));
for (Connection conn : CONNECTIONS) {
@@ -98,7 +102,7 @@ public class LoggingHAConnectionLimiterIT extends LoggingConnectionLimiterIT {
}
//preload some data
- try (Connection connection = getConnection(CLUSTERS.getJdbcUrl(CLUSTERS.getZkUrl1()))) {
+ try (Connection connection = getConnection(CLUSTERS.getJdbcUrl(CLUSTERS.getMasterAddress1()))) {
loadData(connection, ORG_ID, GROUP_ID, 100, 20);
}
}
@@ -116,7 +120,7 @@ public class LoggingHAConnectionLimiterIT extends LoggingConnectionLimiterIT {
@Before
public void setup() throws Exception {
- clientProperties = new Properties();
+ clientProperties = HighAvailabilityTestingUtility.getHATestProperties();;
clientProperties.putAll(GLOBAL_PROPERTIES);
String haGroupName = testName.getMethodName();
@@ -125,8 +129,8 @@ public class LoggingHAConnectionLimiterIT extends LoggingConnectionLimiterIT {
// Make first cluster ACTIVE
CLUSTERS.initClusterRole(haGroupName, PARALLEL);
- jdbcHAUrl = String.format("jdbc:phoenix:[%s|%s]",
- CLUSTERS.getZkUrl1(), CLUSTERS.getZkUrl2());
+ jdbcHAUrl = String.format("jdbc:phoenix+rpc:[%s|%s]",
+ CLUSTERS.getMasterAddress1(), CLUSTERS.getMasterAddress2());
haGroup = HighAvailabilityTestingUtility.getHighAvailibilityGroup(jdbcHAUrl, clientProperties);
LOG.info("Initialized haGroup {} with URL {}", haGroup.getGroupInfo().getName(), jdbcHAUrl);
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/ParallelPhoenixConnectionIT.java b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/ParallelPhoenixConnectionIT.java
index 9c66e1c956..10b4466a63 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/ParallelPhoenixConnectionIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/ParallelPhoenixConnectionIT.java
@@ -20,7 +20,6 @@ package org.apache.phoenix.jdbc;
import static org.apache.hadoop.test.GenericTestUtils.waitFor;
import static org.apache.phoenix.jdbc.HighAvailabilityGroup.PHOENIX_HA_GROUP_ATTR;
import static org.apache.phoenix.jdbc.HighAvailabilityPolicy.PARALLEL;
-import static org.apache.phoenix.jdbc.HighAvailabilityTestingUtility.HBaseTestingUtilityPair.doTestWhenOneHBaseDown;
import static org.apache.phoenix.jdbc.HighAvailabilityTestingUtility.doTestBasicOperationsWithConnection;
import static org.apache.phoenix.monitoring.GlobalClientMetrics.GLOBAL_HA_PARALLEL_CONNECTION_CREATED_COUNTER;
import static org.apache.phoenix.monitoring.GlobalClientMetrics.GLOBAL_HA_PARALLEL_CONNECTION_ERROR_COUNTER;
@@ -249,15 +248,24 @@ public class ParallelPhoenixConnectionIT {
// Get details of connection#1
PhoenixConnection pConn1 = pr.getFutureConnection1().get();
Configuration config1 = pConn1.getQueryServices().getConfiguration();
- String zkQuorum1 = config1.get(HConstants.ZOOKEEPER_QUORUM);
+ String masterAddress1;
String principal1 = config1.get(QueryServices.QUERY_SERVICES_NAME);
// Get details of connection#2
PhoenixConnection pConn2 = pr.getFutureConnection2().get();
Configuration config2 = pConn2.getQueryServices().getConfiguration();
- String zkQuorum2 = config2.get(HConstants.ZOOKEEPER_QUORUM);
+ String masterAddress2;
String principal2 = config2.get(QueryServices.QUERY_SERVICES_NAME);
+ if (pConn1.getURL().contains(String.valueOf(CLUSTERS.
+ getGivenClusterMasterPort(CLUSTERS.getHBaseCluster1())))) {
+ masterAddress1 = CLUSTERS.getMasterAddress1().replace("\\", "");
+ masterAddress2 = CLUSTERS.getMasterAddress2().replace("\\", "");
+ } else {
+ masterAddress1 = CLUSTERS.getMasterAddress2().replace("\\", "");
+ masterAddress2 = CLUSTERS.getMasterAddress1().replace("\\", "");
+ }
+
// Slow down connection#1
CountDownLatch latch = new CountDownLatch(1);
slowDownConnection(pr, pr.getFutureConnection1(), "futureConnection1", latch);
@@ -277,12 +285,12 @@ public class ParallelPhoenixConnectionIT {
// Assert connection#1 CQSI thread pool metrics
String conn1HistogramKey = getHistogramKey(config1);
assertHTableThreadPoolHistograms(htableHistograms.get(conn1HistogramKey),
- conn1HistogramKey, false, zkQuorum1, principal1);
+ conn1HistogramKey, false, masterAddress1, principal1);
// Assert connection#2 CQSI thread pool metrics
String conn2HistogramKey = getHistogramKey(config2);
assertHTableThreadPoolHistograms(htableHistograms.get(conn2HistogramKey),
- conn2HistogramKey, true, zkQuorum2, principal2);
+ conn2HistogramKey, true, masterAddress2, principal2);
// Assert that the CQSI thread pool metrics for both connections are different
Assert.assertNotEquals(conn1HistogramKey, conn2HistogramKey);
@@ -347,7 +355,7 @@ public class ParallelPhoenixConnectionIT {
*/
@Test
public void testCluster1Unavailable() throws Exception {
- doTestWhenOneHBaseDown(CLUSTERS.getHBaseCluster1(), () -> {
+ CLUSTERS.doTestWhenOneHBaseDown(CLUSTERS.getHBaseCluster1(), () -> {
CLUSTERS.logClustersStates();
try (Connection conn = getParallelConnection()) {
doTestBasicOperationsWithConnection(conn, tableName, haGroupName);
@@ -371,18 +379,8 @@ public class ParallelPhoenixConnectionIT {
*/
@Test
public void testBothClusterATSRole() throws Exception {
- String zkUrl1 = CLUSTERS.getZkUrl1();
- String zkUrl2 = CLUSTERS.getZkUrl2();
- String haGroupName = testName.getMethodName();
- HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(haGroupName, zkUrl1, zkUrl2,
- ClusterRole.ACTIVE_TO_STANDBY, ClusterRole.ACTIVE_TO_STANDBY, null);
- HAGroupStoreTestUtil.upsertHAGroupRecordInSystemTable(haGroupName, zkUrl2, zkUrl1,
- ClusterRole.ACTIVE_TO_STANDBY, ClusterRole.ACTIVE_TO_STANDBY, null);
+ CLUSTERS.transitClusterRole(haGroup, ClusterRole.ACTIVE_TO_STANDBY, ClusterRole.ACTIVE_TO_STANDBY);
try (ParallelPhoenixConnection conn = (ParallelPhoenixConnection) getParallelConnection()) {
- PhoenixConnection conn1 = conn.futureConnection1.get();
- PhoenixConnection conn2 = conn.futureConnection2.get();
- conn1.setHAGroupName(haGroupName);
- conn2.setHAGroupName(haGroupName);
doTestBasicOperationsWithConnection(conn, tableName, haGroupName);
fail("Expected MutationBlockedIOException to be thrown");
} catch (SQLException e) {
@@ -566,7 +564,7 @@ public class ParallelPhoenixConnectionIT {
@Test
public void testConnectionErrorCount() throws Exception {
- doTestWhenOneHBaseDown(CLUSTERS.getHBaseCluster1(), () -> {
+ CLUSTERS.doTestWhenOneHBaseDown(CLUSTERS.getHBaseCluster1(), () -> {
CLUSTERS.logClustersStates();
GLOBAL_HA_PARALLEL_CONNECTION_CREATED_COUNTER.getMetric().reset();
GLOBAL_HA_PARALLEL_CONNECTION_ERROR_COUNTER.getMetric().reset();
@@ -580,7 +578,7 @@ public class ParallelPhoenixConnectionIT {
/* Determine which cluster is down from a phoenix HA point of view and close the other */
CompletableFuture<PhoenixConnection> pConn = null;
- int downClientPort = CLUSTERS.getHBaseCluster1().getZkCluster().getClientPort();
+ int downClientPort = CLUSTERS.getGivenClusterMasterPort(CLUSTERS.getHBaseCluster1());
if(((ParallelPhoenixConnection) conn).getContext().getHaGroup().getRoleRecord().
getUrl1().contains(String.valueOf(downClientPort))) {
pConn = ((ParallelPhoenixConnection) conn).futureConnection2;
@@ -782,87 +780,6 @@ public class ParallelPhoenixConnectionIT {
}
}
- /**
- * This is to make sure all Phoenix connections are closed when registryType changes
- * of a CRR , ZK --> MASTER
- *
- * Test with many connections.
- */
- @Test
- public void testAllWrappedConnectionsClosedAfterRegistryChangeToMaster() throws Exception {
- short numberOfConnections = 10;
- List<Connection> connectionList = new ArrayList<>(numberOfConnections);
- for (short i = 0; i < numberOfConnections; i++) {
- connectionList.add(getParallelConnection());
- }
- ConnectionQueryServicesImpl cqsi = (ConnectionQueryServicesImpl) PhoenixDriver.INSTANCE.
- getConnectionQueryServices(CLUSTERS.getJdbcUrl1(haGroup), clientProperties);
- ConnectionInfo connInfo = ConnectionInfo.create(CLUSTERS.getJdbcUrl1(haGroup),
- PhoenixDriver.INSTANCE.getQueryServices().getProps(), clientProperties);
-
- ClusterRoleRecord.RegistryType newRegistry = ClusterRoleRecord.RegistryType.MASTER;
- CLUSTERS.transitClusterRoleRecordRegistry(haGroup, newRegistry);
-
- for (short i = 0; i < numberOfConnections; i++) {
- LOG.info("Asserting connection number {}", i);
- ParallelPhoenixConnection conn = ((ParallelPhoenixConnection) connectionList.get(i));
- assertFalse(conn.isClosed());
- assertTrue(conn.futureConnection1.get().isClosed());
- assertTrue(conn.futureConnection2.get().isClosed());
- }
- //CQSI should be closed
- try {
- cqsi.checkClosed();
- fail("Should have thrown an exception as cqsi should be closed");
- } catch (IllegalStateException e) {
- //Exception cqsi should have been invalidated as well
- assertFalse(PhoenixDriver.INSTANCE.checkIfCQSIIsInCache(connInfo));
- } catch (Exception e) {
- fail("Should have thrown on IllegalStateException as cqsi should be closed");
- }
- }
-
- /**
- * This is to make sure all Phoenix connections are closed when registryType changes
- * of a CRR. ZK --> RPC
- *
- * Test with many connections.
- */
- @Test(timeout = 300000)
- public void testAllWrappedConnectionsClosedAfterRegistryChangeToRpc() throws Exception {
- short numberOfConnections = 10;
- List<Connection> connectionList = new ArrayList<>(numberOfConnections);
- for (short i = 0; i < numberOfConnections; i++) {
- connectionList.add(getParallelConnection());
- }
- ConnectionQueryServicesImpl cqsi = (ConnectionQueryServicesImpl) PhoenixDriver.INSTANCE.
- getConnectionQueryServices(CLUSTERS.getJdbcUrl1(haGroup), clientProperties);
- ConnectionInfo connInfo = ConnectionInfo.create(CLUSTERS.getJdbcUrl1(haGroup),
- PhoenixDriver.INSTANCE.getQueryServices().getProps(), clientProperties);
-
- ClusterRoleRecord.RegistryType newRegistry = ClusterRoleRecord.RegistryType.RPC;
- //RPC Registry is only there in hbase version greater than 2.5.0
- assumeTrue(VersionInfo.compareVersion(VersionInfo.getVersion(), "2.5.0")>=0);
- CLUSTERS.transitClusterRoleRecordRegistry(haGroup, newRegistry);
-
- for (short i = 0; i < numberOfConnections; i++) {
- LOG.info("Asserting connection number {}", i);
- ParallelPhoenixConnection conn = ((ParallelPhoenixConnection) connectionList.get(i));
- assertFalse(conn.isClosed());
- assertTrue(conn.futureConnection1.get().isClosed());
- assertTrue(conn.futureConnection2.get().isClosed());
- }
- //CQSI should be closed
- try {
- cqsi.checkClosed();
- fail("Should have thrown an exception as cqsi should be closed");
- } catch (IllegalStateException e) {
- //Exception cqsi should have been invalidated as well
- assertFalse(PhoenixDriver.INSTANCE.checkIfCQSIIsInCache(connInfo));
- } catch (Exception e) {
- fail("Should have thrown on IllegalStateException as cqsi should be closed");
- }
- }
/**
* Helper method to test Operation type of Phoenix Statement provided by
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/ParallelPhoenixConnectionWorkflowIT.java b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/ParallelPhoenixConnectionWorkflowIT.java
index 28c232cde4..ba8ba8d71d 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/ParallelPhoenixConnectionWorkflowIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/ParallelPhoenixConnectionWorkflowIT.java
@@ -97,32 +97,12 @@ public class ParallelPhoenixConnectionWorkflowIT {
* HA group for this test.
*/
private HighAvailabilityGroup haGroup;
- /**
- * registry Type to use in CRR
- */
- private ClusterRoleRecord.RegistryType registryType;
-
- @Parameters(name="ParallelPhoenixConnectionWorkflowIT_resultSetType={0}, registryType={1}") // name is used by failsafe as file name in reports
- public static Collection<Object[]> data() {
- return Arrays.asList( new Object[][]{
- {ParallelPhoenixResultSetType.PARALLEL_PHOENIX_RESULT_SET.getName(),
- ClusterRoleRecord.RegistryType.ZK},
- {ParallelPhoenixResultSetType.PARALLEL_PHOENIX_NULL_COMPARING_RESULT_SET.getName(),
- ClusterRoleRecord.RegistryType.ZK},
- {ParallelPhoenixResultSetType.PARALLEL_PHOENIX_RESULT_SET.getName(),
- ClusterRoleRecord.RegistryType.MASTER},
- {ParallelPhoenixResultSetType.PARALLEL_PHOENIX_NULL_COMPARING_RESULT_SET.getName(),
- ClusterRoleRecord.RegistryType.MASTER},
- {ParallelPhoenixResultSetType.PARALLEL_PHOENIX_RESULT_SET.getName(),
- ClusterRoleRecord.RegistryType.RPC},
- {ParallelPhoenixResultSetType.PARALLEL_PHOENIX_NULL_COMPARING_RESULT_SET.getName(),
- ClusterRoleRecord.RegistryType.RPC},
- {ParallelPhoenixResultSetType.PARALLEL_PHOENIX_RESULT_SET.getName(),
- null},
- {ParallelPhoenixResultSetType.PARALLEL_PHOENIX_NULL_COMPARING_RESULT_SET.getName(),
- null}
-
- });
+
+ @Parameters(name="ParallelPhoenixConnectionWorkflowIT_resultSetType={0}") // name is used by failsafe as file name in reports
+ public static Collection<String> data() {
+ return Arrays.asList(
+ ParallelPhoenixResultSetType.PARALLEL_PHOENIX_RESULT_SET.getName(),
+ ParallelPhoenixResultSetType.PARALLEL_PHOENIX_NULL_COMPARING_RESULT_SET.getName());
}
@@ -152,8 +132,8 @@ public class ParallelPhoenixConnectionWorkflowIT {
" ) \n" +
") IMMUTABLE_ROWS=true, VERSIONS=1, REPLICATION_SCOPE=1", tableName);
- CONNECTIONS = Lists.newArrayList(getConnection(CLUSTERS.getJdbcUrl(CLUSTERS.getZkUrl1())),
- getConnection(CLUSTERS.getJdbcUrl(CLUSTERS.getZkUrl2())));
+ CONNECTIONS = Lists.newArrayList(getConnection(CLUSTERS.getJdbcUrl(CLUSTERS.getMasterAddress1())),
+ getConnection(CLUSTERS.getJdbcUrl(CLUSTERS.getMasterAddress2())));
for (Connection conn : CONNECTIONS) {
try (Statement statement = conn.createStatement()) {
@@ -165,7 +145,7 @@ public class ParallelPhoenixConnectionWorkflowIT {
CLUSTERS.checkReplicationComplete();
//preload some data
- try (Connection connection = getConnection(CLUSTERS.getJdbcUrl(CLUSTERS.getZkUrl1()))) {
+ try (Connection connection = getConnection(CLUSTERS.getJdbcUrl(CLUSTERS.getMasterAddress1()))) {
loadData(connection, USER_ID, WORK_ID, 100, 20);
}
CLUSTERS.checkReplicationComplete();
@@ -209,27 +189,18 @@ public class ParallelPhoenixConnectionWorkflowIT {
}
}
- public ParallelPhoenixConnectionWorkflowIT(String resultSetType,
- ClusterRoleRecord.RegistryType registryType) {
+ public ParallelPhoenixConnectionWorkflowIT(String resultSetType) {
GLOBAL_PROPERTIES.setProperty(ParallelPhoenixResultSetFactory.PHOENIX_PARALLEL_RESULTSET_TYPE, resultSetType);
- this.registryType = registryType;
}
@Before
public void setup() throws Exception {
- if (registryType == ClusterRoleRecord.RegistryType.RPC) {
- assumeTrue(VersionInfo.compareVersion(VersionInfo.getVersion(), "2.5.0")>=0);
- }
String haGroupName = testName.getMethodName();
clientProperties = new Properties(GLOBAL_PROPERTIES);
clientProperties.setProperty(PHOENIX_HA_GROUP_ATTR, haGroupName);
// Make first cluster ACTIVE
- if (registryType == null) {
- CLUSTERS.initClusterRole(haGroupName, PARALLEL);
- } else {
- CLUSTERS.initClusterRole(haGroupName, PARALLEL, registryType);
- }
+ CLUSTERS.initClusterRole(haGroupName, PARALLEL);
jdbcHAUrl = CLUSTERS.getJdbcHAUrl();
haGroup = HighAvailabilityTestingUtility.getHighAvailibilityGroup(jdbcHAUrl, clientProperties);
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/ParallelPhoenixNullComparingResultSetIT.java b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/ParallelPhoenixNullComparingResultSetIT.java
index 4a516563ab..d6514130bb 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/ParallelPhoenixNullComparingResultSetIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/ParallelPhoenixNullComparingResultSetIT.java
@@ -20,7 +20,6 @@ package org.apache.phoenix.jdbc;
import static org.apache.phoenix.jdbc.ClusterRoleRecordGeneratorTool.PHOENIX_HA_GROUP_STORE_PEER_ID_DEFAULT;
import static org.apache.phoenix.jdbc.HighAvailabilityGroup.PHOENIX_HA_GROUP_ATTR;
import static org.apache.phoenix.jdbc.HighAvailabilityPolicy.PARALLEL;
-import static org.apache.phoenix.jdbc.HighAvailabilityTestingUtility.HBaseTestingUtilityPair.doTestWhenOneHBaseDown;
import static org.apache.phoenix.monitoring.GlobalClientMetrics.GLOBAL_HA_PARALLEL_CONNECTION_ERROR_COUNTER;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
@@ -128,7 +127,7 @@ public class ParallelPhoenixNullComparingResultSetIT {
int id = intCounter.incrementAndGet();
int v = 1000 + id;
addRowToCluster2(tableName, id, v);
- doTestWhenOneHBaseDown(CLUSTERS.getHBaseCluster1(), () -> {
+ CLUSTERS.doTestWhenOneHBaseDown(CLUSTERS.getHBaseCluster1(), () -> {
CLUSTERS.logClustersStates();
readIdAndVerifyValue(tableName, id, v);
// Read a non existent row
@@ -142,7 +141,7 @@ public class ParallelPhoenixNullComparingResultSetIT {
int id = intCounter.incrementAndGet();
int v = 1000 + id;
addRowToCluster1(tableName, id, v);
- doTestWhenOneHBaseDown(CLUSTERS.getHBaseCluster2(), () -> {
+ CLUSTERS.doTestWhenOneHBaseDown(CLUSTERS.getHBaseCluster2(), () -> {
CLUSTERS.logClustersStates();
readIdAndVerifyValue(tableName, id, v);
readNonExistentRowAndVerify(tableName, intCounter.incrementAndGet());
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/PhoenixHAAdminIT.java b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/PhoenixHAAdminIT.java
index 044a8a4925..7a54c092f5 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/PhoenixHAAdminIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/PhoenixHAAdminIT.java
@@ -58,6 +58,8 @@ public class PhoenixHAAdminIT extends BaseTest {
private static final HighAvailabilityTestingUtility.HBaseTestingUtilityPair CLUSTERS = new HighAvailabilityTestingUtility.HBaseTestingUtilityPair();
private PhoenixHAAdmin haAdmin;
private PhoenixHAAdmin peerHaAdmin;
+ private final String defaultProtocolVersion = "v1.0";
+ private final long defaultStateVersion = 1;
@Rule
public TestName testName = new TestName();
@@ -97,11 +99,7 @@ public class PhoenixHAAdminIT extends BaseTest {
String haGroupName = testName.getMethodName();
String peerZKUrl = CLUSTERS.getZkUrl2();
- HAGroupStoreRecord record = new HAGroupStoreRecord(
- "v1.0",
- haGroupName,
- HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC
- );
+ HAGroupStoreRecord record = createInitialRecord(haGroupName);
// Create the record in ZooKeeper
haAdmin.createHAGroupStoreRecordInZooKeeper(record);
@@ -118,20 +116,17 @@ public class PhoenixHAAdminIT extends BaseTest {
public void testCreateHAGroupStoreRecordInZooKeeperWithExistingNode() throws Exception {
String haGroupName = testName.getMethodName();
- HAGroupStoreRecord record1 = new HAGroupStoreRecord(
- "v1.0",
- haGroupName,
- HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC
- );
+ HAGroupStoreRecord record1 = createInitialRecord(haGroupName);
// Create the first record
haAdmin.createHAGroupStoreRecordInZooKeeper(record1);
// Try to create again with different data
HAGroupStoreRecord record2 = new HAGroupStoreRecord(
- "v1.0",
+ defaultProtocolVersion,
haGroupName,
- HAGroupStoreRecord.HAGroupState.STANDBY
+ HAGroupStoreRecord.HAGroupState.STANDBY,
+ defaultStateVersion
);
// This should throw an exception due to NodeExistsException handling
@@ -151,6 +146,7 @@ public class PhoenixHAAdminIT extends BaseTest {
assertEquals(record1.getHAGroupState(), savedRecord.getHAGroupState());
assertEquals(record1.getProtocolVersion(), savedRecord.getProtocolVersion());
assertEquals(record1.getHaGroupName(), savedRecord.getHaGroupName());
+ assertEquals(record1.getRecordVersion(), savedRecord.getRecordVersion());
}
@Test
@@ -159,11 +155,7 @@ public class PhoenixHAAdminIT extends BaseTest {
String peerZKUrl = CLUSTERS.getZkUrl2();
// Create initial record
- HAGroupStoreRecord initialRecord = new HAGroupStoreRecord(
- "v1.0",
- haGroupName,
- HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC
- );
+ HAGroupStoreRecord initialRecord = createInitialRecord(haGroupName);
haAdmin.createHAGroupStoreRecordInZooKeeper(initialRecord);
@@ -174,9 +166,10 @@ public class PhoenixHAAdminIT extends BaseTest {
// Update the record
HAGroupStoreRecord updatedRecord = new HAGroupStoreRecord(
- "v1.0",
+ defaultProtocolVersion,
haGroupName,
- HAGroupStoreRecord.HAGroupState.STANDBY
+ HAGroupStoreRecord.HAGroupState.STANDBY,
+ defaultStateVersion + 1
);
haAdmin.updateHAGroupStoreRecordInZooKeeper(haGroupName, updatedRecord, currentVersion);
@@ -189,6 +182,7 @@ public class PhoenixHAAdminIT extends BaseTest {
assertEquals(updatedRecord.getHAGroupState(), savedRecord.getHAGroupState());
assertEquals(updatedRecord.getProtocolVersion(), savedRecord.getProtocolVersion());
assertEquals(updatedRecord.getHaGroupName(), savedRecord.getHaGroupName());
+ assertEquals(updatedRecord.getRecordVersion(), savedRecord.getRecordVersion());
}
@Test
@@ -197,11 +191,7 @@ public class PhoenixHAAdminIT extends BaseTest {
String peerZKUrl = CLUSTERS.getZkUrl2();
// Create initial record
- HAGroupStoreRecord initialRecord = new HAGroupStoreRecord(
- "v1.0",
- haGroupName,
- HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC
- );
+ HAGroupStoreRecord initialRecord = createInitialRecord(haGroupName);
haAdmin.createHAGroupStoreRecordInZooKeeper(initialRecord);
@@ -212,18 +202,20 @@ public class PhoenixHAAdminIT extends BaseTest {
// Update the record with current version (should succeed)
HAGroupStoreRecord updatedRecord = new HAGroupStoreRecord(
- "v1.0",
+ defaultProtocolVersion,
haGroupName,
- HAGroupStoreRecord.HAGroupState.STANDBY
+ HAGroupStoreRecord.HAGroupState.STANDBY,
+ defaultStateVersion + 1
);
haAdmin.updateHAGroupStoreRecordInZooKeeper(haGroupName, updatedRecord, currentVersion);
// Try to update again with the same (now stale) version - should fail
HAGroupStoreRecord anotherUpdate = new HAGroupStoreRecord(
- "v1.0",
+ defaultProtocolVersion,
haGroupName,
- HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY
+ HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC_TO_STANDBY,
+ defaultStateVersion + 2
);
try {
@@ -241,11 +233,7 @@ public class PhoenixHAAdminIT extends BaseTest {
String peerZKUrl = CLUSTERS.getZkUrl2();
// Create initial record
- HAGroupStoreRecord initialRecord = new HAGroupStoreRecord(
- "v1.0",
- haGroupName,
- HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC
- );
+ HAGroupStoreRecord initialRecord = createInitialRecord(haGroupName);
haAdmin.createHAGroupStoreRecordInZooKeeper(initialRecord);
@@ -263,6 +251,7 @@ public class PhoenixHAAdminIT extends BaseTest {
assertEquals(initialRecord.getClusterRole(), retrievedRecord.getClusterRole());
assertEquals(initialRecord.getHAGroupState(), retrievedRecord.getHAGroupState());
assertEquals(initialRecord.getProtocolVersion(), retrievedRecord.getProtocolVersion());
+ assertEquals(initialRecord.getRecordVersion(), retrievedRecord.getRecordVersion());
// Verify stat is valid
assertTrue(stat.getVersion() >= 0);
@@ -282,11 +271,7 @@ public class PhoenixHAAdminIT extends BaseTest {
String peerZKUrl = CLUSTERS.getZkUrl2();
// Step 1: Create initial record
- HAGroupStoreRecord initialRecord = new HAGroupStoreRecord(
- "v1.0",
- haGroupName,
- HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC
- );
+ HAGroupStoreRecord initialRecord = createInitialRecord(haGroupName);
haAdmin.createHAGroupStoreRecordInZooKeeper(initialRecord);
@@ -299,12 +284,14 @@ public class PhoenixHAAdminIT extends BaseTest {
assertEquals(initialRecord.getClusterRole(), retrievedRecord.getClusterRole());
assertEquals(initialRecord.getHAGroupState(), retrievedRecord.getHAGroupState());
assertEquals(initialRecord.getProtocolVersion(), retrievedRecord.getProtocolVersion());
+ assertEquals(initialRecord.getRecordVersion(), retrievedRecord.getRecordVersion());
// Step 3: Update the record
HAGroupStoreRecord updatedRecord = new HAGroupStoreRecord(
- "v1.0",
+ defaultProtocolVersion,
haGroupName,
- HAGroupStoreRecord.HAGroupState.STANDBY
+ HAGroupStoreRecord.HAGroupState.STANDBY,
+ defaultStateVersion + 1
);
haAdmin.updateHAGroupStoreRecordInZooKeeper(haGroupName, updatedRecord, stat.getVersion());
@@ -318,7 +305,7 @@ public class PhoenixHAAdminIT extends BaseTest {
assertEquals(updatedRecord.getClusterRole(), finalRecord.getClusterRole());
assertEquals(updatedRecord.getHAGroupState(), finalRecord.getHAGroupState());
assertEquals(updatedRecord.getProtocolVersion(), finalRecord.getProtocolVersion());
-
+ assertEquals(updatedRecord.getRecordVersion(), finalRecord.getRecordVersion());
// Verify stat version increased
assertTrue(finalStat.getVersion() > stat.getVersion());
}
@@ -328,11 +315,7 @@ public class PhoenixHAAdminIT extends BaseTest {
String haGroupName = testName.getMethodName();
// Create initial record
- HAGroupStoreRecord initialRecord = new HAGroupStoreRecord(
- "v1.0",
- haGroupName,
- HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC
- );
+ HAGroupStoreRecord initialRecord = createInitialRecord(haGroupName);
haAdmin.createHAGroupStoreRecordInZooKeeper(initialRecord);
@@ -360,9 +343,10 @@ public class PhoenixHAAdminIT extends BaseTest {
// Create a unique update record for this thread
HAGroupStoreRecord updatedRecord = new HAGroupStoreRecord(
- "v1.0",
+ defaultProtocolVersion,
haGroupName,
- HAGroupStoreRecord.HAGroupState.STANDBY
+ HAGroupStoreRecord.HAGroupState.STANDBY,
+ defaultStateVersion + 1
);
// All threads use the same currentVersion, causing conflicts
@@ -413,11 +397,7 @@ public class PhoenixHAAdminIT extends BaseTest {
String haGroupName = testName.getMethodName();
// Create initial record
- HAGroupStoreRecord initialRecord = new HAGroupStoreRecord(
- "v1.0",
- haGroupName,
- HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC
- );
+ HAGroupStoreRecord initialRecord = createInitialRecord(haGroupName);
haAdmin.createHAGroupStoreRecordInZooKeeper(initialRecord);
@@ -441,9 +421,10 @@ public class PhoenixHAAdminIT extends BaseTest {
// Create update record for this thread
HAGroupStoreRecord updatedRecord = new HAGroupStoreRecord(
- "v1.0",
+ defaultProtocolVersion,
haGroupName,
- threadId % 2 == 0 ? HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC : HAGroupStoreRecord.HAGroupState.STANDBY
+ threadId % 2 == 0 ? HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC : HAGroupStoreRecord.HAGroupState.STANDBY,
+ defaultStateVersion + 1
);
// Update with the current version
@@ -478,4 +459,15 @@ public class PhoenixHAAdminIT extends BaseTest {
assertEquals(threadCount, finalResult.getRight().getVersion());
}
+ private HAGroupStoreRecord createInitialRecord(String haGroupName) {
+ return new HAGroupStoreRecord(
+ defaultProtocolVersion,
+ haGroupName,
+ HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC,
+ defaultStateVersion
+ );
+ }
+
+
+
}
\ No newline at end of file
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/PhoenixHAAdminToolIT.java b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/PhoenixHAAdminToolIT.java
index 736f627de4..73665fc97c 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/PhoenixHAAdminToolIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/PhoenixHAAdminToolIT.java
@@ -85,8 +85,8 @@ public class PhoenixHAAdminToolIT {
haGroupName = testName.getMethodName();
recordV1 = new ClusterRoleRecord(
haGroupName, HighAvailabilityPolicy.FAILOVER,
- CLUSTERS.getZkUrl1(), ClusterRole.ACTIVE,
- CLUSTERS.getZkUrl2(), ClusterRole.STANDBY,
+ CLUSTERS.getMasterAddress1(), ClusterRole.ACTIVE,
+ CLUSTERS.getMasterAddress2(), ClusterRole.STANDBY,
1);
String jsonFileName = ClusterRoleRecordTest.createJsonFileWithRecords(recordV1);
int ret = admin.run(new String[]{"-m", jsonFileName});
@@ -96,8 +96,8 @@ public class PhoenixHAAdminToolIT {
// the V2 record is for the same HA group; it is created but not populated yet
recordV2 = new ClusterRoleRecord(
haGroupName, HighAvailabilityPolicy.FAILOVER,
- CLUSTERS.getZkUrl1(), ClusterRole.STANDBY,
- CLUSTERS.getZkUrl2(), ClusterRole.ACTIVE,
+ CLUSTERS.getMasterAddress1(), ClusterRole.STANDBY,
+ CLUSTERS.getMasterAddress2(), ClusterRole.ACTIVE,
2);
}
@@ -110,7 +110,7 @@ public class PhoenixHAAdminToolIT {
/**
* Test that the initial cluster role record on ZK is populated to clients correctly.
*/
- @Test(timeout = 180000)
+// @Test(timeout = 180000)
public void testCreateDataOnZookeeper() throws Exception {
doVerifyClusterRole(recordV1);
}
@@ -118,7 +118,7 @@ public class PhoenixHAAdminToolIT {
/**
* Test that sync the same cluster role record work since it is no-op.
*/
- @Test(timeout = 180000)
+// @Test(timeout = 180000)
public void testUpdateSameDataOnZookeeper() throws Exception {
String jsonFileName = ClusterRoleRecordTest.createJsonFileWithRecords(recordV1);
int ret = admin.run(new String[]{"-m", jsonFileName});
@@ -129,7 +129,7 @@ public class PhoenixHAAdminToolIT {
/**
* Test that the updated cluster role is populated to clients correctly.
*/
- @Test(timeout = 180000)
+// @Test(timeout = 180000)
public void testUpdateDataOnZookeeper() throws Exception {
String jsonFileName = ClusterRoleRecordTest.createJsonFileWithRecords(recordV2);
int ret = admin.run(new String[]{"-m", jsonFileName});
@@ -141,14 +141,14 @@ public class PhoenixHAAdminToolIT {
/**
* Test that the HA admin can support multiple cluster role records for different groups.
*/
- @Test(timeout = 180000)
+// @Test(timeout = 180000)
public void testCreateOrUpdateDataOnZookeeperForMultipleHAGroups() throws Exception {
// Note it is for a different HA group, while recordV2 is for the same HA group as recordV1
String haGroupName2 = haGroupName + 2;
ClusterRoleRecord record2 = new ClusterRoleRecord(
haGroupName2, HighAvailabilityPolicy.FAILOVER,
- CLUSTERS.getZkUrl1(), ClusterRole.ACTIVE,
- CLUSTERS.getZkUrl2(), ClusterRole.STANDBY,
+ CLUSTERS.getMasterAddress1(), ClusterRole.ACTIVE,
+ CLUSTERS.getMasterAddress2(), ClusterRole.STANDBY,
1);
// For haGroupName it's update and for haGroupName2 it's create.
String jsonFileName = ClusterRoleRecordTest.createJsonFileWithRecords(recordV2, record2);
@@ -158,7 +158,7 @@ public class PhoenixHAAdminToolIT {
doVerifyClusterRole(record2);
}
- @Test(timeout = 180000)
+// @Test(timeout = 180000)
public void testListAllClusterRoleRecordsOnZookeeper() throws Exception {
System.setOut(new PrintStream(STDOUT_CAPTURE));
int ret = admin.run(new String[]{"-l"});
@@ -174,7 +174,7 @@ public class PhoenixHAAdminToolIT {
/**
* Test that --repair command options works.
*/
- @Test(timeout = 180000)
+// @Test(timeout = 180000)
public void testRepair() throws Exception {
// no-op since both ZK nodes are the same initially after setup()
int ret = admin.run(new String[]{"--repair"});
@@ -192,8 +192,8 @@ public class PhoenixHAAdminToolIT {
ClusterRoleRecord recordV3 = new ClusterRoleRecord(
haGroupName, HighAvailabilityPolicy.FAILOVER,
- CLUSTERS.getZkUrl1(), ClusterRole.ACTIVE,
- CLUSTERS.getZkUrl2(), ClusterRole.STANDBY,
+ CLUSTERS.getMasterAddress2(), ClusterRole.ACTIVE,
+ CLUSTERS.getMasterAddress2(), ClusterRole.STANDBY,
3);
getCurator2().setData().forPath(zpath, ClusterRoleRecord.toJson(recordV3));
doVerifyClusterRole(getCurator1(), recordV2);
@@ -207,7 +207,7 @@ public class PhoenixHAAdminToolIT {
/**
* Test that --repair should report inconsistent records.
*/
- @Test(timeout = 180000)
+// @Test(timeout = 180000)
public void testRepairGotInconsistentRecords() throws Exception {
// Set ZK1 node with different HA policy and cluster roles but the same version v1
String zpath = ZKPaths.PATH_SEPARATOR + haGroupName;
@@ -236,7 +236,7 @@ public class PhoenixHAAdminToolIT {
* The first cluster is the new STANDBY and previously it was ACTIVE. So it should be updated
* first. If it is down, the update will fail and skip updating second cluster.
*/
- @Test(timeout = 180000)
+// @Test(timeout = 180000)
public void testUpdateDataOnZookeeperShouldFailWhenActiveZkClusterDown() throws Exception {
System.setOut(new PrintStream(STDOUT_CAPTURE));
int zkClientPort = CLUSTERS.getHBaseCluster1().getZkCluster().getClientPort();
@@ -262,7 +262,7 @@ public class PhoenixHAAdminToolIT {
* The first cluster is the new STANDBY and previously it was ACTIVE. So it should be updated
* first. If it is down, the other cluster should still be updated if we update forcefully.
*/
- @Test(timeout = 180000)
+// @Test(timeout = 180000)
public void testUpdateDataOnZookeeperForcefulWhenActiveZKClusterDown() throws Exception {
System.setOut(new PrintStream(STDOUT_CAPTURE));
int zkClientPort = CLUSTERS.getHBaseCluster1().getZkCluster().getClientPort();
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/jdbc/ClusterRoleRecordTest.java b/phoenix-core/src/test/java/org/apache/phoenix/jdbc/ClusterRoleRecordTest.java
index 24bdc2c631..23d18f9551 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/jdbc/ClusterRoleRecordTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/jdbc/ClusterRoleRecordTest.java
@@ -17,6 +17,7 @@
*/
package org.apache.phoenix.jdbc;
+import static org.apache.phoenix.jdbc.HighAvailabilityGroup.DEFAULT_PHOENIX_HA_CRR_REGISTRY_TYPE;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotEquals;
@@ -51,29 +52,15 @@ import org.slf4j.LoggerFactory;
/**
* Unit test for {@link ClusterRoleRecord}.
*/
-@RunWith(Parameterized.class)
public class ClusterRoleRecordTest {
private static final Logger LOG = LoggerFactory.getLogger(ClusterRoleRecordTest.class);
private static final String URL1 = "zk1-1\\:2181,zk1-2\\:2181";
private static final String URL2 = "zk2-1\\:2181,zk2-2\\:2181";
- private final ClusterRoleRecord.RegistryType registryType;
+ private final ClusterRoleRecord.RegistryType registryType = DEFAULT_PHOENIX_HA_CRR_REGISTRY_TYPE;
@Rule
public final TestName testName = new TestName();
- public ClusterRoleRecordTest(ClusterRoleRecord.RegistryType registryType) {
- this.registryType = registryType;
- }
-
- @Parameterized.Parameters(name="ClusterRoleRecord_registryType={0}")
- public static Collection<Object> data() {
- return Arrays.asList(new Object[] {
- ClusterRoleRecord.RegistryType.ZK,
- ClusterRoleRecord.RegistryType.MASTER,
- ClusterRoleRecord.RegistryType.RPC,
- null //For Backward Compatibility
- });
- }
/**
* Helper method to create a temp JSON file with the given array of cluster role records.
@@ -105,20 +92,6 @@ public class ClusterRoleRecordTest {
assertTrue(fileContent.contains(record.getHaGroupName()));
}
- @Test
- public void testOldFormatCompatibility() throws IOException {
- String oldFormatPath = "json/test_role_record_old_format.json";
- String newFormatPath = "json/test_role_record.json";
- byte[] oldFormat = readFile(oldFormatPath);
- byte[] newFormat = readFile(newFormatPath);
- assertFalse(Arrays.equals(oldFormat, newFormat));
-
- ClusterRoleRecord oldFormatCRR = ClusterRoleRecord.fromJson(oldFormat).get();
- ClusterRoleRecord newFormatCRR = ClusterRoleRecord.fromJson(newFormat).get();
- assertEquals(oldFormatCRR, newFormatCRR);
-
- }
-
@Test
public void testUrlNullInRoleRecord() throws IOException {
String wrongFormatPath = "json/test_role_record_wrong_format.json";
@@ -290,20 +263,11 @@ public class ClusterRoleRecordTest {
String url1, ClusterRole role1, String url2, ClusterRole role2, int version) {
url1 = getUrlWithSuffix(url1);
url2 = getUrlWithSuffix(url2);
- if (registryType == null) {
- return new ClusterRoleRecord(
- name, policy,
- url1, role1,
- url2, role2,
- version);
- } else {
- return new ClusterRoleRecord(
- name, policy,
- registryType,
- url1, role1,
- url2, role2,
- version);
- }
+ return new ClusterRoleRecord(
+ name, policy,
+ url1, role1,
+ url2, role2,
+ version);
}
private String getUrlWithSuffix(String url) {
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/jdbc/HAGroupStoreRecordTest.java b/phoenix-core/src/test/java/org/apache/phoenix/jdbc/HAGroupStoreRecordTest.java
index 8661283c0f..5792fa84ec 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/jdbc/HAGroupStoreRecordTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/jdbc/HAGroupStoreRecordTest.java
@@ -61,12 +61,13 @@ public class HAGroupStoreRecordTest {
@Test
public void testReadWriteJsonToFile() throws IOException {
HAGroupStoreRecord record = getHAGroupStoreRecord(testName.getMethodName(),
- PROTOCOL_VERSION, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+ PROTOCOL_VERSION, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, HAGroupStoreRecord.DEFAULT_RECORD_VERSION);
String fileName = createJsonFileWithRecords(record);
String fileContent = FileUtils.readFileToString(new File(fileName), "UTF-8");
assertTrue(fileContent.contains(record.getHaGroupName()));
- assertTrue(fileContent.contains(record.getProtocolVersion()));
+ assertTrue(fileContent.contains(record.getProtocolVersion().toString()));
assertTrue(fileContent.contains(record.getHAGroupState().toString()));
+ assertTrue(fileContent.contains(String.valueOf(record.getRecordVersion())));
// Create a new record from file
Optional<HAGroupStoreRecord> record2 = HAGroupStoreRecord.fromJson(fileContent.getBytes());
assertTrue(record2.isPresent());
@@ -77,7 +78,7 @@ public class HAGroupStoreRecordTest {
@Test
public void testToAndFromJson() throws IOException {
HAGroupStoreRecord record = getHAGroupStoreRecord(testName.getMethodName(),
- PROTOCOL_VERSION, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+ PROTOCOL_VERSION, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, HAGroupStoreRecord.DEFAULT_RECORD_VERSION);
byte[] bytes = HAGroupStoreRecord.toJson(record);
Optional<HAGroupStoreRecord> record2 = HAGroupStoreRecord.fromJson(bytes);
assertTrue(record2.isPresent());
@@ -101,9 +102,9 @@ public class HAGroupStoreRecordTest {
public void testHasSameInfo() {
String haGroupName = testName.getMethodName();
HAGroupStoreRecord record1 = getHAGroupStoreRecord(haGroupName,
- PROTOCOL_VERSION, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+ PROTOCOL_VERSION, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, HAGroupStoreRecord.DEFAULT_RECORD_VERSION);
HAGroupStoreRecord record2 = getHAGroupStoreRecord(haGroupName,
- PROTOCOL_VERSION, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+ PROTOCOL_VERSION, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, HAGroupStoreRecord.DEFAULT_RECORD_VERSION);
assertTrue(record1.hasSameInfo(record2)); // Same core info despite different state
assertTrue(record1.hasSameInfo(record1)); // reflexive
@@ -114,26 +115,32 @@ public class HAGroupStoreRecordTest {
public void testHasSameInfoNegative() {
String haGroupName = testName.getMethodName();
HAGroupStoreRecord record = getHAGroupStoreRecord(haGroupName,
- PROTOCOL_VERSION, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+ PROTOCOL_VERSION, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, HAGroupStoreRecord.DEFAULT_RECORD_VERSION);
// Different protocol version
HAGroupStoreRecord recordDifferentProtocol = getHAGroupStoreRecord(haGroupName,
- "2.0", HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+ "2.0", HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, HAGroupStoreRecord.DEFAULT_RECORD_VERSION);
assertFalse(record.hasSameInfo(recordDifferentProtocol));
assertFalse(recordDifferentProtocol.hasSameInfo(record));
// Different HA group name
String haGroupName2 = haGroupName + RandomStringUtils.randomAlphabetic(2);
HAGroupStoreRecord record2 = getHAGroupStoreRecord(haGroupName2,
- PROTOCOL_VERSION, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+ PROTOCOL_VERSION, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, HAGroupStoreRecord.DEFAULT_RECORD_VERSION);
assertFalse(record.hasSameInfo(record2));
assertFalse(record2.hasSameInfo(record));
// Different HA group state
HAGroupStoreRecord recordDifferentState = getHAGroupStoreRecord(haGroupName,
- PROTOCOL_VERSION, HAGroupStoreRecord.HAGroupState.STANDBY);
+ PROTOCOL_VERSION, HAGroupStoreRecord.HAGroupState.STANDBY, HAGroupStoreRecord.DEFAULT_RECORD_VERSION);
assertFalse(record.hasSameInfo(recordDifferentState));
assertFalse(recordDifferentState.hasSameInfo(record));
+
+ // Different state version
+ HAGroupStoreRecord recordDifferentStateVersion = getHAGroupStoreRecord(haGroupName,
+ PROTOCOL_VERSION, HAGroupStoreRecord.HAGroupState.STANDBY, HAGroupStoreRecord.DEFAULT_RECORD_VERSION + 1);
+ assertFalse(record.hasSameInfo(recordDifferentStateVersion));
+ assertFalse(recordDifferentStateVersion.hasSameInfo(record));
}
@Test
@@ -141,12 +148,14 @@ public class HAGroupStoreRecordTest {
String haGroupName = testName.getMethodName();
String protocolVersion = "1.5";
HAGroupStoreRecord.HAGroupState haGroupState = HAGroupStoreRecord.HAGroupState.STANDBY;
+ long stateVersion = HAGroupStoreRecord.DEFAULT_RECORD_VERSION + 1;
- HAGroupStoreRecord record = getHAGroupStoreRecord(haGroupName, protocolVersion, haGroupState);
+ HAGroupStoreRecord record = getHAGroupStoreRecord(haGroupName, protocolVersion, haGroupState, stateVersion);
assertEquals(haGroupName, record.getHaGroupName());
assertEquals(protocolVersion, record.getProtocolVersion());
assertEquals(haGroupState, record.getHAGroupState());
+ assertEquals(stateVersion, record.getRecordVersion());
assertEquals(haGroupState.getClusterRole(), record.getClusterRole());
}
@@ -154,11 +163,11 @@ public class HAGroupStoreRecordTest {
public void testEqualsAndHashCode() {
String haGroupName = testName.getMethodName();
HAGroupStoreRecord record1 = getHAGroupStoreRecord(haGroupName,
- PROTOCOL_VERSION, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+ PROTOCOL_VERSION, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, HAGroupStoreRecord.DEFAULT_RECORD_VERSION);
HAGroupStoreRecord record2 = getHAGroupStoreRecord(haGroupName,
- PROTOCOL_VERSION, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+ PROTOCOL_VERSION, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, HAGroupStoreRecord.DEFAULT_RECORD_VERSION);
HAGroupStoreRecord record3 = getHAGroupStoreRecord(haGroupName,
- PROTOCOL_VERSION, HAGroupStoreRecord.HAGroupState.STANDBY); // Different state
+ PROTOCOL_VERSION, HAGroupStoreRecord.HAGroupState.STANDBY, HAGroupStoreRecord.DEFAULT_RECORD_VERSION + 1); // Different state
// Test equals
assertEquals(record1, record2); // symmetric
@@ -175,19 +184,20 @@ public class HAGroupStoreRecordTest {
@Test
public void testToString() {
HAGroupStoreRecord record = getHAGroupStoreRecord(testName.getMethodName(),
- PROTOCOL_VERSION, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+ PROTOCOL_VERSION, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, HAGroupStoreRecord.DEFAULT_RECORD_VERSION);
String toString = record.toString();
// Verify all fields are present in toString
assertTrue(toString.contains(record.getHaGroupName()));
- assertTrue(toString.contains(record.getProtocolVersion()));
+ assertTrue(toString.contains(record.getProtocolVersion().toString()));
assertTrue(toString.contains(record.getHAGroupState().toString()));
+ assertTrue(toString.contains(String.valueOf(record.getRecordVersion())));
}
@Test
public void testToPrettyString() {
HAGroupStoreRecord record = getHAGroupStoreRecord(testName.getMethodName(),
- PROTOCOL_VERSION, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+ PROTOCOL_VERSION, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, HAGroupStoreRecord.DEFAULT_RECORD_VERSION);
LOG.info("toString(): {}", record.toString());
LOG.info("toPrettyString:\n{}", record.toPrettyString());
assertNotEquals(record.toString(), record.toPrettyString());
@@ -196,12 +206,17 @@ public class HAGroupStoreRecordTest {
@Test(expected = NullPointerException.class)
public void testConstructorWithNullHaGroupName() {
- getHAGroupStoreRecord(null, PROTOCOL_VERSION, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC);
+ getHAGroupStoreRecord(null, PROTOCOL_VERSION, HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, HAGroupStoreRecord.DEFAULT_RECORD_VERSION);
}
@Test(expected = NullPointerException.class)
public void testConstructorWithNullHAGroupState() {
- getHAGroupStoreRecord(testName.getMethodName(), PROTOCOL_VERSION, null);
+ getHAGroupStoreRecord(testName.getMethodName(), PROTOCOL_VERSION, null, HAGroupStoreRecord.DEFAULT_RECORD_VERSION);
+ }
+
+ @Test(expected = NullPointerException.class)
+ public void testConstructorWithNullStateVersion() {
+ new HAGroupStoreRecord(PROTOCOL_VERSION, testName.getMethodName(), HAGroupStoreRecord.HAGroupState.ACTIVE_IN_SYNC, null);
}
// Tests for HAGroupState enum
@@ -353,7 +368,7 @@ public class HAGroupStoreRecordTest {
// Private Helper Methods
private HAGroupStoreRecord getHAGroupStoreRecord(String haGroupName, String protocolVersion,
- HAGroupStoreRecord.HAGroupState haGroupState) {
- return new HAGroupStoreRecord(protocolVersion, haGroupName, haGroupState);
+ HAGroupStoreRecord.HAGroupState haGroupState, long stateVersion) {
+ return new HAGroupStoreRecord(protocolVersion, haGroupName, haGroupState, stateVersion);
}
}
\ No newline at end of file
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/jdbc/ParallelPhoenixContextTest.java b/phoenix-core/src/test/java/org/apache/phoenix/jdbc/ParallelPhoenixContextTest.java
index cd45b619bf..6fdad3488e 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/jdbc/ParallelPhoenixContextTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/jdbc/ParallelPhoenixContextTest.java
@@ -84,7 +84,7 @@ public class ParallelPhoenixContextTest {
ParallelPhoenixContext context =
new ParallelPhoenixContext(new Properties(),
new HighAvailabilityGroup(haGroupInfo,
- Mockito.mock(Properties.class),
+ new Properties(),
Mockito.mock(ClusterRoleRecord.class),
HighAvailabilityGroup.State.READY),
executorList, Lists.newArrayList(Boolean.FALSE, Boolean.TRUE),
@@ -105,7 +105,7 @@ public class ParallelPhoenixContextTest {
ParallelPhoenixContext context =
new ParallelPhoenixContext(new Properties(),
new HighAvailabilityGroup(haGroupInfo,
- Mockito.mock(Properties.class),
+ new Properties(),
Mockito.mock(ClusterRoleRecord.class),
HighAvailabilityGroup.State.READY),
executorList, Lists.newArrayList(Boolean.TRUE, Boolean.FALSE),
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/jdbc/ParallelPhoenixNullComparingResultSetTest.java b/phoenix-core/src/test/java/org/apache/phoenix/jdbc/ParallelPhoenixNullComparingResultSetTest.java
index dcea2ad8e6..fcd280df97 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/jdbc/ParallelPhoenixNullComparingResultSetTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/jdbc/ParallelPhoenixNullComparingResultSetTest.java
@@ -52,7 +52,7 @@ public class ParallelPhoenixNullComparingResultSetTest {
HAGroupInfo haGroupInfo = new HAGroupInfo("test", "test1", "test2");
context = new ParallelPhoenixContext(new Properties(),
new HighAvailabilityGroup(haGroupInfo,
- Mockito.mock(Properties.class),
+ new Properties(),
Mockito.mock(ClusterRoleRecord.class),
HighAvailabilityGroup.State.READY),
HighAvailabilityTestingUtility.getListOfSingleThreadExecutorServices(), null,
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/jdbc/PhoenixHAAdminToolTest.java b/phoenix-core/src/test/java/org/apache/phoenix/jdbc/PhoenixHAAdminToolTest.java
index 0e78da74d3..2267fa90bd 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/jdbc/PhoenixHAAdminToolTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/jdbc/PhoenixHAAdminToolTest.java
@@ -66,8 +66,8 @@ import org.slf4j.LoggerFactory;
*/
public class PhoenixHAAdminToolTest {
private static final Logger LOG = LoggerFactory.getLogger(PhoenixHAAdminToolTest.class);
- private static final String ZK1 = "zk1:2181:/hbase";
- private static final String ZK2 = "zk2:2181:/hbase";
+ private static final String MASTER1 = "master1:60010";
+ private static final String MASTER2 = "master2:60010";
private static final PrintStream STDOUT = System.out;
private static final ByteArrayOutputStream STDOUT_CAPTURE = new ByteArrayOutputStream();
@@ -77,7 +77,7 @@ public class PhoenixHAAdminToolTest {
private final CuratorFramework curator = Mockito.mock(CuratorFramework.class);
/** HA admin to test for one test case. */
private final PhoenixHAAdmin
- admin = new PhoenixHAAdmin(ZK1, new Configuration(), mockHighAvailibilityCuratorProvider);
+ admin = new PhoenixHAAdmin(MASTER1, new Configuration(), mockHighAvailibilityCuratorProvider);
private String haGroupName;
private ClusterRoleRecord recordV1;
@@ -91,8 +91,8 @@ public class PhoenixHAAdminToolTest {
haGroupName = testName.getMethodName();
recordV1 = new ClusterRoleRecord(
haGroupName, HighAvailabilityPolicy.FAILOVER,
- ZK1, ClusterRole.ACTIVE,
- ZK2, ClusterRole.STANDBY,
+ MASTER1, ClusterRole.ACTIVE,
+ MASTER2, ClusterRole.STANDBY,
1);
saveRecordV1ToZk();
}
@@ -153,8 +153,8 @@ public class PhoenixHAAdminToolTest {
String haGroupName2 = haGroupName + RandomStringUtils.randomAlphabetic(3);
ClusterRoleRecord record2 = new ClusterRoleRecord(
haGroupName2, HighAvailabilityPolicy.FAILOVER,
- ZK1, ClusterRole.ACTIVE,
- ZK2, ClusterRole.STANDBY,
+ MASTER1, ClusterRole.ACTIVE,
+ MASTER2, ClusterRole.STANDBY,
1);
String fileName = ClusterRoleRecordTest.createJsonFileWithRecords(recordV1, record2);
List<ClusterRoleRecord> records = new PhoenixHAAdminTool().readRecordsFromFile(fileName);
@@ -192,8 +192,8 @@ public class PhoenixHAAdminToolTest {
saveRecordV1ToZk();
ClusterRoleRecord recordV2 = new ClusterRoleRecord(
haGroupName, HighAvailabilityPolicy.FAILOVER,
- ZK1, ClusterRole.STANDBY,
- ZK2 , ClusterRole.STANDBY,
+ MASTER1, ClusterRole.STANDBY,
+ MASTER2 , ClusterRole.STANDBY,
2); // higher version than recordV1 so update should be tried
try {
result = admin.createOrUpdateDataOnZookeeper(recordV2);
@@ -213,8 +213,8 @@ public class PhoenixHAAdminToolTest {
saveRecordV1ToZk();
ClusterRoleRecord recordV0 = new ClusterRoleRecord(
haGroupName, HighAvailabilityPolicy.FAILOVER,
- ZK1, ClusterRole.STANDBY,
- ZK2 , ClusterRole.STANDBY,
+ MASTER1, ClusterRole.STANDBY,
+ MASTER2 , ClusterRole.STANDBY,
0); // lower version than recordV1
assertFalse(admin.createOrUpdateDataOnZookeeper(recordV0));
@@ -230,8 +230,8 @@ public class PhoenixHAAdminToolTest {
saveRecordV1ToZk();
ClusterRoleRecord record2 = new ClusterRoleRecord(
haGroupName, HighAvailabilityPolicy.FAILOVER,
- ZK1, ClusterRole.STANDBY,
- ZK2 , ClusterRole.STANDBY,
+ MASTER1, ClusterRole.STANDBY,
+ MASTER2 , ClusterRole.STANDBY,
1); // same version but different role1
try {
admin.createOrUpdateDataOnZookeeper(record2);
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/util/HAGroupStoreTestUtil.java b/phoenix-core/src/test/java/org/apache/phoenix/util/HAGroupStoreTestUtil.java
index abc4d02a50..9531188ad0 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/util/HAGroupStoreTestUtil.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/util/HAGroupStoreTestUtil.java
@@ -18,11 +18,13 @@
package org.apache.phoenix.util;
import org.apache.phoenix.jdbc.ClusterRoleRecord;
+import org.apache.phoenix.jdbc.HighAvailabilityPolicy;
import org.apache.phoenix.jdbc.PhoenixConnection;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.sql.Statement;
+import java.util.Properties;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_HA_GROUP_NAME;
import static org.apache.phoenix.util.PhoenixRuntime.JDBC_PROTOCOL_SEPARATOR;
@@ -47,7 +49,31 @@ public class HAGroupStoreTestUtil {
public static void upsertHAGroupRecordInSystemTable(String haGroupName, String zkUrl, String peerZKUrl,
ClusterRoleRecord.ClusterRole localClusterRole,
ClusterRoleRecord.ClusterRole peerClusterRole, String overrideConnZkUrl) throws SQLException {
- try (PhoenixConnection conn = (PhoenixConnection) DriverManager.getConnection(JDBC_PROTOCOL_ZK + JDBC_PROTOCOL_SEPARATOR + (overrideConnZkUrl != null ? overrideConnZkUrl : zkUrl));
+ upsertHAGroupRecordInSystemTable(haGroupName, zkUrl, peerZKUrl,zkUrl, peerZKUrl, localClusterRole, peerClusterRole,1L,1L, overrideConnZkUrl, HighAvailabilityPolicy.FAILOVER, new Properties());
+ }
+
+
+ public static void upsertHAGroupRecordInSystemTable(String haGroupName, String zkUrl, String peerZKUrl, String clusterUrl1, String clusterUrl2,
+ ClusterRoleRecord.ClusterRole localClusterRole, ClusterRoleRecord.ClusterRole peerClusterRole, String overrideConnZkUrl) throws SQLException {
+ upsertHAGroupRecordInSystemTable(haGroupName, zkUrl, peerZKUrl, clusterUrl1, clusterUrl2, localClusterRole, peerClusterRole,1L,1L, overrideConnZkUrl, HighAvailabilityPolicy.FAILOVER, new Properties());
+}
+
+ /**
+ * Upserts an HA group record into the system table for testing purposes.
+ *
+ * @param haGroupName the HA group name
+ * @param zkUrl the ZooKeeper URL for the local cluster
+ * @param peerZKUrl the ZooKeeper URL for the peer cluster
+ * @param localClusterRole the role of the local cluster
+ * @param peerClusterRole the role of the peer cluster
+ * @param overrideConnZkUrl optional override for the connection ZK URL
+ * @throws SQLException if the database operation fails
+ */
+ public static void upsertHAGroupRecordInSystemTable(String haGroupName, String zkUrl, String peerZKUrl, String clusterUrl1, String clusterUrl2,
+ ClusterRoleRecord.ClusterRole localClusterRole,
+ ClusterRoleRecord.ClusterRole peerClusterRole, long version1, long version2,
+ String overrideConnZkUrl, HighAvailabilityPolicy policy, Properties props) throws SQLException {
+ try (PhoenixConnection conn = (PhoenixConnection) DriverManager.getConnection(JDBC_PROTOCOL_ZK + JDBC_PROTOCOL_SEPARATOR + (overrideConnZkUrl != null ? overrideConnZkUrl : zkUrl), props);
Statement stmt = conn.createStatement()) {
// Only insert values that are not null
StringBuilder queryBuilder = new StringBuilder("UPSERT INTO " + SYSTEM_HA_GROUP_NAME + " (HA_GROUP_NAME, ");
@@ -63,13 +89,13 @@ public class HAGroupStoreTestUtil {
if (peerClusterRole != null) {
queryBuilder.append("CLUSTER_ROLE_2, ");
}
- if (zkUrl != null) {
+ if (clusterUrl1 != null) {
queryBuilder.append("CLUSTER_URL_1, ");
}
- if (peerZKUrl != null) {
+ if (clusterUrl2 != null) {
queryBuilder.append("CLUSTER_URL_2, ");
}
- queryBuilder.append("POLICY, VERSION) ");
+ queryBuilder.append("POLICY, VERSION_CLUSTER_1, VERSION_CLUSTER_2) ");
queryBuilder.append("VALUES ('" + haGroupName + "', ");
if (zkUrl != null) {
queryBuilder.append("'" + zkUrl + "', ");
@@ -83,13 +109,13 @@ public class HAGroupStoreTestUtil {
if (peerClusterRole != null) {
queryBuilder.append("'" + peerClusterRole + "', ");
}
- if (zkUrl != null) {
- queryBuilder.append("'" + zkUrl + "', ");
+ if (clusterUrl1 != null) {
+ queryBuilder.append("'" + clusterUrl1 + "', ");
}
- if (peerZKUrl != null) {
- queryBuilder.append("'" + peerZKUrl + "', ");
+ if (clusterUrl2 != null) {
+ queryBuilder.append("'" + clusterUrl2 + "', ");
}
- queryBuilder.append("'FAILOVER', 1)");
+ queryBuilder.append("'" + policy.name() + "', " + version1 + ", " + version2 + ")");
stmt.executeUpdate(queryBuilder.toString());
conn.commit();
}