You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by st...@apache.org on 2021/03/10 15:29:52 UTC
[phoenix] branch master updated: PHOENIX-6376 Update
MetaDataProtocol.java for Phoenix 5.2
This is an automated email from the ASF dual-hosted git repository.
stoty pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/phoenix.git
The following commit(s) were added to refs/heads/master by this push:
new b6b41ce PHOENIX-6376 Update MetaDataProtocol.java for Phoenix 5.2
b6b41ce is described below
commit b6b41ce954926e1c11c96c26bbfab2a31e9936fc
Author: Xinyi Yan <ya...@apache.org>
AuthorDate: Mon Mar 1 13:20:06 2021 -0800
PHOENIX-6376 Update MetaDataProtocol.java for Phoenix 5.2
includes PHOENIX-6399 Updating BackwardCompatibilityIT supported versions
Co-authored-by: Istvan Toth <st...@apache.org>
---
...BackwardCompatibilityForSplittableSyscatIT.java | 7 +--
.../phoenix/end2end/BackwardCompatibilityIT.java | 31 ++++++------
.../end2end/BackwardCompatibilityTestUtil.java | 56 +++++++++++++++++++---
.../it/resources/compatible_client_versions.json | 15 +++---
.../src/it/resources/scripts/execute_query.sh | 18 +++----
.../phoenix/coprocessor/MetaDataProtocol.java | 10 ++--
6 files changed, 95 insertions(+), 42 deletions(-)
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BackwardCompatibilityForSplittableSyscatIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BackwardCompatibilityForSplittableSyscatIT.java
index 92f4471..e7ee8f6 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BackwardCompatibilityForSplittableSyscatIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BackwardCompatibilityForSplittableSyscatIT.java
@@ -18,6 +18,7 @@
package org.apache.phoenix.end2end;
import org.apache.phoenix.util.PhoenixRuntime;
+import org.apache.phoenix.end2end.BackwardCompatibilityTestUtil.MavenCoordinates;
import org.apache.phoenix.thirdparty.com.google.common.collect.Maps;
import org.junit.Before;
import org.junit.Test;
@@ -45,16 +46,16 @@ import static org.apache.phoenix.end2end.BackwardCompatibilityTestUtil.UpgradePr
@RunWith(Parameterized.class)
@Category(NeedsOwnMiniClusterTest.class)
public class BackwardCompatibilityForSplittableSyscatIT extends SplitSystemCatalogIT {
- private final String compatibleClientVersion;
+ private final MavenCoordinates compatibleClientVersion;
private String zkQuorum;
private String url;
@Parameterized.Parameters(name = "BackwardCompatibilityForSplitableSyscatIT_compatibleClientVersion={0}")
- public static synchronized Collection<String> data() throws Exception {
+ public static synchronized Collection<MavenCoordinates> data() throws Exception {
return computeClientVersions();
}
- public BackwardCompatibilityForSplittableSyscatIT(String compatibleClientVersion) {
+ public BackwardCompatibilityForSplittableSyscatIT(MavenCoordinates compatibleClientVersion) {
this.compatibleClientVersion = compatibleClientVersion;
}
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BackwardCompatibilityIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BackwardCompatibilityIT.java
index a1b2d0e..17a7b9d 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BackwardCompatibilityIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BackwardCompatibilityIT.java
@@ -57,14 +57,17 @@ import org.apache.hadoop.hbase.client.TableDescriptor;
import org.apache.phoenix.compat.hbase.CompatUtil;
import org.apache.phoenix.coprocessor.SystemCatalogRegionObserver;
import org.apache.phoenix.coprocessor.TaskMetaDataEndpoint;
+import org.apache.phoenix.end2end.BackwardCompatibilityTestUtil.MavenCoordinates;
import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
import org.apache.phoenix.jdbc.PhoenixDriver;
import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.query.QueryServicesOptions;
import org.apache.phoenix.schema.SystemTaskSplitPolicy;
import org.apache.phoenix.util.PhoenixRuntime;
+import org.apache.phoenix.util.ServerUtil.ConnectionFactory;
import org.junit.After;
import org.junit.Before;
+import org.junit.Ignore;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.runner.RunWith;
@@ -84,21 +87,21 @@ import org.slf4j.LoggerFactory;
public class BackwardCompatibilityIT {
private static final Logger LOGGER = LoggerFactory.getLogger(
- BackwardCompatibilityIT.class);
+ BackwardCompatibilityIT.class);
- private final String compatibleClientVersion;
+ private final MavenCoordinates compatibleClientVersion;
private static Configuration conf;
private static HBaseTestingUtility hbaseTestUtil;
private static String zkQuorum;
private static String url;
private String tmpDir;
- public BackwardCompatibilityIT(String compatibleClientVersion) {
+ public BackwardCompatibilityIT(MavenCoordinates compatibleClientVersion) {
this.compatibleClientVersion = compatibleClientVersion;
}
@Parameters(name = "BackwardCompatibilityIT_compatibleClientVersion={0}")
- public static synchronized Collection<String> data() throws Exception {
+ public static synchronized Collection<MavenCoordinates> data() throws Exception {
return computeClientVersions();
}
@@ -115,11 +118,11 @@ public class BackwardCompatibilityIT {
DriverManager.registerDriver(PhoenixDriver.INSTANCE);
checkForPreConditions(compatibleClientVersion, conf);
}
-
+
@After
- public void cleanUpAfterTest() throws Exception {
- boolean refCountLeaked = CompatUtil.isAnyStoreRefCountLeaked(
- hbaseTestUtil.getAdmin());
+ public synchronized void cleanUpAfterTest() throws Exception {
+ boolean refCountLeaked = CompatUtil.isAnyStoreRefCountLeaked(hbaseTestUtil.getAdmin());
+ ConnectionFactory.shutdown();
try {
DriverManager.deregisterDriver(PhoenixDriver.INSTANCE);
} finally {
@@ -333,7 +336,7 @@ public class BackwardCompatibilityIT {
executeQueryWithClientVersion(compatibleClientVersion,
CREATE_DIVERGED_VIEW, zkQuorum);
- String[] versionArr = compatibleClientVersion.split("\\.");
+ String[] versionArr = compatibleClientVersion.getVersion().split("\\.");
int majorVersion = Integer.parseInt(versionArr[0]);
int minorVersion = Integer.parseInt(versionArr[1]);
org.apache.hadoop.hbase.client.Connection conn = null;
@@ -375,12 +378,11 @@ public class BackwardCompatibilityIT {
TaskMetaDataEndpoint.class.getName()));
assertExpectedOutput(QUERY_CREATE_DIVERGED_VIEW);
admin.close();
- conn.close();
}
@Test
public void testSystemTaskCreationWithIndexAsyncRebuild() throws Exception {
- String[] versionArr = compatibleClientVersion.split("\\.");
+ String[] versionArr = compatibleClientVersion.getVersion().split("\\.");
int majorVersion = Integer.parseInt(versionArr[0]);
int minorVersion = Integer.parseInt(versionArr[1]);
// index async rebuild support min version check
@@ -397,11 +399,12 @@ public class BackwardCompatibilityIT {
@Test
public void testViewIndexIdCreatedWithOldClient() throws Exception {
executeQueryWithClientVersion(compatibleClientVersion, ADD_VIEW_INDEX, zkQuorum);
- try (org.apache.hadoop.hbase.client.Connection conn =
- hbaseTestUtil.getConnection(); Admin admin = conn.getAdmin()) {
+ org.apache.hadoop.hbase.client.Connection conn = hbaseTestUtil.getConnection();
+ try (Admin admin = conn.getAdmin()) {
HTableDescriptor tableDescriptor = admin.getTableDescriptor(
TableName.valueOf(PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME));
- assertFalse("Coprocessor " + SystemCatalogRegionObserver.class.getName()
+ //The oldest client we test is 5.1.0, which already adds SystemCatalogRegionObserver
+ assertTrue("Coprocessor " + SystemCatalogRegionObserver.class.getName()
+ " has been added with compatible client version: "
+ compatibleClientVersion, tableDescriptor.hasCoprocessor(
SystemCatalogRegionObserver.class.getName()));
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BackwardCompatibilityTestUtil.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BackwardCompatibilityTestUtil.java
index 1ad17f7..e7c7d88 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BackwardCompatibilityTestUtil.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BackwardCompatibilityTestUtil.java
@@ -103,7 +103,7 @@ public final class BackwardCompatibilityTestUtil {
private BackwardCompatibilityTestUtil() {
}
- public static List<String> computeClientVersions() throws Exception {
+ public static List<MavenCoordinates> computeClientVersions() throws Exception {
String hbaseVersion = VersionInfo.getVersion();
Pattern p = Pattern.compile("\\d+\\.\\d+");
Matcher m = p.matcher(hbaseVersion);
@@ -111,7 +111,7 @@ public final class BackwardCompatibilityTestUtil {
if (m.find()) {
hbaseProfile = m.group();
}
- List<String> clientVersions = Lists.newArrayList();
+ List<MavenCoordinates> clientVersions = Lists.newArrayList();
ObjectMapper mapper = new ObjectMapper();
mapper.configure(JsonParser.Feature.ALLOW_COMMENTS, true);
try (InputStream inputStream = BackwardCompatibilityIT.class
@@ -120,14 +120,14 @@ public final class BackwardCompatibilityTestUtil {
JsonNode jsonNode = mapper.readTree(inputStream);
JsonNode HBaseProfile = jsonNode.get(hbaseProfile);
for (final JsonNode clientVersion : HBaseProfile) {
- clientVersions.add(clientVersion.textValue() + "-HBase-" + hbaseProfile);
+ clientVersions.add(mapper.treeToValue(clientVersion, MavenCoordinates.class));
}
}
return clientVersions;
}
// Executes the queries listed in the operation file with a given client version
- public static void executeQueryWithClientVersion(String clientVersion, String operation,
+ public static void executeQueryWithClientVersion(MavenCoordinates clientVersion, String operation,
String zkQuorum) throws Exception {
List<String> cmdParams = Lists.newArrayList();
cmdParams.add(BASH);
@@ -136,7 +136,9 @@ public final class BackwardCompatibilityTestUtil {
assertNotNull(fileUrl);
cmdParams.add(new File(fileUrl.getFile()).getAbsolutePath());
cmdParams.add(zkQuorum);
- cmdParams.add(clientVersion);
+ cmdParams.add(clientVersion.getGroupId());
+ cmdParams.add(clientVersion.getArtifactId());
+ cmdParams.add(clientVersion.getVersion());
fileUrl = BackwardCompatibilityIT.class.getClassLoader()
.getResource(SQL_DIR + operation + SQL_EXTENSION);
@@ -196,11 +198,11 @@ public final class BackwardCompatibilityTestUtil {
}
- public static void checkForPreConditions(String compatibleClientVersion, Configuration conf) throws Exception {
+ public static void checkForPreConditions(MavenCoordinates compatibleClientVersion, Configuration conf) throws Exception {
// For the first code cut of any major version, there wouldn't be any backward compatible
// clients. Hence the test wouldn't run and just return true when the client
// version to be tested is same as current version
- assumeFalse(compatibleClientVersion.contains(MetaDataProtocol.CURRENT_CLIENT_VERSION));
+ assumeFalse(compatibleClientVersion.getVersion().contains(MetaDataProtocol.CURRENT_CLIENT_VERSION));
// Make sure that cluster is clean before test execution with no system tables
try (org.apache.hadoop.hbase.client.Connection conn =
ConnectionFactory.createConnection(conf);
@@ -321,4 +323,44 @@ public final class BackwardCompatibilityTestUtil {
}
}
}
+
+ public static class MavenCoordinates {
+ private String groupId = "org.apache.phoenix";
+ private String artifactId;
+ private String version;
+
+ public MavenCoordinates() {
+ super();
+ }
+
+ public String getGroupId() {
+ return groupId;
+ }
+
+ public void setGroupId(String groupId) {
+ this.groupId = groupId;
+ }
+
+ public String getArtifactId() {
+ return artifactId;
+ }
+
+ public void setArtifactId(String artifactId) {
+ this.artifactId = artifactId;
+ }
+
+ public String getVersion() {
+ return version;
+ }
+
+ public void setVersion(String version) {
+ this.version = version;
+ }
+
+ @Override
+ public String toString() {
+ return groupId + ":" + artifactId + ":" + version;
+ }
+
+ }
}
diff --git a/phoenix-core/src/it/resources/compatible_client_versions.json b/phoenix-core/src/it/resources/compatible_client_versions.json
index aa94642..ed189cc 100644
--- a/phoenix-core/src/it/resources/compatible_client_versions.json
+++ b/phoenix-core/src/it/resources/compatible_client_versions.json
@@ -17,11 +17,12 @@
*/
{
"_comment": "Lists all phoenix compatible client versions against the current branch version for a given hbase profile If hbase profile is 1.3, phoenix client versions 4.14.3 and 4.15.0 are tested against current branch version",
- "1.3": ["4.14.3", "4.15.0"],
- "1.4": ["4.14.3", "4.15.0"],
- "1.5": ["4.15.0"],
- "2.1": ["5.1.0"],
- "2.2": ["5.1.0"],
- "2.3": ["5.1.0"],
- "2.4": ["5.1.0"]
+ "1.3": [ {"artifactId":"phoenix-client", "version":"4.14.3-HBase-1.3"}, {"artifactId":"phoenix-client", "version":"4.15.0-HBase-1.3"}, {"artifactId":"phoenix-client-hbase-1.3", "version":"4.16.0"} ],
+ "1.4": [ {"artifactId":"phoenix-client", "version":"4.14.3-HBase-1.4"}, {"artifactId":"phoenix-client", "version":"4.15.0-HBase-1.4"}, {"artifactId":"phoenix-client-hbase-1.4", "version":"4.16.0"} ],
+ "1.5": [ {"artifactId":"phoenix-client", "version":"4.15-HBase-1.5"}, {"artifactId":"phoenix-client-hbase-1.5", "version":"4.16.0"} ],
+ "1.6": [ {"artifactId":"phoenix-client-hbase-1.6", "version":"4.16.0"} ],
+ "2.1": [ {"artifactId":"phoenix-client-hbase-2.1", "version":"5.1.0"} ],
+ "2.2": [ {"artifactId":"phoenix-client-hbase-2.2", "version":"5.1.0"} ],
+ "2.3": [ {"artifactId":"phoenix-client-hbase-2.3", "version":"5.1.0"} ],
+ "2.4": [ {"artifactId":"phoenix-client-hbase-2.4", "version":"5.1.0"} ]
}
diff --git a/phoenix-core/src/it/resources/scripts/execute_query.sh b/phoenix-core/src/it/resources/scripts/execute_query.sh
index c1fd78c..26e037e 100644
--- a/phoenix-core/src/it/resources/scripts/execute_query.sh
+++ b/phoenix-core/src/it/resources/scripts/execute_query.sh
@@ -20,21 +20,23 @@
# This script is intended to run the sql queries in a file with the given client version
zk_url=$1
-client_version=$2
-sqlfile=$3
-resultfile=$4
-tmp_dir=$5
-maven_home=$6
+client_group_id=$2
+client_artifact_id=$3
+client_version=$4
+sqlfile=$5
+resultfile=$6
+tmp_dir=$7
+maven_home=$8
if [ -n $maven_home ]; then
export PATH=$maven_home/bin:$PATH
fi
-mvn -B dependency:get -Dartifact=org.apache.phoenix:phoenix-client:$client_version
-mvn -B dependency:copy -Dartifact=org.apache.phoenix:phoenix-client:$client_version \
+mvn -B dependency:get -Dartifact=${client_group_id}:${client_artifact_id}:${client_version}
+mvn -B dependency:copy -Dartifact=${client_group_id}:${client_artifact_id}:${client_version} \
-DoutputDirectory=$tmp_dir
-phoenix_client_jar=$tmp_dir/phoenix-client-$client_version.jar
+phoenix_client_jar=$tmp_dir/${client_artifact_id}-${client_version}.jar
java -cp ".:$phoenix_client_jar" sqlline.SqlLine -d org.apache.phoenix.jdbc.PhoenixDriver \
-u jdbc:phoenix:$zk_url -n none -p none --color=false --fastConnect=true --outputformat=csv \
--silent=true --verbose=false --isolation=TRANSACTION_READ_COMMITTED --run=$sqlfile &> $resultfile
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java
index e4fae7e..1add8ae 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java
@@ -65,7 +65,8 @@ import com.google.protobuf.ByteString;
*/
public abstract class MetaDataProtocol extends MetaDataService {
public static final int PHOENIX_MAJOR_VERSION = 5;
- public static final int PHOENIX_MINOR_VERSION = 1;
+ public static final int PHOENIX_MINOR_VERSION = 2;
+
public static final int PHOENIX_PATCH_NUMBER = 0;
public static final int PHOENIX_VERSION =
VersionUtil.encodeVersion(PHOENIX_MAJOR_VERSION, PHOENIX_MINOR_VERSION, PHOENIX_PATCH_NUMBER);
@@ -99,8 +100,10 @@ public abstract class MetaDataProtocol extends MetaDataService {
public static final long MIN_SYSTEM_TABLE_TIMESTAMP_4_15_0 = MIN_TABLE_TIMESTAMP + 29;
public static final long MIN_SYSTEM_TABLE_TIMESTAMP_4_16_0 = MIN_TABLE_TIMESTAMP + 33;
public static final long MIN_SYSTEM_TABLE_TIMESTAMP_5_1_0 = MIN_SYSTEM_TABLE_TIMESTAMP_4_16_0;
+ public static final long MIN_SYSTEM_TABLE_TIMESTAMP_4_17_0 = MIN_TABLE_TIMESTAMP + 34;
+ public static final long MIN_SYSTEM_TABLE_TIMESTAMP_5_2_0 = MIN_SYSTEM_TABLE_TIMESTAMP_4_17_0;
// MIN_SYSTEM_TABLE_TIMESTAMP needs to be set to the max of all the MIN_SYSTEM_TABLE_TIMESTAMP_* constants
- public static final long MIN_SYSTEM_TABLE_TIMESTAMP = MIN_SYSTEM_TABLE_TIMESTAMP_5_1_0;
+ public static final long MIN_SYSTEM_TABLE_TIMESTAMP = MIN_SYSTEM_TABLE_TIMESTAMP_5_2_0;
// Version below which we should disallow usage of mutable secondary indexing.
public static final int MUTABLE_SI_VERSION_THRESHOLD = VersionUtil.encodeVersion("0", "94", "10");
@@ -146,7 +149,8 @@ public abstract class MetaDataProtocol extends MetaDataService {
TIMESTAMP_VERSION_MAP.put(MIN_SYSTEM_TABLE_TIMESTAMP_4_12_0, "4.12.x");
TIMESTAMP_VERSION_MAP.put(MIN_SYSTEM_TABLE_TIMESTAMP_4_13_0, "4.13.x");
TIMESTAMP_VERSION_MAP.put(MIN_SYSTEM_TABLE_TIMESTAMP_5_0_0, "5.0.x");
- TIMESTAMP_VERSION_MAP.put(MIN_SYSTEM_TABLE_TIMESTAMP_5_0_0, "5.1.x");
+ TIMESTAMP_VERSION_MAP.put(MIN_SYSTEM_TABLE_TIMESTAMP_5_1_0, "5.1.x");
+ TIMESTAMP_VERSION_MAP.put(MIN_SYSTEM_TABLE_TIMESTAMP_5_2_0, "5.2.x");
}
public static final String CURRENT_CLIENT_VERSION = PHOENIX_MAJOR_VERSION + "." + PHOENIX_MINOR_VERSION + "." + PHOENIX_PATCH_NUMBER;