You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by sh...@apache.org on 2023/08/31 16:54:15 UTC
[phoenix] branch master updated: PHOENIX-7029: Add support for multiple query services in PhoenixTestDriver (#1664)
This is an automated email from the ASF dual-hosted git repository.
shahrs87 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/phoenix.git
The following commit(s) were added to refs/heads/master by this push:
new ef4989a262 PHOENIX-7029: Add support for multiple query services in PhoenixTestDriver (#1664)
ef4989a262 is described below
commit ef4989a26230f922fe77b2886fb11f2135909b51
Author: palash <pa...@gmail.com>
AuthorDate: Thu Aug 31 09:54:08 2023 -0700
PHOENIX-7029: Add support for multiple query services in PhoenixTestDriver (#1664)
---
.../org/apache/phoenix/end2end/CreateTableIT.java | 2 +-
.../phoenix/iterate/ChunkedResultIteratorIT.java | 2 +-
.../apache/phoenix/jdbc/PhoenixTestDriverIT.java | 124 +++++++++++++++++++++
.../query/SkipSystemTablesExistenceCheckIT.java | 7 +-
.../apache/phoenix/jdbc/PhoenixEmbeddedDriver.java | 9 +-
.../phoenix/query/ConnectionQueryServicesImpl.java | 17 +--
.../apache/phoenix/compile/QueryMetaDataTest.java | 3 +-
.../org/apache/phoenix/jdbc/PhoenixTestDriver.java | 25 ++++-
.../java/org/apache/phoenix/query/BaseTest.java | 2 +-
9 files changed, 163 insertions(+), 28 deletions(-)
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/CreateTableIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/CreateTableIT.java
index 2b30f25b81..ab37d5794f 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/CreateTableIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/CreateTableIT.java
@@ -1150,7 +1150,7 @@ public class CreateTableIT extends ParallelStatsDisabledIT {
String fullTableName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, tableName);
String fullIndexeName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, indexName);
// Check system tables priorities.
- try (Admin admin = driver.getConnectionQueryServices(null, null).getAdmin();
+ try (Admin admin = driver.getConnectionQueryServices(getUrl(), new Properties()).getAdmin();
Connection c = DriverManager.getConnection(getUrl())) {
ResultSet rs = c.getMetaData().getTables("",
"\""+ PhoenixDatabaseMetaData.SYSTEM_CATALOG_SCHEMA + "\"",
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/iterate/ChunkedResultIteratorIT.java b/phoenix-core/src/it/java/org/apache/phoenix/iterate/ChunkedResultIteratorIT.java
index 7ba7e2b78d..d9a4155722 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/iterate/ChunkedResultIteratorIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/iterate/ChunkedResultIteratorIT.java
@@ -52,7 +52,7 @@ public class ChunkedResultIteratorIT
Properties props = new Properties();
props.setProperty(QueryServices.RENEW_LEASE_ENABLED, "false");
props.setProperty(QueryServices.SCAN_RESULT_CHUNK_SIZE, "2");
- Connection conn = DriverManager.getConnection(PHOENIX_JDBC_URL, props);
+ Connection conn = DriverManager.getConnection(getUrl(), props);
String tableName = generateUniqueName();
conn.createStatement().execute("CREATE TABLE " + tableName
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/PhoenixTestDriverIT.java b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/PhoenixTestDriverIT.java
new file mode 100644
index 0000000000..9dd0d741d9
--- /dev/null
+++ b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/PhoenixTestDriverIT.java
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.jdbc;
+
+import org.apache.phoenix.end2end.NeedsOwnMiniClusterTest;
+import org.apache.phoenix.query.BaseTest;
+import org.apache.phoenix.query.ConnectionQueryServices;
+import org.apache.phoenix.schema.PMetaData;
+import org.apache.phoenix.schema.PTableKey;
+import org.apache.phoenix.schema.PTableRef;
+import org.apache.phoenix.schema.TableNotFoundException;
+import org.apache.phoenix.thirdparty.com.google.common.collect.Maps;
+import org.apache.phoenix.util.PropertiesUtil;
+import org.apache.phoenix.util.QueryUtil;
+import org.apache.phoenix.util.ReadOnlyProps;
+import org.apache.phoenix.util.TestUtil;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.util.Map;
+import java.util.Properties;
+
+@Category(NeedsOwnMiniClusterTest.class)
+public class PhoenixTestDriverIT extends BaseTest {
+
+ @BeforeClass
+ public static synchronized void doSetup() throws Exception {
+ Map<String, String> props = Maps.newHashMapWithExpectedSize(1);
+ setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator()));
+ }
+
+ /**
+ * Test that connections created using the same url have the same CQSI object.
+ */
+ @Test
+ public void testSameCQSI() throws SQLException {
+ Properties props = PropertiesUtil.deepCopy(TestUtil.TEST_PROPERTIES);
+ String url = QueryUtil.getConnectionUrl(props, config, "client1");
+ try (Connection conn1 = DriverManager.getConnection(url);
+ Connection conn2 = DriverManager.getConnection(url)) {
+ ConnectionQueryServices cqs1 = conn1.unwrap(PhoenixConnection.class).getQueryServices();
+ ConnectionQueryServices cqs2 = conn2.unwrap(PhoenixConnection.class).getQueryServices();
+ Assert.assertNotNull(cqs1);
+ Assert.assertNotNull(cqs2);
+ Assert.assertEquals("Connections using the same URL should have the same CQSI object.", cqs1, cqs2);
+ }
+ }
+
+ /**
+ * Test that connections created using different urls have different CQSI objects.
+ */
+ @Test
+ public void testDifferentCQSI() throws SQLException {
+ Properties props = PropertiesUtil.deepCopy(TestUtil.TEST_PROPERTIES);
+ String url1 = QueryUtil.getConnectionUrl(props, config, "client1");
+ String url2 = QueryUtil.getConnectionUrl(props, config, "client2");
+ try (Connection conn1 = DriverManager.getConnection(url1);
+ Connection conn2 = DriverManager.getConnection(url2)) {
+ ConnectionQueryServices cqs1 = conn1.unwrap(PhoenixConnection.class).getQueryServices();
+ ConnectionQueryServices cqs2 = conn2.unwrap(PhoenixConnection.class).getQueryServices();
+ Assert.assertNotNull(cqs1);
+ Assert.assertNotNull(cqs2);
+ Assert.assertNotEquals("Connections using different URL should have different CQSI objects.", cqs1, cqs2);
+ }
+ }
+
+ /**
+ * Create 2 connections using URLs with different principals.
+ * Create a table using one connection and verify that the other connection's cache
+ * does not have this table's metadata.
+ */
+ @Test
+ public void testDifferentCQSICache() throws SQLException {
+ Properties props = PropertiesUtil.deepCopy(TestUtil.TEST_PROPERTIES);
+ String url1 = QueryUtil.getConnectionUrl(props, config, "client1");
+ String url2 = QueryUtil.getConnectionUrl(props, config, "client2");
+
+ // create a table with url1
+ String tableName = generateUniqueName();
+ try (Connection conn1 = DriverManager.getConnection(url1)) {
+ conn1.createStatement().execute("CREATE TABLE " + tableName
+ + "(k INTEGER NOT NULL PRIMARY KEY, v1 INTEGER)");
+
+ // this connection's cqsi cache should have the table metadata
+ PMetaData cache = conn1.unwrap(PhoenixConnection.class).getQueryServices().getMetaDataCache();
+ PTableRef pTableRef = cache.getTableRef(new PTableKey(null, tableName));
+ Assert.assertNotNull(pTableRef);
+ }
+ catch (TableNotFoundException e) {
+ Assert.fail("Table should have been found in CQSI cache.");
+ }
+
+ // table metadata should not be present in the other cqsi cache
+ Connection conn2 = DriverManager.getConnection(url2);
+ PMetaData cache = conn2.unwrap(PhoenixConnection.class).getQueryServices().getMetaDataCache();
+ try {
+ cache.getTableRef(new PTableKey(null, tableName));
+ Assert.fail("Table should not have been found in CQSI cache.");
+ }
+ catch (TableNotFoundException e) {
+ // expected since this connection was created using a different CQSI.
+ }
+ }
+}
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/query/SkipSystemTablesExistenceCheckIT.java b/phoenix-core/src/it/java/org/apache/phoenix/query/SkipSystemTablesExistenceCheckIT.java
index 96a9641314..2768b4d6ef 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/query/SkipSystemTablesExistenceCheckIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/query/SkipSystemTablesExistenceCheckIT.java
@@ -31,7 +31,6 @@ import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.Properties;
-import static org.apache.phoenix.util.TestUtil.PHOENIX_JDBC_URL;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
@@ -42,7 +41,7 @@ public class SkipSystemTablesExistenceCheckIT extends ParallelStatsDisabledIT {
@Test
public void testTableResultIterator() throws Exception {
- Connection conn = DriverManager.getConnection(PHOENIX_JDBC_URL);
+ Connection conn = DriverManager.getConnection(getUrl());
PhoenixConnection phoenixConnection = conn.unwrap(PhoenixConnection.class);
String tableName = generateUniqueName();
@@ -60,9 +59,9 @@ public class SkipSystemTablesExistenceCheckIT extends ParallelStatsDisabledIT {
ConnectionQueryServicesImpl queryServices = ((ConnectionQueryServicesImpl)phoenixConnection.getQueryServices());
phoenixConnection.close();
queryServices.setInitialized(false);
- queryServices.init(PHOENIX_JDBC_URL, props);
+ queryServices.init(getUrl(), props);
assertTrue(queryServices.isInitialized());
- conn = DriverManager.getConnection(PHOENIX_JDBC_URL, props);
+ conn = DriverManager.getConnection(getUrl(), props);
scanTable(conn, tableName);
}
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixEmbeddedDriver.java b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixEmbeddedDriver.java
index 1fb237e840..d97e88844c 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixEmbeddedDriver.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixEmbeddedDriver.java
@@ -650,8 +650,13 @@ public abstract class PhoenixEmbeddedDriver implements Driver, SQLCloseable {
}
public String toUrl() {
- return PhoenixRuntime.JDBC_PROTOCOL + PhoenixRuntime.JDBC_PROTOCOL_SEPARATOR
- + toString();
+ // Some tests like PhoenixConfigurationUtilTest add JDBC_PROTOCOL to zookeeper quorum
+ if (toString().contains(PhoenixRuntime.JDBC_PROTOCOL)) {
+ return toString();
+ } else {
+ return PhoenixRuntime.JDBC_PROTOCOL + PhoenixRuntime.JDBC_PROTOCOL_SEPARATOR
+ + toString();
+ }
}
private static ConnectionInfo defaultConnectionInfo(String url) throws SQLException {
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
index 3ae740c6d8..aec2f1960b 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
@@ -3555,6 +3555,9 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
success = true;
return null;
}
+ nSequenceSaltBuckets = ConnectionQueryServicesImpl.this.props.getInt(
+ QueryServices.SEQUENCE_SALT_BUCKETS_ATTRIB,
+ QueryServicesOptions.DEFAULT_SEQUENCE_TABLE_SALT_BUCKETS);
boolean isDoNotUpgradePropSet = UpgradeUtil.isNoUpgradeSet(props);
Properties scnProps = PropertiesUtil.deepCopy(props);
scnProps.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB,
@@ -3770,10 +3773,6 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
private void createOtherSystemTables(PhoenixConnection metaConnection) throws SQLException, IOException {
try {
-
- nSequenceSaltBuckets = ConnectionQueryServicesImpl.this.props.getInt(
- QueryServices.SEQUENCE_SALT_BUCKETS_ATTRIB,
- QueryServicesOptions.DEFAULT_SEQUENCE_TABLE_SALT_BUCKETS);
metaConnection.createStatement().execute(getSystemSequenceTableDDL(nSequenceSaltBuckets));
// When creating the table above, DDL statements are
// used. However, the CFD level properties are not set
@@ -4415,13 +4414,9 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
public PhoenixConnection upgradeSystemSequence(
PhoenixConnection metaConnection,
Map<String, String> systemTableToSnapshotMap) throws SQLException, IOException {
- int nSaltBuckets = ConnectionQueryServicesImpl.this.props.getInt(
- QueryServices.SEQUENCE_SALT_BUCKETS_ATTRIB,
- QueryServicesOptions.DEFAULT_SEQUENCE_TABLE_SALT_BUCKETS);
try (Statement statement = metaConnection.createStatement()) {
- String createSequenceTable = getSystemSequenceTableDDL(nSaltBuckets);
+ String createSequenceTable = getSystemSequenceTableDDL(nSequenceSaltBuckets);
statement.executeUpdate(createSequenceTable);
- nSequenceSaltBuckets = nSaltBuckets;
} catch (NewerTableAlreadyExistsException e) {
// Ignore, as this will happen if the SYSTEM.SEQUENCE already exists at this fixed
// timestamp.
@@ -4454,7 +4449,8 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
// If the table timestamp is before 4.2.1 then run the upgrade script
if (currentServerSideTableTimeStamp <
MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_2_1) {
- if (UpgradeUtil.upgradeSequenceTable(metaConnection, nSaltBuckets, e.getTable())) {
+ if (UpgradeUtil.upgradeSequenceTable(metaConnection, nSequenceSaltBuckets,
+ e.getTable())) {
metaConnection.removeTable(null,
PhoenixDatabaseMetaData.SYSTEM_SEQUENCE_SCHEMA,
PhoenixDatabaseMetaData.SYSTEM_SEQUENCE_TABLE,
@@ -4466,7 +4462,6 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
clearTableRegionCache(TableName.valueOf(
PhoenixDatabaseMetaData.SYSTEM_SEQUENCE_NAME_BYTES));
}
- nSequenceSaltBuckets = nSaltBuckets;
} else {
nSequenceSaltBuckets = getSaltBuckets(e);
}
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryMetaDataTest.java b/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryMetaDataTest.java
index 73ba2a46ef..902162d815 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryMetaDataTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryMetaDataTest.java
@@ -17,7 +17,6 @@
*/
package org.apache.phoenix.compile;
-import static org.apache.phoenix.util.TestUtil.PHOENIX_JDBC_URL;
import static org.junit.Assert.assertEquals;
import java.math.BigDecimal;
@@ -433,7 +432,7 @@ public class QueryMetaDataTest extends BaseConnectionlessQueryTest {
@Test
public void testBindParamMetaDataForNestedRVC() throws Exception {
String query = "SELECT organization_id, entity_id, a_string FROM aTable WHERE (organization_id, (entity_id, a_string)) >= (?, (?, ?))";
- Connection conn = DriverManager.getConnection(PHOENIX_JDBC_URL, PropertiesUtil.deepCopy(TestUtil.TEST_PROPERTIES));
+ Connection conn = DriverManager.getConnection(getUrl(), PropertiesUtil.deepCopy(TestUtil.TEST_PROPERTIES));
PreparedStatement statement = conn.prepareStatement(query);
ParameterMetaData pmd = statement.getParameterMetaData();
assertEquals(3, pmd.getParameterCount());
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/jdbc/PhoenixTestDriver.java b/phoenix-core/src/test/java/org/apache/phoenix/jdbc/PhoenixTestDriver.java
index f9fa9f8acd..8146368304 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/jdbc/PhoenixTestDriver.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/jdbc/PhoenixTestDriver.java
@@ -19,6 +19,8 @@ package org.apache.phoenix.jdbc;
import java.sql.Connection;
import java.sql.SQLException;
+import java.util.HashMap;
+import java.util.Map;
import java.util.Properties;
import java.util.concurrent.ThreadPoolExecutor;
@@ -44,9 +46,7 @@ import org.apache.phoenix.util.ReadOnlyProps;
*/
@ThreadSafe
public class PhoenixTestDriver extends PhoenixEmbeddedDriver {
-
- @GuardedBy("this")
- private ConnectionQueryServices connectionQueryServices;
+
private final ReadOnlyProps overrideProps;
@GuardedBy("this")
@@ -55,6 +55,10 @@ public class PhoenixTestDriver extends PhoenixEmbeddedDriver {
@GuardedBy("this")
private boolean closed = false;
+ @GuardedBy("this")
+ private final Map<ConnectionInfo, ConnectionQueryServices>
+ connectionQueryServicesMap = new HashMap<>();
+
public PhoenixTestDriver() {
this(ReadOnlyProps.EMPTY_PROPS);
}
@@ -86,14 +90,21 @@ public class PhoenixTestDriver extends PhoenixEmbeddedDriver {
@Override // public for testing
public synchronized ConnectionQueryServices getConnectionQueryServices(String url, Properties info) throws SQLException {
checkClosed();
- if (connectionQueryServices != null) { return connectionQueryServices; }
ConnectionInfo connInfo = ConnectionInfo.create(url);
+ ConnectionQueryServices connectionQueryServices = connectionQueryServicesMap.get(connInfo);
+ if (connectionQueryServices != null) {
+ return connectionQueryServices;
+ }
+ if (info == null) {
+ info = new Properties();
+ }
if (connInfo.isConnectionless()) {
connectionQueryServices = new ConnectionlessQueryServicesImpl(queryServices, connInfo, info);
} else {
connectionQueryServices = new ConnectionQueryServicesTestImpl(queryServices, connInfo, info);
}
connectionQueryServices.init(url, info);
+ connectionQueryServicesMap.put(connInfo, connectionQueryServices);
return connectionQueryServices;
}
@@ -110,14 +121,16 @@ public class PhoenixTestDriver extends PhoenixEmbeddedDriver {
}
closed = true;
try {
- if (connectionQueryServices != null) connectionQueryServices.close();
+ for (ConnectionQueryServices cqs : connectionQueryServicesMap.values()) {
+ cqs.close();
+ }
} finally {
ThreadPoolExecutor executor = queryServices.getExecutor();
try {
queryServices.close();
} finally {
if (executor != null) executor.shutdownNow();
- connectionQueryServices = null;
+ connectionQueryServicesMap.clear();;
}
}
}
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java b/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java
index ca7d08fa1e..e092d856bd 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java
@@ -1584,7 +1584,7 @@ public abstract class BaseTest {
*/
protected static synchronized void disableAndDropNonSystemTables() throws Exception {
if (driver == null) return;
- Admin admin = driver.getConnectionQueryServices(null, null).getAdmin();
+ Admin admin = driver.getConnectionQueryServices(getUrl(), new Properties()).getAdmin();
try {
List<TableDescriptor> tables = admin.listTableDescriptors();
for (TableDescriptor table : tables) {