You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by ya...@apache.org on 2020/10/31 06:17:04 UTC
[phoenix] 01/02: PHOENIX-6179 : Relax maxLookBackAge check during
an upgrade
This is an automated email from the ASF dual-hosted git repository.
yanxinyi pushed a commit to branch 4.x
in repository https://gitbox.apache.org/repos/asf/phoenix.git
commit 9ffd740f2ee851abc485a9c283c741dce105dab4
Author: Viraj Jasani <vj...@apache.org>
AuthorDate: Tue Oct 27 23:29:15 2020 +0530
PHOENIX-6179 : Relax maxLookBackAge check during an upgrade
Signed-off-by: Xinyi Yan <ya...@apache.org>
---
.../phoenix/end2end/BackwardCompatibilityIT.java | 76 +++++++++++++++++-----
.../org/apache/phoenix/compile/QueryCompiler.java | 5 ++
2 files changed, 65 insertions(+), 16 deletions(-)
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BackwardCompatibilityIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BackwardCompatibilityIT.java
index d20610a..731d045 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BackwardCompatibilityIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BackwardCompatibilityIT.java
@@ -53,6 +53,7 @@ import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.regionserver.ScanInfoUtil;
import org.apache.hadoop.hbase.util.VersionInfo;
import org.apache.phoenix.coprocessor.MetaDataProtocol;
import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
@@ -179,7 +180,7 @@ public class BackwardCompatibilityIT {
public void testUpsertWithOldClient() throws Exception {
// Insert data with old client and read with new client
executeQueryWithClientVersion(compatibleClientVersion, CREATE_ADD);
- executeQueriesWithCurrentVersion(QUERY_CREATE_ADD);
+ executeQueriesWithCurrentVersion(QUERY_CREATE_ADD, false);
assertExpectedOutput(QUERY_CREATE_ADD);
}
@@ -187,7 +188,16 @@ public class BackwardCompatibilityIT {
public void testCreateDivergedViewWithOldClientReadFromNewClient() throws Exception {
// Create a base table, view and make it diverge from an old client
executeQueryWithClientVersion(compatibleClientVersion, CREATE_DIVERGED_VIEW);
- executeQueriesWithCurrentVersion(QUERY_CREATE_DIVERGED_VIEW);
+ executeQueriesWithCurrentVersion(QUERY_CREATE_DIVERGED_VIEW, false);
+ assertExpectedOutput(QUERY_CREATE_DIVERGED_VIEW);
+ }
+
+ @Test
+ public void testCreateDivergedViewWithOldClientReadWithMaxLookBackAge()
+ throws Exception {
+ // Create a base table, view and make it diverge from an old client
+ executeQueryWithClientVersion(compatibleClientVersion, CREATE_DIVERGED_VIEW);
+ executeQueriesWithCurrentVersion(QUERY_CREATE_DIVERGED_VIEW, true);
assertExpectedOutput(QUERY_CREATE_DIVERGED_VIEW);
}
@@ -214,15 +224,15 @@ public class BackwardCompatibilityIT {
@Test
public void testCreateDivergedViewWithNewClientReadFromOldClient() throws Exception {
- executeQueriesWithCurrentVersion(CREATE_DIVERGED_VIEW);
+ executeQueriesWithCurrentVersion(CREATE_DIVERGED_VIEW, false);
executeQueryWithClientVersion(compatibleClientVersion, QUERY_CREATE_DIVERGED_VIEW);
assertExpectedOutput(QUERY_CREATE_DIVERGED_VIEW);
}
@Test
public void testCreateDivergedViewWithNewClientReadFromNewClient() throws Exception {
- executeQueriesWithCurrentVersion(CREATE_DIVERGED_VIEW);
- executeQueriesWithCurrentVersion(QUERY_CREATE_DIVERGED_VIEW);
+ executeQueriesWithCurrentVersion(CREATE_DIVERGED_VIEW, false);
+ executeQueriesWithCurrentVersion(QUERY_CREATE_DIVERGED_VIEW, false);
assertExpectedOutput(QUERY_CREATE_DIVERGED_VIEW);
}
@@ -237,7 +247,7 @@ public class BackwardCompatibilityIT {
@Test
public void testSelectWithOldClient() throws Exception {
// Insert data with new client and read with old client
- executeQueriesWithCurrentVersion(CREATE_ADD);
+ executeQueriesWithCurrentVersion(CREATE_ADD, false);
executeQueryWithClientVersion(compatibleClientVersion, QUERY_CREATE_ADD);
assertExpectedOutput(QUERY_CREATE_ADD);
}
@@ -249,18 +259,43 @@ public class BackwardCompatibilityIT {
* 3. New Client reads the data inserted by the old client
* 4. New Client inserts more data into the tables created by old client
* 5. Old Client reads the data inserted by new client
+ * Use phoenix.max.lookback.age.seconds config and ensure that upgrade
+ * is not impacted by the config.
*
* @throws Exception thrown if any errors encountered during query execution or file IO
*/
@Test
+ public void testSelectUpsertWithNewClientWithMaxLookBackAge() throws Exception {
+ // Insert data with old client and read with new client
+ executeQueryWithClientVersion(compatibleClientVersion, CREATE_ADD);
+ executeQueriesWithCurrentVersion(QUERY_CREATE_ADD, true);
+ assertExpectedOutput(QUERY_CREATE_ADD);
+
+ // Insert more data with new client and read with old client
+ executeQueriesWithCurrentVersion(ADD_DATA, true);
+ executeQueryWithClientVersion(compatibleClientVersion, QUERY_ADD_DATA);
+ assertExpectedOutput(QUERY_ADD_DATA);
+ }
+
+ /**
+ * Scenario:
+ * 1. Old Client connects to the updated server
+ * 2. Old Client creates tables and inserts data
+ * 3. New Client reads the data inserted by the old client
+ * 4. New Client inserts more data into the tables created by old client
+ * 5. Old Client reads the data inserted by new client
+ *
+ * @throws Exception thrown if any errors encountered during query execution or file IO
+ */
+ @Test
public void testSelectUpsertWithNewClient() throws Exception {
// Insert data with old client and read with new client
executeQueryWithClientVersion(compatibleClientVersion, CREATE_ADD);
- executeQueriesWithCurrentVersion(QUERY_CREATE_ADD);
+ executeQueriesWithCurrentVersion(QUERY_CREATE_ADD, false);
assertExpectedOutput(QUERY_CREATE_ADD);
// Insert more data with new client and read with old client
- executeQueriesWithCurrentVersion(ADD_DATA);
+ executeQueriesWithCurrentVersion(ADD_DATA, false);
executeQueryWithClientVersion(compatibleClientVersion, QUERY_ADD_DATA);
assertExpectedOutput(QUERY_ADD_DATA);
}
@@ -278,13 +313,13 @@ public class BackwardCompatibilityIT {
@Test
public void testSelectUpsertWithOldClient() throws Exception {
// Insert data with new client and read with old client
- executeQueriesWithCurrentVersion(CREATE_ADD);
+ executeQueriesWithCurrentVersion(CREATE_ADD, false);
executeQueryWithClientVersion(compatibleClientVersion, QUERY_CREATE_ADD);
assertExpectedOutput(QUERY_CREATE_ADD);
// Insert more data with old client and read with new client
executeQueryWithClientVersion(compatibleClientVersion, ADD_DATA);
- executeQueriesWithCurrentVersion(QUERY_ADD_DATA);
+ executeQueriesWithCurrentVersion(QUERY_ADD_DATA, false);
assertExpectedOutput(QUERY_ADD_DATA);
}
@@ -301,7 +336,7 @@ public class BackwardCompatibilityIT {
public void testUpsertDeleteWithOldClient() throws Exception {
// Insert data with old client and read with new client
executeQueryWithClientVersion(compatibleClientVersion, CREATE_ADD);
- executeQueriesWithCurrentVersion(QUERY_CREATE_ADD);
+ executeQueriesWithCurrentVersion(QUERY_CREATE_ADD, false);
assertExpectedOutput(QUERY_CREATE_ADD);
// Deletes with the old client
@@ -322,13 +357,13 @@ public class BackwardCompatibilityIT {
@Test
public void testUpsertDeleteWithNewClient() throws Exception {
// Insert data with old client and read with new client
- executeQueriesWithCurrentVersion(CREATE_ADD);
+ executeQueriesWithCurrentVersion(CREATE_ADD, false);
executeQueryWithClientVersion(compatibleClientVersion, QUERY_CREATE_ADD);
assertExpectedOutput(QUERY_CREATE_ADD);
// Deletes with the new client
- executeQueriesWithCurrentVersion(ADD_DELETE);
- executeQueriesWithCurrentVersion(QUERY_ADD_DELETE);
+ executeQueriesWithCurrentVersion(ADD_DELETE, false);
+ executeQueriesWithCurrentVersion(QUERY_ADD_DELETE, false);
assertExpectedOutput(QUERY_ADD_DELETE);
}
@@ -336,7 +371,7 @@ public class BackwardCompatibilityIT {
public void testUpdatedSplitPolicyForSysTask() throws Exception {
executeQueryWithClientVersion(compatibleClientVersion,
CREATE_DIVERGED_VIEW);
- executeQueriesWithCurrentVersion(QUERY_CREATE_DIVERGED_VIEW);
+ executeQueriesWithCurrentVersion(QUERY_CREATE_DIVERGED_VIEW, false);
try (org.apache.hadoop.hbase.client.Connection conn =
hbaseTestUtil.getConnection(); Admin admin = conn.getAdmin()) {
HTableDescriptor tableDescriptor = admin.getTableDescriptor(
@@ -433,8 +468,17 @@ public class BackwardCompatibilityIT {
}
// Executes the SQL commands listed in the given operation file from the sql_files directory
- private void executeQueriesWithCurrentVersion(String operation) throws Exception {
+ private void executeQueriesWithCurrentVersion(String operation,
+ boolean setMaxLookBackAge) throws Exception {
Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ if (setMaxLookBackAge) {
+ props.put(
+ QueryServices.GLOBAL_INDEX_ROW_AGE_THRESHOLD_TO_DELETE_MS_ATTRIB,
+ Long.toString(0));
+ props.put(ScanInfoUtil.PHOENIX_MAX_LOOKBACK_AGE_CONF_KEY,
+ Integer.toString(15));
+ }
+
try (Connection conn = DriverManager.getConnection(url, props)) {
StringBuilder sb = new StringBuilder();
try (BufferedReader reader =
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/QueryCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/QueryCompiler.java
index 5f75e8b..58565de 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/QueryCompiler.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/QueryCompiler.java
@@ -170,6 +170,11 @@ public class QueryCompiler {
private void verifySCN() throws SQLException {
PhoenixConnection conn = statement.getConnection();
+ if (conn.isRunningUpgrade()) {
+ // PHOENIX-6179 : if upgrade is going on, we don't need to
+ // perform MaxLookBackAge check
+ return;
+ }
Long scn = conn.getSCN();
if (scn == null) {
return;