You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by sk...@apache.org on 2020/01/14 19:07:56 UTC
[phoenix] branch 4.15-HBase-1.5 updated: PHOENIX-5644 and
PHOENIX-5651 addendum patch
This is an automated email from the ASF dual-hosted git repository.
skadam pushed a commit to branch 4.15-HBase-1.5
in repository https://gitbox.apache.org/repos/asf/phoenix.git
The following commit(s) were added to refs/heads/4.15-HBase-1.5 by this push:
new 7eb7ed7 PHOENIX-5644 and PHOENIX-5651 addendum patch
7eb7ed7 is described below
commit 7eb7ed77d8bf2ec4574f1ac9aa4d7df28a639354
Author: s.kadam <s....@apache.org>
AuthorDate: Mon Jan 13 10:51:24 2020 -0800
PHOENIX-5644 and PHOENIX-5651 addendum patch
---
.../apache/phoenix/end2end/IndexUpgradeToolIT.java | 95 ++++++++++++++++++++++
.../end2end/ParameterizedIndexUpgradeToolIT.java | 31 +------
.../index/IndexScrutinyMapperForTest.java | 17 ++++
.../phoenix/mapreduce/index/IndexUpgradeTool.java | 38 ++++-----
4 files changed, 131 insertions(+), 50 deletions(-)
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexUpgradeToolIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexUpgradeToolIT.java
new file mode 100644
index 0000000..56161a7
--- /dev/null
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexUpgradeToolIT.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.end2end;
+
+import org.apache.phoenix.mapreduce.index.IndexUpgradeTool;
+import org.apache.phoenix.query.BaseTest;
+import org.apache.phoenix.util.ReadOnlyProps;
+import org.apache.phoenix.util.SchemaUtil;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+
+public class IndexUpgradeToolIT extends BaseTest {
+
+ public static final String
+ VERIFY_COUNT_ASSERT_MESSAGE = "view-index count in system table doesn't match";
+
+ @Test
+ public void verifyViewAndViewIndexes() throws Exception {
+ String tableName = generateUniqueName();
+ String schemaName = generateUniqueName();
+ Map<String, String> props = Collections.emptyMap();
+ setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator()));
+
+ try (Connection conn = DriverManager.getConnection(getUrl(), new Properties())) {
+ prepareForTest(conn, schemaName, tableName);
+ IndexUpgradeTool iut = new IndexUpgradeTool();
+ String viewQuery = iut.getViewSql(tableName, schemaName);
+ ResultSet rs = conn.createStatement().executeQuery(viewQuery);
+ int countViews = 0;
+ List<String> views = new ArrayList<>();
+ List<Integer> indexCount = new ArrayList<>();
+ while (rs.next()) {
+ views.add(rs.getString(1));
+ countViews++;
+ }
+ Assert.assertEquals("view count in system table doesn't match", 2, countViews);
+ for (int i = 0; i < views.size(); i++) {
+ String viewName = SchemaUtil.getTableNameFromFullName(views.get(i));
+ String viewIndexQuery = iut.getViewIndexesSql(viewName, schemaName, null);
+ rs = conn.createStatement().executeQuery(viewIndexQuery);
+ int indexes = 0;
+ while (rs.next()) {
+ indexes++;
+ }
+ indexCount.add(indexes);
+ }
+ Assert.assertEquals(VERIFY_COUNT_ASSERT_MESSAGE, 2, (int) indexCount.get(0));
+ Assert.assertEquals(VERIFY_COUNT_ASSERT_MESSAGE, 1, (int) indexCount.get(1));
+ }
+ }
+
+ private void prepareForTest(Connection conn, String schemaName, String tableName)
+ throws SQLException {
+ String fullTableName = SchemaUtil.getTableName(schemaName, tableName);
+ conn.createStatement().execute("CREATE TABLE "+fullTableName+" (id bigint NOT NULL "
+ + "PRIMARY KEY, a.name varchar, sal bigint, address varchar)");
+
+ for (int i = 0; i<2; i++) {
+ String view = generateUniqueName();
+ String fullViewName = SchemaUtil.getTableName(schemaName, view);
+ conn.createStatement().execute("CREATE VIEW "+ fullViewName+ " (view_column varchar)"
+ + " AS SELECT * FROM " +fullTableName + " WHERE a.name = 'a'");
+ for(int j=i; j<2; j++) {
+ String index = generateUniqueName();
+ conn.createStatement().execute("CREATE INDEX " + index + " ON "
+ + fullViewName + " (view_column)");
+ }
+ }
+ }
+}
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ParameterizedIndexUpgradeToolIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ParameterizedIndexUpgradeToolIT.java
index fe8ee3c..8646985 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ParameterizedIndexUpgradeToolIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ParameterizedIndexUpgradeToolIT.java
@@ -89,7 +89,6 @@ public class ParameterizedIndexUpgradeToolIT extends BaseTest {
private static Map<String, String> serverProps = Maps.newHashMapWithExpectedSize(1),
clientProps = Maps.newHashMapWithExpectedSize(1);
- public static final String VERIFY_COUNT_ASSERT_MESSAGE = "view-index count in system table doesn't match";
private final boolean mutable;
private final boolean upgrade;
@@ -313,7 +312,7 @@ public class ParameterizedIndexUpgradeToolIT extends BaseTest {
//testing actual run
validate(false);
Assert.assertEquals("Index upgrade tool didn't wait for client cache to expire",
- true, iut.getWaited());
+ true, iut.getIsWaitComplete());
}
@Test
@@ -353,34 +352,6 @@ public class ParameterizedIndexUpgradeToolIT extends BaseTest {
}
}
- @Test
- public void verifyViewAndViewIndexes() throws SQLException {
- String tableName = "MOCK1";
- String schemaName = "TEST";
- String viewQuery = iut.getViewSql(tableName, schemaName);
- ResultSet rs = conn.createStatement().executeQuery(viewQuery);
- int countViews = 0;
- List<String> views = new ArrayList<>();
- List<Integer> indexCount = new ArrayList<>();
- while (rs.next()) {
- views.add(rs.getString(1));
- countViews++;
- }
- Assert.assertEquals("view count in system table doesn't match", 2, countViews);
- for (int i=0; i < views.size(); i++) {
- String viewName = SchemaUtil.getTableNameFromFullName(views.get(i));
- String viewIndexQuery = iut.getViewIndexesSql(viewName, schemaName, null);
- rs = conn.createStatement().executeQuery(viewIndexQuery);
- int indexes = 0;
- while (rs.next()) {
- indexes++;
- }
- indexCount.add(indexes);
- }
- Assert.assertEquals(VERIFY_COUNT_ASSERT_MESSAGE, 1, (int) indexCount.get(0));
- Assert.assertEquals(VERIFY_COUNT_ASSERT_MESSAGE, 2, (int) indexCount.get(1));
- }
-
@After
public void cleanup() throws IOException, SQLException {
//TEST.MOCK1,TEST1.MOCK2,TEST.MOCK3
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexScrutinyMapperForTest.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexScrutinyMapperForTest.java
index bdd5e45..99d50ee 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexScrutinyMapperForTest.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexScrutinyMapperForTest.java
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.phoenix.mapreduce.index;
import org.apache.phoenix.util.EnvironmentEdge;
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexUpgradeTool.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexUpgradeTool.java
index 6ea8e3b..8246d51 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexUpgradeTool.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexUpgradeTool.java
@@ -119,7 +119,7 @@ public class IndexUpgradeTool extends Configured implements Tool {
private String inputFile;
private boolean test = false;
- private boolean waited = false;
+ private boolean isWaitComplete = false;
public void setDryRun(boolean dryRun) {
this.dryRun = dryRun;
@@ -138,7 +138,7 @@ public class IndexUpgradeTool extends Configured implements Tool {
}
public void setTest(boolean test) { this.test = test; }
- public boolean getWaited() { return this.waited; }
+ public boolean getIsWaitComplete() { return this.isWaitComplete; }
public boolean getDryRun() {
return this.dryRun;
}
@@ -397,7 +397,7 @@ public class IndexUpgradeTool extends Configured implements Tool {
private void handleFailure(ConnectionQueryServices queryServices,
String dataTableFullName,
ArrayList<String> tableList) {
- LOGGER.info("Performing error handling to revert the steps taken during " +operation);
+ LOGGER.info("Performing error handling to revert the steps taken during " + operation);
HashSet<String> indexes = tablesAndIndexes.get(dataTableFullName);
try (Admin admin = queryServices.getAdmin()) {
upgrade = !upgrade;
@@ -422,24 +422,24 @@ public class IndexUpgradeTool extends Configured implements Tool {
private void enableImmutableTables(ConnectionQueryServices queryServices,
ArrayList<String> immutableList,
long startWaitTime) {
- long endWaitTime = EnvironmentEdgeManager.currentTimeMillis();
- long waitMore = getWaitMoreTime(endWaitTime, startWaitTime);
- while (waitMore>0) {
- // If the table is immutable, we need to wait for clients to purge
- // their caches of table metadata
- LOGGER.info("waiting for more " + waitMore + " ms for client cache "
- + "to expire for immutable tables");
+
+ while(true) {
+ long waitMore = getWaitMoreTime(startWaitTime);
+ if (waitMore <= 0) {
+ isWaitComplete = true;
+ break;
+ }
try {
- startWaitTime = EnvironmentEdgeManager.currentTimeMillis();
+ // If the table is immutable, we need to wait for clients to purge
+ // their caches of table metadata
Thread.sleep(waitMore);
- waited = true;
- } catch (InterruptedException e) {
- endWaitTime = EnvironmentEdgeManager.currentTimeMillis();
- waitMore = getWaitMoreTime(endWaitTime, startWaitTime);
+ isWaitComplete = true;
+ } catch(InterruptedException e) {
LOGGER.warning("Sleep before starting index rebuild is interrupted. "
+ "Attempting to sleep again! " + e.getMessage());
}
}
+
for (String dataTableFullName: immutableList) {
try (Admin admin = queryServices.getAdmin()) {
HashSet<String> indexes = tablesAndIndexes.get(dataTableFullName);
@@ -461,12 +461,10 @@ public class IndexUpgradeTool extends Configured implements Tool {
tableList.size()), ",");
}
- private long getWaitMoreTime(long endWaitTime, long startWaitTime) {
+ private long getWaitMoreTime(long startWaitTime) {
int waitTime = GLOBAL_INDEX_CHECKER_ENABLED_MAP_EXPIRATION_MIN+1;
- if(test) {
- return 1;
- }
- if(dryRun) {
+ long endWaitTime = EnvironmentEdgeManager.currentTimeMillis();
+ if(test || dryRun) {
return 0; //no wait
}
return (((waitTime) * 60000) - Math.abs(endWaitTime-startWaitTime));