You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by sk...@apache.org on 2020/01/29 19:59:09 UTC
[phoenix] branch 4.x-HBase-1.3 updated: PHOENIX-5689: TableNotFoundException in IndexUpgradeTool in case of v… (#692)
This is an automated email from the ASF dual-hosted git repository.
skadam pushed a commit to branch 4.x-HBase-1.3
in repository https://gitbox.apache.org/repos/asf/phoenix.git
The following commit(s) were added to refs/heads/4.x-HBase-1.3 by this push:
new 9b34743 PHOENIX-5689: TableNotFoundException in IndexUpgradeTool in case of v… (#692)
9b34743 is described below
commit 9b347438ee57f1d77d54b47d39afe3f6795f6457
Author: Swaroopa Kadam <sw...@gmail.com>
AuthorDate: Wed Jan 29 11:58:04 2020 -0800
PHOENIX-5689: TableNotFoundException in IndexUpgradeTool in case of v… (#692)
PHOENIX-5689: TableNotFoundException in IndexUpgradeTool in case of view indexes
---
.../apache/phoenix/end2end/IndexUpgradeToolIT.java | 81 +++++++++++++++++-----
.../end2end/ParameterizedIndexUpgradeToolIT.java | 39 +++++++++--
.../phoenix/mapreduce/index/IndexUpgradeTool.java | 80 ++++++---------------
3 files changed, 118 insertions(+), 82 deletions(-)
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexUpgradeToolIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexUpgradeToolIT.java
index 56161a7..586147f 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexUpgradeToolIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexUpgradeToolIT.java
@@ -19,76 +19,121 @@ package org.apache.phoenix.end2end;
import org.apache.phoenix.mapreduce.index.IndexUpgradeTool;
import org.apache.phoenix.query.BaseTest;
+import org.apache.phoenix.util.PhoenixRuntime;
+import org.apache.phoenix.util.PropertiesUtil;
import org.apache.phoenix.util.ReadOnlyProps;
import org.apache.phoenix.util.SchemaUtil;
import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Properties;
+import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
+
+@RunWith(Parameterized.class)
public class IndexUpgradeToolIT extends BaseTest {
public static final String
VERIFY_COUNT_ASSERT_MESSAGE = "view-index count in system table doesn't match";
+ private final boolean multiTenant;
+ private String tenantId = null;
+
+ public IndexUpgradeToolIT(boolean multiTenant) {
+ this.multiTenant = multiTenant;
+ }
+
+ @Parameters(name="isMultiTenant = {0}")
+ public static synchronized Collection<Boolean[]> data() {
+ return Arrays.asList(new Boolean[][] {
+ { true },{ false }
+ });
+ }
+
+ @BeforeClass
+ public static void setup() throws Exception {
+ Map<String, String> props = Collections.emptyMap();
+ setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator()));
+ }
@Test
public void verifyViewAndViewIndexes() throws Exception {
String tableName = generateUniqueName();
String schemaName = generateUniqueName();
- Map<String, String> props = Collections.emptyMap();
- setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator()));
-
- try (Connection conn = DriverManager.getConnection(getUrl(), new Properties())) {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ if (multiTenant) {
+ tenantId = generateUniqueName();
+ props.setProperty(PhoenixRuntime.TENANT_ID_ATTRIB, tenantId);
+ }
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
prepareForTest(conn, schemaName, tableName);
- IndexUpgradeTool iut = new IndexUpgradeTool();
- String viewQuery = iut.getViewSql(tableName, schemaName);
+ String viewQuery = IndexUpgradeTool.getViewSql(tableName, schemaName);
ResultSet rs = conn.createStatement().executeQuery(viewQuery);
- int countViews = 0;
List<String> views = new ArrayList<>();
- List<Integer> indexCount = new ArrayList<>();
+ List<String> tenants = new ArrayList<>();
while (rs.next()) {
+ //1st column has the view name and 2nd column has the Tenant ID
views.add(rs.getString(1));
- countViews++;
+ if(multiTenant) {
+ Assert.assertNotNull(rs.getString(2));
+ }
+ tenants.add(rs.getString(2));
}
- Assert.assertEquals("view count in system table doesn't match", 2, countViews);
+ Assert.assertEquals("view count in system table doesn't match", 2, views.size());
+
for (int i = 0; i < views.size(); i++) {
String viewName = SchemaUtil.getTableNameFromFullName(views.get(i));
- String viewIndexQuery = iut.getViewIndexesSql(viewName, schemaName, null);
+ String viewIndexQuery = IndexUpgradeTool.getViewIndexesSql(viewName, schemaName,
+ tenants.get(i));
rs = conn.createStatement().executeQuery(viewIndexQuery);
int indexes = 0;
while (rs.next()) {
indexes++;
}
- indexCount.add(indexes);
+ // first (i=0) TSV has 2 indexes, and second(i=1) TSV has 1 index.
+ Assert.assertEquals(VERIFY_COUNT_ASSERT_MESSAGE, 2-i, indexes);
}
- Assert.assertEquals(VERIFY_COUNT_ASSERT_MESSAGE, 2, (int) indexCount.get(0));
- Assert.assertEquals(VERIFY_COUNT_ASSERT_MESSAGE, 1, (int) indexCount.get(1));
}
}
private void prepareForTest(Connection conn, String schemaName, String tableName)
throws SQLException {
String fullTableName = SchemaUtil.getTableName(schemaName, tableName);
- conn.createStatement().execute("CREATE TABLE "+fullTableName+" (id bigint NOT NULL "
- + "PRIMARY KEY, a.name varchar, sal bigint, address varchar)");
+ Connection globalConn = conn;
+ if (multiTenant) {
+ Properties prop = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ globalConn = DriverManager.getConnection(getUrl(), prop);
+ }
+ globalConn.createStatement().execute("CREATE TABLE " + fullTableName + " (" + (
+ multiTenant ? "TENANT_ID VARCHAR(15) NOT NULL, " : "") + "id bigint NOT NULL,"
+ + " a.name varchar, sal bigint, address varchar CONSTRAINT PK_1 PRIMARY KEY "
+ + "(" + (multiTenant ? "TENANT_ID, " : "") + "ID)) "
+ + (multiTenant ? "MULTI_TENANT=true" : ""));
for (int i = 0; i<2; i++) {
String view = generateUniqueName();
String fullViewName = SchemaUtil.getTableName(schemaName, view);
- conn.createStatement().execute("CREATE VIEW "+ fullViewName+ " (view_column varchar)"
+ conn.createStatement().execute("CREATE VIEW "+ fullViewName
+ " AS SELECT * FROM " +fullTableName + " WHERE a.name = 'a'");
for(int j=i; j<2; j++) {
String index = generateUniqueName();
conn.createStatement().execute("CREATE INDEX " + index + " ON "
- + fullViewName + " (view_column)");
+ + fullViewName + " (address)");
}
}
}
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ParameterizedIndexUpgradeToolIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ParameterizedIndexUpgradeToolIT.java
index 8646985..099a2c6 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ParameterizedIndexUpgradeToolIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ParameterizedIndexUpgradeToolIT.java
@@ -44,7 +44,11 @@ import org.junit.experimental.categories.Category;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized.Parameters;
import org.junit.runners.Parameterized;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Captor;
+import org.mockito.Mock;
import org.mockito.Mockito;
+import org.mockito.MockitoAnnotations;
import java.io.BufferedWriter;
import java.io.File;
@@ -66,6 +70,7 @@ import static org.apache.phoenix.mapreduce.index.IndexUpgradeTool.ROLLBACK_OP;
import static org.apache.phoenix.mapreduce.index.IndexUpgradeTool.UPGRADE_OP;
import static org.apache.phoenix.query.QueryServices.DROP_METADATA_ATTRIB;
import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
+import static org.mockito.Mockito.times;
@RunWith(Parameterized.class)
@Category(NeedsOwnMiniClusterTest.class)
@@ -101,8 +106,16 @@ public class ParameterizedIndexUpgradeToolIT extends BaseTest {
private Admin admin;
private IndexUpgradeTool iut;
+ @Mock
+ private IndexTool indexToolMock;
+
+ @Captor
+ private ArgumentCaptor<String []> argCapture;
+
@Before
public void setup () throws Exception {
+ MockitoAnnotations.initMocks(this);
+
optionsBuilder = new StringBuilder();
setClusterProperties();
@@ -123,7 +136,7 @@ public class ParameterizedIndexUpgradeToolIT extends BaseTest {
admin = queryServices.getAdmin();
iut = new IndexUpgradeTool(upgrade ? UPGRADE_OP : ROLLBACK_OP, INPUT_LIST,
null, "/tmp/index_upgrade_" + UUID.randomUUID().toString(),
- true, Mockito.mock(IndexTool.class));
+ true, indexToolMock);
iut.setConf(getUtility().getConfiguration());
iut.setTest(true);
if (!mutable) {
@@ -311,8 +324,22 @@ public class ParameterizedIndexUpgradeToolIT extends BaseTest {
iut.executeTool();
//testing actual run
validate(false);
- Assert.assertEquals("Index upgrade tool didn't wait for client cache to expire",
- true, iut.getIsWaitComplete());
+ // testing if tool waited in case of immutable tables
+ if (!mutable) {
+ Assert.assertEquals("Index upgrade tool didn't wait for client cache to expire "
+ + "for immutable tables", true, iut.getIsWaitComplete());
+ } else {
+ Assert.assertEquals("Index upgrade tool waited for client cache to expire "
+ + "for mutable tables", false, iut.getIsWaitComplete());
+ }
+ if(upgrade) {
+ //verifying if index tool was started
+ Mockito.verify(indexToolMock,
+ times(11)) // for every index/view-index except index on transaction table
+ .run(argCapture.capture());
+ } else {
+ Mockito.verifyZeroInteractions(indexToolMock);
+ }
}
@Test
@@ -354,6 +381,9 @@ public class ParameterizedIndexUpgradeToolIT extends BaseTest {
@After
public void cleanup() throws IOException, SQLException {
+ if (conn == null) {
+ return;
+ }
//TEST.MOCK1,TEST1.MOCK2,TEST.MOCK3
conn.createStatement().execute("DROP INDEX INDEX1 ON TEST.MOCK1");
conn.createStatement().execute("DROP INDEX INDEX2 ON TEST.MOCK1");
@@ -390,8 +420,9 @@ public class ParameterizedIndexUpgradeToolIT extends BaseTest {
conn.close();
connTenant.close();
assertTableNotExists("TEST.MOCK1");
- assertTableNotExists("TEST.MOCK2");
+ assertTableNotExists("TEST1.MOCK2");
assertTableNotExists("TEST.MOCK3");
+ assertTableNotExists("TRANSACTIONAL_TABLE");
assertTableNotExists("TEST.MULTI_TENANT_TABLE");
}
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexUpgradeTool.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexUpgradeTool.java
index 729a147..5812c69 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexUpgradeTool.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexUpgradeTool.java
@@ -350,7 +350,7 @@ public class IndexUpgradeTool extends Configured implements Tool {
}
} catch (SQLException e) {
LOGGER.severe("Something went wrong while getting the PTable "
- + dataTableFullName + " "+e);
+ + dataTableFullName + " " + e);
return -1;
}
}
@@ -363,6 +363,9 @@ public class IndexUpgradeTool extends Configured implements Tool {
private long executeToolForImmutableTables(ConnectionQueryServices queryServices,
ArrayList<String> immutableList) {
+ if (immutableList.isEmpty()) {
+ return 0;
+ }
LOGGER.info("Started " + operation + " for immutable tables");
for (String dataTableFullName : immutableList) {
try (Admin admin = queryServices.getAdmin()) {
@@ -385,6 +388,9 @@ public class IndexUpgradeTool extends Configured implements Tool {
ConnectionQueryServices queryServices,
Configuration conf,
ArrayList<String> mutableTables) {
+ if (mutableTables.isEmpty()) {
+ return;
+ }
LOGGER.info("Started " + operation + " for mutable tables");
for (String dataTableFullName : mutableTables) {
try (Admin admin = queryServices.getAdmin()) {
@@ -432,7 +438,9 @@ public class IndexUpgradeTool extends Configured implements Tool {
private void enableImmutableTables(ConnectionQueryServices queryServices,
ArrayList<String> immutableList,
long startWaitTime) {
-
+ if (immutableList.isEmpty()) {
+ return;
+ }
while(true) {
long waitMore = getWaitMoreTime(startWaitTime);
if (waitMore <= 0) {
@@ -560,7 +568,7 @@ public class IndexUpgradeTool extends Configured implements Tool {
indexingTool.setConf(conf);
}
- startIndexRebuilds(conn, dataTableFullName, rebuildMap, indexingTool);
+ startIndexRebuilds(rebuildMap, indexingTool);
} catch (SQLException e) {
LOGGER.severe("Failed to prepare the map for index rebuilds " + e);
@@ -636,9 +644,7 @@ public class IndexUpgradeTool extends Configured implements Tool {
}
}
- private int startIndexRebuilds(Connection conn,
- String dataTable,
- HashMap<String, IndexInfo> indexInfos,
+ private int startIndexRebuilds(HashMap<String, IndexInfo> indexInfos,
IndexTool indexingTool) {
for(Map.Entry<String, IndexInfo> entry : indexInfos.entrySet()) {
@@ -652,51 +658,15 @@ public class IndexUpgradeTool extends Configured implements Tool {
(GLOBAL_INDEX_ID.equals(tenantId)?"":"_"+tenantId) +"_"
+ UUID.randomUUID().toString();
String[] args = getIndexToolArgValues(schema, baseTable, indexName, outFile, tenantId);
- Connection newConnection = conn;
- Connection tenantConnection = null;
try {
LOGGER.info("Rebuilding index: " + StringUtils.join( args,","));
if (!dryRun) {
- // If the index is in DISABLED state, indexTool will fail.
- // First to ALTER REBUILD ASYNC.
- // ALTER REBUILD ASYNC will set the index state to BUILDING
- // which is safe to make ACTIVE later.
- if (!Strings.isNullOrEmpty(tenantId) && !GLOBAL_INDEX_ID.equals(tenantId)) {
- Configuration conf = HBaseConfiguration.addHbaseResources(getConf());
- conf.set(PhoenixRuntime.TENANT_ID_ATTRIB, tenantId);
- newConnection = ConnectionUtil.getInputConnection(conf);
- tenantConnection = newConnection;
- }
-
- PTable indexPTable = PhoenixRuntime.getTable(newConnection,
- indexInfo.getPhysicalIndexTableName());
- if (indexPTable.getIndexState() == PIndexState.DISABLE) {
- String dataTableFullName = dataTable;
- if (!dataTableFullName.contains(":") && !dataTableFullName.contains(".")) {
- dataTableFullName = SchemaUtil.getTableName(schema, dataTable);
- }
- String
- stmt =
- String.format("ALTER INDEX %s ON %s REBUILD ASYNC", indexName,
- dataTableFullName);
- newConnection.createStatement().execute(stmt);
- }
-
indexingTool.run(args);
}
} catch (Exception e) {
LOGGER.severe("Something went wrong while building the index "
+ index + " " + e);
return -1;
- } finally {
- try {
- // Close tenant connection
- if (tenantConnection != null) {
- tenantConnection.close();
- }
- } catch (SQLException e) {
- LOGGER.warning("Couldn't close tenant connection. Ignoring");
- }
}
}
return 0;
@@ -775,8 +745,7 @@ public class IndexUpgradeTool extends Configured implements Tool {
}
String indexTableName = SchemaUtil.getTableNameFromFullName(physicalIndexName);
String pIndexName = SchemaUtil.getTableName(schemaName, indexTableName);
- IndexInfo indexInfo = new IndexInfo(schemaName, tableName,
- GLOBAL_INDEX_ID, pIndexName, pIndexName);
+ IndexInfo indexInfo = new IndexInfo(schemaName, tableName, GLOBAL_INDEX_ID, pIndexName);
indexInfos.put(physicalIndexName, indexInfo);
}
@@ -792,8 +761,7 @@ public class IndexUpgradeTool extends Configured implements Tool {
tenantId);
for (String viewIndex : viewIndexes) {
IndexInfo indexInfo = new IndexInfo(schemaName, viewName,
- tenantId == null ? GLOBAL_INDEX_ID : tenantId, viewIndex,
- viewIndexPhysicalName);
+ tenantId == null ? GLOBAL_INDEX_ID : tenantId, viewIndex);
indexInfos.put(viewIndex, indexInfo);
}
}
@@ -803,8 +771,9 @@ public class IndexUpgradeTool extends Configured implements Tool {
}
@VisibleForTesting
- public String getViewSql(String tableName, String schemaName) {
- return "SELECT DISTINCT COLUMN_FAMILY, TENANT_ID FROM "
+ public static String getViewSql(String tableName, String schemaName) {
+ //column_family has the view name and column_name has the Tenant ID
+ return "SELECT DISTINCT COLUMN_FAMILY, COLUMN_NAME FROM "
+ "SYSTEM.CHILD_LINK "
+ "WHERE TABLE_NAME = \'" + tableName + "\'"
+ (!Strings.isNullOrEmpty(schemaName) ? " AND TABLE_SCHEM = \'"
@@ -827,7 +796,7 @@ public class IndexUpgradeTool extends Configured implements Tool {
}
@VisibleForTesting
- public String getViewIndexesSql(String viewName, String schemaName, String tenantId) {
+ public static String getViewIndexesSql(String viewName, String schemaName, String tenantId) {
return "SELECT DISTINCT COLUMN_FAMILY FROM "
+ "SYSTEM.CATALOG "
+ "WHERE TABLE_NAME = \'" + viewName + "\'"
@@ -842,24 +811,19 @@ public class IndexUpgradeTool extends Configured implements Tool {
final private String baseTable;
final private String tenantId;
final private String indexName;
- final private String physicalIndexTableName;
- public IndexInfo(String schemaName, String baseTable, String tenantId, String indexName,
- String physicalIndexTableName) {
+ public IndexInfo(String schemaName, String baseTable, String tenantId, String indexName) {
this.schemaName = schemaName;
this.baseTable = baseTable;
this.tenantId = tenantId;
this.indexName = indexName;
- this.physicalIndexTableName = physicalIndexTableName;
}
public String getSchemaName() {
return schemaName;
}
- public String getBaseTable() {
- return baseTable;
- }
+ public String getBaseTable() { return baseTable; }
public String getTenantId() {
return tenantId;
@@ -868,10 +832,6 @@ public class IndexUpgradeTool extends Configured implements Tool {
public String getIndexName() {
return indexName;
}
-
- public String getPhysicalIndexTableName() {
- return physicalIndexTableName;
- }
}
public static void main (String[] args) throws Exception {