You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by sk...@apache.org on 2019/07/22 22:46:08 UTC
[phoenix] branch 4.14-HBase-1.3 updated: PHOENIX-5395:
IndexUpgradeTool passes data table instead of view for post upgrade view
index rebuild
This is an automated email from the ASF dual-hosted git repository.
skadam pushed a commit to branch 4.14-HBase-1.3
in repository https://gitbox.apache.org/repos/asf/phoenix.git
The following commit(s) were added to refs/heads/4.14-HBase-1.3 by this push:
new efe6cc1 PHOENIX-5395: IndexUpgradeTool passes data table instead of view for post upgrade view index rebuild
efe6cc1 is described below
commit efe6cc188e94ffc9ee9ecec7d4eaf0b9f6853264
Author: s.kadam <sk...@apache.org>
AuthorDate: Mon Jul 22 15:45:24 2019 -0700
PHOENIX-5395: IndexUpgradeTool passes data table instead of view for post upgrade view index rebuild
---
.../phoenix/mapreduce/index/IndexUpgradeTool.java | 198 +++++++++++++++------
1 file changed, 141 insertions(+), 57 deletions(-)
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexUpgradeTool.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexUpgradeTool.java
index 0da4153..7855c40 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexUpgradeTool.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexUpgradeTool.java
@@ -92,7 +92,8 @@ public class IndexUpgradeTool extends Configured {
"Log file path where the logs are written");
private static final Option INDEX_SYNC_REBUILD_OPTION = new Option("sr", "index-sync-rebuild",
false,
- "[Optional]Whether or not synchronously rebuild the indexes; default rebuild asynchronous");
+ "[Optional]Whether or not synchronously rebuild the indexes; "
+ + "default rebuild asynchronous");
public static final String UPGRADE_OP = "upgrade";
public static final String ROLLBACK_OP = "rollback";
@@ -100,7 +101,7 @@ public class IndexUpgradeTool extends Configured {
private IndexTool indexingTool;
private HashMap<String, HashSet<String>> tablesAndIndexes = new HashMap<>();
- private HashMap<String, HashMap<String,String>> rebuildMap = new HashMap<>();
+ private HashMap<String, HashMap<String,IndexInfo>> rebuildMap = new HashMap<>();
private HashMap<String, String> prop = new HashMap<>();
private boolean dryRun, upgrade, syncRebuild;
@@ -128,7 +129,6 @@ public class IndexUpgradeTool extends Configured {
}
public void setTest(boolean test) { this.test = test; }
-
public boolean getDryRun() {
return this.dryRun;
}
@@ -339,13 +339,7 @@ public class IndexUpgradeTool extends Configured {
disableTable(admin, dataTableFullName, indexes);
modifyTable(admin, dataTableFullName, indexes);
enableTable(admin, dataTableFullName, indexes);
- if (upgrade) {
- if(!test) {
- indexingTool = new IndexTool();
- }
- indexingTool.setConf(conf);
- rebuildIndexes(dataTableFullName, indexingTool);
- }
+ rebuildIndexes(conn, conf, dataTableFullName);
} catch (IOException | SQLException | InterruptedException e) {
LOGGER.severe("Something went wrong while executing " + operation + " steps " + e);
return -1;
@@ -354,17 +348,6 @@ public class IndexUpgradeTool extends Configured {
return 0;
}
- private void modifyTable(Admin admin, String dataTableFullName, HashSet<String> indexes)
- throws IOException {
- if (upgrade) {
- modifyIndexTable(admin, indexes);
- modifyDataTable(admin, dataTableFullName);
- } else {
- modifyDataTable(admin, dataTableFullName);
- modifyIndexTable(admin, indexes);
- }
- }
-
private void disableTable(Admin admin, String dataTable, HashSet<String>indexes)
throws IOException {
if (admin.isTableEnabled(TableName.valueOf(dataTable))) {
@@ -387,6 +370,17 @@ public class IndexUpgradeTool extends Configured {
}
}
+ private void modifyTable(Admin admin, String dataTableFullName, HashSet<String> indexes)
+ throws IOException {
+ if (upgrade) {
+ modifyIndexTable(admin, indexes);
+ modifyDataTable(admin, dataTableFullName);
+ } else {
+ modifyDataTable(admin, dataTableFullName);
+ modifyIndexTable(admin, indexes);
+ }
+ }
+
private void enableTable(Admin admin, String dataTable, HashSet<String>indexes)
throws IOException {
if (!admin.isTableEnabled(TableName.valueOf(dataTable))) {
@@ -409,6 +403,17 @@ public class IndexUpgradeTool extends Configured {
}
}
+ private void rebuildIndexes(Connection conn, Configuration conf, String dataTableFullName) {
+ if (upgrade) {
+ prepareToRebuildIndexes(conn, dataTableFullName);
+ if(!test) {
+ indexingTool = new IndexTool();
+ }
+ indexingTool.setConf(conf);
+ rebuildIndexes(dataTableFullName, indexingTool);
+ }
+ }
+
private void modifyDataTable(Admin admin, String tableName)
throws IOException {
HTableDescriptor tableDesc = admin.getTableDescriptor(TableName.valueOf(tableName));
@@ -465,29 +470,20 @@ public class IndexUpgradeTool extends Configured {
}
private int rebuildIndexes(String dataTable, IndexTool indexingTool) {
- String schema = SchemaUtil.getSchemaNameFromFullName(dataTable);
- String table = SchemaUtil.getTableNameFromFullName(dataTable);
- for(Map.Entry<String, String> indexMap : rebuildMap.get(dataTable).entrySet()) {
+ for(Map.Entry<String, IndexInfo> indexMap : rebuildMap.get(dataTable).entrySet()) {
String index = indexMap.getKey();
- String tenantId = indexMap.getValue();
+ IndexInfo indexInfo = indexMap.getValue();
String indexName = SchemaUtil.getTableNameFromFullName(index);
- String outFile = "/tmp/index_rebuild_" + indexName +
+ String tenantId = indexInfo.getTenantId();
+ String baseTable = indexInfo.getBaseTable();
+ String schema = indexInfo.getSchemaName();
+ String outFile = "/tmp/index_rebuild_" +schema+"_"+ indexName +
(GLOBAL_INDEX_ID.equals(tenantId)?"":"_"+tenantId)
+"_"+ UUID.randomUUID().toString();
- String[] args =
- { "-s", schema, "-dt", table, "-it", indexName, "-direct", "-op", outFile };
- ArrayList<String> list = new ArrayList<>(Arrays.asList(args));
- if (!GLOBAL_INDEX_ID.equals(tenantId)) {
- list.add("-tenant");
- list.add(tenantId);
- }
- if (syncRebuild) {
- list.add("-runfg");
- }
- args = list.toArray(new String[list.size()]);
+ String[] args = getIndexToolArgValues(schema, baseTable, indexName, outFile, tenantId);
try {
- LOGGER.info("Rebuilding index " + indexName);
+ LOGGER.info("Rebuilding index: " + StringUtils.join( args,","));
if (!dryRun) {
indexingTool.run(args);
}
@@ -499,6 +495,21 @@ public class IndexUpgradeTool extends Configured {
return 0;
}
+ private String[] getIndexToolArgValues(String schema, String baseTable, String indexName,
+ String outFile, String tenantId) {
+ String args[] = { "-s", schema, "-dt", baseTable, "-it", indexName,
+ "-direct", "-op", outFile };
+ ArrayList<String> list = new ArrayList<>(Arrays.asList(args));
+ if (!GLOBAL_INDEX_ID.equals(tenantId)) {
+ list.add("-tenant");
+ list.add(tenantId);
+ }
+ if (syncRebuild) {
+ list.add("-runfg");
+ }
+ return list.toArray(new String[list.size()]);
+ }
+
private boolean extractTablesAndIndexes(PhoenixConnection conn) {
String [] tables = inputTables.trim().split(",");
PTable dataTable = null;
@@ -507,35 +518,19 @@ public class IndexUpgradeTool extends Configured {
HashSet<String> physicalIndexes = new HashSet<>();
dataTable = PhoenixRuntime.getTableNoCache(conn, tableName);
String physicalTableName = dataTable.getPhysicalName().getString();
- HashMap<String, String> rebuildIndexes = new HashMap<>();
-
if (!dataTable.isTransactional() && dataTable.getType().equals(PTableType.TABLE)) {
for (PTable indexTable : dataTable.getIndexes()) {
if (indexTable.getIndexType().equals(PTable.IndexType.GLOBAL)) {
- physicalIndexes.add(indexTable.getPhysicalName().getString());
- rebuildIndexes.put(indexTable.getPhysicalName().getString(),
- GLOBAL_INDEX_ID);
+ String physicalIndexName = indexTable.getPhysicalName().getString();
+ physicalIndexes.add(physicalIndexName);
}
}
-
if (MetaDataUtil.hasViewIndexTable(conn, dataTable.getPhysicalName())) {
String viewIndexPhysicalName = MetaDataUtil
.getViewIndexPhysicalName(physicalTableName);
physicalIndexes.add(viewIndexPhysicalName);
-
- ResultSet rs =
- conn.createStatement().executeQuery(
- "SELECT DISTINCT TABLE_NAME, TENANT_ID FROM "
- + "SYSTEM.CATALOG WHERE COLUMN_FAMILY = \'"
- + viewIndexPhysicalName
- +"\' AND TABLE_TYPE = \'i\'");
- while (rs.next()) {
- String viewIndexName = rs.getString(1);
- String tenantId = rs.getString(2);
- rebuildIndexes.put(viewIndexName, tenantId);
- }
}
- rebuildMap.put(physicalTableName, rebuildIndexes);
+ //for upgrade or rollback
tablesAndIndexes.put(physicalTableName, physicalIndexes);
} else {
LOGGER.info("Skipping Table " + tableName + " because it is "+
@@ -551,4 +546,93 @@ public class IndexUpgradeTool extends Configured {
return false;
}
}
+
+ private void prepareToRebuildIndexes(Connection conn, String dataTableFullName) {
+ try {
+ HashMap<String, IndexInfo> rebuildIndexes = new HashMap<>();
+
+ HashSet<String> physicalIndexes = tablesAndIndexes.get(dataTableFullName);
+
+ String viewIndexPhysicalName = MetaDataUtil
+ .getViewIndexPhysicalName(dataTableFullName);
+ boolean hasViewIndex = physicalIndexes.contains(viewIndexPhysicalName);
+
+ String schemaName = SchemaUtil.getSchemaNameFromFullName(dataTableFullName);
+ String tableName = SchemaUtil.getTableNameFromFullName(dataTableFullName);
+
+ for (String physicalIndexName : physicalIndexes) {
+ if (physicalIndexName.equals(viewIndexPhysicalName)) {
+ continue;
+ }
+ String indexTableName = SchemaUtil.getTableNameFromFullName(physicalIndexName);
+ String pIndexName = SchemaUtil.getTableName(schemaName, indexTableName);
+ IndexInfo indexInfo = new IndexInfo(schemaName, tableName,
+ GLOBAL_INDEX_ID, pIndexName);
+ rebuildIndexes.put(physicalIndexName, indexInfo);
+ }
+
+ if (hasViewIndex) {
+ ResultSet
+ rs =
+ conn.createStatement().executeQuery(
+ "SELECT DISTINCT TABLE_NAME, TENANT_ID FROM "
+ + "SYSTEM.CATALOG WHERE COLUMN_FAMILY = \'"
+ + viewIndexPhysicalName
+ + "\' AND TABLE_TYPE = \'i\' AND " + "LINK_TYPE = "
+ + PTable.LinkType.PHYSICAL_TABLE.getSerializedValue());
+ while (rs.next()) {
+ String viewIndexName = rs.getString(1);
+ String tenantId = rs.getString(2);
+ ResultSet
+ innerRS =
+ conn.createStatement().executeQuery(
+ "SELECT DISTINCT TABLE_NAME FROM "
+ + "SYSTEM.CATALOG WHERE COLUMN_FAMILY = \'"
+ + viewIndexName
+ + "\' AND TABLE_TYPE = \'i\' AND " + "LINK_TYPE = "
+ + PTable.LinkType.INDEX_TABLE.getSerializedValue());
+ innerRS.next();
+ String viewName = innerRS.getString(1);
+ IndexInfo indexInfo = new IndexInfo(schemaName, viewName, tenantId == null ? GLOBAL_INDEX_ID: tenantId,
+ viewIndexName);
+ rebuildIndexes.put(viewIndexName, indexInfo);
+ }
+ }
+ //for rebuilding indexes in case of upgrade.
+ rebuildMap.put(dataTableFullName, rebuildIndexes);
+ } catch (SQLException e) {
+ LOGGER.severe("Failed to prepare the map for index rebuilds "+e);
+ throw new RuntimeException("Failed to prepare the map for index rebuilds");
+ }
+ }
+
+ private class IndexInfo {
+ final private String schemaName;
+ final private String baseTable;
+ final private String tenantId;
+ final private String indexName;
+
+ public IndexInfo(String schemaName, String baseTable, String tenantId, String indexName) {
+ this.schemaName = schemaName;
+ this.baseTable = baseTable;
+ this.tenantId = tenantId;
+ this.indexName = indexName;
+ }
+
+ public String getSchemaName() {
+ return schemaName;
+ }
+
+ public String getBaseTable() {
+ return baseTable;
+ }
+
+ public String getTenantId() {
+ return tenantId;
+ }
+
+ public String getIndexName() {
+ return indexName;
+ }
+ }
}
\ No newline at end of file