You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by ka...@apache.org on 2019/10/17 17:25:59 UTC
[phoenix] branch 4.x-HBase-1.5 updated: PHOENIX-5522 If index is
disabled, IndexUpgradeTool should not fail during indexTool run
This is an automated email from the ASF dual-hosted git repository.
kadir pushed a commit to branch 4.x-HBase-1.5
in repository https://gitbox.apache.org/repos/asf/phoenix.git
The following commit(s) were added to refs/heads/4.x-HBase-1.5 by this push:
new 1e01da6 PHOENIX-5522 If index is disabled, IndexUpgradeTool should not fail during indexTool run
1e01da6 is described below
commit 1e01da694010f8e32572fe431718e895c83da8ae
Author: Gokcen Iskender <gi...@salesforce.com>
AuthorDate: Fri Oct 11 15:00:46 2019 -0700
PHOENIX-5522 If index is disabled, IndexUpgradeTool should not fail during indexTool run
Signed-off-by: Kadir <ko...@salesforce.com>
---
.../end2end/ParameterizedIndexUpgradeToolIT.java | 77 ++++++++++++++--------
.../phoenix/mapreduce/index/IndexUpgradeTool.java | 53 +++++++++++++--
2 files changed, 96 insertions(+), 34 deletions(-)
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ParameterizedIndexUpgradeToolIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ParameterizedIndexUpgradeToolIT.java
index ceea647..1648368 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ParameterizedIndexUpgradeToolIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ParameterizedIndexUpgradeToolIT.java
@@ -29,7 +29,10 @@ import org.apache.phoenix.mapreduce.index.IndexUpgradeTool;
import org.apache.phoenix.query.BaseTest;
import org.apache.phoenix.query.ConnectionQueryServices;
import org.apache.phoenix.query.QueryServices;
+import org.apache.phoenix.util.PhoenixRuntime;
+import org.apache.phoenix.util.PropertiesUtil;
import org.apache.phoenix.util.ReadOnlyProps;
+import org.apache.phoenix.util.SchemaUtil;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -55,19 +58,21 @@ import java.util.UUID;
import static org.apache.phoenix.mapreduce.index.IndexUpgradeTool.ROLLBACK_OP;
import static org.apache.phoenix.mapreduce.index.IndexUpgradeTool.UPGRADE_OP;
+import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
@RunWith(Parameterized.class)
@Category(NeedsOwnMiniClusterTest.class)
public class ParameterizedIndexUpgradeToolIT extends BaseTest {
private static final String [] INDEXES_LIST = {"TEST.INDEX1", "TEST.INDEX2", "TEST1.INDEX3",
"TEST1.INDEX2","TEST1.INDEX1","TEST.INDEX3", "_IDX_TEST.MOCK1", "_IDX_TEST1.MOCK2"};
- private static final String [] INDEXES_LIST_NAMESPACE = {"TEST:INDEX1", "TEST:INDEX2",
- "TEST1:INDEX3", "TEST1:INDEX2","TEST1:INDEX1","TEST:INDEX3", "TEST:_IDX_MOCK1",
- "TEST1:_IDX_MOCK2"};
- private static final String [] TABLE_LIST = {"TEST.MOCK1","TEST1.MOCK2","TEST.MOCK3"};
- private static final String [] TABLE_LIST_NAMESPACE = {"TEST:MOCK1","TEST1:MOCK2","TEST:MOCK3"};
-
- private static final String INPUT_LIST = "TEST.MOCK1,TEST1.MOCK2,TEST.MOCK3";
+ private static final String [] INDEXES_LIST_NAMESPACE = {"TEST:INDEX1", "TEST:INDEX2"
+ , "TEST1:INDEX3", "TEST1:INDEX2","TEST1:INDEX1"
+ , "TEST:INDEX3", "TEST:_IDX_MOCK1", "TEST1:_IDX_MOCK2"};
+ private static final String [] TABLE_LIST = {"TEST.MOCK1","TEST1.MOCK2","TEST.MOCK3","TEST.MULTI_TENANT_TABLE"};
+ private static final String [] TABLE_LIST_NAMESPACE = {"TEST:MOCK1","TEST1:MOCK2","TEST:MOCK3",
+ "TEST:MULTI_TENANT_TABLE"};
+
+ private static final String INPUT_LIST = "TEST.MOCK1,TEST1.MOCK2,TEST.MOCK3,TEST.MULTI_TENANT_TABLE";
private static final String INPUT_FILE = "/tmp/input_file_index_upgrade.csv";
private static Map<String, String> serverProps = Maps.newHashMapWithExpectedSize(1),
@@ -80,6 +85,7 @@ public class ParameterizedIndexUpgradeToolIT extends BaseTest {
private StringBuilder optionsBuilder;
private String tableDDLOptions;
private Connection conn;
+ private Connection connTenant;
private Admin admin;
private IndexUpgradeTool iut;
@@ -94,6 +100,12 @@ public class ParameterizedIndexUpgradeToolIT extends BaseTest {
conn = DriverManager.getConnection(getUrl(), new Properties());
conn.setAutoCommit(true);
+
+ String tenantId = generateUniqueName();
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ props.setProperty(PhoenixRuntime.TENANT_ID_ATTRIB, tenantId);
+ connTenant = DriverManager.getConnection(getUrl(), props);
+
ConnectionQueryServices queryServices = conn.unwrap(PhoenixConnection.class)
.getQueryServices();
admin = queryServices.getAdmin();
@@ -135,11 +147,16 @@ public class ParameterizedIndexUpgradeToolIT extends BaseTest {
conn.createStatement().execute("CREATE SCHEMA TEST1");
}
conn.createStatement().execute("CREATE TABLE TEST.MOCK1 (id bigint NOT NULL "
- + "PRIMARY KEY, a.name varchar, sal bigint, address varchar)"+tableDDLOptions);
+ + "PRIMARY KEY, a.name varchar, sal bigint, address varchar)" + tableDDLOptions);
conn.createStatement().execute("CREATE TABLE TEST1.MOCK2 (id bigint NOT NULL "
- + "PRIMARY KEY, name varchar, city varchar, phone bigint)"+tableDDLOptions);
+ + "PRIMARY KEY, name varchar, city varchar, phone bigint)" + tableDDLOptions);
conn.createStatement().execute("CREATE TABLE TEST.MOCK3 (id bigint NOT NULL "
- + "PRIMARY KEY, name varchar, age bigint)"+tableDDLOptions);
+ + "PRIMARY KEY, name varchar, age bigint)" + tableDDLOptions);
+ String
+ createTblStr =
+ "CREATE TABLE TEST.MULTI_TENANT_TABLE " + " (TENANT_ID VARCHAR(15) NOT NULL,ID INTEGER NOT NULL"
+ + ", NAME VARCHAR, CONSTRAINT PK_1 PRIMARY KEY (TENANT_ID, ID)) MULTI_TENANT=true";
+ conn.createStatement().execute(createTblStr);
//views
conn.createStatement().execute("CREATE VIEW TEST.MOCK1_VIEW (view_column varchar) "
@@ -149,14 +166,10 @@ public class ParameterizedIndexUpgradeToolIT extends BaseTest {
conn.createStatement().execute("CREATE VIEW TEST1.MOCK2_VIEW (view_column varchar,"
+ " state varchar) AS SELECT * FROM TEST1.MOCK2 WHERE name = 'c'");
//view-indexes
- conn.createStatement().execute("CREATE INDEX MOCK1_INDEX1 ON TEST.MOCK1_VIEW1 "
- + "(view_column)");
- conn.createStatement().execute("CREATE INDEX MOCK1_INDEX2 ON TEST.MOCK1_VIEW1 "
- + "(zip)");
- conn.createStatement().execute("CREATE INDEX MOCK2_INDEX1 ON TEST1.MOCK2_VIEW "
- + "(state, city)");
- conn.createStatement().execute("CREATE INDEX MOCK1_INDEX3 ON TEST.MOCK1_VIEW "
- + "(view_column)");
+ conn.createStatement().execute("CREATE INDEX MOCK1_INDEX1 ON TEST.MOCK1_VIEW1 " + "(view_column)");
+ conn.createStatement().execute("CREATE INDEX MOCK1_INDEX2 ON TEST.MOCK1_VIEW1 " + "(zip)");
+ conn.createStatement().execute("CREATE INDEX MOCK2_INDEX1 ON TEST1.MOCK2_VIEW " + "(state, city)");
+ conn.createStatement().execute("CREATE INDEX MOCK1_INDEX3 ON TEST.MOCK1_VIEW " + "(view_column)");
//indexes
conn.createStatement().execute("CREATE INDEX INDEX1 ON TEST.MOCK1 (sal, a.name)");
conn.createStatement().execute("CREATE INDEX INDEX2 ON TEST.MOCK1 (a.name)");
@@ -164,6 +177,15 @@ public class ParameterizedIndexUpgradeToolIT extends BaseTest {
conn.createStatement().execute("CREATE INDEX INDEX2 ON TEST1.MOCK2 (phone)");
conn.createStatement().execute("CREATE INDEX INDEX3 ON TEST1.MOCK2 (name)");
conn.createStatement().execute("CREATE INDEX INDEX3 ON TEST.MOCK3 (age, name)");
+
+ // Tenant ones
+ connTenant.createStatement().execute(
+ "CREATE VIEW TEST.TEST_TENANT_VIEW AS SELECT * FROM TEST.MULTI_TENANT_TABLE");
+ connTenant.createStatement().execute("CREATE INDEX MULTI_TENANT_INDEX ON TEST.TEST_TENANT_VIEW (NAME)");
+
+ conn.createStatement().execute("ALTER INDEX MOCK1_INDEX2 ON TEST.MOCK1_VIEW1 DISABLE");
+ connTenant.createStatement().execute("ALTER INDEX MULTI_TENANT_INDEX ON TEST.TEST_TENANT_VIEW DISABLE");
+ conn.createStatement().execute("ALTER INDEX INDEX2 ON TEST.MOCK1 DISABLE");
}
private void validate(boolean pre) throws IOException {
@@ -223,19 +245,14 @@ public class ParameterizedIndexUpgradeToolIT extends BaseTest {
@Parameters(name ="IndexUpgradeToolIT_mutable={0},upgrade={1},isNamespaceEnabled={2}")
public static Collection<Object[]> data() {
return Arrays.asList(new Object[][] {
- {false, false, true},
- {true, false, true},
- {false, true, true},
- {true, true, true},
- {false, false, false},
- {true, false, false},
- {false, true, false},
- {true, true, false}
+ {false, false, true},
+ {true, false, false},
+ {false, true, false},
+ {true, true, true}
});
}
- public ParameterizedIndexUpgradeToolIT(boolean mutable, boolean upgrade, boolean
- isNamespaceEnabled) {
+ public ParameterizedIndexUpgradeToolIT(boolean mutable, boolean upgrade, boolean isNamespaceEnabled) {
this.mutable = mutable;
this.upgrade = upgrade;
this.isNamespaceEnabled = isNamespaceEnabled;
@@ -288,6 +305,7 @@ public class ParameterizedIndexUpgradeToolIT extends BaseTest {
conn.createStatement().execute("DROP INDEX INDEX2 ON TEST1.MOCK2");
conn.createStatement().execute("DROP INDEX INDEX3 ON TEST1.MOCK2");
conn.createStatement().execute("DROP INDEX INDEX3 ON TEST.MOCK3");
+ connTenant.createStatement().execute("DROP INDEX MULTI_TENANT_INDEX ON TEST.TEST_TENANT_VIEW");
conn.createStatement().execute("DROP INDEX MOCK1_INDEX3 ON TEST.MOCK1_VIEW");
conn.createStatement().execute("DROP INDEX MOCK1_INDEX1 ON TEST.MOCK1_VIEW1");
@@ -297,15 +315,18 @@ public class ParameterizedIndexUpgradeToolIT extends BaseTest {
conn.createStatement().execute("DROP VIEW TEST.MOCK1_VIEW");
conn.createStatement().execute("DROP VIEW TEST.MOCK1_VIEW1");
conn.createStatement().execute("DROP VIEW TEST1.MOCK2_VIEW");
+ connTenant.createStatement().execute("DROP VIEW TEST.TEST_TENANT_VIEW");
conn.createStatement().execute("DROP TABLE TEST.MOCK1");
conn.createStatement().execute("DROP TABLE TEST1.MOCK2");
conn.createStatement().execute("DROP TABLE TEST.MOCK3");
+ conn.createStatement().execute("DROP TABLE TEST.MULTI_TENANT_TABLE");
if (isNamespaceEnabled) {
conn.createStatement().execute("DROP SCHEMA TEST");
conn.createStatement().execute("DROP SCHEMA TEST1");
}
conn.close();
+ connTenant.close();
}
}
\ No newline at end of file
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexUpgradeTool.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexUpgradeTool.java
index 5d0ebed..d0323a0 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexUpgradeTool.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexUpgradeTool.java
@@ -18,6 +18,7 @@
package org.apache.phoenix.mapreduce.index;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Strings;
import com.google.gson.Gson;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
@@ -46,6 +47,7 @@ import org.apache.phoenix.mapreduce.util.ConnectionUtil;
import org.apache.phoenix.query.ConnectionQueryServices;
import org.apache.phoenix.query.QueryServicesOptions;
+import org.apache.phoenix.schema.PIndexState;
import org.apache.phoenix.schema.PTable;
import org.apache.phoenix.schema.PTableType;
import org.apache.phoenix.util.MetaDataUtil;
@@ -416,7 +418,7 @@ public class IndexUpgradeTool extends Configured implements Tool {
indexingTool = new IndexTool();
}
indexingTool.setConf(conf);
- rebuildIndexes(dataTableFullName, indexingTool);
+ rebuildIndexes(conn, dataTableFullName, indexingTool);
}
}
@@ -476,7 +478,7 @@ public class IndexUpgradeTool extends Configured implements Tool {
}
}
- private int rebuildIndexes(String dataTable, IndexTool indexingTool) {
+ private int rebuildIndexes(Connection conn, String dataTable, IndexTool indexingTool) {
for(Map.Entry<String, IndexInfo> indexMap : rebuildMap.get(dataTable).entrySet()) {
String index = indexMap.getKey();
IndexInfo indexInfo = indexMap.getValue();
@@ -488,16 +490,48 @@ public class IndexUpgradeTool extends Configured implements Tool {
(GLOBAL_INDEX_ID.equals(tenantId)?"":"_"+tenantId) +"_"
+ UUID.randomUUID().toString();
String[] args = getIndexToolArgValues(schema, baseTable, indexName, outFile, tenantId);
-
+ Connection newConnection = conn;
+ Connection tenantConnection = null;
try {
LOGGER.info("Rebuilding index: " + StringUtils.join( args,","));
if (!dryRun) {
+ // If the index is in DISABLED state, indexTool will fail. First to ALTER REBUILD ASYNC.
+ // ALTER REBUILD ASYNC will set the index state to BUILDING which is safe to make ACTIVE later.
+ if (!Strings.isNullOrEmpty(tenantId) && !GLOBAL_INDEX_ID.equals(tenantId)) {
+ Configuration conf = HBaseConfiguration.addHbaseResources(getConf());
+ conf.set(PhoenixRuntime.TENANT_ID_ATTRIB, tenantId);
+ newConnection = ConnectionUtil.getInputConnection(conf);
+ tenantConnection = newConnection;
+ }
+
+ PTable indexPTable = PhoenixRuntime.getTable(newConnection, indexInfo.getPhysicalIndexTableName());
+ if (indexPTable.getIndexState() == PIndexState.DISABLE) {
+ String dataTableFullName = dataTable;
+ if (!dataTableFullName.contains(":") && !dataTableFullName.contains(".")) {
+ dataTableFullName = SchemaUtil.getTableName(schema, dataTable);
+ }
+ String
+ stmt =
+ String.format("ALTER INDEX %s ON %s REBUILD ASYNC", indexName,
+ dataTableFullName);
+ newConnection.createStatement().execute(stmt);
+ }
+
indexingTool.run(args);
}
} catch (Exception e) {
LOGGER.severe("Something went wrong while building the index "
+ index + " " + e);
return -1;
+ } finally {
+ try {
+ // Close tenant connection
+ if (tenantConnection != null) {
+ tenantConnection.close();
+ }
+ } catch (SQLException e) {
+ LOGGER.warning("Couldn't close tenant connection. Ignoring");
+ }
}
}
return 0;
@@ -575,7 +609,7 @@ public class IndexUpgradeTool extends Configured implements Tool {
String indexTableName = SchemaUtil.getTableNameFromFullName(physicalIndexName);
String pIndexName = SchemaUtil.getTableName(schemaName, indexTableName);
IndexInfo indexInfo = new IndexInfo(schemaName, tableName,
- GLOBAL_INDEX_ID, pIndexName);
+ GLOBAL_INDEX_ID, pIndexName, pIndexName);
rebuildIndexes.put(physicalIndexName, indexInfo);
}
@@ -596,7 +630,7 @@ public class IndexUpgradeTool extends Configured implements Tool {
tenantId);
for (String viewIndex : viewIndexes) {
IndexInfo indexInfo = new IndexInfo(schemaName, viewName,
- tenantId == null ? GLOBAL_INDEX_ID : tenantId, viewIndex);
+ tenantId == null ? GLOBAL_INDEX_ID : tenantId, viewIndex, viewIndexPhysicalName);
rebuildIndexes.put(viewIndex, indexInfo);
}
}
@@ -640,12 +674,15 @@ public class IndexUpgradeTool extends Configured implements Tool {
final private String baseTable;
final private String tenantId;
final private String indexName;
+ final private String physicalIndexTableName;
- public IndexInfo(String schemaName, String baseTable, String tenantId, String indexName) {
+ public IndexInfo(String schemaName, String baseTable, String tenantId, String indexName,
+ String physicalIndexTableName) {
this.schemaName = schemaName;
this.baseTable = baseTable;
this.tenantId = tenantId;
this.indexName = indexName;
+ this.physicalIndexTableName = physicalIndexTableName;
}
public String getSchemaName() {
@@ -663,6 +700,10 @@ public class IndexUpgradeTool extends Configured implements Tool {
public String getIndexName() {
return indexName;
}
+
+ public String getPhysicalIndexTableName() {
+ return physicalIndexTableName;
+ }
}
public static void main (String[] args) throws Exception {