You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by st...@apache.org on 2021/11/23 10:44:22 UTC
[phoenix] branch master updated: PHOENIX-6601 Fix IndexTools bugs with namespace mapping
This is an automated email from the ASF dual-hosted git repository.
stoty pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/phoenix.git
The following commit(s) were added to refs/heads/master by this push:
new 12bc597 PHOENIX-6601 Fix IndexTools bugs with namespace mapping
12bc597 is described below
commit 12bc597d2a06de88ea39645d09a002db8c853945
Author: Istvan Toth <st...@apache.org>
AuthorDate: Mon Nov 22 13:00:59 2021 +0100
PHOENIX-6601 Fix IndexTools bugs with namespace mapping
---
.../phoenix/end2end/IndexBuildTimestampIT.java | 19 ++-
.../org/apache/phoenix/end2end/IndexToolIT.java | 150 +++++++++++++--------
.../apache/phoenix/mapreduce/index/IndexTool.java | 17 +--
3 files changed, 122 insertions(+), 64 deletions(-)
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexBuildTimestampIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexBuildTimestampIT.java
index 6c36838..faa6cb0 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexBuildTimestampIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexBuildTimestampIT.java
@@ -28,6 +28,7 @@ import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.Collection;
import java.util.List;
+import java.util.Map;
import java.util.Properties;
import org.apache.hadoop.hbase.client.ResultScanner;
@@ -37,18 +38,22 @@ import org.apache.phoenix.jdbc.PhoenixConnection;
import org.apache.phoenix.jdbc.PhoenixResultSet;
import org.apache.phoenix.query.BaseTest;
import org.apache.phoenix.query.QueryServices;
+import org.apache.phoenix.query.QueryServicesOptions;
import org.apache.phoenix.schema.PTable;
import org.apache.phoenix.util.EnvironmentEdge;
import org.apache.phoenix.util.EnvironmentEdgeManager;
import org.apache.phoenix.util.PhoenixRuntime;
import org.apache.phoenix.util.QueryUtil;
+import org.apache.phoenix.util.ReadOnlyProps;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameters;
+
import org.apache.phoenix.thirdparty.com.google.common.collect.Lists;
+import org.apache.phoenix.thirdparty.com.google.common.collect.Maps;
@Category(NeedsOwnMiniClusterTest.class)
@RunWith(Parameterized.class)
@@ -73,7 +78,19 @@ public class IndexBuildTimestampIT extends BaseTest {
@BeforeClass
public static synchronized void setup() throws Exception {
- IndexToolIT.setup();
+ Map<String, String> serverProps = Maps.newHashMapWithExpectedSize(2);
+ serverProps.put(QueryServices.STATS_GUIDEPOST_WIDTH_BYTES_ATTRIB, Long.toString(20));
+ serverProps.put(QueryServices.MAX_SERVER_METADATA_CACHE_TIME_TO_LIVE_MS_ATTRIB, Long.toString(5));
+ serverProps.put(QueryServices.EXTRA_JDBC_ARGUMENTS_ATTRIB,
+ QueryServicesOptions.DEFAULT_EXTRA_JDBC_ARGUMENTS);
+ serverProps.put(QueryServices.INDEX_REBUILD_PAGE_SIZE_IN_ROWS, Long.toString(8));
+ Map<String, String> clientProps = Maps.newHashMapWithExpectedSize(2);
+ clientProps.put(QueryServices.USE_STATS_FOR_PARALLELIZATION, Boolean.toString(true));
+ clientProps.put(QueryServices.STATS_UPDATE_FREQ_MS_ATTRIB, Long.toString(5));
+ clientProps.put(QueryServices.TRANSACTIONS_ENABLED, Boolean.TRUE.toString());
+ clientProps.put(QueryServices.FORCE_ROW_KEY_ORDER_ATTRIB, Boolean.TRUE.toString());
+ setUpTestDriver(new ReadOnlyProps(serverProps.entrySet().iterator()),
+ new ReadOnlyProps(clientProps.entrySet().iterator()));
}
@Parameters(
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolIT.java
index c2de883..b121c3f 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolIT.java
@@ -29,7 +29,6 @@ import org.apache.hadoop.hbase.HBaseIOException;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.client.Mutation;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
@@ -55,7 +54,6 @@ import org.apache.phoenix.query.BaseTest;
import org.apache.phoenix.query.ConnectionQueryServices;
import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.query.QueryServicesOptions;
-import org.apache.phoenix.schema.PIndexState;
import org.apache.phoenix.schema.PTable;
import org.apache.phoenix.transaction.PhoenixTransactionProvider.Feature;
import org.apache.phoenix.transaction.TransactionFactory;
@@ -65,11 +63,11 @@ import org.apache.phoenix.util.QueryUtil;
import org.apache.phoenix.util.ReadOnlyProps;
import org.apache.phoenix.util.SchemaUtil;
import org.apache.phoenix.util.TestUtil;
-import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.BeforeParam;
import org.junit.runners.Parameterized.Parameters;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -112,6 +110,8 @@ import static org.junit.Assert.fail;
public class IndexToolIT extends BaseTest {
private static final Logger LOGGER = LoggerFactory.getLogger(IndexToolIT.class);
+ private static String tmpDir;
+
private final boolean localIndex;
private final boolean mutable;
private final boolean transactional;
@@ -120,15 +120,17 @@ public class IndexToolIT extends BaseTest {
private final String indexDDLOptions;
private final boolean useSnapshot;
private final boolean useTenantId;
+ private final boolean namespaceMapped;
public IndexToolIT(String transactionProvider, boolean mutable, boolean localIndex,
- boolean directApi, boolean useSnapshot, boolean useTenantId) {
+ boolean directApi, boolean useSnapshot, boolean useTenantId, boolean namespaceMapped) {
this.localIndex = localIndex;
this.mutable = mutable;
this.transactional = transactionProvider != null;
this.directApi = directApi;
this.useSnapshot = useSnapshot;
this.useTenantId = useTenantId;
+ this.namespaceMapped = namespaceMapped;
StringBuilder optionBuilder = new StringBuilder();
if (!mutable) {
optionBuilder.append(" IMMUTABLE_ROWS=true ");
@@ -152,58 +154,81 @@ public class IndexToolIT extends BaseTest {
this.indexDDLOptions = indexOptionBuilder.toString();
}
- @BeforeClass
- public static synchronized void setup() throws Exception {
- Map<String, String> serverProps = Maps.newHashMapWithExpectedSize(2);
- serverProps.put(QueryServices.STATS_GUIDEPOST_WIDTH_BYTES_ATTRIB, Long.toString(20));
- serverProps.put(QueryServices.MAX_SERVER_METADATA_CACHE_TIME_TO_LIVE_MS_ATTRIB, Long.toString(5));
- serverProps.put(QueryServices.EXTRA_JDBC_ARGUMENTS_ATTRIB,
- QueryServicesOptions.DEFAULT_EXTRA_JDBC_ARGUMENTS);
- serverProps.put(QueryServices.INDEX_REBUILD_PAGE_SIZE_IN_ROWS, Long.toString(8));
- Map<String, String> clientProps = Maps.newHashMapWithExpectedSize(2);
- clientProps.put(QueryServices.USE_STATS_FOR_PARALLELIZATION, Boolean.toString(true));
- clientProps.put(QueryServices.STATS_UPDATE_FREQ_MS_ATTRIB, Long.toString(5));
- clientProps.put(QueryServices.TRANSACTIONS_ENABLED, Boolean.TRUE.toString());
- clientProps.put(QueryServices.FORCE_ROW_KEY_ORDER_ATTRIB, Boolean.TRUE.toString());
- setUpTestDriver(new ReadOnlyProps(serverProps.entrySet().iterator()),
- new ReadOnlyProps(clientProps.entrySet().iterator()));
+ @BeforeParam
+ public static synchronized void setup(
+ String transactionProvider, boolean mutable, boolean localIndex,
+ boolean directApi, boolean useSnapshot, boolean useTenantId, boolean namespaceMapped)
+ throws Exception {
+ if (clusterInitialized && Boolean.valueOf(namespaceMapped).equals(utility.getConfiguration()
+ .getBoolean(QueryServices.IS_NAMESPACE_MAPPING_ENABLED, true))) {
+ //Perf optimization: no need to re-initialize the minicluster
+ return;
+ } else {
+ //This has been known to cause flakeyness
+ //Maybe we're gonna have to refactor the namespace
+ if(clusterInitialized) {
+ tearDownMiniCluster(0);
+ System.setProperty("java.io.tmpdir", tmpDir);
+ } else {
+ tmpDir = System.getProperty("java.io.tmpdir");
+ }
+ Map<String, String> serverProps = Maps.newHashMapWithExpectedSize(2);
+ serverProps.put(QueryServices.STATS_GUIDEPOST_WIDTH_BYTES_ATTRIB, Long.toString(20));
+ serverProps.put(QueryServices.MAX_SERVER_METADATA_CACHE_TIME_TO_LIVE_MS_ATTRIB, Long.toString(5));
+ serverProps.put(QueryServices.EXTRA_JDBC_ARGUMENTS_ATTRIB,
+ QueryServicesOptions.DEFAULT_EXTRA_JDBC_ARGUMENTS);
+ serverProps.put(QueryServices.INDEX_REBUILD_PAGE_SIZE_IN_ROWS, Long.toString(8));
+ serverProps.put(QueryServices.IS_NAMESPACE_MAPPING_ENABLED,
+ Boolean.toString(namespaceMapped));
+ Map<String, String> clientProps = Maps.newHashMapWithExpectedSize(2);
+ clientProps.put(QueryServices.USE_STATS_FOR_PARALLELIZATION, Boolean.toString(true));
+ clientProps.put(QueryServices.STATS_UPDATE_FREQ_MS_ATTRIB, Long.toString(5));
+ clientProps.put(QueryServices.TRANSACTIONS_ENABLED, Boolean.TRUE.toString());
+ clientProps.put(QueryServices.FORCE_ROW_KEY_ORDER_ATTRIB, Boolean.TRUE.toString());
+ clientProps.put(QueryServices.IS_NAMESPACE_MAPPING_ENABLED,
+ Boolean.toString(namespaceMapped));
+ setUpTestDriver(new ReadOnlyProps(serverProps.entrySet().iterator()),
+ new ReadOnlyProps(clientProps.entrySet().iterator()));
+ }
}
@Parameters(
name = "transactionProvider={0},mutable={1},localIndex={2},directApi={3}," +
- "useSnapshot={4},useTenant={5}")
+ "useSnapshot={4},useTenant={5},namespaceMapped={6}")
public static synchronized Collection<Object[]> data() {
List<Object[]> list = Lists.newArrayListWithExpectedSize(48);
boolean[] Booleans = new boolean[] { false, true };
- for (String transactionProvider : new String[] {"TEPHRA", "OMID", null}) {
- if(transactionProvider !=null &&
- !TransactionFactory.Provider.valueOf(transactionProvider).runTests()) {
- continue;
- }
- for (boolean mutable : Booleans) {
- for (boolean localIndex : Booleans) {
- if (!localIndex
- || transactionProvider == null
- || !TransactionFactory.getTransactionProvider(
- TransactionFactory.Provider.valueOf(transactionProvider))
- .isUnsupported(Feature.ALLOW_LOCAL_INDEX)) {
- if (localIndex) {
- for (boolean directApi : Booleans) {
+ for (boolean namespaceMapped : Booleans) {
+ for (String transactionProvider : new String[] {"TEPHRA", "OMID", null}) {
+ if(transactionProvider !=null &&
+ !TransactionFactory.Provider.valueOf(transactionProvider).runTests()) {
+ continue;
+ }
+ for (boolean mutable : Booleans) {
+ for (boolean localIndex : Booleans) {
+ if (!localIndex
+ || transactionProvider == null
+ || !TransactionFactory.getTransactionProvider(
+ TransactionFactory.Provider.valueOf(transactionProvider))
+ .isUnsupported(Feature.ALLOW_LOCAL_INDEX)) {
+ if (localIndex) {
+ for (boolean directApi : Booleans) {
+ list.add(new Object[]{transactionProvider, mutable, localIndex,
+ directApi, false, false, namespaceMapped});
+ }
+ }
+ else {
+ // Due to PHOENIX-5375 and PHOENIX-5376, the snapshot and bulk load options are ignored for global indexes
list.add(new Object[]{transactionProvider, mutable, localIndex,
- directApi, false, false});
+ true, false, false, namespaceMapped});
}
}
- else {
- // Due to PHOENIX-5375 and PHOENIX-5376, the snapshot and bulk load options are ignored for global indexes
- list.add(new Object[]{transactionProvider, mutable, localIndex,
- true, false, false});
- }
}
}
}
}
// Add the usetenantId
- list.add(new Object[] { null, false, false, true, false, true});
+ list.add(new Object[] { null, false, false, true, false, true, false});
return TestUtil.filterTxParamData(list,0);
}
@@ -230,6 +255,9 @@ public class IndexToolIT extends BaseTest {
Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
Connection conn = DriverManager.getConnection(getUrl(), props);
try {
+ if(namespaceMapped) {
+ conn.createStatement().execute("CREATE SCHEMA " + schemaName);
+ }
String stmString1 =
"CREATE TABLE " + dataTableFullName
+ " (ID INTEGER NOT NULL PRIMARY KEY, NAME VARCHAR, ZIP INTEGER) "
@@ -289,9 +317,9 @@ public class IndexToolIT extends BaseTest {
assertEquals("FULL SCAN ",
explainPlanAttributes.getExplainScanType());
assertEquals(dataTableFullName,
- explainPlanAttributes.getTableName());
+ explainPlanAttributes.getTableName().replaceAll(":", "."));
assertEquals("SERVER FILTER BY (LPAD(UPPER(NAME, 'en_US'), 8, 'x') || '_xyz') = 'xxUNAME2_xyz'",
- explainPlanAttributes.getServerWhereFilter());
+ explainPlanAttributes.getServerWhereFilter().replaceAll(":", "."));
ResultSet rs = stmt1.executeQuery(selectSql);
assertTrue(rs.next());
@@ -321,7 +349,7 @@ public class IndexToolIT extends BaseTest {
expectedTableName = indexTableFullName;
}
assertEquals(expectedTableName,
- explainPlanAttributes.getTableName());
+ explainPlanAttributes.getTableName().replaceAll(":", "."));
rs = conn.createStatement().executeQuery(selectSql);
assertTrue(rs.next());
@@ -549,6 +577,9 @@ public class IndexToolIT extends BaseTest {
String indexTableName = generateUniqueName();
try (Connection conn =
DriverManager.getConnection(getUrl(), PropertiesUtil.deepCopy(TEST_PROPERTIES))) {
+ if(namespaceMapped) {
+ conn.createStatement().execute("CREATE SCHEMA " + schemaName);
+ }
String dataDDL =
"CREATE TABLE " + dataTableFullName + "(\n"
+ "ID VARCHAR NOT NULL PRIMARY KEY,\n"
@@ -592,13 +623,13 @@ public class IndexToolIT extends BaseTest {
String schemaName = generateUniqueName();
String dataTableName = generateUniqueName();
String dataTableFullName = SchemaUtil.getTableName(schemaName, dataTableName);
- final TableName dataTN = TableName.valueOf(dataTableFullName);
String indexTableName = generateUniqueName();
- String indexTableFullName = SchemaUtil.getTableName(schemaName, indexTableName);
- TableName indexTN = TableName.valueOf(indexTableFullName);
try (Connection conn =
DriverManager.getConnection(getUrl(), PropertiesUtil.deepCopy(TEST_PROPERTIES));
Admin admin = conn.unwrap(PhoenixConnection.class).getQueryServices().getAdmin()) {
+ if(namespaceMapped) {
+ conn.createStatement().execute("CREATE SCHEMA " + schemaName);
+ }
String dataDDL =
"CREATE TABLE " + dataTableFullName + "(\n"
+ "ID VARCHAR NOT NULL PRIMARY KEY,\n"
@@ -618,11 +649,14 @@ public class IndexToolIT extends BaseTest {
for (String prefix : carNumPrefixes) {
splitPoints[--numSplits] = Bytes.toBytes(prefix);
}
- HTableDescriptor dataTD = admin.getTableDescriptor(dataTN);
- admin.disableTable(dataTN);
- admin.deleteTable(dataTN);
+ TableName hDataName = TableName.valueOf(
+ SchemaUtil.getPhysicalHBaseTableName(schemaName, dataTableName, namespaceMapped)
+ .getBytes());
+ HTableDescriptor dataTD = admin.getTableDescriptor(hDataName);
+ admin.disableTable(hDataName);
+ admin.deleteTable(hDataName);
admin.createTable(dataTD, splitPoints);
- assertEquals(targetNumRegions, admin.getTableRegions(dataTN).size());
+ assertEquals(targetNumRegions, admin.getTableRegions(hDataName).size());
// insert data where index column values start with a, b, c, d
int idCounter = 1;
@@ -649,10 +683,13 @@ public class IndexToolIT extends BaseTest {
// run with 50% sampling rate, split if data table more than 3 regions
runIndexTool(directApi, useSnapshot, schemaName, dataTableName, indexTableName, "-sp", "50", "-spa", "3");
- assertEquals(targetNumRegions, admin.getTableRegions(indexTN).size());
+ TableName hIndexName = TableName.valueOf(
+ SchemaUtil.getPhysicalHBaseTableName(schemaName, indexTableName, namespaceMapped)
+ .getBytes());
+ assertEquals(targetNumRegions, admin.getTableRegions(hIndexName).size());
List<Cell> values = new ArrayList<>();
// every index region should have been written to, if the index table was properly split uniformly
- for (HRegion region : getUtility().getHBaseCluster().getRegions(indexTN)) {
+ for (HRegion region : getUtility().getHBaseCluster().getRegions(hIndexName)) {
values.clear();
RegionScanner scanner = region.getScanner(new Scan());
scanner.next(values);
@@ -684,6 +721,9 @@ public class IndexToolIT extends BaseTest {
Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
Connection conn = DriverManager.getConnection(getUrl(), props);
try {
+ if(namespaceMapped) {
+ conn.createStatement().execute("CREATE SCHEMA " + sSchemaName);
+ }
String stmString1 =
"CREATE TABLE " + qDataTableName
+ " (ID INTEGER NOT NULL PRIMARY KEY, NAME VARCHAR, ZIP INTEGER) "
@@ -739,7 +779,7 @@ public class IndexToolIT extends BaseTest {
assertEquals(String.format(
"CLIENT PARALLEL 1-WAY FULL SCAN OVER %s\n"
+ " SERVER FILTER BY (LPAD(UPPER(NAME, 'en_US'), 8, 'x') || '_xyz') = 'xxUNAME2_xyz'",
- dataTableNameForExplain), actualExplainPlan);
+ dataTableNameForExplain), actualExplainPlan.replaceAll(":", "."));
rs = stmt1.executeQuery(selectSql);
assertTrue(rs.next());
@@ -789,7 +829,7 @@ public class IndexToolIT extends BaseTest {
: indexTableFullName);
}
assertTrue(actualExplainPlan + "\n expected to contain \n" + expectedExplainPlan,
- actualExplainPlan.contains(expectedExplainPlan));
+ actualExplainPlan.replaceAll(":", ".").contains(expectedExplainPlan));
}
public static CounterGroup getMRJobCounters(IndexTool indexTool) throws IOException {
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexTool.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexTool.java
index 34d8b1c..439485b 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexTool.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexTool.java
@@ -666,9 +666,9 @@ public class IndexTool extends Configured implements Tool {
String snapshotName;
try {
admin = pConnection.getQueryServices().getAdmin();
- String pdataTableName = pDataTable.getName().getString();
- snapshotName = new StringBuilder(pdataTableName).append("-Snapshot").toString();
- admin.snapshot(snapshotName, TableName.valueOf(pdataTableName));
+ TableName hDdataTableName = TableName.valueOf(pDataTable.getPhysicalName().getBytes());
+ snapshotName = new StringBuilder(hDdataTableName.toString()).append("-Snapshot").toString();
+ admin.snapshot(snapshotName, hDdataTableName);
} finally {
if (admin != null) {
admin.close();
@@ -1046,9 +1046,10 @@ public class IndexTool extends Configured implements Tool {
throws SQLException, IOException, IllegalArgumentException {
int numRegions;
+ TableName hDataName = TableName.valueOf(pDataTable.getPhysicalName().getBytes());
try (org.apache.hadoop.hbase.client.Connection tempHConn = getTemporaryHConnection(pConnection);
RegionLocator regionLocator =
- tempHConn.getRegionLocator(TableName.valueOf(qDataTable))) {
+ tempHConn.getRegionLocator(hDataName)) {
numRegions = regionLocator.getStartKeys().length;
if (autosplit && (numRegions <= autosplitNumRegions)) {
LOGGER.info(String.format(
@@ -1089,10 +1090,10 @@ public class IndexTool extends Configured implements Tool {
splitPoints[splitIdx++] = b.getRightBoundExclusive();
}
// drop table and recreate with appropriate splits
- TableName indexTN = TableName.valueOf(pIndexTable.getPhysicalName().getBytes());
- HTableDescriptor descriptor = admin.getTableDescriptor(indexTN);
- admin.disableTable(indexTN);
- admin.deleteTable(indexTN);
+ TableName hIndexName = TableName.valueOf(pIndexTable.getPhysicalName().getBytes());
+ HTableDescriptor descriptor = admin.getTableDescriptor(hIndexName);
+ admin.disableTable(hIndexName);
+ admin.deleteTable(hIndexName);
admin.createTable(descriptor, splitPoints);
}
}