You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by ka...@apache.org on 2020/12/11 00:39:16 UTC
[phoenix] branch master updated: PHOENIX-6058 When maxLookback is
not enabled, IndexTool should not do deep verification (#901)
This is an automated email from the ASF dual-hosted git repository.
kadir pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/phoenix.git
The following commit(s) were added to refs/heads/master by this push:
new 338578b PHOENIX-6058 When maxLookback is not enabled, IndexTool should not do deep verification (#901)
338578b is described below
commit 338578ba56daa34ddaa96c9b49700963721a9d19
Author: kadirozde <37...@users.noreply.github.com>
AuthorDate: Wed Dec 9 13:19:15 2020 -0800
PHOENIX-6058 When maxLookback is not enabled, IndexTool should not do deep verification (#901)
---
.../apache/phoenix/end2end/IndexExtendedIT.java | 16 ++--
.../end2end/IndexToolForNonTxGlobalIndexIT.java | 92 ++++++++++++++--------
.../end2end/IndexVerificationOldDesignIT.java | 2 +
.../coprocessor/GlobalIndexRegionScanner.java | 6 +-
.../index/PhoenixIndexImportDirectReducer.java | 4 +
5 files changed, 78 insertions(+), 42 deletions(-)
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexExtendedIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexExtendedIT.java
index cd2a733..b3f092b 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexExtendedIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexExtendedIT.java
@@ -36,6 +36,7 @@ import java.util.Properties;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.phoenix.compat.hbase.coprocessor.CompatBaseScannerRegionObserver;
import org.apache.phoenix.coprocessor.IndexRebuildRegionScanner;
import org.apache.phoenix.jdbc.PhoenixConnection;
import org.apache.phoenix.mapreduce.index.IndexTool;
@@ -86,6 +87,7 @@ public class IndexExtendedIT extends BaseTest {
public static synchronized void doSetup() throws Exception {
Map<String, String> serverProps = Maps.newHashMapWithExpectedSize(2);
serverProps.put(QueryServices.EXTRA_JDBC_ARGUMENTS_ATTRIB, QueryServicesOptions.DEFAULT_EXTRA_JDBC_ARGUMENTS);
+ serverProps.put(CompatBaseScannerRegionObserver.PHOENIX_MAX_LOOKBACK_AGE_CONF_KEY, Integer.toString(60*60)); // An hour
Map<String, String> clientProps = Maps.newHashMapWithExpectedSize(2);
clientProps.put(QueryServices.TRANSACTIONS_ENABLED, Boolean.TRUE.toString());
clientProps.put(QueryServices.FORCE_ROW_KEY_ORDER_ATTRIB, Boolean.TRUE.toString());
@@ -349,15 +351,15 @@ public class IndexExtendedIT extends BaseTest {
// Verify that the index table is not in the ACTIVE state
assertFalse(checkIndexState(conn, indexFullName, PIndexState.ACTIVE, 0L));
- // Run the index MR job and verify that the index table rebuild fails
- IndexToolIT.runIndexTool(true, false, schemaName, dataTableName,
- indexTableName, null, -1, IndexTool.IndexVerifyType.AFTER);
+ if (CompatBaseScannerRegionObserver.isMaxLookbackTimeEnabled(getUtility().getConfiguration())) {
+ // Run the index MR job and verify that the index table rebuild fails
+ IndexToolIT.runIndexTool(true, false, schemaName, dataTableName,
+ indexTableName, null, -1, IndexTool.IndexVerifyType.AFTER);
+ // job failed, verify that the index table is still not in the ACTIVE state
+ assertFalse(checkIndexState(conn, indexFullName, PIndexState.ACTIVE, 0L));
+ }
IndexRebuildRegionScanner.setIgnoreIndexRebuildForTesting(false);
-
- // job failed, verify that the index table is still not in the ACTIVE state
- assertFalse(checkIndexState(conn, indexFullName, PIndexState.ACTIVE, 0L));
-
// Run the index MR job and verify that the index table rebuild succeeds
IndexToolIT.runIndexTool(true, false, schemaName, dataTableName,
indexTableName, null, 0, IndexTool.IndexVerifyType.AFTER);
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolForNonTxGlobalIndexIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolForNonTxGlobalIndexIT.java
index f4a3834..7865eb9 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolForNonTxGlobalIndexIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolForNonTxGlobalIndexIT.java
@@ -266,18 +266,22 @@ public class IndexToolForNonTxGlobalIndexIT extends BaseUniqueNamesOwnClusterIT
conn.createStatement()
.execute(String.format("CREATE INDEX %s ON %s (NAME) INCLUDE (CODE) ASYNC",
indexTableName, dataTableFullName));
- IndexToolIT.runIndexTool(directApi, useSnapshot, schemaName, dataTableName, indexTableName, null, 0,
+ IndexTool indexTool = IndexToolIT.runIndexTool(directApi, useSnapshot, schemaName, dataTableName, indexTableName, null, 0,
IndexTool.IndexVerifyType.ONLY);
- Cell cell =
- IndexToolIT.getErrorMessageFromIndexToolOutputTable(conn, dataTableFullName,
- indexTableFullName);
- try {
- String expectedErrorMsg = IndexRebuildRegionScanner.ERROR_MESSAGE_MISSING_INDEX_ROW;
- String actualErrorMsg = Bytes
- .toString(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength());
- assertEquals(expectedErrorMsg, actualErrorMsg);
- } catch(Exception ex) {
- Assert.fail("Fail to parsing the error message from IndexToolOutputTable");
+ if (CompatBaseScannerRegionObserver.isMaxLookbackTimeEnabled(getUtility().getConfiguration())) {
+ Cell cell =
+ IndexToolIT.getErrorMessageFromIndexToolOutputTable(conn, dataTableFullName,
+ indexTableFullName);
+ try {
+ String expectedErrorMsg = IndexRebuildRegionScanner.ERROR_MESSAGE_MISSING_INDEX_ROW;
+ String actualErrorMsg = Bytes
+ .toString(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength());
+ assertEquals(expectedErrorMsg, actualErrorMsg);
+ } catch (Exception ex) {
+ Assert.fail("Fail to parsing the error message from IndexToolOutputTable");
+ }
+ } else {
+ assertEquals(1, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_BEYOND_MAXLOOKBACK_MISSING_INDEX_ROW_COUNT).getValue());
}
// Run the index tool to populate the index while verifying rows
@@ -409,8 +413,13 @@ public class IndexToolForNonTxGlobalIndexIT extends BaseUniqueNamesOwnClusterIT
assertEquals(0, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_VALID_INDEX_ROW_COUNT).getValue());
assertEquals(0, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_EXPIRED_INDEX_ROW_COUNT).getValue());
assertEquals(0, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_INVALID_INDEX_ROW_COUNT).getValue());
- assertEquals(NROWS, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_MISSING_INDEX_ROW_COUNT).getValue());
- assertEquals(0, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_BEYOND_MAXLOOKBACK_MISSING_INDEX_ROW_COUNT).getValue());
+ if (CompatBaseScannerRegionObserver.isMaxLookbackTimeEnabled(getUtility().getConfiguration())) {
+ assertEquals(NROWS, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_MISSING_INDEX_ROW_COUNT).getValue());
+ assertEquals(0, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_BEYOND_MAXLOOKBACK_MISSING_INDEX_ROW_COUNT).getValue());
+ } else {
+ assertEquals(0, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_MISSING_INDEX_ROW_COUNT).getValue());
+ assertEquals(NROWS, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_BEYOND_MAXLOOKBACK_MISSING_INDEX_ROW_COUNT).getValue());
+ }
assertEquals(0, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_BEYOND_MAXLOOKBACK_INVALID_INDEX_ROW_COUNT).getValue());
long actualRowCount = IndexScrutiny.scrutinizeIndex(conn, dataTableFullName, indexTableFullName);
assertEquals(NROWS, actualRowCount);
@@ -577,17 +586,23 @@ public class IndexToolForNonTxGlobalIndexIT extends BaseUniqueNamesOwnClusterIT
IndexRebuildRegionScanner.setIgnoreIndexRebuildForTesting(true);
conn.createStatement().execute(String.format(
"CREATE INDEX %s ON %s (NAME) INCLUDE (ZIP) ASYNC", indexTableName, viewFullName));
- // Run the index MR job and verify that the index table rebuild fails
- IndexToolIT.runIndexTool(directApi, useSnapshot, schemaName, viewName, indexTableName,
- null, -1, IndexTool.IndexVerifyType.AFTER);
- // The index tool output table should report that there is a missing index row
- Cell cell = IndexToolIT.getErrorMessageFromIndexToolOutputTable(conn, dataTableFullName, "_IDX_" + dataTableFullName);
- try {
- String expectedErrorMsg = IndexRebuildRegionScanner.ERROR_MESSAGE_MISSING_INDEX_ROW;
- String actualErrorMsg = Bytes.toString(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength());
- assertEquals(expectedErrorMsg, actualErrorMsg);
- } catch(Exception ex){
- Assert.fail("Fail to parsing the error message from IndexToolOutputTable");
+ if (CompatBaseScannerRegionObserver.isMaxLookbackTimeEnabled(getUtility().getConfiguration())) {
+ // Run the index MR job and verify that the index table rebuild fails
+ IndexToolIT.runIndexTool(directApi, useSnapshot, schemaName, viewName, indexTableName,
+ null, -1, IndexTool.IndexVerifyType.AFTER);
+ // The index tool output table should report that there is a missing index row
+ Cell cell = IndexToolIT.getErrorMessageFromIndexToolOutputTable(conn, dataTableFullName, "_IDX_" + dataTableFullName);
+ try {
+ String expectedErrorMsg = IndexRebuildRegionScanner.ERROR_MESSAGE_MISSING_INDEX_ROW;
+ String actualErrorMsg = Bytes.toString(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength());
+ assertEquals(expectedErrorMsg, actualErrorMsg);
+ } catch (Exception ex) {
+ Assert.fail("Fail to parsing the error message from IndexToolOutputTable");
+ }
+ } else {
+ IndexTool indexTool = IndexToolIT.runIndexTool(directApi, useSnapshot, schemaName, viewName, indexTableName,
+ null, 0, IndexTool.IndexVerifyType.AFTER);
+ assertEquals(1, indexTool.getJob().getCounters().findCounter(AFTER_REBUILD_BEYOND_MAXLOOKBACK_MISSING_INDEX_ROW_COUNT).getValue());
}
IndexRebuildRegionScanner.setIgnoreIndexRebuildForTesting(false);
}
@@ -614,13 +629,15 @@ public class IndexToolForNonTxGlobalIndexIT extends BaseUniqueNamesOwnClusterIT
// called PHOENIX_INDEX_TOOL
IndexToolIT.runIndexTool(directApi, useSnapshot, schemaName, dataTableName, indexTableName,
null, 0, IndexTool.IndexVerifyType.ONLY);
- Cell cell = IndexToolIT.getErrorMessageFromIndexToolOutputTable(conn, dataTableFullName, indexTableFullName);
- try {
- String expectedErrorMsg = IndexRebuildRegionScanner.ERROR_MESSAGE_MISSING_INDEX_ROW;
- String actualErrorMsg = Bytes.toString(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength());
- assertEquals(expectedErrorMsg, actualErrorMsg);
- } catch(Exception ex) {
- Assert.fail("Fail to parsing the error message from IndexToolOutputTable");
+ if (CompatBaseScannerRegionObserver.isMaxLookbackTimeEnabled(getUtility().getConfiguration())) {
+ Cell cell = IndexToolIT.getErrorMessageFromIndexToolOutputTable(conn, dataTableFullName, indexTableFullName);
+ try {
+ String expectedErrorMsg = IndexRebuildRegionScanner.ERROR_MESSAGE_MISSING_INDEX_ROW;
+ String actualErrorMsg = Bytes.toString(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength());
+ assertEquals(expectedErrorMsg, actualErrorMsg);
+ } catch (Exception ex) {
+ Assert.fail("Fail to parsing the error message from IndexToolOutputTable");
+ }
}
// VERIFY option should not change the index state.
@@ -1005,8 +1022,10 @@ public class IndexToolForNonTxGlobalIndexIT extends BaseUniqueNamesOwnClusterIT
//now clear out all the rebuilt index rows
truncateIndexAndIndexToolTables(indexTableFullName);
+ boolean MaxLookbackEnabled =
+ CompatBaseScannerRegionObserver.isMaxLookbackTimeEnabled(getUtility().getConfiguration());
//now check that disabling logging AFTER leaves only the BEFORE logs on a BOTH run
- assertDisableLogging(conn, 2, IndexTool.IndexVerifyType.BOTH,
+ assertDisableLogging(conn, MaxLookbackEnabled ? 2 : 0, IndexTool.IndexVerifyType.BOTH,
IndexTool.IndexDisableLoggingType.AFTER,
IndexVerificationOutputRepository.PHASE_BEFORE_VALUE, schemaName,
dataTableName, indexTableName,
@@ -1143,8 +1162,13 @@ public class IndexToolForNonTxGlobalIndexIT extends BaseUniqueNamesOwnClusterIT
assertEquals(0, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_VALID_INDEX_ROW_COUNT).getValue());
assertEquals(0, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_EXPIRED_INDEX_ROW_COUNT).getValue());
assertEquals(0, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_INVALID_INDEX_ROW_COUNT).getValue());
- assertEquals(NROWS, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_MISSING_INDEX_ROW_COUNT).getValue());
- assertEquals(0, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_BEYOND_MAXLOOKBACK_MISSING_INDEX_ROW_COUNT).getValue());
+ if (CompatBaseScannerRegionObserver.isMaxLookbackTimeEnabled(getUtility().getConfiguration())) {
+ assertEquals(NROWS, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_MISSING_INDEX_ROW_COUNT).getValue());
+ assertEquals(0, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_BEYOND_MAXLOOKBACK_MISSING_INDEX_ROW_COUNT).getValue());
+ } else {
+ assertEquals(0, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_MISSING_INDEX_ROW_COUNT).getValue());
+ assertEquals(NROWS, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_BEYOND_MAXLOOKBACK_MISSING_INDEX_ROW_COUNT).getValue());
+ }
assertEquals(0, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_BEYOND_MAXLOOKBACK_INVALID_INDEX_ROW_COUNT).getValue());
}
}
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexVerificationOldDesignIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexVerificationOldDesignIT.java
index 81745c8..322d0bb 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexVerificationOldDesignIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexVerificationOldDesignIT.java
@@ -17,6 +17,7 @@
*/
package org.apache.phoenix.end2end;
+import org.apache.phoenix.compat.hbase.coprocessor.CompatBaseScannerRegionObserver;
import org.apache.phoenix.thirdparty.com.google.common.collect.Maps;
import org.apache.phoenix.mapreduce.index.IndexTool;
import org.apache.phoenix.query.QueryServices;
@@ -55,6 +56,7 @@ public class IndexVerificationOldDesignIT extends BaseUniqueNamesOwnClusterIT {
serverProps.put(QueryServices.EXTRA_JDBC_ARGUMENTS_ATTRIB,
QueryServicesOptions.DEFAULT_EXTRA_JDBC_ARGUMENTS);
serverProps.put(QueryServices.INDEX_REBUILD_PAGE_SIZE_IN_ROWS, Long.toString(8));
+ serverProps.put(CompatBaseScannerRegionObserver.PHOENIX_MAX_LOOKBACK_AGE_CONF_KEY, Integer.toString(60*60)); // An hour
Map<String, String> clientProps = Maps.newHashMapWithExpectedSize(2);
clientProps.put(QueryServices.USE_STATS_FOR_PARALLELIZATION, Boolean.toString(true));
clientProps.put(QueryServices.STATS_UPDATE_FREQ_MS_ATTRIB, Long.toString(5));
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/GlobalIndexRegionScanner.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/GlobalIndexRegionScanner.java
index 9c62245..6865f1a 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/GlobalIndexRegionScanner.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/GlobalIndexRegionScanner.java
@@ -312,7 +312,11 @@ public abstract class GlobalIndexRegionScanner extends BaseRegionScanner {
protected static boolean isTimestampBeyondMaxLookBack(long maxLookBackInMills,
long currentTime, long tsToCheck) {
if (!CompatBaseScannerRegionObserver.isMaxLookbackTimeEnabled(maxLookBackInMills)) {
- return false;
+ // By definition, if the max lookback feature is not enabled, then delete markers and rows
+ // version can be removed by compaction any time, and thus there is no window in which these mutations are
+ // preserved, i.e., the max lookback window size is zero. This means all the mutations are effectively
+ // beyond the zero size max lookback window.
+ return true;
}
return tsToCheck < (currentTime - maxLookBackInMills);
}
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/PhoenixIndexImportDirectReducer.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/PhoenixIndexImportDirectReducer.java
index 8ec3603..b9a0cc6 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/PhoenixIndexImportDirectReducer.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/PhoenixIndexImportDirectReducer.java
@@ -108,6 +108,10 @@ public class PhoenixIndexImportDirectReducer extends
setValue(verificationResult.getAfterIndexHasExtraCellsCount());
context.getCounter(PhoenixIndexToolJobCounters.AFTER_REBUILD_INVALID_INDEX_ROW_COUNT_COZ_MISSING_CELLS).
setValue(verificationResult.getAfterIndexHasMissingCellsCount());
+ context.getCounter(PhoenixIndexToolJobCounters.AFTER_REBUILD_BEYOND_MAXLOOKBACK_MISSING_INDEX_ROW_COUNT).
+ setValue(verificationResult.getAfterRebuildBeyondMaxLookBackMissingIndexRowCount());
+ context.getCounter(PhoenixIndexToolJobCounters.AFTER_REBUILD_BEYOND_MAXLOOKBACK_INVALID_INDEX_ROW_COUNT).
+ setValue(verificationResult.getAfterRebuildBeyondMaxLookBackInvalidIndexRowCount());
}
if (verificationResult.isVerificationFailed()) {
throw new IOException("Index verification failed! " + verificationResult);