You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by ac...@apache.org on 2020/07/11 06:56:29 UTC
[phoenix] branch 4.x updated: PHOENIX-6001 Incremental
rebuild/verification can result in missing rows and false positives
This is an automated email from the ASF dual-hosted git repository.
achouhan pushed a commit to branch 4.x
in repository https://gitbox.apache.org/repos/asf/phoenix.git
The following commit(s) were added to refs/heads/4.x by this push:
new 43e18ce PHOENIX-6001 Incremental rebuild/verification can result in missing rows and false positives
43e18ce is described below
commit 43e18ce761987139eec17c1647fa01fe092f4307
Author: Abhishek Singh Chouhan <ac...@apache.org>
AuthorDate: Fri Jul 10 14:23:34 2020 -0700
PHOENIX-6001 Incremental rebuild/verification can result in missing rows and false positives
---
.../end2end/IndexToolForNonTxGlobalIndexIT.java | 54 ++++++++++++++++++++++
.../coprocessor/IndexRebuildRegionScanner.java | 4 +-
2 files changed, 57 insertions(+), 1 deletion(-)
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolForNonTxGlobalIndexIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolForNonTxGlobalIndexIT.java
index 527dc87..a2e5da0 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolForNonTxGlobalIndexIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolForNonTxGlobalIndexIT.java
@@ -1175,6 +1175,60 @@ public class IndexToolForNonTxGlobalIndexIT extends BaseUniqueNamesOwnClusterIT
}
}
+ @Test
+ public void testIncrementalRebuildWithPageSize() throws Exception {
+ String schemaName = generateUniqueName();
+ String dataTableName = generateUniqueName();
+ String fullDataTableName = SchemaUtil.getTableName(schemaName, dataTableName);
+ String indexTableName = generateUniqueName();
+ try (Connection conn = DriverManager.getConnection(getUrl())) {
+ long minTs = EnvironmentEdgeManager.currentTimeMillis();
+ conn.createStatement().execute(
+ "CREATE TABLE " + fullDataTableName + "(k VARCHAR PRIMARY KEY, v VARCHAR)");
+ conn.createStatement().execute("UPSERT INTO " + fullDataTableName + " VALUES('a','aa')");
+ conn.createStatement().execute("UPSERT INTO " + fullDataTableName + " VALUES('b','bb')");
+ conn.commit();
+ conn.createStatement().execute("DELETE FROM " + fullDataTableName + " WHERE k = 'a'");
+ conn.createStatement().execute("DELETE FROM " + fullDataTableName + " WHERE k = 'b'");
+ conn.commit();
+ conn.createStatement().execute("UPSERT INTO " + fullDataTableName + " VALUES('a','aaa')");
+ conn.createStatement().execute("UPSERT INTO " + fullDataTableName + " VALUES('b','bbb')");
+ conn.createStatement().execute("DELETE FROM " + fullDataTableName + " WHERE k = 'c'");
+ conn.commit();
+ conn.createStatement().execute(String.format("CREATE INDEX %s ON %s (v) ASYNC",
+ indexTableName, fullDataTableName));
+ // Run the index MR job and verify that the index table is built correctly
+ Configuration conf = new Configuration(getUtility().getConfiguration());
+ conf.set(QueryServices.INDEX_REBUILD_PAGE_SIZE_IN_ROWS, Long.toString(2));
+ IndexTool indexTool =
+ IndexToolIT.runIndexTool(conf, directApi, useSnapshot, schemaName,
+ dataTableName, indexTableName, null, 0, IndexTool.IndexVerifyType.AFTER,
+ "-st", String.valueOf(minTs), "-et",
+ String.valueOf(EnvironmentEdgeManager.currentTimeMillis()));
+ assertEquals(3, indexTool.getJob().getCounters().findCounter(INPUT_RECORDS).getValue());
+ assertEquals(3,
+ indexTool.getJob().getCounters().findCounter(SCANNED_DATA_ROW_COUNT).getValue());
+ assertEquals(4,
+ indexTool.getJob().getCounters().findCounter(REBUILT_INDEX_ROW_COUNT).getValue());
+ assertEquals(4, indexTool.getJob().getCounters()
+ .findCounter(AFTER_REBUILD_VALID_INDEX_ROW_COUNT).getValue());
+ assertEquals(0, indexTool.getJob().getCounters()
+ .findCounter(AFTER_REBUILD_EXPIRED_INDEX_ROW_COUNT).getValue());
+ assertEquals(0, indexTool.getJob().getCounters()
+ .findCounter(AFTER_REBUILD_INVALID_INDEX_ROW_COUNT).getValue());
+ assertEquals(0, indexTool.getJob().getCounters()
+ .findCounter(AFTER_REBUILD_MISSING_INDEX_ROW_COUNT).getValue());
+ assertEquals(0,
+ indexTool.getJob().getCounters()
+ .findCounter(AFTER_REBUILD_BEYOND_MAXLOOKBACK_MISSING_INDEX_ROW_COUNT)
+ .getValue());
+ assertEquals(0,
+ indexTool.getJob().getCounters()
+ .findCounter(AFTER_REBUILD_BEYOND_MAXLOOKBACK_INVALID_INDEX_ROW_COUNT)
+ .getValue());
+ }
+ }
+
@Test
public void testUpdatablePKFilterViewIndexRebuild() throws Exception {
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/IndexRebuildRegionScanner.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/IndexRebuildRegionScanner.java
index 36a1426..f8bac4e 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/IndexRebuildRegionScanner.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/IndexRebuildRegionScanner.java
@@ -1359,6 +1359,9 @@ public class IndexRebuildRegionScanner extends GlobalIndexRegionScanner {
if (indexRowKeyforReadRepair != null) {
dataRowCount = singleRowRebuildReturnCode;
}
+ if (minTimestamp != 0) {
+ nextStartKey = ByteUtil.calculateTheClosestNextRowKeyForPrefix(lastCell.getRow());
+ }
byte[] rowCountBytes = PLong.INSTANCE.toBytes(Long.valueOf(dataRowCount));
final Cell aggKeyValue;
if (lastCell == null) {
@@ -1419,7 +1422,6 @@ public class IndexRebuildRegionScanner extends GlobalIndexRegionScanner {
if (keys.isEmpty()) {
return innerScanner;
}
- nextStartKey = ByteUtil.calculateTheClosestNextRowKeyForPrefix(keys.get(keys.size() - 1).getLowerRange());
ScanRanges scanRanges = ScanRanges.createPointLookup(keys);
scanRanges.initializeScan(incrScan);
SkipScanFilter skipScanFilter = scanRanges.getSkipScanFilter();