You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by ac...@apache.org on 2020/07/11 06:56:08 UTC
[phoenix] branch master updated: PHOENIX-6001 Incremental
rebuild/verification can result in missing rows and false positives
This is an automated email from the ASF dual-hosted git repository.
achouhan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/phoenix.git
The following commit(s) were added to refs/heads/master by this push:
new aafc481 PHOENIX-6001 Incremental rebuild/verification can result in missing rows and false positives
aafc481 is described below
commit aafc4810bec9a9176410e463aca79f46fa59f409
Author: Abhishek Singh Chouhan <ac...@apache.org>
AuthorDate: Fri Jul 10 18:23:43 2020 -0700
PHOENIX-6001 Incremental rebuild/verification can result in missing rows and false positives
---
.../end2end/IndexToolForNonTxGlobalIndexIT.java | 49 ++++++++++++++++++++++
.../coprocessor/IndexRebuildRegionScanner.java | 5 ++-
2 files changed, 52 insertions(+), 2 deletions(-)
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolForNonTxGlobalIndexIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolForNonTxGlobalIndexIT.java
index 9347f53..f89bcb3 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolForNonTxGlobalIndexIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolForNonTxGlobalIndexIT.java
@@ -323,6 +323,55 @@ public class IndexToolForNonTxGlobalIndexIT extends BaseUniqueNamesOwnClusterIT
// TODO: Enable once we move to these versions
@Ignore
@Test
+ public void testIncrementalRebuildWithPageSize() throws Exception {
+ String schemaName = generateUniqueName();
+ String dataTableName = generateUniqueName();
+ String fullDataTableName = SchemaUtil.getTableName(schemaName, dataTableName);
+ String indexTableName = generateUniqueName();
+ try (Connection conn = DriverManager.getConnection(getUrl())) {
+ long minTs = EnvironmentEdgeManager.currentTimeMillis();
+ conn.createStatement().execute(
+ "CREATE TABLE " + fullDataTableName + "(k VARCHAR PRIMARY KEY, v VARCHAR)");
+ conn.createStatement().execute("UPSERT INTO " + fullDataTableName + " VALUES('a','aa')");
+ conn.createStatement().execute("UPSERT INTO " + fullDataTableName + " VALUES('b','bb')");
+ conn.commit();
+ conn.createStatement().execute("DELETE FROM " + fullDataTableName + " WHERE k = 'a'");
+ conn.createStatement().execute("DELETE FROM " + fullDataTableName + " WHERE k = 'b'");
+ conn.commit();
+ conn.createStatement().execute("UPSERT INTO " + fullDataTableName + " VALUES('a','aaa')");
+ conn.createStatement().execute("UPSERT INTO " + fullDataTableName + " VALUES('b','bbb')");
+ conn.createStatement().execute("DELETE FROM " + fullDataTableName + " WHERE k = 'c'");
+ conn.commit();
+ conn.createStatement().execute(String.format("CREATE INDEX %s ON %s (v) ASYNC",
+ indexTableName, fullDataTableName));
+ // Run the index MR job and verify that the index table is built correctly
+ Configuration conf = new Configuration(getUtility().getConfiguration());
+ conf.set(QueryServices.INDEX_REBUILD_PAGE_SIZE_IN_ROWS, Long.toString(2));
+ IndexTool indexTool =
+ IndexToolIT.runIndexTool(conf, directApi, useSnapshot, schemaName,
+ dataTableName, indexTableName, null, 0, IndexTool.IndexVerifyType.AFTER,
+ "-st", String.valueOf(minTs), "-et",
+ String.valueOf(EnvironmentEdgeManager.currentTimeMillis()));
+ assertEquals(3, indexTool.getJob().getCounters().findCounter(INPUT_RECORDS).getValue());
+ assertEquals(3,
+ indexTool.getJob().getCounters().findCounter(SCANNED_DATA_ROW_COUNT).getValue());
+ assertEquals(4,
+ indexTool.getJob().getCounters().findCounter(REBUILT_INDEX_ROW_COUNT).getValue());
+ assertEquals(4, indexTool.getJob().getCounters()
+ .findCounter(AFTER_REBUILD_VALID_INDEX_ROW_COUNT).getValue());
+ assertEquals(0, indexTool.getJob().getCounters()
+ .findCounter(AFTER_REBUILD_EXPIRED_INDEX_ROW_COUNT).getValue());
+ assertEquals(0, indexTool.getJob().getCounters()
+ .findCounter(AFTER_REBUILD_INVALID_INDEX_ROW_COUNT).getValue());
+ assertEquals(0, indexTool.getJob().getCounters()
+ .findCounter(AFTER_REBUILD_MISSING_INDEX_ROW_COUNT).getValue());
+ }
+ }
+
+ // This test will only work with HBASE-22710 which is in 2.2.5+
+ // TODO: Enable once we move to these versions
+ @Ignore
+ @Test
public void testUpdatablePKFilterViewIndexRebuild() throws Exception {
if (!mutable) {
return;
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/IndexRebuildRegionScanner.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/IndexRebuildRegionScanner.java
index 9cad0f0..9a96124 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/IndexRebuildRegionScanner.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/IndexRebuildRegionScanner.java
@@ -1351,6 +1351,9 @@ public class IndexRebuildRegionScanner extends GlobalIndexRegionScanner {
if (indexRowKey != null) {
rowCount = singleRowRebuildReturnCode;
}
+ if (minTimestamp != 0) {
+ nextStartKey = ByteUtil.calculateTheClosestNextRowKeyForPrefix(CellUtil.cloneRow(lastCell));
+ }
byte[] rowCountBytes = PLong.INSTANCE.toBytes(Long.valueOf(rowCount));
final Cell aggKeyValue;
if (lastCell == null) {
@@ -1411,8 +1414,6 @@ public class IndexRebuildRegionScanner extends GlobalIndexRegionScanner {
if (keys.isEmpty()) {
return innerScanner;
}
- nextStartKey =
- ByteUtil.calculateTheClosestNextRowKeyForPrefix(keys.get(keys.size() - 1).getLowerRange());
ScanRanges scanRanges = ScanRanges.createPointLookup(keys);
scanRanges.initializeScan(incrScan);
SkipScanFilter skipScanFilter = scanRanges.getSkipScanFilter();