You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by gj...@apache.org on 2020/09/14 17:30:43 UTC
[phoenix] branch master updated: PHOENIX-6067 - 5.x Global
Secondary Index Parity with 4.x
This is an automated email from the ASF dual-hosted git repository.
gjacoby pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/phoenix.git
The following commit(s) were added to refs/heads/master by this push:
new b9fb245 PHOENIX-6067 - 5.x Global Secondary Index Parity with 4.x
b9fb245 is described below
commit b9fb2452342bf3064b33245a09878530e953c6f8
Author: Geoffrey Jacoby <gj...@apache.org>
AuthorDate: Thu Sep 3 14:31:49 2020 -0700
PHOENIX-6067 - 5.x Global Secondary Index Parity with 4.x
---
.../end2end/ConcurrentMutationsExtendedIT.java | 102 +-
.../apache/phoenix/end2end/IndexExtendedIT.java | 6 +-
.../phoenix/end2end/IndexScrutinyToolBaseIT.java | 7 +-
.../end2end/IndexScrutinyToolForTenantIT.java | 9 +-
.../phoenix/end2end/IndexScrutinyToolIT.java | 91 +-
.../end2end/IndexScrutinyWithMaxLookbackIT.java | 317 +++++
.../end2end/IndexToolForNonTxGlobalIndexIT.java | 1287 +++++++++++---------
.../phoenix/end2end/IndexToolTimeRangeIT.java | 6 +
.../org/apache/phoenix/end2end/MaxLookbackIT.java | 29 +-
.../NonParameterizedIndexScrutinyToolIT.java | 8 +-
.../apache/phoenix/end2end/index/DropColumnIT.java | 20 +-
.../end2end/index/GlobalIndexCheckerIT.java | 131 +-
.../index/IndexVerificationOutputRepositoryIT.java | 96 +-
.../phoenix/compile/ServerBuildIndexCompiler.java | 2 +-
.../coprocessor/BaseScannerRegionObserver.java | 2 +
.../coprocessor/GlobalIndexRegionScanner.java | 88 +-
.../coprocessor/IndexRebuildRegionScanner.java | 912 +++++++-------
.../coprocessor/IndexToolVerificationResult.java | 143 ++-
.../phoenix/coprocessor/IndexerRegionScanner.java | 159 ++-
.../UngroupedAggregateRegionObserver.java | 389 +++---
.../apache/phoenix/index/GlobalIndexChecker.java | 131 +-
.../PhoenixServerBuildIndexInputFormat.java | 7 +
.../mapreduce/index/IndexScrutinyMapper.java | 61 +-
.../mapreduce/index/IndexScrutinyTableOutput.java | 49 +-
.../phoenix/mapreduce/index/IndexScrutinyTool.java | 61 +-
.../apache/phoenix/mapreduce/index/IndexTool.java | 6 +
.../index/IndexVerificationOutputRepository.java | 69 +-
.../index/IndexVerificationOutputRow.java | 45 +-
.../index/IndexVerificationResultRepository.java | 72 +-
.../index/PhoenixIndexImportDirectReducer.java | 4 +
.../index/PhoenixIndexToolJobCounters.java | 8 +-
.../index/PhoenixScrutinyJobCounters.java | 8 +-
.../java/org/apache/phoenix/util/IndexUtil.java | 85 +-
.../org/apache/phoenix/index/IndexToolTest.java | 50 +-
.../apache/phoenix/index/IndexUpgradeToolTest.java | 4 +
.../phoenix/index/VerifySingleIndexRowTest.java | 368 ++++--
.../index/IndexScrutinyTableOutputTest.java | 45 +-
.../java/org/apache/phoenix/util/TestUtil.java | 8 +-
.../compat/hbase/HbaseCompatCapabilities.java | 5 +
.../compat/hbase/HbaseCompatCapabilities.java | 5 +
.../compat/hbase/HbaseCompatCapabilities.java | 5 +
41 files changed, 3208 insertions(+), 1692 deletions(-)
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ConcurrentMutationsExtendedIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ConcurrentMutationsExtendedIT.java
index 9389d0c..704df4e 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ConcurrentMutationsExtendedIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ConcurrentMutationsExtendedIT.java
@@ -17,34 +17,50 @@
*/
package org.apache.phoenix.end2end;
+import com.google.common.collect.Maps;
import org.apache.hadoop.hbase.DoNotRetryIOException;
import org.apache.hadoop.hbase.HBaseIOException;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.Mutation;
-import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.coprocessor.ObserverContext;
import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
import org.apache.hadoop.hbase.coprocessor.SimpleRegionObserver;
import org.apache.hadoop.hbase.regionserver.MiniBatchOperationInProgress;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.phoenix.compat.hbase.coprocessor.CompatBaseScannerRegionObserver;
import org.apache.phoenix.jdbc.PhoenixConnection;
import org.apache.phoenix.mapreduce.index.IndexTool;
+import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.schema.PTable;
import org.apache.phoenix.util.*;
-import org.junit.Ignore;
+import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.runner.RunWith;
import java.io.IOException;
import java.sql.*;
+import java.util.Map;
import java.util.Properties;
import java.util.Random;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
+import static org.apache.phoenix.mapreduce.index.PhoenixIndexToolJobCounters.AFTER_REBUILD_BEYOND_MAXLOOKBACK_INVALID_INDEX_ROW_COUNT;
+import static org.apache.phoenix.mapreduce.index.PhoenixIndexToolJobCounters.AFTER_REBUILD_BEYOND_MAXLOOKBACK_MISSING_INDEX_ROW_COUNT;
+import static org.apache.phoenix.mapreduce.index.PhoenixIndexToolJobCounters.AFTER_REBUILD_INVALID_INDEX_ROW_COUNT;
+import static org.apache.phoenix.mapreduce.index.PhoenixIndexToolJobCounters.AFTER_REBUILD_MISSING_INDEX_ROW_COUNT;
+import static org.apache.phoenix.mapreduce.index.PhoenixIndexToolJobCounters.AFTER_REBUILD_VALID_INDEX_ROW_COUNT;
+import static org.apache.phoenix.mapreduce.index.PhoenixIndexToolJobCounters.BEFORE_REBUILD_BEYOND_MAXLOOKBACK_INVALID_INDEX_ROW_COUNT;
+import static org.apache.phoenix.mapreduce.index.PhoenixIndexToolJobCounters.BEFORE_REBUILD_BEYOND_MAXLOOKBACK_MISSING_INDEX_ROW_COUNT;
+import static org.apache.phoenix.mapreduce.index.PhoenixIndexToolJobCounters.BEFORE_REBUILD_INVALID_INDEX_ROW_COUNT;
+import static org.apache.phoenix.mapreduce.index.PhoenixIndexToolJobCounters.BEFORE_REBUILD_MISSING_INDEX_ROW_COUNT;
+import static org.apache.phoenix.mapreduce.index.PhoenixIndexToolJobCounters.BEFORE_REBUILD_OLD_INDEX_ROW_COUNT;
+import static org.apache.phoenix.mapreduce.index.PhoenixIndexToolJobCounters.BEFORE_REBUILD_UNKNOWN_INDEX_ROW_COUNT;
+import static org.apache.phoenix.mapreduce.index.PhoenixIndexToolJobCounters.BEFORE_REBUILD_UNVERIFIED_INDEX_ROW_COUNT;
+import static org.apache.phoenix.mapreduce.index.PhoenixIndexToolJobCounters.REBUILT_INDEX_ROW_COUNT;
import static org.junit.Assert.*;
@RunWith(RunUntilFailure.class) @Category(NeedsOwnMiniClusterTest.class)
@@ -54,21 +70,57 @@ public class ConcurrentMutationsExtendedIT extends ParallelStatsDisabledIT {
private static final String MVCC_LOCK_TEST_TABLE_PREFIX = "MVCCLOCKTEST_";
private static final String LOCK_TEST_TABLE_PREFIX = "LOCKTEST_";
private static final int ROW_LOCK_WAIT_TIME = 10000;
-
+ private static final int MAX_LOOKBACK_AGE = 1000000;
private final Object lock = new Object();
+ @BeforeClass
+ public static synchronized void doSetup() throws Exception {
+ Map<String, String> props = Maps.newHashMapWithExpectedSize(1);
+ props.put(QueryServices.GLOBAL_INDEX_ROW_AGE_THRESHOLD_TO_DELETE_MS_ATTRIB, Long.toString(0));
+ props.put(CompatBaseScannerRegionObserver.PHOENIX_MAX_LOOKBACK_AGE_CONF_KEY,
+ Integer.toString(MAX_LOOKBACK_AGE));
+ setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator()));
+ }
+
private long verifyIndexTable(String tableName, String indexName, Connection conn) throws Exception {
// This checks the state of every raw index row without rebuilding any row
- IndexToolIT.runIndexTool(true, false, "", tableName, indexName, null,
- 0, IndexTool.IndexVerifyType.ONLY);
+ IndexTool indexTool = IndexToolIT.runIndexTool(true, false, "", tableName,
+ indexName, null, 0, IndexTool.IndexVerifyType.ONLY);
+ System.out.println(indexTool.getJob().getCounters());
+ assertEquals(0, indexTool.getJob().getCounters().findCounter(REBUILT_INDEX_ROW_COUNT).getValue());
+ assertEquals(0, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_INVALID_INDEX_ROW_COUNT).getValue());
+ assertEquals(0, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_MISSING_INDEX_ROW_COUNT).getValue());
+ assertEquals(0, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_BEYOND_MAXLOOKBACK_MISSING_INDEX_ROW_COUNT).getValue());
+ assertEquals(0, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_BEYOND_MAXLOOKBACK_INVALID_INDEX_ROW_COUNT).getValue());
+ assertEquals(0, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_OLD_INDEX_ROW_COUNT).getValue());
+ assertEquals(0, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_UNKNOWN_INDEX_ROW_COUNT).getValue());
+
// This checks the state of an index row after it is repaired
long actualRowCount = IndexScrutiny.scrutinizeIndex(conn, tableName, indexName);
// We want to check the index rows again as they may be modified by the read repair
- IndexToolIT.runIndexTool(true, false, "", tableName, indexName, null,
- 0, IndexTool.IndexVerifyType.ONLY);
+ indexTool = IndexToolIT.runIndexTool(true, false, "", tableName, indexName,
+ null, 0, IndexTool.IndexVerifyType.ONLY);
+ System.out.println(indexTool.getJob().getCounters());
+
+ assertEquals(0, indexTool.getJob().getCounters().findCounter(REBUILT_INDEX_ROW_COUNT).getValue());
+ assertEquals(0, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_INVALID_INDEX_ROW_COUNT).getValue());
+ assertEquals(0, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_MISSING_INDEX_ROW_COUNT).getValue());
+ assertEquals(0, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_BEYOND_MAXLOOKBACK_MISSING_INDEX_ROW_COUNT).getValue());
+ assertEquals(0, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_BEYOND_MAXLOOKBACK_INVALID_INDEX_ROW_COUNT).getValue());
+ // The index scrutiny run will trigger index repair on all unverified rows and they rows will be made verified
+ // deleted (since the age threshold is set to zero ms for these tests
+ assertEquals(0, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_UNVERIFIED_INDEX_ROW_COUNT).getValue());
+ assertEquals(0, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_OLD_INDEX_ROW_COUNT).getValue());
+ assertEquals(0, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_UNKNOWN_INDEX_ROW_COUNT).getValue());
// Now we rebuild the entire index table and expect that it is still good after the full rebuild
- IndexToolIT.runIndexTool(true, false, "", tableName, indexName, null,
- 0, IndexTool.IndexVerifyType.AFTER);
+ indexTool = IndexToolIT.runIndexTool(true, false, "", tableName, indexName,
+ null, 0, IndexTool.IndexVerifyType.AFTER);
+ assertEquals(indexTool.getJob().getCounters().findCounter(AFTER_REBUILD_VALID_INDEX_ROW_COUNT).getValue(),
+ indexTool.getJob().getCounters().findCounter(REBUILT_INDEX_ROW_COUNT).getValue());
+ assertEquals(0, indexTool.getJob().getCounters().findCounter(AFTER_REBUILD_INVALID_INDEX_ROW_COUNT).getValue());
+ assertEquals(0, indexTool.getJob().getCounters().findCounter(AFTER_REBUILD_MISSING_INDEX_ROW_COUNT).getValue());
+ assertEquals(0, indexTool.getJob().getCounters().findCounter(AFTER_REBUILD_BEYOND_MAXLOOKBACK_MISSING_INDEX_ROW_COUNT).getValue());
+ assertEquals(0, indexTool.getJob().getCounters().findCounter(AFTER_REBUILD_BEYOND_MAXLOOKBACK_INVALID_INDEX_ROW_COUNT).getValue());
// Truncate, rebuild and verify the index table
PTable pIndexTable = PhoenixRuntime.getTable(conn, indexName);
TableName physicalTableName = TableName.valueOf(pIndexTable.getPhysicalName().getBytes());
@@ -77,8 +129,12 @@ public class ConcurrentMutationsExtendedIT extends ParallelStatsDisabledIT {
admin.disableTable(physicalTableName);
admin.truncateTable(physicalTableName, true);
}
- IndexToolIT.runIndexTool(true, false, "", tableName, indexName, null,
- 0, IndexTool.IndexVerifyType.AFTER);
+ indexTool = IndexToolIT.runIndexTool(true, false, "", tableName, indexName,
+ null, 0, IndexTool.IndexVerifyType.AFTER);
+ assertEquals(0, indexTool.getJob().getCounters().findCounter(AFTER_REBUILD_INVALID_INDEX_ROW_COUNT).getValue());
+ assertEquals(0, indexTool.getJob().getCounters().findCounter(AFTER_REBUILD_MISSING_INDEX_ROW_COUNT).getValue());
+ assertEquals(0, indexTool.getJob().getCounters().findCounter(AFTER_REBUILD_BEYOND_MAXLOOKBACK_MISSING_INDEX_ROW_COUNT).getValue());
+ assertEquals(0, indexTool.getJob().getCounters().findCounter(AFTER_REBUILD_BEYOND_MAXLOOKBACK_INVALID_INDEX_ROW_COUNT).getValue());
long actualRowCountAfterCompaction = IndexScrutiny.scrutinizeIndex(conn, tableName, indexName);
assertEquals(actualRowCount, actualRowCountAfterCompaction);
return actualRowCount;
@@ -102,15 +158,10 @@ public class ConcurrentMutationsExtendedIT extends ParallelStatsDisabledIT {
for (int i = 0; i < 50; i++) {
Thread.sleep(20);
synchronized (lock) {
- PhoenixConnection conn = null;
- try {
- conn =
- DriverManager.getConnection(getUrl(), props)
- .unwrap(PhoenixConnection.class);
+ try (PhoenixConnection conn = DriverManager.getConnection(getUrl(), props)
+ .unwrap(PhoenixConnection.class)) {
conn.setAutoCommit(true);
conn.createStatement().execute("DELETE FROM " + tableName);
- } finally {
- if (conn != null) conn.close();
}
}
}
@@ -133,19 +184,14 @@ public class ConcurrentMutationsExtendedIT extends ParallelStatsDisabledIT {
int nRowsToUpsert = 1000;
for (int i = 0; i < nRowsToUpsert; i++) {
synchronized (lock) {
- PhoenixConnection conn = null;
- try {
- conn =
- DriverManager.getConnection(getUrl(), props)
- .unwrap(PhoenixConnection.class);
+ try (PhoenixConnection conn = DriverManager.getConnection(getUrl(), props)
+ .unwrap(PhoenixConnection.class)) {
conn.createStatement().execute(
- "UPSERT INTO " + tableName + " VALUES (" + (i % 10)
- + ", 0, 1)");
+ "UPSERT INTO " + tableName + " VALUES (" + (i % 10)
+ + ", 0, 1)");
if ((i % 20) == 0 || i == nRowsToUpsert - 1) {
conn.commit();
}
- } finally {
- if (conn != null) conn.close();
}
}
}
@@ -239,7 +285,6 @@ public class ConcurrentMutationsExtendedIT extends ParallelStatsDisabledIT {
conn.createStatement().execute("CREATE TABLE " + tableName
+ "(k1 INTEGER NOT NULL, k2 INTEGER NOT NULL, a.v1 INTEGER, b.v2 INTEGER, c.v3 INTEGER, d.v4 INTEGER," +
"CONSTRAINT pk PRIMARY KEY (k1,k2)) COLUMN_ENCODED_BYTES = 0, VERSIONS=1");
- TestUtil.addCoprocessor(conn, tableName, DelayingRegionObserver.class);
conn.createStatement().execute("CREATE INDEX " + indexName + " ON " + tableName + "(v1) INCLUDE(v2, v3)");
final CountDownLatch doneSignal = new CountDownLatch(nThreads);
Runnable[] runnables = new Runnable[nThreads];
@@ -291,7 +336,6 @@ public class ConcurrentMutationsExtendedIT extends ParallelStatsDisabledIT {
conn.createStatement().execute("CREATE TABLE " + tableName
+ "(k1 INTEGER NOT NULL, k2 INTEGER NOT NULL, a.v1 INTEGER, b.v2 INTEGER, c.v3 INTEGER, d.v4 INTEGER," +
"CONSTRAINT pk PRIMARY KEY (k1,k2)) COLUMN_ENCODED_BYTES = 0, VERSIONS=1");
- TestUtil.addCoprocessor(conn, tableName, DelayingRegionObserver.class);
conn.createStatement().execute("CREATE INDEX " + indexName + " ON " + tableName + "(v1) INCLUDE(v2, v3)");
final CountDownLatch doneSignal = new CountDownLatch(nThreads);
Runnable[] runnables = new Runnable[nThreads];
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexExtendedIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexExtendedIT.java
index bf78cef..a791066 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexExtendedIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexExtendedIT.java
@@ -36,7 +36,7 @@ import java.util.Properties;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.phoenix.hbase.index.IndexRegionObserver;
+import org.apache.phoenix.coprocessor.IndexRebuildRegionScanner;
import org.apache.phoenix.jdbc.PhoenixConnection;
import org.apache.phoenix.mapreduce.index.IndexTool;
import org.apache.phoenix.query.BaseTest;
@@ -341,7 +341,7 @@ public class IndexExtendedIT extends BaseTest {
// Configure IndexRegionObserver to fail the first write phase. This should not
// lead to any change on index and thus index verify during index rebuild should fail
- IndexRegionObserver.setIgnoreIndexRebuildForTesting(true);
+ IndexRebuildRegionScanner.setIgnoreIndexRebuildForTesting(true);
stmt.execute(String.format(
"CREATE INDEX %s ON %s (NAME) INCLUDE (ZIP) ASYNC",
indexTableName, dataTableFullName));
@@ -353,7 +353,7 @@ public class IndexExtendedIT extends BaseTest {
IndexToolIT.runIndexTool(true, false, schemaName, dataTableName,
indexTableName, null, -1, IndexTool.IndexVerifyType.AFTER);
- IndexRegionObserver.setIgnoreIndexRebuildForTesting(false);
+ IndexRebuildRegionScanner.setIgnoreIndexRebuildForTesting(false);
// job failed, verify that the index table is still not in the ACTIVE state
assertFalse(checkIndexState(conn, indexFullName, PIndexState.ACTIVE, 0L));
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexScrutinyToolBaseIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexScrutinyToolBaseIT.java
index 54a5408..980c0fa 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexScrutinyToolBaseIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexScrutinyToolBaseIT.java
@@ -16,7 +16,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.mapreduce.Counters;
import org.apache.hadoop.mapreduce.Job;
-import org.apache.phoenix.mapreduce.index.IndexScrutinyMapperForTest;
+import org.apache.phoenix.mapreduce.index.IndexScrutinyMapper;
import org.apache.phoenix.mapreduce.index.IndexScrutinyTool;
import org.apache.phoenix.mapreduce.index.IndexScrutinyTool.OutputFormat;
import org.apache.phoenix.mapreduce.index.IndexScrutinyTool.SourceTable;
@@ -54,8 +54,9 @@ public class IndexScrutinyToolBaseIT extends BaseTest {
new ReadOnlyProps(clientProps.entrySet().iterator()));
}
- protected List<Job> runScrutiny(String[] cmdArgs) throws Exception {
- IndexScrutinyTool scrutiny = new IndexScrutinyTool(IndexScrutinyMapperForTest.class);
+ protected List<Job> runScrutiny(Class<? extends IndexScrutinyMapper> mapperClass,
+ String[] cmdArgs) throws Exception {
+ IndexScrutinyTool scrutiny = new IndexScrutinyTool(mapperClass);
Configuration conf = new Configuration(getUtility().getConfiguration());
scrutiny.setConf(conf);
int status = scrutiny.run(cmdArgs);
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexScrutinyToolForTenantIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexScrutinyToolForTenantIT.java
index 351af9c..f4e1ce7 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexScrutinyToolForTenantIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexScrutinyToolForTenantIT.java
@@ -16,6 +16,7 @@ import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.mapreduce.Counters;
import org.apache.hadoop.mapreduce.Job;
import org.apache.phoenix.jdbc.PhoenixConnection;
+import org.apache.phoenix.mapreduce.index.IndexScrutinyMapperForTest;
import org.apache.phoenix.mapreduce.index.IndexScrutinyTool;
import org.apache.phoenix.mapreduce.index.IndexScrutinyTool.OutputFormat;
import org.apache.phoenix.mapreduce.index.IndexScrutinyTool.SourceTable;
@@ -115,7 +116,7 @@ public class IndexScrutinyToolForTenantIT extends IndexScrutinyToolBaseIT {
getArgValues("", tenantViewName, indexNameTenant, 10L, SourceTable.INDEX_TABLE_SOURCE, false, null, null, tenantId,
EnvironmentEdgeManager.currentTimeMillis());
- List<Job> completedJobs = runScrutiny(argValues);
+ List<Job> completedJobs = runScrutiny(IndexScrutinyMapperForTest.class, argValues);
// Sunny case, both index and view are equal. 1 row
assertEquals(1, completedJobs.size());
for (Job job : completedJobs) {
@@ -145,7 +146,7 @@ public class IndexScrutinyToolForTenantIT extends IndexScrutinyToolBaseIT {
argValues =
getArgValues("", globalViewName, indexNameGlobal, 10L, SourceTable.INDEX_TABLE_SOURCE, false, null, null, null,
EnvironmentEdgeManager.currentTimeMillis());
- List<Job> completedJobs = runScrutiny(argValues);
+ List<Job> completedJobs = runScrutiny(IndexScrutinyMapperForTest.class, argValues);
// Sunny case, both index and view are equal. 1 row
assertEquals(1, completedJobs.size());
for (Job job : completedJobs) {
@@ -200,7 +201,7 @@ public class IndexScrutinyToolForTenantIT extends IndexScrutinyToolBaseIT {
String[]
argValues =
getArgValues("", tenantViewName, indexNameTenant, 10L, SourceTable.BOTH, false, null, null, tenantId, EnvironmentEdgeManager.currentTimeMillis());
- List<Job> completedJobs = runScrutiny(argValues);
+ List<Job> completedJobs = runScrutiny(IndexScrutinyMapperForTest.class, argValues);
assertEquals(2, completedJobs.size());
for (Job job : completedJobs) {
@@ -246,7 +247,7 @@ public class IndexScrutinyToolForTenantIT extends IndexScrutinyToolBaseIT {
argValues =
getArgValues("", tenantViewName, indexNameTenant, 10L, SourceTable.DATA_TABLE_SOURCE, true, outputFormat, null,
tenantId, EnvironmentEdgeManager.currentTimeMillis());
- List<Job> completedJobs = runScrutiny(argValues);
+ List<Job> completedJobs = runScrutiny(IndexScrutinyMapperForTest.class, argValues);
assertEquals(1, completedJobs.size());
for (Job job : completedJobs) {
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexScrutinyToolIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexScrutinyToolIT.java
index 799ce43..a56ebb3 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexScrutinyToolIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexScrutinyToolIT.java
@@ -47,6 +47,7 @@ import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.mapreduce.Counters;
import org.apache.hadoop.mapreduce.Job;
import org.apache.phoenix.mapreduce.CsvBulkImportUtil;
+import org.apache.phoenix.mapreduce.index.IndexScrutinyMapperForTest;
import org.apache.phoenix.mapreduce.index.IndexScrutinyTableOutput;
import org.apache.phoenix.mapreduce.index.IndexScrutinyTool;
import org.apache.phoenix.mapreduce.index.IndexScrutinyTool.OutputFormat;
@@ -60,6 +61,7 @@ import org.apache.phoenix.util.PropertiesUtil;
import org.apache.phoenix.util.SchemaUtil;
import org.junit.After;
import org.junit.Before;
+
import org.junit.Ignore;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@@ -74,6 +76,28 @@ import com.google.common.collect.Lists;
@Category(NeedsOwnMiniClusterTest.class)
@RunWith(Parameterized.class)
public class IndexScrutinyToolIT extends IndexScrutinyToolBaseIT {
+ public static final String MISSING_ROWS_QUERY_TEMPLATE =
+ "SELECT \"SOURCE_TABLE\" , \"TARGET_TABLE\" , \"SCRUTINY_EXECUTE_TIME\" , " +
+ "\"SOURCE_ROW_PK_HASH\" , \"SOURCE_TS\" , \"TARGET_TS\" , \"HAS_TARGET_ROW\" , " +
+ "\"BEYOND_MAX_LOOKBACK\" , \"ID\" , \"NAME\" , \"EMPLOY_DATE\" , \"ZIP\" , \":ID\" , " +
+ "\"0:NAME\" , \"0:EMPLOY_DATE\" , \"0:ZIP\" " +
+ "FROM PHOENIX_INDEX_SCRUTINY(\"ID\" INTEGER,\"NAME\" VARCHAR," +
+ "\"EMPLOY_DATE\" TIMESTAMP,\"ZIP\" INTEGER,\":ID\" INTEGER,\"0:NAME\" VARCHAR," +
+ "\"0:EMPLOY_DATE\" DECIMAL,\"0:ZIP\" INTEGER) WHERE (\"SOURCE_TABLE\",\"TARGET_TABLE\"," +
+ "\"SCRUTINY_EXECUTE_TIME\", \"HAS_TARGET_ROW\") IN (('[TableName]','[IndexName]'," +
+ "[ScrutinyTs],[HasTargetRow]))";
+ public static final String BEYOND_MAX_LOOKBACK_QUERY_TEMPLATE =
+ "SELECT \"SOURCE_TABLE\" , \"TARGET_TABLE\" , \"SCRUTINY_EXECUTE_TIME\" , " +
+ "\"SOURCE_ROW_PK_HASH\" , \"SOURCE_TS\" , \"TARGET_TS\" , \"HAS_TARGET_ROW\" , " +
+ "\"BEYOND_MAX_LOOKBACK\" , \"ID\" , \"NAME\" , \"EMPLOY_DATE\" , \"ZIP\" , \":ID\" , " +
+ "\"0:NAME\" , \"0:EMPLOY_DATE\" , \"0:ZIP\" " +
+ "FROM PHOENIX_INDEX_SCRUTINY(\"ID\" INTEGER,\"NAME\" VARCHAR," +
+ "\"EMPLOY_DATE\" TIMESTAMP,\"ZIP\" INTEGER,\":ID\" INTEGER,\"0:NAME\" VARCHAR," +
+ "\"0:EMPLOY_DATE\" DECIMAL,\"0:ZIP\" INTEGER) WHERE (\"SOURCE_TABLE\",\"TARGET_TABLE\"," +
+ "\"SCRUTINY_EXECUTE_TIME\", \"HAS_TARGET_ROW\", \"BEYOND_MAX_LOOKBACK\") " +
+ "IN (('[TableName]','[IndexName]'," +
+ "[ScrutinyTs],false,true))";
+
private String dataTableDdl;
private String indexTableDdl;
@@ -401,7 +425,7 @@ public class IndexScrutinyToolIT extends IndexScrutinyToolBaseIT {
String[]
argValues =
getArgValues(schemaName, dataTableName, indexTableName, 10L, SourceTable.DATA_TABLE_SOURCE, true, OutputFormat.FILE, null);
- runScrutiny(argValues);
+ runScrutiny(IndexScrutinyMapperForTest.class, argValues);
// check the output files
Path outputPath = CsvBulkImportUtil.getOutputPath(new Path(outputDir), dataTableFullName);
@@ -439,7 +463,6 @@ public class IndexScrutinyToolIT extends IndexScrutinyToolBaseIT {
+ new Timestamp(testTime).toString() + ", 9999]", lineIterator.next());
assertEquals("[3, name-3, " + new Timestamp(testTime).toString() + ", 95123]\tTarget row not found",
lineIterator.next());
-
}
/**
@@ -450,7 +473,7 @@ public class IndexScrutinyToolIT extends IndexScrutinyToolBaseIT {
String[]
argValues =
getArgValues(schemaName, dataTableName, indexTableName, 10L, SourceTable.DATA_TABLE_SOURCE, true, OutputFormat.TABLE, null);
- List<Job> completedJobs = runScrutiny(argValues);
+ List<Job> completedJobs = runScrutiny(IndexScrutinyMapperForTest.class, argValues);
// check that the output table contains the invalid rows
long
@@ -460,6 +483,10 @@ public class IndexScrutinyToolIT extends IndexScrutinyToolBaseIT {
invalidRowsQuery =
IndexScrutinyTableOutput
.getSqlQueryAllInvalidRows(conn, getColNames(), scrutinyTimeMillis);
+ String missingRowsQuery = getMissingRowsQuery(scrutinyTimeMillis);
+ String invalidColsQuery = getInvalidColsQuery(scrutinyTimeMillis);
+ String beyondMaxLookbackQuery = getBeyondMaxLookbackQuery(scrutinyTimeMillis);
+
ResultSet rs = conn.createStatement().executeQuery(invalidRowsQuery + " ORDER BY ID asc");
assertTrue(rs.next());
assertEquals(dataTableFullName, rs.getString(IndexScrutinyTableOutput.SOURCE_TABLE_COL_NAME));
@@ -473,12 +500,37 @@ public class IndexScrutinyToolIT extends IndexScrutinyToolBaseIT {
assertEquals(dataTableFullName, rs.getString(IndexScrutinyTableOutput.SOURCE_TABLE_COL_NAME));
assertEquals(indexTableFullName, rs.getString(IndexScrutinyTableOutput.TARGET_TABLE_COL_NAME));
assertFalse(rs.getBoolean("HAS_TARGET_ROW"));
+ assertFalse(rs.getBoolean("BEYOND_MAX_LOOKBACK"));
assertEquals(3, rs.getInt("ID"));
assertEquals(null, rs.getObject(":ID")); // null for missing target row
assertFalse(rs.next());
// check that the job results were written correctly to the metadata table
- assertMetadataTableValues(argValues, scrutinyTimeMillis, invalidRowsQuery);
+ assertMetadataTableValues(argValues, scrutinyTimeMillis, invalidRowsQuery, missingRowsQuery,
+ invalidColsQuery, beyondMaxLookbackQuery);
+ }
+
+ private String getMissingRowsQuery(long scrutinyTimeMillis) {
+ return MISSING_ROWS_QUERY_TEMPLATE.
+ replace("[TableName]", dataTableFullName).
+ replace("[IndexName]", indexTableFullName).
+ replace("[ScrutinyTs]", Long.toString(scrutinyTimeMillis)).
+ replace("[HasTargetRow]", "false");
+ }
+
+ private String getInvalidColsQuery(long scrutinyTimeMillis) {
+ return MISSING_ROWS_QUERY_TEMPLATE.
+ replace("[TableName]", dataTableFullName).
+ replace("[IndexName]", indexTableFullName).
+ replace("[ScrutinyTs]", Long.toString(scrutinyTimeMillis)).
+ replace("[HasTargetRow]", "true");
+ }
+
+ private String getBeyondMaxLookbackQuery(long scrutinyTimeMillis) {
+ return BEYOND_MAX_LOOKBACK_QUERY_TEMPLATE.
+ replace("[TableName]", dataTableFullName).
+ replace("[IndexName]", indexTableFullName).
+ replace("[ScrutinyTs]", Long.toString(scrutinyTimeMillis));
}
/**
@@ -490,7 +542,7 @@ public class IndexScrutinyToolIT extends IndexScrutinyToolBaseIT {
String[]
argValues =
getArgValues(schemaName, dataTableName, indexTableName, 10L, SourceTable.DATA_TABLE_SOURCE, true, OutputFormat.TABLE, new Long(1));
- List<Job> completedJobs = runScrutiny(argValues);
+ List<Job> completedJobs = runScrutiny(IndexScrutinyMapperForTest.class, argValues);
long
scrutinyTimeMillis =
PhoenixConfigurationUtil.getScrutinyExecuteTimestamp(completedJobs.get(0).getConfiguration());
@@ -535,7 +587,11 @@ public class IndexScrutinyToolIT extends IndexScrutinyToolBaseIT {
conn.commit();
}
- private void assertMetadataTableValues(String[] argValues, long scrutinyTimeMillis, String invalidRowsQuery) throws SQLException {
+ private void assertMetadataTableValues(String[] argValues, long scrutinyTimeMillis,
+ String invalidRowsQuery,
+ String missingRowsQuery,
+ String invalidColValuesQuery,
+ String beyondMaxLookbackQuery) throws SQLException {
ResultSet rs;
ResultSet
metadataRs =
@@ -545,11 +601,13 @@ public class IndexScrutinyToolIT extends IndexScrutinyToolBaseIT {
assertTrue(metadataRs.next());
List<? extends Object>
expected =
- Arrays.asList(dataTableFullName, indexTableFullName, scrutinyTimeMillis, SourceTable.DATA_TABLE_SOURCE.name(), Arrays.toString(argValues), 3L, 0L,
+ Arrays.asList(dataTableFullName, indexTableFullName, scrutinyTimeMillis,
+ SourceTable.DATA_TABLE_SOURCE.name(), Arrays.toString(argValues), 3L, 0L,
1L, 2L, 1L, 1L,
"[\"ID\" INTEGER, \"NAME\" VARCHAR, \"EMPLOY_DATE\" TIMESTAMP, \"ZIP\" INTEGER]",
"[\":ID\" INTEGER, \"0:NAME\" VARCHAR, \"0:EMPLOY_DATE\" DECIMAL, \"0:ZIP\" INTEGER]",
- invalidRowsQuery);
+ invalidRowsQuery, missingRowsQuery, invalidColValuesQuery,
+ beyondMaxLookbackQuery, 0L);
if (dataTableDdl.contains("SALT_BUCKETS")) {
expected =
Arrays.asList(dataTableFullName, indexTableFullName, scrutinyTimeMillis,
@@ -557,9 +615,9 @@ public class IndexScrutinyToolIT extends IndexScrutinyToolBaseIT {
2L, 1L, 2L,
"[\"ID\" INTEGER, \"NAME\" VARCHAR, \"EMPLOY_DATE\" TIMESTAMP, \"ZIP\" INTEGER]",
"[\":ID\" INTEGER, \"0:NAME\" VARCHAR, \"0:EMPLOY_DATE\" DECIMAL, \"0:ZIP\" INTEGER]",
- invalidRowsQuery);
+ invalidRowsQuery, missingRowsQuery,
+ invalidColValuesQuery, beyondMaxLookbackQuery, 0L);
}
-
assertRsValues(metadataRs, expected);
String missingTargetQuery = metadataRs.getString("INVALID_ROWS_QUERY_MISSING_TARGET");
rs = conn.createStatement().executeQuery(missingTargetQuery);
@@ -571,13 +629,19 @@ public class IndexScrutinyToolIT extends IndexScrutinyToolBaseIT {
assertTrue(rs.next());
assertEquals(2, rs.getInt("ID"));
assertFalse(rs.next());
+ String lookbackQuery = metadataRs.getString("INVALID_ROWS_QUERY_BEYOND_MAX_LOOKBACK");
+ rs = conn.createStatement().executeQuery(lookbackQuery);
+ assertFalse(rs.next());
}
// assert the result set contains the expected values in the given order
private void assertRsValues(ResultSet rs, List<? extends Object> expected) throws SQLException {
for (int i = 0; i < expected.size(); i++) {
- assertEquals(expected.get(i), rs.getObject(i + 1));
+ assertEquals("Failed comparing 1-based column " + (i + 1) +
+ " out of " + expected.size(),
+ expected.get(i), rs.getObject(i + 1));
}
+
}
private void generateUniqueTableNames() {
@@ -609,7 +673,8 @@ public class IndexScrutinyToolIT extends IndexScrutinyToolBaseIT {
}
private List<Job> runScrutinyCurrentSCN(String schemaName, String dataTableName, String indexTableName, Long scrutinyTS) throws Exception {
- return runScrutiny(getArgValues(schemaName, dataTableName, indexTableName, null, SourceTable.BOTH,
+ return runScrutiny(IndexScrutinyMapperForTest.class,
+ getArgValues(schemaName, dataTableName, indexTableName, null, SourceTable.BOTH,
false, null, null, null, scrutinyTS));
}
@@ -629,7 +694,7 @@ public class IndexScrutinyToolIT extends IndexScrutinyToolBaseIT {
cmdArgs =
getArgValues(schemaName, dataTableName, indexTableName, batchSize, sourceTable,
false, null, null, null, Long.MAX_VALUE);
- return runScrutiny(cmdArgs);
+ return runScrutiny(IndexScrutinyMapperForTest.class, cmdArgs);
}
private void upsertRow(PreparedStatement stmt, int id, String name, int zip) throws SQLException {
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexScrutinyWithMaxLookbackIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexScrutinyWithMaxLookbackIT.java
new file mode 100644
index 0000000..702f99a
--- /dev/null
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexScrutinyWithMaxLookbackIT.java
@@ -0,0 +1,317 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.end2end;
+
+import com.google.common.collect.Maps;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.mapreduce.Counters;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.phoenix.compat.hbase.HbaseCompatCapabilities;
+import org.apache.phoenix.compat.hbase.coprocessor.CompatBaseScannerRegionObserver;
+import org.apache.phoenix.mapreduce.index.IndexScrutinyMapper;
+import org.apache.phoenix.mapreduce.index.IndexScrutinyTableOutput;
+import org.apache.phoenix.mapreduce.index.IndexScrutinyTool;
+import org.apache.phoenix.query.QueryServices;
+import org.apache.phoenix.schema.TableNotFoundException;
+import org.apache.phoenix.util.EnvironmentEdgeManager;
+import org.apache.phoenix.util.ManualEnvironmentEdge;
+import org.apache.phoenix.util.MetaDataUtil;
+import org.apache.phoenix.util.PropertiesUtil;
+import org.apache.phoenix.util.ReadOnlyProps;
+import org.apache.phoenix.util.SchemaUtil;
+import org.apache.phoenix.util.TestUtil;
+import org.junit.Assume;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.List;
+import java.util.Map;
+
+import static org.apache.phoenix.mapreduce.index.PhoenixScrutinyJobCounters.INVALID_ROW_COUNT;
+import static org.apache.phoenix.mapreduce.index.PhoenixScrutinyJobCounters.BEYOND_MAX_LOOKBACK_COUNT;
+import static org.apache.phoenix.mapreduce.index.PhoenixScrutinyJobCounters.VALID_ROW_COUNT;
+import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+public class IndexScrutinyWithMaxLookbackIT extends IndexScrutinyToolBaseIT {
+
+ private static PreparedStatement upsertDataStmt;
+ private static String dataTableFullName;
+ private static String schema;
+ private static String dataTableName;
+ private static String indexTableName;
+ private static String indexTableFullName;
+ private static String viewName;
+ private static boolean isViewIndex;
+ private static ManualEnvironmentEdge testClock;
+ public static final String UPSERT_DATA = "UPSERT INTO %s VALUES (?, ?, ?)";
+ public static final String DELETE_SCRUTINY_METADATA = "DELETE FROM "
+ + IndexScrutinyTableOutput.OUTPUT_METADATA_TABLE_NAME;
+ public static final String DELETE_SCRUTINY_OUTPUT = "DELETE FROM " +
+ IndexScrutinyTableOutput.OUTPUT_TABLE_NAME;
+ public static final int MAX_LOOKBACK = 6;
+ private long scrutinyTs;
+
+
+ @BeforeClass
+ public static synchronized void doSetup() throws Exception {
+ Map<String, String> props = Maps.newHashMapWithExpectedSize(2);
+ props.put(QueryServices.GLOBAL_INDEX_ROW_AGE_THRESHOLD_TO_DELETE_MS_ATTRIB, Long.toString(0));
+ props.put(CompatBaseScannerRegionObserver.PHOENIX_MAX_LOOKBACK_AGE_CONF_KEY,
+ Integer.toString(MAX_LOOKBACK));
+ setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator()));
+ }
+
+ @Before
+ public void setupTest() throws SQLException {
+ try(Connection conn = DriverManager.getConnection(getUrl(),
+ PropertiesUtil.deepCopy(TEST_PROPERTIES))){
+ conn.createStatement().execute(DELETE_SCRUTINY_METADATA);
+ conn.createStatement().execute(DELETE_SCRUTINY_OUTPUT);
+ conn.commit();
+ } catch (TableNotFoundException tnfe){
+ //This will happen the first time
+ }
+ }
+
+ @Test
+ public void testScrutinyOnRowsBeyondMaxLookBack() throws Exception {
+ Assume.assumeTrue(HbaseCompatCapabilities.isLookbackBeyondDeletesSupported());
+ setupTables();
+ try {
+ upsertDataAndScrutinize(dataTableName, dataTableFullName, testClock);
+ assertBeyondMaxLookbackOutput(dataTableFullName, indexTableFullName);
+ } finally {
+ EnvironmentEdgeManager.reset();
+ }
+
+ }
+
+ @Test
+ public void testScrutinyOnRowsBeyondMaxLookback_viewIndex() throws Exception {
+ Assume.assumeTrue(HbaseCompatCapabilities.isLookbackBeyondDeletesSupported());
+ schema = "S"+generateUniqueName();
+ dataTableName = "T"+generateUniqueName();
+ dataTableFullName = SchemaUtil.getTableName(schema,dataTableName);
+ indexTableName = "VI"+generateUniqueName();
+ isViewIndex = true;
+ viewName = "V"+generateUniqueName();
+ String viewFullName = SchemaUtil.getTableName(schema,viewName);
+ String indexFullName = SchemaUtil.getTableName(schema, indexTableName);
+ String dataTableDDL = "CREATE TABLE %s (ID INTEGER NOT NULL PRIMARY KEY, NAME VARCHAR, "
+ + "ZIP INTEGER) COLUMN_ENCODED_BYTES = 0, VERSIONS = 1 ";
+ String viewDDL = "CREATE VIEW %s AS SELECT * FROM %s";
+ String indexTableDDL = "CREATE INDEX %s ON %s (NAME) INCLUDE (ZIP) VERSIONS = 1";
+ testClock = new ManualEnvironmentEdge();
+
+ try (Connection conn =
+ DriverManager.getConnection(getUrl(), PropertiesUtil.deepCopy(TEST_PROPERTIES))) {
+ conn.createStatement().execute(String.format(dataTableDDL, dataTableFullName));
+ conn.createStatement().execute(String.format(viewDDL, viewFullName, dataTableFullName));
+ conn.createStatement().execute(String.format(indexTableDDL, indexTableName,
+ viewFullName));
+ conn.commit();
+ }
+ upsertDataAndScrutinize(viewName, viewFullName, testClock);
+ assertBeyondMaxLookbackOutput(viewFullName, indexFullName);
+ }
+
+ private void assertBeyondMaxLookbackOutput(String dataTableName, String indexTableName)
+ throws Exception {
+ try (Connection conn =
+ DriverManager.getConnection(getUrl(), PropertiesUtil.deepCopy(TEST_PROPERTIES))) {
+ ResultSet metadataRS = IndexScrutinyTableOutput.queryAllLatestMetadata(conn, dataTableName,
+ indexTableName);
+ assertTrue("No results from scrutiny metadata query!", metadataRS.next());
+ assertEquals(1, metadataRS.getLong("BEYOND_MAX_LOOKBACK_COUNT"));
+ String beyondLookbackOutputSql =
+ metadataRS.getString("INVALID_ROWS_QUERY_BEYOND_MAX_LOOKBACK");
+ assertNotNull(beyondLookbackOutputSql);
+ ResultSet outputRS = conn.createStatement().executeQuery(beyondLookbackOutputSql);
+ assertTrue("No results from scrutiny beyond max lookback output query!",
+ outputRS.next());
+ assertTrue("Beyond max lookback flag not set",
+ outputRS.getBoolean("BEYOND_MAX_LOOKBACK"));
+ assertFalse("Too many rows output from scrutiny beyond max lookback output query!",
+ outputRS.next());
+ }
+ }
+
+ @Test
+ public void testScrutinyOnDeletedRowsBeyondMaxLookBack() throws Exception {
+ Assume.assumeTrue(HbaseCompatCapabilities.isLookbackBeyondDeletesSupported());
+ setupTables();
+ try {
+ upsertDataThenDeleteAndScrutinize(dataTableName, dataTableFullName, testClock);
+ assertBeyondMaxLookbackOutput(dataTableFullName, indexTableFullName);
+ } finally {
+ EnvironmentEdgeManager.reset();
+ }
+ }
+
+ private void setupTables() throws SQLException {
+ schema = "S" + generateUniqueName();
+ dataTableName = "T" + generateUniqueName();
+ indexTableName = "I" + generateUniqueName();
+ dataTableFullName = SchemaUtil.getTableName(schema, dataTableName);
+ indexTableFullName = SchemaUtil.getTableName(schema, indexTableName);
+ isViewIndex = false;
+ String dataTableDDL = "CREATE TABLE %s (ID INTEGER NOT NULL PRIMARY KEY, NAME VARCHAR, "
+ + "ZIP INTEGER) COLUMN_ENCODED_BYTES=0, VERSIONS=1";
+ String indexTableDDL = "CREATE INDEX %s ON %s (NAME) INCLUDE (ZIP)";
+ testClock = new ManualEnvironmentEdge();
+
+ try (Connection conn =
+ DriverManager.getConnection(getUrl(), PropertiesUtil.deepCopy(TEST_PROPERTIES))) {
+ conn.createStatement().execute(String.format(dataTableDDL, dataTableFullName));
+ conn.createStatement().execute(String.format(indexTableDDL, indexTableName,
+ dataTableFullName));
+ conn.commit();
+ }
+ }
+
+ private void upsertDataAndScrutinize(String tableName, String tableFullName,
+ ManualEnvironmentEdge testClock)
+ throws Exception {
+ try(Connection conn =
+ DriverManager.getConnection(getUrl(), PropertiesUtil.deepCopy(TEST_PROPERTIES))) {
+ // insert two rows
+ populateTable(tableFullName, conn);
+ long afterInsertSCN = EnvironmentEdgeManager.currentTimeMillis() + 1;
+ testClock.setValue(afterInsertSCN);
+ EnvironmentEdgeManager.injectEdge(testClock);
+ //move forward to the time we want to scrutinize, which is less than max lookback age
+ //for the initial inserts
+ testClock.incrementValue(MAX_LOOKBACK /2 * 1000);
+ scrutinyTs = EnvironmentEdgeManager.currentTimeMillis();
+ updateIndexRows(conn);
+ //now go past max lookback age for the initial 2 inserts
+ testClock.incrementValue(MAX_LOOKBACK /2 * 1000);
+ List<Job> completedJobs = runScrutiny(schema, tableName, indexTableName, scrutinyTs);
+ Job job = completedJobs.get(0);
+ assertTrue(job.isSuccessful());
+ assertCounters(job.getCounters());
+ }
+ }
+
+ private void populateTable(String tableFullName, Connection conn) throws SQLException {
+ upsertDataStmt = getUpsertDataStmt(tableFullName, conn);
+
+ NonParameterizedIndexScrutinyToolIT.upsertRow(upsertDataStmt, 1, "name-1", 98051);
+ NonParameterizedIndexScrutinyToolIT.upsertRow(upsertDataStmt, 2, "name-2", 98052);
+ conn.commit();
+ }
+
+ private void updateIndexRows(Connection conn) throws SQLException {
+ String tableName = isViewIndex ?
+ SchemaUtil.getTableName(schema, viewName) : dataTableFullName;
+ PreparedStatement stmt = getUpsertDataStmt(tableName, conn);
+ NonParameterizedIndexScrutinyToolIT.upsertRow(stmt, 1, "name-1", 38139);
+ conn.commit();
+ }
+
+ private void upsertDataThenDeleteAndScrutinize(String tableName, String tableFullName,
+ ManualEnvironmentEdge testClock)
+ throws Exception {
+ try(Connection conn =
+ DriverManager.getConnection(getUrl(), PropertiesUtil.deepCopy(TEST_PROPERTIES))) {
+ populateTable(tableFullName, conn);
+ long afterInsertSCN = EnvironmentEdgeManager.currentTimeMillis() + 1;
+ testClock.setValue(afterInsertSCN);
+ EnvironmentEdgeManager.injectEdge(testClock);
+ deleteIndexRows(conn);
+ //move forward to the time we want to scrutinize, which is less than max lookback age
+ //for the initial inserts
+ testClock.incrementValue(MAX_LOOKBACK /2 * 1000);
+ scrutinyTs = EnvironmentEdgeManager.currentTimeMillis();
+ //now go past max lookback age for the initial 2 inserts
+ testClock.incrementValue(MAX_LOOKBACK /2 * 1000);
+
+ List<Job> completedJobs = runScrutiny(schema, tableName, indexTableName, scrutinyTs);
+ Job job = completedJobs.get(0);
+ assertTrue(job.isSuccessful());
+ assertCounters(job.getCounters());
+ }
+ }
+
+ private void deleteIndexRows(Connection conn) throws SQLException {
+ String deleteSql = "DELETE FROM " + SchemaUtil.getTableName(schema, indexTableName) + " " +
+ "LIMIT 1";
+ conn.createStatement().execute(deleteSql);
+ conn.commit();
+ }
+
+ private static PreparedStatement getUpsertDataStmt(String tableFullName, Connection conn) throws SQLException {
+ return conn.prepareStatement(String.format(UPSERT_DATA, tableFullName));
+ }
+
+ private void assertCounters(Counters counters) {
+ assertEquals(1, getCounterValue(counters, VALID_ROW_COUNT));
+ assertEquals(1, getCounterValue(counters, BEYOND_MAX_LOOKBACK_COUNT));
+ assertEquals(0, getCounterValue(counters, INVALID_ROW_COUNT));
+ }
+
+ private List<Job> runScrutiny(String schemaName, String dataTableName, String indexTableName,
+ Long scrutinyTs)
+ throws Exception {
+ return runScrutiny(schemaName, dataTableName, indexTableName, null, null, scrutinyTs);
+ }
+
+ private List<Job> runScrutiny(String schemaName, String dataTableName, String indexTableName,
+ Long batchSize, IndexScrutinyTool.SourceTable sourceTable,
+ Long scrutinyTs) throws Exception {
+ final String[]
+ cmdArgs =
+ getArgValues(schemaName, dataTableName, indexTableName, batchSize, sourceTable,
+ true, IndexScrutinyTool.OutputFormat.TABLE,
+ null, null, scrutinyTs);
+ return runScrutiny(MaxLookbackIndexScrutinyMapper.class, cmdArgs);
+ }
+
+ private static class MaxLookbackIndexScrutinyMapper extends IndexScrutinyMapper {
+ @Override
+ public void postSetup(){
+ try {
+ String tableToCompact;
+ if (isViewIndex){
+ String physicalDataTableName =
+ SchemaUtil.getPhysicalHBaseTableName(schema, dataTableName, false).getString();
+ tableToCompact = MetaDataUtil.getViewIndexPhysicalName(physicalDataTableName);
+ } else {
+ tableToCompact = SchemaUtil.getPhysicalHBaseTableName(schema, indexTableName, false).getString();
+ }
+ TableName indexTable = TableName.valueOf(tableToCompact);
+ testClock.incrementValue(1);
+ getUtility().getAdmin().flush(indexTable);
+ TestUtil.majorCompact(getUtility(), indexTable);
+ } catch (Exception e){
+ throw new RuntimeException(e);
+ }
+
+ }
+ }
+}
\ No newline at end of file
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolForNonTxGlobalIndexIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolForNonTxGlobalIndexIT.java
index 53ee745..a4547be 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolForNonTxGlobalIndexIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolForNonTxGlobalIndexIT.java
@@ -18,6 +18,7 @@
package org.apache.phoenix.end2end;
import com.google.common.collect.Maps;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
@@ -42,9 +43,10 @@ import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Pair;
+import org.apache.hadoop.mapreduce.Counters;
import org.apache.phoenix.compat.hbase.HbaseCompatCapabilities;
+import org.apache.phoenix.compat.hbase.coprocessor.CompatBaseScannerRegionObserver;
import org.apache.phoenix.coprocessor.IndexRebuildRegionScanner;
-import org.apache.phoenix.hbase.index.IndexRegionObserver;
import org.apache.phoenix.jdbc.PhoenixConnection;
import org.apache.phoenix.mapreduce.index.IndexTool;
import org.apache.phoenix.mapreduce.index.IndexVerificationOutputRepository;
@@ -63,6 +65,7 @@ import org.apache.phoenix.util.SchemaUtil;
import org.apache.phoenix.util.TestUtil;
import org.junit.After;
import org.junit.Assert;
+import org.junit.Assume;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Rule;
@@ -83,24 +86,25 @@ import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Properties;
-import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
import static org.apache.phoenix.mapreduce.PhoenixJobCounters.INPUT_RECORDS;
import static org.apache.phoenix.mapreduce.index.IndexVerificationResultRepository.INDEX_TOOL_RUN_STATUS_BYTES;
import static org.apache.phoenix.mapreduce.index.IndexVerificationResultRepository.RESULT_TABLE_COLUMN_FAMILY;
+import static org.apache.phoenix.mapreduce.index.IndexVerificationResultRepository.RESULT_TABLE_NAME;
import static org.apache.phoenix.mapreduce.index.IndexVerificationResultRepository.RESULT_TABLE_NAME_BYTES;
import static org.apache.phoenix.mapreduce.index.IndexVerificationResultRepository.ROW_KEY_SEPARATOR;
import static org.apache.phoenix.mapreduce.index.IndexVerificationResultRepository.RUN_STATUS_EXECUTED;
import static org.apache.phoenix.mapreduce.index.IndexVerificationResultRepository.RUN_STATUS_SKIPPED;
+import static org.apache.phoenix.mapreduce.index.PhoenixIndexToolJobCounters.AFTER_REBUILD_BEYOND_MAXLOOKBACK_INVALID_INDEX_ROW_COUNT;
+import static org.apache.phoenix.mapreduce.index.PhoenixIndexToolJobCounters.AFTER_REBUILD_BEYOND_MAXLOOKBACK_MISSING_INDEX_ROW_COUNT;
import static org.apache.phoenix.mapreduce.index.PhoenixIndexToolJobCounters.AFTER_REBUILD_EXPIRED_INDEX_ROW_COUNT;
import static org.apache.phoenix.mapreduce.index.PhoenixIndexToolJobCounters.AFTER_REBUILD_INVALID_INDEX_ROW_COUNT;
import static org.apache.phoenix.mapreduce.index.PhoenixIndexToolJobCounters.AFTER_REBUILD_MISSING_INDEX_ROW_COUNT;
import static org.apache.phoenix.mapreduce.index.PhoenixIndexToolJobCounters.AFTER_REBUILD_VALID_INDEX_ROW_COUNT;
+import static org.apache.phoenix.mapreduce.index.PhoenixIndexToolJobCounters.BEFORE_REBUILD_BEYOND_MAXLOOKBACK_INVALID_INDEX_ROW_COUNT;
+import static org.apache.phoenix.mapreduce.index.PhoenixIndexToolJobCounters.BEFORE_REBUILD_BEYOND_MAXLOOKBACK_MISSING_INDEX_ROW_COUNT;
import static org.apache.phoenix.mapreduce.index.PhoenixIndexToolJobCounters.BEFORE_REBUILD_EXPIRED_INDEX_ROW_COUNT;
import static org.apache.phoenix.mapreduce.index.PhoenixIndexToolJobCounters.BEFORE_REBUILD_INVALID_INDEX_ROW_COUNT;
-import static org.apache.phoenix.mapreduce.index.PhoenixIndexToolJobCounters.BEFORE_REBUILD_INVALID_INDEX_ROW_COUNT_COZ_EXTRA_CELLS;
-import static org.apache.phoenix.mapreduce.index.PhoenixIndexToolJobCounters.BEFORE_REBUILD_INVALID_INDEX_ROW_COUNT_COZ_MISSING_CELLS;
import static org.apache.phoenix.mapreduce.index.PhoenixIndexToolJobCounters.BEFORE_REBUILD_MISSING_INDEX_ROW_COUNT;
import static org.apache.phoenix.mapreduce.index.PhoenixIndexToolJobCounters.BEFORE_REBUILD_OLD_INDEX_ROW_COUNT;
import static org.apache.phoenix.mapreduce.index.PhoenixIndexToolJobCounters.BEFORE_REBUILD_UNKNOWN_INDEX_ROW_COUNT;
@@ -113,15 +117,17 @@ import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
@RunWith(Parameterized.class)
public class IndexToolForNonTxGlobalIndexIT extends BaseUniqueNamesOwnClusterIT {
+ public static final int MAX_LOOKBACK_AGE = 3600;
private final String tableDDLOptions;
- private boolean directApi = true;
- private boolean useSnapshot = false;
- private boolean mutable;
+ private final boolean directApi = true;
+ private final boolean useSnapshot = false;
+ private final boolean mutable;
@Rule
public ExpectedException exceptionRule = ExpectedException.none();
@@ -144,17 +150,23 @@ public class IndexToolForNonTxGlobalIndexIT extends BaseUniqueNamesOwnClusterIT
@BeforeClass
public static synchronized void setup() throws Exception {
- Map<String, String> serverProps = Maps.newHashMapWithExpectedSize(2);
+ Map<String, String> serverProps = Maps.newHashMapWithExpectedSize(8);
serverProps.put(QueryServices.STATS_GUIDEPOST_WIDTH_BYTES_ATTRIB, Long.toString(20));
serverProps.put(QueryServices.MAX_SERVER_METADATA_CACHE_TIME_TO_LIVE_MS_ATTRIB, Long.toString(5));
serverProps.put(QueryServices.EXTRA_JDBC_ARGUMENTS_ATTRIB,
QueryServicesOptions.DEFAULT_EXTRA_JDBC_ARGUMENTS);
serverProps.put(QueryServices.INDEX_REBUILD_PAGE_SIZE_IN_ROWS, Long.toString(8));
+ serverProps.put(CompatBaseScannerRegionObserver.PHOENIX_MAX_LOOKBACK_AGE_CONF_KEY,
+ Long.toString(MAX_LOOKBACK_AGE));
serverProps.put(QueryServices.TASK_HANDLING_INTERVAL_MS_ATTRIB,
Long.toString(Long.MAX_VALUE));
serverProps.put(QueryServices.TASK_HANDLING_INITIAL_DELAY_MS_ATTRIB,
Long.toString(Long.MAX_VALUE));
- Map<String, String> clientProps = Maps.newHashMapWithExpectedSize(2);
+ serverProps.put(QueryServices.GLOBAL_INDEX_ROW_AGE_THRESHOLD_TO_DELETE_MS_ATTRIB, Long.toString(0));
+ //In HBase 2.2.6 and HBase 2.3.0+, helps region server assignments when under an injected
+ // edge clock. See HBASE-24511.
+ serverProps.put("hbase.regionserver.rpc.retry.interval", Long.toString(0));
+ Map<String, String> clientProps = Maps.newHashMapWithExpectedSize(4);
clientProps.put(QueryServices.USE_STATS_FOR_PARALLELIZATION, Boolean.toString(true));
clientProps.put(QueryServices.STATS_UPDATE_FREQ_MS_ATTRIB, Long.toString(5));
clientProps.put(QueryServices.TRANSACTIONS_ENABLED, Boolean.TRUE.toString());
@@ -232,48 +244,166 @@ public class IndexToolForNonTxGlobalIndexIT extends BaseUniqueNamesOwnClusterIT
assertEquals(0, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_EXPIRED_INDEX_ROW_COUNT).getValue());
assertEquals(0, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_INVALID_INDEX_ROW_COUNT).getValue());
assertEquals(0, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_MISSING_INDEX_ROW_COUNT).getValue());
- assertEquals(0, indexTool.getJob().getCounters().findCounter(
- BEFORE_REBUILD_INVALID_INDEX_ROW_COUNT_COZ_EXTRA_CELLS).getValue());
- assertEquals(0, indexTool.getJob().getCounters().findCounter(
- BEFORE_REBUILD_INVALID_INDEX_ROW_COUNT_COZ_MISSING_CELLS).getValue());
+ assertEquals(0, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_BEYOND_MAXLOOKBACK_INVALID_INDEX_ROW_COUNT).getValue());
+ assertEquals(0, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_BEYOND_MAXLOOKBACK_MISSING_INDEX_ROW_COUNT).getValue());
}
}
@Test
- public void testOverrideIndexRebuildPageSizeFromIndexTool() throws Exception {
+ public void testIndexToolVerifyWithExpiredIndexRows() throws Exception {
String schemaName = generateUniqueName();
String dataTableName = generateUniqueName();
String dataTableFullName = SchemaUtil.getTableName(schemaName, dataTableName);
String indexTableName = generateUniqueName();
+ String indexTableFullName = SchemaUtil.getTableName(schemaName, indexTableName);
Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ conn.createStatement().execute("CREATE TABLE " + dataTableFullName
+ + " (ID INTEGER NOT NULL PRIMARY KEY, NAME VARCHAR, CODE VARCHAR) COLUMN_ENCODED_BYTES=0");
+ // Insert a row
+ conn.createStatement()
+ .execute("upsert into " + dataTableFullName + " values (1, 'Phoenix', 'A')");
+ conn.commit();
+ conn.createStatement()
+ .execute(String.format("CREATE INDEX %s ON %s (NAME) INCLUDE (CODE) ASYNC",
+ indexTableName, dataTableFullName));
+ IndexToolIT.runIndexTool(directApi, useSnapshot, schemaName, dataTableName, indexTableName, null, 0,
+ IndexTool.IndexVerifyType.ONLY);
+ Cell cell =
+ IndexToolIT.getErrorMessageFromIndexToolOutputTable(conn, dataTableFullName,
+ indexTableFullName);
+ try {
+ String expectedErrorMsg = IndexRebuildRegionScanner.ERROR_MESSAGE_MISSING_INDEX_ROW;
+ String actualErrorMsg = Bytes
+ .toString(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength());
+ assertEquals(expectedErrorMsg, actualErrorMsg);
+ } catch(Exception ex) {
+ Assert.fail("Fail to parsing the error message from IndexToolOutputTable");
+ }
- try(Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ // Run the index tool to populate the index while verifying rows
+ IndexToolIT.runIndexTool(directApi, useSnapshot, schemaName, dataTableName, indexTableName, null, 0,
+ IndexTool.IndexVerifyType.AFTER);
+
+ // Set ttl of index table ridiculously low so that all data is expired
+ Admin admin = conn.unwrap(PhoenixConnection.class).getQueryServices().getAdmin();
+ TableName indexTable = TableName.valueOf(indexTableFullName);
+ ColumnFamilyDescriptor desc =
+ admin.getDescriptor(indexTable).getColumnFamilies()[0];
+ ColumnFamilyDescriptor newDesc =
+ ColumnFamilyDescriptorBuilder.newBuilder(desc).setTimeToLive(1).build();
+ admin.modifyColumnFamily(indexTable, newDesc);
+ Thread.sleep(1000);
+ Pair<Integer, Integer> status = admin.getAlterStatus(indexTable);
+ int retry = 0;
+ while (retry < 20 && status.getFirst() != 0) {
+ Thread.sleep(2000);
+ status = admin.getAlterStatus(indexTable);
+ }
+ assertEquals(0, (int) status.getFirst());
+
+ TableName indexToolOutputTable = TableName.valueOf(IndexVerificationOutputRepository.OUTPUT_TABLE_NAME_BYTES);
+ admin.disableTable(indexToolOutputTable);
+ admin.deleteTable(indexToolOutputTable);
+ // Run the index tool using the only-verify option, verify it gives no mismatch
+ IndexToolIT.runIndexTool(directApi, useSnapshot, schemaName, dataTableName, indexTableName, null, 0,
+ IndexTool.IndexVerifyType.ONLY);
+ Scan scan = new Scan();
+ Table hIndexToolTable =
+ conn.unwrap(PhoenixConnection.class).getQueryServices()
+ .getTable(indexToolOutputTable.getName());
+ Result r = hIndexToolTable.getScanner(scan).next();
+ assertNull(r);
+ }
+ }
+
+ @Test
+ public void testSecondaryGlobalIndexFailure() throws Exception {
+ if (!mutable) {
+ return; //nothing in this test is mutable specific, so no need to run twice
+ }
+ String schemaName = generateUniqueName();
+ String dataTableName = generateUniqueName();
+ String dataTableFullName = SchemaUtil.getTableName(schemaName, dataTableName);
+ String indexTableName = generateUniqueName();
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
String stmString1 =
- "CREATE TABLE " + dataTableFullName
- + " (ID INTEGER NOT NULL PRIMARY KEY, NAME VARCHAR, ZIP INTEGER) "
- + tableDDLOptions;
+ "CREATE TABLE " + dataTableFullName
+ + " (ID INTEGER NOT NULL PRIMARY KEY, NAME VARCHAR, ZIP INTEGER) "
+ + tableDDLOptions;
+ conn.createStatement().execute(stmString1);
+ conn.commit();
+
+ String upsertQuery = String.format("UPSERT INTO %s VALUES(?, ?, ?)", dataTableFullName);
+ PreparedStatement stmt1 = conn.prepareStatement(upsertQuery);
+
+ // Insert two rows
+ IndexToolIT.upsertRow(stmt1, 1);
+ IndexToolIT.upsertRow(stmt1, 2);
+ conn.commit();
+
+ String stmtString2 =
+ String.format(
+ "CREATE INDEX %s ON %s (LPAD(UPPER(NAME, 'en_US'),8,'x')||'_xyz') ", indexTableName, dataTableFullName);
+ conn.createStatement().execute(stmtString2);
+ conn.commit();
+ String qIndexTableName = SchemaUtil.getQualifiedTableName(schemaName, indexTableName);
+
+ // Verify that the index table is in the ACTIVE state
+ assertEquals(PIndexState.ACTIVE, TestUtil.getIndexState(conn, qIndexTableName));
+
+ ConnectionQueryServices queryServices = conn.unwrap(PhoenixConnection.class).getQueryServices();
+ Admin admin = queryServices.getAdmin();
+ TableName tn = TableName.valueOf(Bytes.toBytes(dataTableFullName));
+ TableDescriptor td = admin.getDescriptor(tn);
+ CoprocessorDescriptor cd =
+ CoprocessorDescriptorBuilder.newBuilder(FastFailRegionObserver.class.getName()).
+ setPriority(1).build();
+ TableDescriptor newTD =
+ TableDescriptorBuilder.newBuilder(td).
+ setCoprocessor(cd).build();
+ //add the fast fail coproc and make sure it goes first
+ admin.modifyTable(newTD);
+ // Run the index MR job and it should fail (return -1)
+ IndexToolIT.runIndexTool(directApi, useSnapshot, schemaName, dataTableName, indexTableName,
+ null, -1, new String[0]);
+
+ // Verify that the index table should be still in the ACTIVE state
+ assertEquals(PIndexState.ACTIVE, TestUtil.getIndexState(conn, qIndexTableName));
+ }
+ }
+
+ @Test
+ public void testBuildSecondaryIndexAndScrutinize() throws Exception {
+ String schemaName = generateUniqueName();
+ String dataTableName = generateUniqueName();
+ String dataTableFullName = SchemaUtil.getTableName(schemaName, dataTableName);
+ String indexTableName = generateUniqueName();
+ String indexTableFullName = SchemaUtil.getTableName(schemaName, indexTableName);
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ String stmString1 =
+ "CREATE TABLE " + dataTableFullName
+ + " (ID INTEGER NOT NULL PRIMARY KEY, NAME VARCHAR, ZIP INTEGER) "
+ + tableDDLOptions;
conn.createStatement().execute(stmString1);
String upsertQuery = String.format("UPSERT INTO %s VALUES(?, ?, ?)", dataTableFullName);
PreparedStatement stmt1 = conn.prepareStatement(upsertQuery);
// Insert NROWS rows
- final int NROWS = 16;
+ final int NROWS = 1000;
for (int i = 0; i < NROWS; i++) {
IndexToolIT.upsertRow(stmt1, i);
}
conn.commit();
-
String stmtString2 =
String.format(
"CREATE INDEX %s ON %s (NAME) INCLUDE (ZIP) ASYNC ", indexTableName, dataTableFullName);
conn.createStatement().execute(stmtString2);
// Run the index MR job and verify that the index table is built correctly
- Configuration conf = new Configuration(getUtility().getConfiguration());
- conf.set(QueryServices.INDEX_REBUILD_PAGE_SIZE_IN_ROWS, Long.toString(2));
- IndexTool indexTool = IndexToolIT.runIndexTool(conf, directApi, useSnapshot,
- schemaName, dataTableName, indexTableName, null, 0,
- IndexTool.IndexVerifyType.BEFORE, IndexTool.IndexDisableLoggingType.NONE, new String[0]);
+ IndexTool indexTool = IndexToolIT.runIndexTool(directApi, useSnapshot, schemaName, dataTableName, indexTableName, null, 0, IndexTool.IndexVerifyType.BEFORE, new String[0]);
assertEquals(NROWS, indexTool.getJob().getCounters().findCounter(INPUT_RECORDS).getValue());
assertEquals(NROWS, indexTool.getJob().getCounters().findCounter(SCANNED_DATA_ROW_COUNT).getValue());
assertEquals(NROWS, indexTool.getJob().getCounters().findCounter(REBUILT_INDEX_ROW_COUNT).getValue());
@@ -281,405 +411,49 @@ public class IndexToolForNonTxGlobalIndexIT extends BaseUniqueNamesOwnClusterIT
assertEquals(0, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_EXPIRED_INDEX_ROW_COUNT).getValue());
assertEquals(0, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_INVALID_INDEX_ROW_COUNT).getValue());
assertEquals(NROWS, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_MISSING_INDEX_ROW_COUNT).getValue());
- }
- }
-
- @Test
- public void testPointDeleteRebuildWithPageSize() throws Exception {
- String schemaName = generateUniqueName();
- String dataTableName = generateUniqueName();
- String fullDataTableName = SchemaUtil.getTableName(schemaName, dataTableName);
- String indexTableName = generateUniqueName();
- try (Connection conn = DriverManager.getConnection(getUrl())) {
- conn.createStatement().execute(
- "CREATE TABLE " + fullDataTableName + "(k VARCHAR PRIMARY KEY, v VARCHAR)");
- conn.createStatement().execute("DELETE FROM " + fullDataTableName + " WHERE k = 'a'");
- conn.createStatement().execute("DELETE FROM " + fullDataTableName + " WHERE k = 'b'");
- conn.createStatement().execute("DELETE FROM " + fullDataTableName + " WHERE k = 'c'");
- conn.commit();
- conn.createStatement().execute(String.format("CREATE INDEX %s ON %s (v) ASYNC",
- indexTableName, fullDataTableName));
- // Run the index MR job and verify that the index table is built correctly
- Configuration conf = new Configuration(getUtility().getConfiguration());
- conf.set(QueryServices.INDEX_REBUILD_PAGE_SIZE_IN_ROWS, Long.toString(1));
- IndexTool indexTool =
- IndexToolIT.runIndexTool(conf, directApi, useSnapshot, schemaName,
- dataTableName, indexTableName, null, 0, IndexTool.IndexVerifyType.BEFORE,
- IndexTool.IndexDisableLoggingType.NONE, new String[0]);
- assertEquals(3, indexTool.getJob().getCounters().findCounter(INPUT_RECORDS).getValue());
- assertEquals(3,
- indexTool.getJob().getCounters().findCounter(SCANNED_DATA_ROW_COUNT).getValue());
- assertEquals(0,
- indexTool.getJob().getCounters().findCounter(REBUILT_INDEX_ROW_COUNT).getValue());
- assertEquals(0, indexTool.getJob().getCounters()
- .findCounter(BEFORE_REBUILD_VALID_INDEX_ROW_COUNT).getValue());
- assertEquals(0, indexTool.getJob().getCounters()
- .findCounter(BEFORE_REBUILD_EXPIRED_INDEX_ROW_COUNT).getValue());
- assertEquals(0, indexTool.getJob().getCounters()
- .findCounter(BEFORE_REBUILD_INVALID_INDEX_ROW_COUNT).getValue());
- assertEquals(0, indexTool.getJob().getCounters()
- .findCounter(BEFORE_REBUILD_MISSING_INDEX_ROW_COUNT).getValue());
- }
- }
+ assertEquals(0, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_BEYOND_MAXLOOKBACK_MISSING_INDEX_ROW_COUNT).getValue());
+ assertEquals(0, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_BEYOND_MAXLOOKBACK_INVALID_INDEX_ROW_COUNT).getValue());
+ long actualRowCount = IndexScrutiny.scrutinizeIndex(conn, dataTableFullName, indexTableFullName);
+ assertEquals(NROWS, actualRowCount);
- // This test will only work with HBASE-22710 which is in 2.2.5+
- // TODO: Enable once we move to these versions
- @Ignore
- @Test
- public void testIncrementalRebuildWithPageSize() throws Exception {
- String schemaName = generateUniqueName();
- String dataTableName = generateUniqueName();
- String fullDataTableName = SchemaUtil.getTableName(schemaName, dataTableName);
- String indexTableName = generateUniqueName();
- try (Connection conn = DriverManager.getConnection(getUrl())) {
- long minTs = EnvironmentEdgeManager.currentTimeMillis();
- conn.createStatement().execute(
- "CREATE TABLE " + fullDataTableName + "(k VARCHAR PRIMARY KEY, v VARCHAR)");
- conn.createStatement().execute("UPSERT INTO " + fullDataTableName + " VALUES('a','aa')");
- conn.createStatement().execute("UPSERT INTO " + fullDataTableName + " VALUES('b','bb')");
- conn.commit();
- conn.createStatement().execute("DELETE FROM " + fullDataTableName + " WHERE k = 'a'");
- conn.createStatement().execute("DELETE FROM " + fullDataTableName + " WHERE k = 'b'");
- conn.commit();
- conn.createStatement().execute("UPSERT INTO " + fullDataTableName + " VALUES('a','aaa')");
- conn.createStatement().execute("UPSERT INTO " + fullDataTableName + " VALUES('b','bbb')");
- conn.createStatement().execute("DELETE FROM " + fullDataTableName + " WHERE k = 'c'");
+ // Add more rows and make sure that these rows will be visible to IndexTool
+ for (int i = NROWS; i < 2 * NROWS; i++) {
+ IndexToolIT.upsertRow(stmt1, i);
+ }
conn.commit();
- conn.createStatement().execute(String.format("CREATE INDEX %s ON %s (v) ASYNC",
- indexTableName, fullDataTableName));
- // Run the index MR job and verify that the index table is built correctly
- Configuration conf = new Configuration(getUtility().getConfiguration());
- conf.set(QueryServices.INDEX_REBUILD_PAGE_SIZE_IN_ROWS, Long.toString(2));
- IndexTool indexTool =
- IndexToolIT.runIndexTool(conf, directApi, useSnapshot, schemaName,
- dataTableName, indexTableName, null, 0, IndexTool.IndexVerifyType.AFTER,
- IndexTool.IndexDisableLoggingType.NONE,
- "-st", String.valueOf(minTs), "-et",
- String.valueOf(EnvironmentEdgeManager.currentTimeMillis()));
- assertEquals(3, indexTool.getJob().getCounters().findCounter(INPUT_RECORDS).getValue());
- assertEquals(3,
- indexTool.getJob().getCounters().findCounter(SCANNED_DATA_ROW_COUNT).getValue());
- assertEquals(4,
- indexTool.getJob().getCounters().findCounter(REBUILT_INDEX_ROW_COUNT).getValue());
- assertEquals(4, indexTool.getJob().getCounters()
- .findCounter(AFTER_REBUILD_VALID_INDEX_ROW_COUNT).getValue());
- assertEquals(0, indexTool.getJob().getCounters()
- .findCounter(AFTER_REBUILD_EXPIRED_INDEX_ROW_COUNT).getValue());
- assertEquals(0, indexTool.getJob().getCounters()
- .findCounter(AFTER_REBUILD_INVALID_INDEX_ROW_COUNT).getValue());
- assertEquals(0, indexTool.getJob().getCounters()
- .findCounter(AFTER_REBUILD_MISSING_INDEX_ROW_COUNT).getValue());
+ indexTool = IndexToolIT.runIndexTool(directApi, useSnapshot, schemaName, dataTableName, indexTableName, null, 0, IndexTool.IndexVerifyType.BOTH, new String[0]);
+ assertEquals(2 * NROWS, indexTool.getJob().getCounters().findCounter(INPUT_RECORDS).getValue());
+ assertEquals(2 * NROWS, indexTool.getJob().getCounters().findCounter(SCANNED_DATA_ROW_COUNT).getValue());
+ assertEquals(0, indexTool.getJob().getCounters().findCounter(REBUILT_INDEX_ROW_COUNT).getValue());
+ assertEquals(2 * NROWS, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_VALID_INDEX_ROW_COUNT).getValue());
+ assertEquals(0, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_EXPIRED_INDEX_ROW_COUNT).getValue());
+ assertEquals(0, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_INVALID_INDEX_ROW_COUNT).getValue());
+ assertEquals(0, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_MISSING_INDEX_ROW_COUNT).getValue());
+ assertEquals(0, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_BEYOND_MAXLOOKBACK_MISSING_INDEX_ROW_COUNT).getValue());
+ assertEquals(0, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_BEYOND_MAXLOOKBACK_INVALID_INDEX_ROW_COUNT).getValue());
+ assertEquals(0, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_UNVERIFIED_INDEX_ROW_COUNT).getValue());
+ assertEquals(0, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_OLD_INDEX_ROW_COUNT).getValue());
+ assertEquals(0, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_UNKNOWN_INDEX_ROW_COUNT).getValue());
+ assertEquals(0, indexTool.getJob().getCounters().findCounter(AFTER_REBUILD_VALID_INDEX_ROW_COUNT).getValue());
+ assertEquals(0, indexTool.getJob().getCounters().findCounter(AFTER_REBUILD_EXPIRED_INDEX_ROW_COUNT).getValue());
+ assertEquals(0, indexTool.getJob().getCounters().findCounter(AFTER_REBUILD_INVALID_INDEX_ROW_COUNT).getValue());
+ assertEquals(0, indexTool.getJob().getCounters().findCounter(AFTER_REBUILD_MISSING_INDEX_ROW_COUNT).getValue());
+ assertEquals(0, indexTool.getJob().getCounters().findCounter(AFTER_REBUILD_BEYOND_MAXLOOKBACK_MISSING_INDEX_ROW_COUNT).getValue());
+ assertEquals(0, indexTool.getJob().getCounters().findCounter(AFTER_REBUILD_BEYOND_MAXLOOKBACK_INVALID_INDEX_ROW_COUNT).getValue());
+ actualRowCount = IndexScrutiny.scrutinizeIndex(conn, dataTableFullName, indexTableFullName);
+ assertEquals(2 * NROWS, actualRowCount);
}
}
- // This test will only work with HBASE-22710 which is in 2.2.5+
- // TODO: Enable once we move to these versions
- @Ignore
+ @Ignore("HBase 1.7 is required for this test")
@Test
- public void testUpdatablePKFilterViewIndexRebuild() throws Exception {
+ public void testCleanUpOldDesignIndexRows() throws Exception {
if (!mutable) {
return;
}
String schemaName = generateUniqueName();
String dataTableName = generateUniqueName();
String dataTableFullName = SchemaUtil.getTableName(schemaName, dataTableName);
- String view1Name = generateUniqueName();
- String view1FullName = SchemaUtil.getTableName(schemaName, view1Name);
- String view2Name = generateUniqueName();
- String view2FullName = SchemaUtil.getTableName(schemaName, view2Name);
- String indexTableName = generateUniqueName();
- String indexTableFullName = SchemaUtil.getTableName(schemaName, indexTableName);
- Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
-
- try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
- // Create Table and Views. Note the view is on a non leading PK data table column
- String createTable =
- "CREATE TABLE IF NOT EXISTS " + dataTableFullName + " (\n"
- + " ORGANIZATION_ID VARCHAR NOT NULL,\n"
- + " KEY_PREFIX CHAR(3) NOT NULL,\n" + " CREATED_BY VARCHAR,\n"
- + " CONSTRAINT PK PRIMARY KEY (\n" + " ORGANIZATION_ID,\n"
- + " KEY_PREFIX\n" + " )\n"
- + ") VERSIONS=1, COLUMN_ENCODED_BYTES=0";
- conn.createStatement().execute(createTable);
- String createView1 =
- "CREATE VIEW IF NOT EXISTS " + view1FullName + " (\n"
- + " VIEW_COLA VARCHAR NOT NULL,\n"
- + " VIEW_COLB CHAR(1) CONSTRAINT PKVIEW PRIMARY KEY (\n"
- + " VIEW_COLA\n" + " )) AS \n" + " SELECT * FROM " + dataTableFullName
- + " WHERE KEY_PREFIX = 'aaa'";
- conn.createStatement().execute(createView1);
- String createView2 =
- "CREATE VIEW IF NOT EXISTS " + view2FullName + " (\n"
- + " VIEW_COL1 VARCHAR NOT NULL,\n"
- + " VIEW_COL2 CHAR(1) CONSTRAINT PKVIEW PRIMARY KEY (\n"
- + " VIEW_COL1\n" + " )) AS \n" + " SELECT * FROM " + dataTableFullName
- + " WHERE KEY_PREFIX = 'ccc'";
- conn.createStatement().execute(createView2);
-
- // We want to verify if deletes and set null result in expected rebuild of view index
- conn.createStatement().execute("UPSERT INTO " + view1FullName
- + "(ORGANIZATION_ID, VIEW_COLA, VIEW_COLB) VALUES('ORG1', 'A', 'G')");
- conn.createStatement().execute("UPSERT INTO " + view1FullName
- + "(ORGANIZATION_ID, VIEW_COLA, VIEW_COLB) VALUES('ORG1', 'C', 'I')");
- conn.createStatement().execute("UPSERT INTO " + view1FullName
- + "(ORGANIZATION_ID, VIEW_COLA, VIEW_COLB) VALUES('ORG1', 'D', 'J')");
-
- conn.createStatement().execute("UPSERT INTO " + view2FullName
- + "(ORGANIZATION_ID, VIEW_COL1, VIEW_COL2) VALUES('ORG2', 'B', 'H')");
- conn.commit();
- conn.createStatement().execute("DELETE FROM " + view1FullName
- + " WHERE ORGANIZATION_ID = 'ORG1' AND VIEW_COLA = 'C'");
- conn.createStatement().execute("UPSERT INTO " + view1FullName
- + "(ORGANIZATION_ID, VIEW_COLA, VIEW_COLB) VALUES('ORG1', 'D', NULL)");
- conn.commit();
-
- String createViewIndex =
- "CREATE INDEX IF NOT EXISTS " + indexTableName + " ON " + view1FullName
- + " (VIEW_COLB) ASYNC";
- conn.createStatement().execute(createViewIndex);
- conn.commit();
- // Rebuild using index tool
- IndexToolIT.runIndexTool(directApi, useSnapshot, schemaName, view1Name, indexTableName);
- ResultSet rs =
- conn.createStatement()
- .executeQuery("SELECT COUNT(*) FROM " + indexTableFullName);
- rs.next();
- assertEquals(2, rs.getInt(1));
-
- Pair<Integer, Integer> putsAndDeletes =
- countPutsAndDeletes("_IDX_" + dataTableFullName);
- assertEquals(4, (int) putsAndDeletes.getFirst());
- assertEquals(2, (int) putsAndDeletes.getSecond());
- }
- }
-
- // This test will only work with HBASE-22710 which is in 2.2.5+
- @Ignore
- @Test
- public void testUpdatableNonPkFilterViewIndexRebuild() throws Exception {
- if (!mutable) {
- return;
- }
- String schemaName = generateUniqueName();
- String dataTableName = generateUniqueName();
- String dataTableFullName = SchemaUtil.getTableName(schemaName, dataTableName);
- String view1Name = generateUniqueName();
- String view1FullName = SchemaUtil.getTableName(schemaName, view1Name);
- String view2Name = generateUniqueName();
- String view2FullName = SchemaUtil.getTableName(schemaName, view2Name);
- String indexTableName = generateUniqueName();
- String indexTableFullName = SchemaUtil.getTableName(schemaName, indexTableName);
- Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
-
- try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
- // Create Table and Views. Note the view is on a non PK data table column
- String createTable =
- "CREATE TABLE IF NOT EXISTS " + dataTableFullName + " (\n"
- + " ORGANIZATION_ID VARCHAR NOT NULL,\n"
- + " KEY_PREFIX CHAR(3) NOT NULL,\n" + " CREATED_BY VARCHAR,\n"
- + " CONSTRAINT PK PRIMARY KEY (\n" + " ORGANIZATION_ID,\n"
- + " KEY_PREFIX\n" + " )\n"
- + ") VERSIONS=1, COLUMN_ENCODED_BYTES=0";
- conn.createStatement().execute(createTable);
- String createView1 =
- "CREATE VIEW IF NOT EXISTS " + view1FullName + " (\n"
- + " VIEW_COLA VARCHAR NOT NULL,\n"
- + " VIEW_COLB CHAR(1) CONSTRAINT PKVIEW PRIMARY KEY (\n"
- + " VIEW_COLA\n" + " )) AS \n" + " SELECT * FROM " + dataTableFullName
- + " WHERE CREATED_BY = 'foo'";
- conn.createStatement().execute(createView1);
- String createView2 =
- "CREATE VIEW IF NOT EXISTS " + view2FullName + " (\n"
- + " VIEW_COL1 VARCHAR NOT NULL,\n"
- + " VIEW_COL2 CHAR(1) CONSTRAINT PKVIEW PRIMARY KEY (\n"
- + " VIEW_COL1\n" + " )) AS \n" + " SELECT * FROM " + dataTableFullName
- + " WHERE CREATED_BY = 'bar'";
- conn.createStatement().execute(createView2);
-
- // We want to verify if deletes and set null result in expected rebuild of view index
- conn.createStatement().execute("UPSERT INTO " + view1FullName
- + "(ORGANIZATION_ID, KEY_PREFIX, VIEW_COLA, VIEW_COLB) VALUES('ORG1', 'aaa', 'A', 'G')");
- conn.createStatement().execute("UPSERT INTO " + view1FullName
- + "(ORGANIZATION_ID, KEY_PREFIX, VIEW_COLA, VIEW_COLB) VALUES('ORG1', 'ccc', 'C', 'I')");
- conn.createStatement().execute("UPSERT INTO " + view1FullName
- + "(ORGANIZATION_ID, KEY_PREFIX, VIEW_COLA, VIEW_COLB) VALUES('ORG1', 'ddd', 'D', 'J')");
-
- conn.createStatement().execute("UPSERT INTO " + view2FullName
- + "(ORGANIZATION_ID, KEY_PREFIX, VIEW_COL1, VIEW_COL2) VALUES('ORG2', 'bbb', 'B', 'H')");
- conn.commit();
- conn.createStatement().execute("DELETE FROM " + view1FullName
- + " WHERE ORGANIZATION_ID = 'ORG1' AND VIEW_COLA = 'C'");
- conn.createStatement().execute("UPSERT INTO " + view1FullName
- + "(ORGANIZATION_ID, KEY_PREFIX, VIEW_COLA, VIEW_COLB) VALUES('ORG1', 'ddd', 'D', NULL)");
- conn.commit();
-
- String createViewIndex =
- "CREATE INDEX IF NOT EXISTS " + indexTableName + " ON " + view1FullName
- + " (VIEW_COLB) ASYNC";
- conn.createStatement().execute(createViewIndex);
- conn.commit();
- // Rebuild using index tool
- IndexToolIT.runIndexTool(directApi, useSnapshot, schemaName, view1Name, indexTableName);
- ResultSet rs =
- conn.createStatement()
- .executeQuery("SELECT COUNT(*) FROM " + indexTableFullName);
- rs.next();
- assertEquals(2, rs.getInt(1));
-
- Pair<Integer, Integer> putsAndDeletes =
- countPutsAndDeletes("_IDX_" + dataTableFullName);
- assertEquals(4, (int) putsAndDeletes.getFirst());
- assertEquals(2, (int) putsAndDeletes.getSecond());
- }
- }
-
- private Pair<Integer, Integer> countPutsAndDeletes(String tableName) throws Exception {
- int numPuts = 0;
- int numDeletes = 0;
- try (org.apache.hadoop.hbase.client.Connection hcon =
- ConnectionFactory.createConnection(config)) {
- Table htable = hcon.getTable(TableName.valueOf(tableName));
- Scan scan = new Scan();
- scan.setRaw(true);
- ResultScanner scanner = htable.getScanner(scan);
-
- for (Result result = scanner.next(); result != null; result = scanner.next()) {
- for (Cell cell : result.rawCells()) {
- if (cell.getType() == Cell.Type.Put) {
- numPuts++;
- } else if (cell.getType() == Cell.Type.DeleteFamily) {
- numDeletes++;
- }
- }
- }
- }
- return new Pair<Integer, Integer>(numPuts, numDeletes);
- }
-
- @Test
- public void testIndexToolVerifyWithExpiredIndexRows() throws Exception {
- String schemaName = generateUniqueName();
- String dataTableName = generateUniqueName();
- String dataTableFullName = SchemaUtil.getTableName(schemaName, dataTableName);
- String indexTableName = generateUniqueName();
- String indexTableFullName = SchemaUtil.getTableName(schemaName, indexTableName);
- Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
- try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
- conn.createStatement().execute("CREATE TABLE " + dataTableFullName
- + " (ID INTEGER NOT NULL PRIMARY KEY, NAME VARCHAR, CODE VARCHAR) COLUMN_ENCODED_BYTES=0");
- // Insert a row
- conn.createStatement()
- .execute("upsert into " + dataTableFullName + " values (1, 'Phoenix', 'A')");
- conn.commit();
- conn.createStatement()
- .execute(String.format("CREATE INDEX %s ON %s (NAME) INCLUDE (CODE) ASYNC",
- indexTableName, dataTableFullName));
- IndexToolIT.runIndexTool(directApi, useSnapshot, schemaName, dataTableName, indexTableName, null, 0,
- IndexTool.IndexVerifyType.ONLY);
- Cell cell =
- IndexToolIT.getErrorMessageFromIndexToolOutputTable(conn, dataTableFullName,
- indexTableFullName);
- try {
- String expectedErrorMsg = IndexRebuildRegionScanner.ERROR_MESSAGE_MISSING_INDEX_ROW;
- String actualErrorMsg = Bytes
- .toString(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength());
- assertTrue(expectedErrorMsg.equals(actualErrorMsg));
- } catch(Exception ex) {
- Assert.fail("Fail to parsing the error message from IndexToolOutputTable");
- }
-
- // Run the index tool to populate the index while verifying rows
- IndexToolIT.runIndexTool(directApi, useSnapshot, schemaName, dataTableName, indexTableName, null, 0,
- IndexTool.IndexVerifyType.AFTER);
-
- // Set ttl of index table ridiculously low so that all data is expired
- Admin admin = conn.unwrap(PhoenixConnection.class).getQueryServices().getAdmin();
- TableName indexTable = TableName.valueOf(indexTableFullName);
- ColumnFamilyDescriptor desc = admin.getDescriptor(indexTable).getColumnFamilies()[0];
- ColumnFamilyDescriptorBuilder builder =
- ColumnFamilyDescriptorBuilder.newBuilder(desc).setTimeToLive(1);
- Future<Void> future = admin.modifyColumnFamilyAsync(indexTable, builder.build());
- Thread.sleep(1000);
- future.get(40, TimeUnit.SECONDS);
-
- TableName indexToolOutputTable = TableName.valueOf(IndexVerificationOutputRepository.OUTPUT_TABLE_NAME_BYTES);
- admin.disableTable(indexToolOutputTable);
- admin.deleteTable(indexToolOutputTable);
- // Run the index tool using the only-verify option, verify it gives no mismatch
- IndexToolIT.runIndexTool(directApi, useSnapshot, schemaName, dataTableName, indexTableName, null, 0,
- IndexTool.IndexVerifyType.ONLY);
- Scan scan = new Scan();
- Table hIndexToolTable =
- conn.unwrap(PhoenixConnection.class).getQueryServices()
- .getTable(indexToolOutputTable.getName());
- Result r = hIndexToolTable.getScanner(scan).next();
- assertTrue(r == null);
- }
- }
-
- @Test
- public void testSecondaryGlobalIndexFailure() throws Exception {
- if (!mutable) {
- return; //nothing in this test is mutable specific, so no need to run twice
- }
- String schemaName = generateUniqueName();
- String dataTableName = generateUniqueName();
- String dataTableFullName = SchemaUtil.getTableName(schemaName, dataTableName);
- String indexTableName = generateUniqueName();
- Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
- try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
- String stmString1 =
- "CREATE TABLE " + dataTableFullName
- + " (ID INTEGER NOT NULL PRIMARY KEY, NAME VARCHAR, ZIP INTEGER) "
- + tableDDLOptions;
- conn.createStatement().execute(stmString1);
- conn.commit();
-
- String upsertQuery = String.format("UPSERT INTO %s VALUES(?, ?, ?)", dataTableFullName);
- PreparedStatement stmt1 = conn.prepareStatement(upsertQuery);
-
- // Insert two rows
- IndexToolIT.upsertRow(stmt1, 1);
- IndexToolIT.upsertRow(stmt1, 2);
- conn.commit();
-
- String stmtString2 =
- String.format(
- "CREATE INDEX %s ON %s (LPAD(UPPER(NAME, 'en_US'),8,'x')||'_xyz') ", indexTableName, dataTableFullName);
- conn.createStatement().execute(stmtString2);
- conn.commit();
- String qIndexTableName = SchemaUtil.getQualifiedTableName(schemaName, indexTableName);
-
- // Verify that the index table is in the ACTIVE state
- assertEquals(PIndexState.ACTIVE, TestUtil.getIndexState(conn, qIndexTableName));
-
- ConnectionQueryServices queryServices = conn.unwrap(PhoenixConnection.class).getQueryServices();
- Admin admin = queryServices.getAdmin();
- TableName tn = TableName.valueOf(Bytes.toBytes(dataTableFullName));
- TableDescriptor td =
- admin.getDescriptor(tn);
- //add the fast fail coproc and make sure it goes first
- CoprocessorDescriptor cd =
- CoprocessorDescriptorBuilder.
- newBuilder(FastFailRegionObserver.class.getName()).
- setPriority(1).build();
- TableDescriptor newTd =
- TableDescriptorBuilder.newBuilder(td).setCoprocessor(cd).build();
- admin.modifyTable(tn, newTd);
- // Run the index MR job and it should fail (return -1)
- IndexToolIT.runIndexTool(directApi, useSnapshot, schemaName, dataTableName, indexTableName,
- null, -1, new String[0]);
-
- // Verify that the index table should be still in the ACTIVE state
- assertEquals(PIndexState.ACTIVE, TestUtil.getIndexState(conn, qIndexTableName));
- }
- }
-
- @Test
- public void testBuildSecondaryIndexAndScrutinize() throws Exception {
- String schemaName = generateUniqueName();
- String dataTableName = generateUniqueName();
- String dataTableFullName = SchemaUtil.getTableName(schemaName, dataTableName);
String indexTableName = generateUniqueName();
String indexTableFullName = SchemaUtil.getTableName(schemaName, indexTableName);
Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
@@ -692,9 +466,10 @@ public class IndexToolForNonTxGlobalIndexIT extends BaseUniqueNamesOwnClusterIT
String upsertQuery = String.format("UPSERT INTO %s VALUES(?, ?, ?)", dataTableFullName);
PreparedStatement stmt1 = conn.prepareStatement(upsertQuery);
- // Insert NROWS rows
- final int NROWS = 1000;
- for (int i = 0; i < NROWS; i++) {
+ // Insert N_ROWS rows
+ final int N_ROWS = 100;
+ final int N_OLD_ROWS = 10;
+ for (int i = 0; i < N_ROWS; i++) {
IndexToolIT.upsertRow(stmt1, i);
}
conn.commit();
@@ -705,38 +480,38 @@ public class IndexToolForNonTxGlobalIndexIT extends BaseUniqueNamesOwnClusterIT
// Run the index MR job and verify that the index table is built correctly
IndexTool indexTool = IndexToolIT.runIndexTool(directApi, useSnapshot, schemaName, dataTableName, indexTableName, null, 0, IndexTool.IndexVerifyType.BEFORE, new String[0]);
- assertEquals(NROWS, indexTool.getJob().getCounters().findCounter(INPUT_RECORDS).getValue());
- assertEquals(NROWS, indexTool.getJob().getCounters().findCounter(SCANNED_DATA_ROW_COUNT).getValue());
- assertEquals(NROWS, indexTool.getJob().getCounters().findCounter(REBUILT_INDEX_ROW_COUNT).getValue());
+ assertEquals(N_ROWS, indexTool.getJob().getCounters().findCounter(INPUT_RECORDS).getValue());
+ assertEquals(N_ROWS, indexTool.getJob().getCounters().findCounter(SCANNED_DATA_ROW_COUNT).getValue());
+ assertEquals(N_ROWS, indexTool.getJob().getCounters().findCounter(REBUILT_INDEX_ROW_COUNT).getValue());
assertEquals(0, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_VALID_INDEX_ROW_COUNT).getValue());
assertEquals(0, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_EXPIRED_INDEX_ROW_COUNT).getValue());
assertEquals(0, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_INVALID_INDEX_ROW_COUNT).getValue());
- assertEquals(NROWS, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_MISSING_INDEX_ROW_COUNT).getValue());
- long actualRowCount = IndexScrutiny.scrutinizeIndex(conn, dataTableFullName, indexTableFullName);
- assertEquals(NROWS, actualRowCount);
-
- // Add more rows and make sure that these rows will be visible to IndexTool
- for (int i = NROWS; i < 2 * NROWS; i++) {
- IndexToolIT.upsertRow(stmt1, i);
- }
- conn.commit();
- indexTool = IndexToolIT.runIndexTool(directApi, useSnapshot, schemaName, dataTableName, indexTableName, null, 0, IndexTool.IndexVerifyType.BOTH, new String[0]);
- assertEquals(2 * NROWS, indexTool.getJob().getCounters().findCounter(INPUT_RECORDS).getValue());
- assertEquals(2 * NROWS, indexTool.getJob().getCounters().findCounter(SCANNED_DATA_ROW_COUNT).getValue());
- assertEquals(0, indexTool.getJob().getCounters().findCounter(REBUILT_INDEX_ROW_COUNT).getValue());
- assertEquals(2 * NROWS, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_VALID_INDEX_ROW_COUNT).getValue());
- assertEquals(0, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_EXPIRED_INDEX_ROW_COUNT).getValue());
- assertEquals(0, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_INVALID_INDEX_ROW_COUNT).getValue());
- assertEquals(0, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_MISSING_INDEX_ROW_COUNT).getValue());
+ assertEquals(N_ROWS, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_MISSING_INDEX_ROW_COUNT).getValue());
+ assertEquals(0, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_BEYOND_MAXLOOKBACK_MISSING_INDEX_ROW_COUNT).getValue());
+ assertEquals(0, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_BEYOND_MAXLOOKBACK_INVALID_INDEX_ROW_COUNT).getValue());
assertEquals(0, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_UNVERIFIED_INDEX_ROW_COUNT).getValue());
assertEquals(0, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_OLD_INDEX_ROW_COUNT).getValue());
assertEquals(0, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_UNKNOWN_INDEX_ROW_COUNT).getValue());
- assertEquals(0, indexTool.getJob().getCounters().findCounter(AFTER_REBUILD_VALID_INDEX_ROW_COUNT).getValue());
- assertEquals(0, indexTool.getJob().getCounters().findCounter(AFTER_REBUILD_EXPIRED_INDEX_ROW_COUNT).getValue());
- assertEquals(0, indexTool.getJob().getCounters().findCounter(AFTER_REBUILD_INVALID_INDEX_ROW_COUNT).getValue());
- assertEquals(0, indexTool.getJob().getCounters().findCounter(AFTER_REBUILD_MISSING_INDEX_ROW_COUNT).getValue());
+ long actualRowCount = IndexScrutiny.scrutinizeIndex(conn, dataTableFullName, indexTableFullName);
+ assertEquals(N_ROWS, actualRowCount);
+ // Read N_OLD_ROWS index rows and rewrite them back directly. This will overwrite existing rows with newer
+ // timestamps and set the empty column to value "x". The will make them old design rows (for mutable indexes)
+ String stmtString3 =
+ String.format(
+ "UPSERT INTO %s SELECT * FROM %s LIMIT %d", indexTableFullName, indexTableFullName, N_OLD_ROWS);
+ conn.createStatement().execute(stmtString3);
+ conn.commit();
+ // Verify that IndexTool reports that there are old design index rows
+ indexTool = IndexToolIT.runIndexTool(directApi, useSnapshot, schemaName, dataTableName, indexTableName, null, 0, IndexTool.IndexVerifyType.ONLY, new String[0]);
+ assertEquals(N_OLD_ROWS, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_OLD_INDEX_ROW_COUNT).getValue());
+ // Clean up all old design rows
+ indexTool = IndexToolIT.runIndexTool(directApi, useSnapshot, schemaName, dataTableName, indexTableName, null, 0, IndexTool.IndexVerifyType.BEFORE, new String[0]);
+ assertEquals(N_OLD_ROWS, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_OLD_INDEX_ROW_COUNT).getValue());
+ // Verify that IndexTool does not report them anymore
+ indexTool = IndexToolIT.runIndexTool(directApi, useSnapshot, schemaName, dataTableName, indexTableName, null, 0, IndexTool.IndexVerifyType.BEFORE, new String[0]);
+ assertEquals(0, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_OLD_INDEX_ROW_COUNT).getValue());
actualRowCount = IndexScrutiny.scrutinizeIndex(conn, dataTableFullName, indexTableFullName);
- assertEquals(2 * NROWS, actualRowCount);
+ assertEquals(N_ROWS, actualRowCount);
}
}
@@ -800,7 +575,7 @@ public class IndexToolForNonTxGlobalIndexIT extends BaseUniqueNamesOwnClusterIT
conn.commit();
// Configure IndexRegionObserver to fail the first write phase. This should not
// lead to any change on index and thus the index verify during index rebuild should fail
- IndexRegionObserver.setIgnoreIndexRebuildForTesting(true);
+ IndexRebuildRegionScanner.setIgnoreIndexRebuildForTesting(true);
conn.createStatement().execute(String.format(
"CREATE INDEX %s ON %s (NAME) INCLUDE (ZIP) ASYNC", indexTableName, viewFullName));
// Run the index MR job and verify that the index table rebuild fails
@@ -811,11 +586,11 @@ public class IndexToolForNonTxGlobalIndexIT extends BaseUniqueNamesOwnClusterIT
try {
String expectedErrorMsg = IndexRebuildRegionScanner.ERROR_MESSAGE_MISSING_INDEX_ROW;
String actualErrorMsg = Bytes.toString(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength());
- assertTrue(expectedErrorMsg.equals(actualErrorMsg));
+ assertEquals(expectedErrorMsg, actualErrorMsg);
} catch(Exception ex){
Assert.fail("Fail to parsing the error message from IndexToolOutputTable");
}
- IndexRegionObserver.setIgnoreIndexRebuildForTesting(false);
+ IndexRebuildRegionScanner.setIgnoreIndexRebuildForTesting(false);
}
}
@@ -844,7 +619,7 @@ public class IndexToolForNonTxGlobalIndexIT extends BaseUniqueNamesOwnClusterIT
try {
String expectedErrorMsg = IndexRebuildRegionScanner.ERROR_MESSAGE_MISSING_INDEX_ROW;
String actualErrorMsg = Bytes.toString(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength());
- assertTrue(expectedErrorMsg.equals(actualErrorMsg));
+ assertEquals(expectedErrorMsg, actualErrorMsg);
} catch(Exception ex) {
Assert.fail("Fail to parsing the error message from IndexToolOutputTable");
}
@@ -865,6 +640,7 @@ public class IndexToolForNonTxGlobalIndexIT extends BaseUniqueNamesOwnClusterIT
@Test
public void testIndexToolForIncrementalRebuild() throws Exception {
+ Assume.assumeTrue(HbaseCompatCapabilities.isRawFilterSupported());
String schemaName = generateUniqueName();
String dataTableName = generateUniqueName();
String dataTableFullName = SchemaUtil.getTableName(schemaName, dataTableName);
@@ -889,8 +665,12 @@ public class IndexToolForNonTxGlobalIndexIT extends BaseUniqueNamesOwnClusterIT
expectedStatus.add(RUN_STATUS_EXECUTED);
expectedStatus.add(RUN_STATUS_EXECUTED);
expectedStatus.add(RUN_STATUS_EXECUTED);
-
- verifyRunStatusFromResultTable(conn, scn, indexTableFullName, 3, expectedStatus);
+ try {
+ verifyRunStatusFromResultTable(conn, scn, indexTableFullName, 3, expectedStatus);
+ } catch (AssertionError ae) {
+ TestUtil.dumpTable(conn, TableName.valueOf(RESULT_TABLE_NAME));
+ throw ae;
+ }
deleteOneRowFromResultTable(conn, scn, indexTableFullName);
@@ -927,6 +707,7 @@ public class IndexToolForNonTxGlobalIndexIT extends BaseUniqueNamesOwnClusterIT
@Test
public void testIndexToolForIncrementalVerify() throws Exception {
+ Assume.assumeTrue(HbaseCompatCapabilities.isRawFilterSupported());
ManualEnvironmentEdge customEdge = new ManualEnvironmentEdge();
String schemaName = generateUniqueName();
String dataTableName = generateUniqueName();
@@ -939,16 +720,6 @@ public class IndexToolForNonTxGlobalIndexIT extends BaseUniqueNamesOwnClusterIT
Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
conn.setAutoCommit(true);
- IndexVerificationOutputRepository outputRepository =
- new IndexVerificationOutputRepository();
- //need to make sure that the output and result tables are created before we start
- //overriding the environment edge, because there's a bug in HBase 2.x where table
- //creation hangs forever when an edge is an injected
- outputRepository.createOutputTable(conn);
- IndexVerificationResultRepository resultRepository =
- new IndexVerificationResultRepository();
- resultRepository.createResultTable(conn);
-
conn.createStatement().execute("CREATE TABLE "+dataTableFullName+" "
+ "(key1 BIGINT NOT NULL, key2 BIGINT NOT NULL, val1 VARCHAR, val2 BIGINT, "
+ "val3 BIGINT, val4 DOUBLE, val5 BIGINT, val6 VARCHAR "
@@ -1024,14 +795,10 @@ public class IndexToolForNonTxGlobalIndexIT extends BaseUniqueNamesOwnClusterIT
customEdge.incrementValue(waitForUpsert);
return;
}
- //In HBase 2.0-2.2, we can't see Puts behind Deletes even on lookback / SCN queries. Starting in 2.3 we can
- //That changes the counts we expect from index tool verification
- int putBehindDeleteMarkerCount = HbaseCompatCapabilities.isLookbackBeyondDeletesSupported() ? 1 :0;
-
// regular job without delete row
it = IndexToolIT.runIndexTool(directApi, useSnapshot, schemaName, dataTableName, indexTableName,
null, 0, IndexTool.IndexVerifyType.ONLY, "-st", String.valueOf(t0),"-et", String.valueOf(t4));
- verifyCounters(it, 2, 2 + putBehindDeleteMarkerCount);
+ verifyCounters(it, 2, 3);
customEdge.incrementValue(waitForUpsert);
// job with 2 rows
@@ -1043,13 +810,13 @@ public class IndexToolForNonTxGlobalIndexIT extends BaseUniqueNamesOwnClusterIT
// job with update on only one row
it = IndexToolIT.runIndexTool(directApi, useSnapshot, schemaName, dataTableName, indexTableName,
null, 0, IndexTool.IndexVerifyType.ONLY, "-st", String.valueOf(t1),"-et", String.valueOf(t3));
- verifyCounters(it, 1, 1 + putBehindDeleteMarkerCount);
+ verifyCounters(it, 1, 2);
customEdge.incrementValue(waitForUpsert);
// job with update on only one row
it = IndexToolIT.runIndexTool(directApi, useSnapshot, schemaName, dataTableName, indexTableName,
null, 0, IndexTool.IndexVerifyType.ONLY, "-st", String.valueOf(t2),"-et", String.valueOf(t4));
- verifyCounters(it, 1, 1 + putBehindDeleteMarkerCount);
+ verifyCounters(it, 1, 2);
customEdge.incrementValue(waitForUpsert);
// job with update on only one row
@@ -1070,6 +837,7 @@ public class IndexToolForNonTxGlobalIndexIT extends BaseUniqueNamesOwnClusterIT
@Test
public void testIndexToolForIncrementalVerify_viewIndex() throws Exception {
+ Assume.assumeTrue(HbaseCompatCapabilities.isRawFilterSupported());
ManualEnvironmentEdge customeEdge = new ManualEnvironmentEdge();
String schemaName = generateUniqueName();
String dataTableName = generateUniqueName();
@@ -1112,13 +880,15 @@ public class IndexToolForNonTxGlobalIndexIT extends BaseUniqueNamesOwnClusterIT
long t5 = customeEdge.currentTime();
IndexTool it;
// regular job with delete row
- it = IndexToolIT.runIndexTool(directApi, useSnapshot, schemaName, viewName,
+ it =
+ IndexToolIT.runIndexTool(directApi, useSnapshot, schemaName, viewName,
viewIndexName, null, 0, IndexTool.IndexVerifyType.ONLY, "-st", String.valueOf(t1),
"-et", String.valueOf(t4));
verifyCounters(it, 2, 2);
// job with 1 row
- it = IndexToolIT.runIndexTool(directApi, useSnapshot, schemaName, viewName,
+ it =
+ IndexToolIT.runIndexTool(directApi, useSnapshot, schemaName, viewName,
viewIndexName, null, 0, IndexTool.IndexVerifyType.ONLY, "-st", String.valueOf(t1),
"-et", String.valueOf(t2));
verifyCounters(it, 1, 1);
@@ -1136,15 +906,19 @@ public class IndexToolForNonTxGlobalIndexIT extends BaseUniqueNamesOwnClusterIT
viewIndexName, null, 0, IndexTool.IndexVerifyType.ONLY, "-st", String.valueOf(t1),
"-et", String.valueOf(t3));
verifyCounters(it, 2, 2);
-/* Disabled pending completion of PHOENIX-5989, because the view filter doesn't include
- a PK column, so the delete is getting filtered out of the verification scan
- // job with update on only one row
+
+ // job with update on only one row - commented out because it requires PHOENIX-5989
+ // because the view index verification code doesn't recognize the delete marker as
+ // being part of the view, because the view filters on a non-PK field
+ /*
it =
IndexToolIT.runIndexTool(directApi, useSnapshot, schemaName, viewName,
viewIndexName, null, 0, IndexTool.IndexVerifyType.ONLY, "-st", String.valueOf(t3),
"-et", String.valueOf(t4));
+ //0,0 because the view index verification code doesn't recognize the delete marker as
+ // being part of the view, because the view filters on a non-PK field
verifyCounters(it, 1, 1);
-*/
+ */
// job with no new updates on any row
it =
IndexToolIT.runIndexTool(directApi, useSnapshot, schemaName, viewName,
@@ -1156,113 +930,515 @@ public class IndexToolForNonTxGlobalIndexIT extends BaseUniqueNamesOwnClusterIT
}
}
- private void verifyCounters(IndexTool it, int scanned, int valid) throws IOException {
- assertEquals(scanned, it.getJob().getCounters().findCounter(SCANNED_DATA_ROW_COUNT).getValue());
- assertEquals(0, it.getJob().getCounters().findCounter(REBUILT_INDEX_ROW_COUNT).getValue());
- assertEquals(valid, it.getJob().getCounters().findCounter(BEFORE_REBUILD_VALID_INDEX_ROW_COUNT).getValue());
- assertEquals(0, it.getJob().getCounters().findCounter(BEFORE_REBUILD_EXPIRED_INDEX_ROW_COUNT).getValue());
- assertEquals(0, it.getJob().getCounters().findCounter(BEFORE_REBUILD_INVALID_INDEX_ROW_COUNT).getValue());
- assertEquals(0, it.getJob().getCounters().findCounter(BEFORE_REBUILD_MISSING_INDEX_ROW_COUNT).getValue());
- assertEquals(0, it.getJob().getCounters().findCounter(BEFORE_REBUILD_UNVERIFIED_INDEX_ROW_COUNT).getValue());
- assertEquals(0, it.getJob().getCounters().findCounter(BEFORE_REBUILD_OLD_INDEX_ROW_COUNT).getValue());
- assertEquals(0, it.getJob().getCounters().findCounter(BEFORE_REBUILD_UNKNOWN_INDEX_ROW_COUNT).getValue());
- }
+ private void verifyCounters(IndexTool it, int scanned, int valid) throws IOException {
+ assertEquals(scanned, it.getJob().getCounters().findCounter(SCANNED_DATA_ROW_COUNT).getValue());
+ assertEquals(0, it.getJob().getCounters().findCounter(REBUILT_INDEX_ROW_COUNT).getValue());
+ assertEquals(valid, it.getJob().getCounters().findCounter(BEFORE_REBUILD_VALID_INDEX_ROW_COUNT).getValue());
+ assertEquals(0, it.getJob().getCounters().findCounter(BEFORE_REBUILD_EXPIRED_INDEX_ROW_COUNT).getValue());
+ assertEquals(0, it.getJob().getCounters().findCounter(BEFORE_REBUILD_INVALID_INDEX_ROW_COUNT).getValue());
+ assertEquals(0, it.getJob().getCounters().findCounter(BEFORE_REBUILD_MISSING_INDEX_ROW_COUNT).getValue());
+ assertEquals(0, it.getJob().getCounters().findCounter(BEFORE_REBUILD_BEYOND_MAXLOOKBACK_MISSING_INDEX_ROW_COUNT).getValue());
+ assertEquals(0, it.getJob().getCounters().findCounter(BEFORE_REBUILD_BEYOND_MAXLOOKBACK_INVALID_INDEX_ROW_COUNT).getValue());
+ assertEquals(0, it.getJob().getCounters().findCounter(BEFORE_REBUILD_UNVERIFIED_INDEX_ROW_COUNT).getValue());
+ assertEquals(0, it.getJob().getCounters().findCounter(BEFORE_REBUILD_OLD_INDEX_ROW_COUNT).getValue());
+ assertEquals(0, it.getJob().getCounters().findCounter(BEFORE_REBUILD_UNKNOWN_INDEX_ROW_COUNT).getValue());
+ }
+
+ @Test
+ public void testDisableOutputLogging() throws Exception {
+ if (!mutable || useSnapshot) {
+ return;
+ }
+
+ String schemaName = generateUniqueName();
+ String dataTableName = generateUniqueName();
+ String dataTableFullName = SchemaUtil.getTableName(schemaName, dataTableName);
+ String indexTableName = generateUniqueName();
+ String indexTableFullName = SchemaUtil.getTableName(schemaName, indexTableName);
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+
+ try(Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ deleteAllRows(conn,
+ TableName.valueOf(IndexVerificationOutputRepository.OUTPUT_TABLE_NAME));
+ String stmString1 =
+ "CREATE TABLE " + dataTableFullName
+ + " (ID INTEGER NOT NULL PRIMARY KEY, NAME VARCHAR, ZIP INTEGER) "
+ + tableDDLOptions;
+ conn.createStatement().execute(stmString1);
+ String upsertQuery = String.format("UPSERT INTO %s VALUES(?, ?, ?)", dataTableFullName);
+ PreparedStatement stmt1 = conn.prepareStatement(upsertQuery);
+
+ // insert two rows
+ IndexToolIT.upsertRow(stmt1, 1);
+ IndexToolIT.upsertRow(stmt1, 2);
+ conn.commit();
+
+ //create ASYNC
+ String stmtString2 =
+ String.format(
+ "CREATE INDEX %s ON %s (LPAD(UPPER(NAME, 'en_US'),8,'x')||'_xyz') ASYNC ",
+ indexTableName, dataTableFullName);
+ conn.createStatement().execute(stmtString2);
+ conn.commit();
+
+ // run the index MR job as ONLY so the index doesn't get rebuilt. Should be 2 missing
+ //rows. We pass in --disable-logging BEFORE to silence the output logging to
+ // PHOENIX_INDEX_TOOL, since ONLY logs BEFORE the (non-existent in this case)
+ // rebuild
+ assertDisableLogging(conn, 0, IndexTool.IndexVerifyType.ONLY,
+ IndexTool.IndexDisableLoggingType.BEFORE, null, schemaName, dataTableName, indexTableName,
+ indexTableFullName, 0);
+
+ truncateIndexAndIndexToolTables(indexTableFullName);
+
+ // disabling logging AFTER on an AFTER run should leave no output rows
+ assertDisableLogging(conn, 0, IndexTool.IndexVerifyType.AFTER,
+ IndexTool.IndexDisableLoggingType.AFTER, null, schemaName, dataTableName,
+ indexTableName,
+ indexTableFullName, 0);
+
+ truncateIndexAndIndexToolTables(indexTableFullName);
+
+ //disabling logging BEFORE on a BEFORE run should leave no output rows
+ assertDisableLogging(conn, 0, IndexTool.IndexVerifyType.BEFORE,
+ IndexTool.IndexDisableLoggingType.BEFORE, null, schemaName, dataTableName, indexTableName,
+ indexTableFullName, 0);
+ //now clear out all the rebuilt index rows
+ truncateIndexAndIndexToolTables(indexTableFullName);
+
+ //now check that disabling logging AFTER leaves only the BEFORE logs on a BOTH run
+ assertDisableLogging(conn, 2, IndexTool.IndexVerifyType.BOTH,
+ IndexTool.IndexDisableLoggingType.AFTER,
+ IndexVerificationOutputRepository.PHASE_BEFORE_VALUE, schemaName,
+ dataTableName, indexTableName,
+ indexTableFullName, 0);
+
+ //clear out both the output table and the index
+ truncateIndexAndIndexToolTables(indexTableFullName);
+
+ //now check that disabling logging BEFORE creates only the AFTER logs on a BOTH run
+ assertDisableLogging(conn, 0, IndexTool.IndexVerifyType.BOTH,
+ IndexTool.IndexDisableLoggingType.BEFORE,
+ IndexVerificationOutputRepository.PHASE_AFTER_VALUE, schemaName,
+ dataTableName, indexTableName,
+ indexTableFullName, 0);
+
+ truncateIndexAndIndexToolTables(indexTableFullName);
+
+ //now check that disabling logging BOTH creates no logs on a BOTH run
+ assertDisableLogging(conn, 0, IndexTool.IndexVerifyType.BOTH,
+ IndexTool.IndexDisableLoggingType.BOTH,
+ IndexVerificationOutputRepository.PHASE_BEFORE_VALUE, schemaName,
+ dataTableName, indexTableName,
+ indexTableFullName, 0);
+
+ }
+ }
+
+ @Test
+ public void testEnableOutputLoggingForMaxLookback() throws Exception {
+ Assume.assumeTrue(HbaseCompatCapabilities.isMaxLookbackTimeSupported());
+ //by default we don't log invalid or missing rows past max lookback age to the
+ // PHOENIX_INDEX_TOOL table. Verify that we can flip that logging on from the client-side
+ // using a system property (such as from the command line) and have it log rows on the
+ // server-side
+ if (!mutable) {
+ return;
+ }
+ String schemaName = generateUniqueName();
+ String dataTableName = generateUniqueName();
+ String dataTableFullName = SchemaUtil.getTableName(schemaName, dataTableName);
+ String indexTableName = generateUniqueName();
+ String indexTableFullName = SchemaUtil.getTableName(schemaName, indexTableName);
+
+ try(Connection conn = DriverManager.getConnection(getUrl())) {
+
+ deleteAllRows(conn,
+ TableName.valueOf(IndexVerificationOutputRepository.OUTPUT_TABLE_NAME));
+ String stmString1 =
+ "CREATE TABLE " + dataTableFullName
+ + " (ID INTEGER NOT NULL PRIMARY KEY, NAME VARCHAR, ZIP INTEGER) ";
+ conn.createStatement().execute(stmString1);
+ conn.commit();
+
+ String upsertQuery = String.format("UPSERT INTO %s VALUES(?, ?, ?)", dataTableFullName);
+ PreparedStatement stmt1 = conn.prepareStatement(upsertQuery);
+
+ // insert two rows
+ IndexToolIT.upsertRow(stmt1, 1);
+ IndexToolIT.upsertRow(stmt1, 2);
+ conn.commit();
+ //now create an index async so it won't have the two rows in the base table
+
+ String stmtString2 =
+ String.format(
+ "CREATE INDEX %s ON %s (LPAD(UPPER(NAME, 'en_US'),8,'x')||'_xyz') ASYNC ",
+ indexTableName, dataTableFullName);
+ conn.createStatement().execute(stmtString2);
+ conn.commit();
+ ManualEnvironmentEdge injectEdge = new ManualEnvironmentEdge();
+ injectEdge.setValue(EnvironmentEdgeManager.currentTimeMillis());
+ EnvironmentEdgeManager.injectEdge(injectEdge);
+ injectEdge.incrementValue(1L);
+ injectEdge.incrementValue(MAX_LOOKBACK_AGE * 1000);
+ deleteAllRows(conn, TableName.valueOf(IndexVerificationOutputRepository.OUTPUT_TABLE_NAME));
+ getUtility().getConfiguration().
+ set(IndexRebuildRegionScanner.PHOENIX_INDEX_MR_LOG_BEYOND_MAX_LOOKBACK_ERRORS, "true");
+ IndexTool it = IndexToolIT.runIndexTool(directApi, useSnapshot, schemaName,
+ dataTableName,
+ indexTableName, null, 0, IndexTool.IndexVerifyType.ONLY);
+ TestUtil.dumpTable(conn, TableName.valueOf(IndexVerificationOutputRepository.OUTPUT_TABLE_NAME));
+ Counters counters = it.getJob().getCounters();
+ System.out.println(counters.toString());
+ assertEquals(2L,
+ counters.findCounter(BEFORE_REBUILD_BEYOND_MAXLOOKBACK_MISSING_INDEX_ROW_COUNT).getValue());
+ injectEdge.incrementValue(1L);
+ IndexVerificationOutputRepository outputRepository =
+ new IndexVerificationOutputRepository(Bytes.toBytes(indexTableFullName), conn);
+ List<IndexVerificationOutputRow> outputRows = outputRepository.getAllOutputRows();
+ assertEquals(2, outputRows.size());
+ } finally {
+ EnvironmentEdgeManager.reset();
+ truncateIndexToolTables(); //have to truncate rather than delete in the after handler
+ // because we wrote in "the future" and the after handler will be in the present
+ }
+ }
+
+ @Test
+ public void testOverrideIndexRebuildPageSizeFromIndexTool() throws Exception {
+ String schemaName = generateUniqueName();
+ String dataTableName = generateUniqueName();
+ String dataTableFullName = SchemaUtil.getTableName(schemaName, dataTableName);
+ String indexTableName = generateUniqueName();
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+
+ try(Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ String stmString1 =
+ "CREATE TABLE " + dataTableFullName
+ + " (ID INTEGER NOT NULL PRIMARY KEY, NAME VARCHAR, ZIP INTEGER) "
+ + tableDDLOptions;
+ conn.createStatement().execute(stmString1);
+ String upsertQuery = String.format("UPSERT INTO %s VALUES(?, ?, ?)", dataTableFullName);
+ PreparedStatement stmt1 = conn.prepareStatement(upsertQuery);
+
+ // Insert NROWS rows
+ final int NROWS = 16;
+ for (int i = 0; i < NROWS; i++) {
+ IndexToolIT.upsertRow(stmt1, i);
+ }
+ conn.commit();
+
+ String stmtString2 =
+ String.format(
+ "CREATE INDEX %s ON %s (NAME) INCLUDE (ZIP) ASYNC ", indexTableName, dataTableFullName);
+ conn.createStatement().execute(stmtString2);
+
+ // Run the index MR job and verify that the index table is built correctly
+ Configuration conf = new Configuration(getUtility().getConfiguration());
+ conf.set(QueryServices.INDEX_REBUILD_PAGE_SIZE_IN_ROWS, Long.toString(2));
+ IndexTool indexTool = IndexToolIT.runIndexTool(directApi, useSnapshot, schemaName, dataTableName, indexTableName, null, 0, IndexTool.IndexVerifyType.BEFORE, new String[0]);
+ assertEquals(NROWS, indexTool.getJob().getCounters().findCounter(INPUT_RECORDS).getValue());
+ assertEquals(NROWS, indexTool.getJob().getCounters().findCounter(SCANNED_DATA_ROW_COUNT).getValue());
+ assertEquals(NROWS, indexTool.getJob().getCounters().findCounter(REBUILT_INDEX_ROW_COUNT).getValue());
+ assertEquals(0, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_VALID_INDEX_ROW_COUNT).getValue());
+ assertEquals(0, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_EXPIRED_INDEX_ROW_COUNT).getValue());
+ assertEquals(0, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_INVALID_INDEX_ROW_COUNT).getValue());
+ assertEquals(NROWS, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_MISSING_INDEX_ROW_COUNT).getValue());
+ assertEquals(0, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_BEYOND_MAXLOOKBACK_MISSING_INDEX_ROW_COUNT).getValue());
+ assertEquals(0, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_BEYOND_MAXLOOKBACK_INVALID_INDEX_ROW_COUNT).getValue());
+ }
+ }
+
+ @Test
+ public void testPointDeleteRebuildWithPageSize() throws Exception {
+ String schemaName = generateUniqueName();
+ String dataTableName = generateUniqueName();
+ String fullDataTableName = SchemaUtil.getTableName(schemaName, dataTableName);
+ String indexTableName = generateUniqueName();
+ try (Connection conn = DriverManager.getConnection(getUrl())) {
+ conn.createStatement().execute(
+ "CREATE TABLE " + fullDataTableName + "(k VARCHAR PRIMARY KEY, v VARCHAR)");
+ conn.createStatement().execute("DELETE FROM " + fullDataTableName + " WHERE k = 'a'");
+ conn.createStatement().execute("DELETE FROM " + fullDataTableName + " WHERE k = 'b'");
+ conn.createStatement().execute("DELETE FROM " + fullDataTableName + " WHERE k = 'c'");
+ conn.commit();
+ conn.createStatement().execute(String.format("CREATE INDEX %s ON %s (v) ASYNC",
+ indexTableName, fullDataTableName));
+ // Run the index MR job and verify that the index table is built correctly
+ Configuration conf = new Configuration(getUtility().getConfiguration());
+ conf.set(QueryServices.INDEX_REBUILD_PAGE_SIZE_IN_ROWS, Long.toString(1));
+ IndexTool indexTool =
+ IndexToolIT.runIndexTool(directApi, useSnapshot, schemaName,
+ dataTableName, indexTableName, null, 0, IndexTool.IndexVerifyType.BEFORE,
+ new String[0]);
+ assertEquals(3, indexTool.getJob().getCounters().findCounter(INPUT_RECORDS).getValue());
+ assertEquals(3,
+ indexTool.getJob().getCounters().findCounter(SCANNED_DATA_ROW_COUNT).getValue());
+ assertEquals(0,
+ indexTool.getJob().getCounters().findCounter(REBUILT_INDEX_ROW_COUNT).getValue());
+ assertEquals(0, indexTool.getJob().getCounters()
+ .findCounter(BEFORE_REBUILD_VALID_INDEX_ROW_COUNT).getValue());
+ assertEquals(0, indexTool.getJob().getCounters()
+ .findCounter(BEFORE_REBUILD_EXPIRED_INDEX_ROW_COUNT).getValue());
+ assertEquals(0, indexTool.getJob().getCounters()
+ .findCounter(BEFORE_REBUILD_INVALID_INDEX_ROW_COUNT).getValue());
+ assertEquals(0, indexTool.getJob().getCounters()
+ .findCounter(BEFORE_REBUILD_MISSING_INDEX_ROW_COUNT).getValue());
+ assertEquals(0,
+ indexTool.getJob().getCounters()
+ .findCounter(BEFORE_REBUILD_BEYOND_MAXLOOKBACK_MISSING_INDEX_ROW_COUNT)
+ .getValue());
+ assertEquals(0,
+ indexTool.getJob().getCounters()
+ .findCounter(BEFORE_REBUILD_BEYOND_MAXLOOKBACK_INVALID_INDEX_ROW_COUNT)
+ .getValue());
+ }
+ }
+
+ @Test
+ public void testIncrementalRebuildWithPageSize() throws Exception {
+ Assume.assumeTrue(HbaseCompatCapabilities.isRawFilterSupported());
+ String schemaName = generateUniqueName();
+ String dataTableName = generateUniqueName();
+ String fullDataTableName = SchemaUtil.getTableName(schemaName, dataTableName);
+ String indexTableName = generateUniqueName();
+ try (Connection conn = DriverManager.getConnection(getUrl())) {
+ long minTs = EnvironmentEdgeManager.currentTimeMillis();
+ conn.createStatement().execute(
+ "CREATE TABLE " + fullDataTableName + "(k VARCHAR PRIMARY KEY, v VARCHAR)");
+ conn.createStatement().execute("UPSERT INTO " + fullDataTableName + " VALUES('a','aa')");
+ conn.createStatement().execute("UPSERT INTO " + fullDataTableName + " VALUES('b','bb')");
+ conn.commit();
+ conn.createStatement().execute("DELETE FROM " + fullDataTableName + " WHERE k = 'a'");
+ conn.createStatement().execute("DELETE FROM " + fullDataTableName + " WHERE k = 'b'");
+ conn.commit();
+ conn.createStatement().execute("UPSERT INTO " + fullDataTableName + " VALUES('a','aaa')");
+ conn.createStatement().execute("UPSERT INTO " + fullDataTableName + " VALUES('b','bbb')");
+ conn.createStatement().execute("DELETE FROM " + fullDataTableName + " WHERE k = 'c'");
+ conn.commit();
+ conn.createStatement().execute(String.format("CREATE INDEX %s ON %s (v) ASYNC",
+ indexTableName, fullDataTableName));
+ // Run the index MR job and verify that the index table is built correctly
+ Configuration conf = new Configuration(getUtility().getConfiguration());
+ conf.set(QueryServices.INDEX_REBUILD_PAGE_SIZE_IN_ROWS, Long.toString(2));
+ IndexTool indexTool =
+ IndexToolIT.runIndexTool(directApi, useSnapshot, schemaName,
+ dataTableName, indexTableName, null, 0, IndexTool.IndexVerifyType.AFTER,
+ "-st", String.valueOf(minTs), "-et",
+ String.valueOf(EnvironmentEdgeManager.currentTimeMillis()));
+ assertEquals(3, indexTool.getJob().getCounters().findCounter(INPUT_RECORDS).getValue());
+ assertEquals(3,
+ indexTool.getJob().getCounters().findCounter(SCANNED_DATA_ROW_COUNT).getValue());
+ assertEquals(4,
+ indexTool.getJob().getCounters().findCounter(REBUILT_INDEX_ROW_COUNT).getValue());
+ assertEquals(4, indexTool.getJob().getCounters()
+ .findCounter(AFTER_REBUILD_VALID_INDEX_ROW_COUNT).getValue());
+ assertEquals(0, indexTool.getJob().getCounters()
+ .findCounter(AFTER_REBUILD_EXPIRED_INDEX_ROW_COUNT).getValue());
+ assertEquals(0, indexTool.getJob().getCounters()
+ .findCounter(AFTER_REBUILD_INVALID_INDEX_ROW_COUNT).getValue());
+ assertEquals(0, indexTool.getJob().getCounters()
+ .findCounter(AFTER_REBUILD_MISSING_INDEX_ROW_COUNT).getValue());
+ assertEquals(0,
+ indexTool.getJob().getCounters()
+ .findCounter(AFTER_REBUILD_BEYOND_MAXLOOKBACK_MISSING_INDEX_ROW_COUNT)
+ .getValue());
+ assertEquals(0,
+ indexTool.getJob().getCounters()
+ .findCounter(AFTER_REBUILD_BEYOND_MAXLOOKBACK_INVALID_INDEX_ROW_COUNT)
+ .getValue());
+ }
+ }
+
@Test
- public void testDisableOutputLogging() throws Exception {
- if (!mutable || useSnapshot) {
+ public void testUpdatablePKFilterViewIndexRebuild() throws Exception {
+ Assume.assumeTrue(HbaseCompatCapabilities.isRawFilterSupported());
+ if (!mutable) {
return;
}
-
String schemaName = generateUniqueName();
String dataTableName = generateUniqueName();
String dataTableFullName = SchemaUtil.getTableName(schemaName, dataTableName);
+ String view1Name = generateUniqueName();
+ String view1FullName = SchemaUtil.getTableName(schemaName, view1Name);
+ String view2Name = generateUniqueName();
+ String view2FullName = SchemaUtil.getTableName(schemaName, view2Name);
String indexTableName = generateUniqueName();
String indexTableFullName = SchemaUtil.getTableName(schemaName, indexTableName);
Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
- try(Connection conn = DriverManager.getConnection(getUrl(), props)) {
- deleteAllRows(conn,
- TableName.valueOf(IndexVerificationOutputRepository.OUTPUT_TABLE_NAME));
- String stmString1 =
- "CREATE TABLE " + dataTableFullName
- + " (ID INTEGER NOT NULL PRIMARY KEY, NAME VARCHAR, ZIP INTEGER) "
- + tableDDLOptions;
- conn.createStatement().execute(stmString1);
- String upsertQuery = String.format("UPSERT INTO %s VALUES(?, ?, ?)", dataTableFullName);
- PreparedStatement stmt1 = conn.prepareStatement(upsertQuery);
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ // Create Table and Views. Note the view is on a non leading PK data table column
+ String createTable =
+ "CREATE TABLE IF NOT EXISTS " + dataTableFullName + " (\n"
+ + " ORGANIZATION_ID VARCHAR NOT NULL,\n"
+ + " KEY_PREFIX CHAR(3) NOT NULL,\n" + " CREATED_BY VARCHAR,\n"
+ + " CONSTRAINT PK PRIMARY KEY (\n" + " ORGANIZATION_ID,\n"
+ + " KEY_PREFIX\n" + " )\n"
+ + ") VERSIONS=1, COLUMN_ENCODED_BYTES=0";
+ conn.createStatement().execute(createTable);
+ String createView1 =
+ "CREATE VIEW IF NOT EXISTS " + view1FullName + " (\n"
+ + " VIEW_COLA VARCHAR NOT NULL,\n"
+ + " VIEW_COLB CHAR(1) CONSTRAINT PKVIEW PRIMARY KEY (\n"
+ + " VIEW_COLA\n" + " )) AS \n" + " SELECT * FROM " + dataTableFullName
+ + " WHERE KEY_PREFIX = 'aaa'";
+ conn.createStatement().execute(createView1);
+ String createView2 =
+ "CREATE VIEW IF NOT EXISTS " + view2FullName + " (\n"
+ + " VIEW_COL1 VARCHAR NOT NULL,\n"
+ + " VIEW_COL2 CHAR(1) CONSTRAINT PKVIEW PRIMARY KEY (\n"
+ + " VIEW_COL1\n" + " )) AS \n" + " SELECT * FROM " + dataTableFullName
+ + " WHERE KEY_PREFIX = 'ccc'";
+ conn.createStatement().execute(createView2);
- // insert two rows
- IndexToolIT.upsertRow(stmt1, 1);
- IndexToolIT.upsertRow(stmt1, 2);
+ // We want to verify if deletes and set null result in expected rebuild of view index
+ conn.createStatement().execute("UPSERT INTO " + view1FullName
+ + "(ORGANIZATION_ID, VIEW_COLA, VIEW_COLB) VALUES('ORG1', 'A', 'G')");
+ conn.createStatement().execute("UPSERT INTO " + view1FullName
+ + "(ORGANIZATION_ID, VIEW_COLA, VIEW_COLB) VALUES('ORG1', 'C', 'I')");
+ conn.createStatement().execute("UPSERT INTO " + view1FullName
+ + "(ORGANIZATION_ID, VIEW_COLA, VIEW_COLB) VALUES('ORG1', 'D', 'J')");
+
+ conn.createStatement().execute("UPSERT INTO " + view2FullName
+ + "(ORGANIZATION_ID, VIEW_COL1, VIEW_COL2) VALUES('ORG2', 'B', 'H')");
+ conn.commit();
+ conn.createStatement().execute("DELETE FROM " + view1FullName
+ + " WHERE ORGANIZATION_ID = 'ORG1' AND VIEW_COLA = 'C'");
+ conn.createStatement().execute("UPSERT INTO " + view1FullName
+ + "(ORGANIZATION_ID, VIEW_COLA, VIEW_COLB) VALUES('ORG1', 'D', NULL)");
conn.commit();
- //create ASYNC
- String stmtString2 =
- String.format(
- "CREATE INDEX %s ON %s (LPAD(UPPER(NAME, 'en_US'),8,'x')||'_xyz') ASYNC ",
- indexTableName, dataTableFullName);
- conn.createStatement().execute(stmtString2);
+ String createViewIndex =
+ "CREATE INDEX IF NOT EXISTS " + indexTableName + " ON " + view1FullName
+ + " (VIEW_COLB) ASYNC";
+ conn.createStatement().execute(createViewIndex);
conn.commit();
+ // Rebuild using index tool
+ IndexToolIT.runIndexTool(directApi, useSnapshot, schemaName, view1Name, indexTableName);
+ ResultSet rs =
+ conn.createStatement()
+ .executeQuery("SELECT COUNT(*) FROM " + indexTableFullName);
+ rs.next();
+ assertEquals(2, rs.getInt(1));
- // run the index MR job as ONLY so the index doesn't get rebuilt. Should be 2 missing
- //rows. We pass in --disable-logging BEFORE to silence the output logging to
- // PHOENIX_INDEX_TOOL, since ONLY logs BEFORE the (non-existent in this case)
- // rebuild
- assertDisableLogging(conn, 0, IndexTool.IndexVerifyType.ONLY,
- IndexTool.IndexDisableLoggingType.BEFORE, null, schemaName, dataTableName, indexTableName,
- indexTableFullName, 0);
+ Pair<Integer, Integer> putsAndDeletes =
+ countPutsAndDeletes("_IDX_" + dataTableFullName);
+ assertEquals(4, (int) putsAndDeletes.getFirst());
+ assertEquals(2, (int) putsAndDeletes.getSecond());
+ }
+ }
- // disabling logging AFTER on an AFTER run should leave no output rows
- assertDisableLogging(conn, 0, IndexTool.IndexVerifyType.AFTER,
- IndexTool.IndexDisableLoggingType.AFTER, null, schemaName, dataTableName,
- indexTableName,
- indexTableFullName, 0);
+ @Test
+ public void testUpdatableNonPkFilterViewIndexRebuild() throws Exception {
+ Assume.assumeTrue(HbaseCompatCapabilities.isRawFilterSupported());
+ if (!mutable) {
+ return;
+ }
+ String schemaName = generateUniqueName();
+ String dataTableName = generateUniqueName();
+ String dataTableFullName = SchemaUtil.getTableName(schemaName, dataTableName);
+ String view1Name = generateUniqueName();
+ String view1FullName = SchemaUtil.getTableName(schemaName, view1Name);
+ String view2Name = generateUniqueName();
+ String view2FullName = SchemaUtil.getTableName(schemaName, view2Name);
+ String indexTableName = generateUniqueName();
+ String indexTableFullName = SchemaUtil.getTableName(schemaName, indexTableName);
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
- //disabling logging BEFORE on a BEFORE run should leave no output rows
- assertDisableLogging(conn, 0, IndexTool.IndexVerifyType.BEFORE,
- IndexTool.IndexDisableLoggingType.BEFORE, null, schemaName, dataTableName, indexTableName,
- indexTableFullName, 0);
- //now clear out all the rebuilt index rows
- deleteAllRows(conn, TableName.valueOf(indexTableFullName));
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ // Create Table and Views. Note the view is on a non PK data table column
+ String createTable =
+ "CREATE TABLE IF NOT EXISTS " + dataTableFullName + " (\n"
+ + " ORGANIZATION_ID VARCHAR NOT NULL,\n"
+ + " KEY_PREFIX CHAR(3) NOT NULL,\n" + " CREATED_BY VARCHAR,\n"
+ + " CONSTRAINT PK PRIMARY KEY (\n" + " ORGANIZATION_ID,\n"
+ + " KEY_PREFIX\n" + " )\n"
+ + ") VERSIONS=1, COLUMN_ENCODED_BYTES=0";
+ conn.createStatement().execute(createTable);
+ String createView1 =
+ "CREATE VIEW IF NOT EXISTS " + view1FullName + " (\n"
+ + " VIEW_COLA VARCHAR NOT NULL,\n"
+ + " VIEW_COLB CHAR(1) CONSTRAINT PKVIEW PRIMARY KEY (\n"
+ + " VIEW_COLA\n" + " )) AS \n" + " SELECT * FROM " + dataTableFullName
+ + " WHERE CREATED_BY = 'foo'";
+ conn.createStatement().execute(createView1);
+ String createView2 =
+ "CREATE VIEW IF NOT EXISTS " + view2FullName + " (\n"
+ + " VIEW_COL1 VARCHAR NOT NULL,\n"
+ + " VIEW_COL2 CHAR(1) CONSTRAINT PKVIEW PRIMARY KEY (\n"
+ + " VIEW_COL1\n" + " )) AS \n" + " SELECT * FROM " + dataTableFullName
+ + " WHERE CREATED_BY = 'bar'";
+ conn.createStatement().execute(createView2);
- //now check that disabling logging AFTER leaves only the BEFORE logs on a BOTH run
- assertDisableLogging(conn, 2, IndexTool.IndexVerifyType.BOTH,
- IndexTool.IndexDisableLoggingType.AFTER,
- IndexVerificationOutputRepository.PHASE_BEFORE_VALUE, schemaName,
- dataTableName, indexTableName,
- indexTableFullName, 0);
+ // We want to verify if deletes and set null result in expected rebuild of view index
+ conn.createStatement().execute("UPSERT INTO " + view1FullName
+ + "(ORGANIZATION_ID, KEY_PREFIX, VIEW_COLA, VIEW_COLB) VALUES('ORG1', 'aaa', 'A', 'G')");
+ conn.createStatement().execute("UPSERT INTO " + view1FullName
+ + "(ORGANIZATION_ID, KEY_PREFIX, VIEW_COLA, VIEW_COLB) VALUES('ORG1', 'ccc', 'C', 'I')");
+ conn.createStatement().execute("UPSERT INTO " + view1FullName
+ + "(ORGANIZATION_ID, KEY_PREFIX, VIEW_COLA, VIEW_COLB) VALUES('ORG1', 'ddd', 'D', 'J')");
- //clear out both the output table and the index
- deleteAllRows(conn,
- TableName.valueOf(IndexVerificationOutputRepository.OUTPUT_TABLE_NAME));
- deleteAllRows(conn, TableName.valueOf(indexTableFullName));
+ conn.createStatement().execute("UPSERT INTO " + view2FullName
+ + "(ORGANIZATION_ID, KEY_PREFIX, VIEW_COL1, VIEW_COL2) VALUES('ORG2', 'bbb', 'B', 'H')");
+ conn.commit();
+ conn.createStatement().execute("DELETE FROM " + view1FullName
+ + " WHERE ORGANIZATION_ID = 'ORG1' AND VIEW_COLA = 'C'");
+ conn.createStatement().execute("UPSERT INTO " + view1FullName
+ + "(ORGANIZATION_ID, KEY_PREFIX, VIEW_COLA, VIEW_COLB) VALUES('ORG1', 'ddd', 'D', NULL)");
+ conn.commit();
- //now check that disabling logging BEFORE creates only the AFTER logs on a BOTH run
- assertDisableLogging(conn, 0, IndexTool.IndexVerifyType.BOTH,
- IndexTool.IndexDisableLoggingType.BEFORE,
- IndexVerificationOutputRepository.PHASE_AFTER_VALUE, schemaName,
- dataTableName, indexTableName,
- indexTableFullName, 0);
+ String createViewIndex =
+ "CREATE INDEX IF NOT EXISTS " + indexTableName + " ON " + view1FullName
+ + " (VIEW_COLB) ASYNC";
+ conn.createStatement().execute(createViewIndex);
+ conn.commit();
+ // Rebuild using index tool
+ IndexToolIT.runIndexTool(directApi, useSnapshot, schemaName, view1Name, indexTableName);
+ ResultSet rs =
+ conn.createStatement()
+ .executeQuery("SELECT COUNT(*) FROM " + indexTableFullName);
+ rs.next();
+ assertEquals(2, rs.getInt(1));
- deleteAllRows(conn, TableName.valueOf(IndexVerificationOutputRepository.OUTPUT_TABLE_NAME));
- deleteAllRows(conn, TableName.valueOf(indexTableFullName));
+ Pair<Integer, Integer> putsAndDeletes =
+ countPutsAndDeletes("_IDX_" + dataTableFullName);
+ assertEquals(4, (int) putsAndDeletes.getFirst());
+ assertEquals(2, (int) putsAndDeletes.getSecond());
+ }
+ }
- //now check that disabling logging BOTH creates no logs on a BOTH run
- assertDisableLogging(conn, 0, IndexTool.IndexVerifyType.BOTH,
- IndexTool.IndexDisableLoggingType.BOTH,
- IndexVerificationOutputRepository.PHASE_BEFORE_VALUE, schemaName,
- dataTableName, indexTableName,
- indexTableFullName, 0);
+ private Pair<Integer, Integer> countPutsAndDeletes(String tableName) throws Exception {
+ int numPuts = 0;
+ int numDeletes = 0;
+ try (org.apache.hadoop.hbase.client.Connection hcon =
+ ConnectionFactory.createConnection(config)) {
+ Table htable = hcon.getTable(TableName.valueOf(tableName));
+ Scan scan = new Scan();
+ scan.setRaw(true);
+ ResultScanner scanner = htable.getScanner(scan);
+ for (Result result = scanner.next(); result != null; result = scanner.next()) {
+ for (Cell cell : result.rawCells()) {
+ if (cell.getType().equals(Cell.Type.Put)) {
+ numPuts++;
+ } else if (cell.getType().equals(Cell.Type.DeleteFamily)) {
+ numDeletes++;
+ }
+ }
+ }
}
+ return new Pair<Integer, Integer>(numPuts, numDeletes);
}
public void deleteAllRows(Connection conn, TableName tableName) throws SQLException,
IOException, InterruptedException {
Scan scan = new Scan();
- Table table = conn.unwrap(PhoenixConnection.class).getQueryServices().
- getTable(tableName.getName());
+ Admin admin = conn.unwrap(PhoenixConnection.class).getQueryServices().
+ getAdmin();
+ org.apache.hadoop.hbase.client.Connection hbaseConn = admin.getConnection();
+ Table table = hbaseConn.getTable(tableName);
boolean deletedRows = false;
try (ResultScanner scanner = table.getScanner(scan)) {
for (Result r : scanner) {
@@ -1276,11 +1452,24 @@ public class IndexToolForNonTxGlobalIndexIT extends BaseUniqueNamesOwnClusterIT
}
//don't flush/compact if we didn't write anything, because we'll hang forever
if (deletedRows) {
- getUtility().getHBaseAdmin().flush(tableName);
+ getUtility().getAdmin().flush(tableName);
TestUtil.majorCompact(getUtility(), tableName);
}
}
+ private void truncateIndexAndIndexToolTables(String indexTableFullName) throws IOException {
+ truncateIndexToolTables();
+ getUtility().getAdmin().disableTable(TableName.valueOf(indexTableFullName));
+ getUtility().getAdmin().truncateTable(TableName.valueOf(indexTableFullName), true);
+ }
+
+ private void truncateIndexToolTables() throws IOException {
+ getUtility().getAdmin().disableTable(TableName.valueOf(IndexVerificationOutputRepository.OUTPUT_TABLE_NAME));
+ getUtility().getAdmin().truncateTable(TableName.valueOf(IndexVerificationOutputRepository.OUTPUT_TABLE_NAME), true);
+ getUtility().getAdmin().disableTable(TableName.valueOf(RESULT_TABLE_NAME));
+ getUtility().getAdmin().truncateTable(TableName.valueOf(RESULT_TABLE_NAME), true);
+ }
+
private void assertDisableLogging(Connection conn, int expectedRows,
IndexTool.IndexVerifyType verifyType,
IndexTool.IndexDisableLoggingType disableLoggingType,
@@ -1331,7 +1520,7 @@ public class IndexToolForNonTxGlobalIndexIT extends BaseUniqueNamesOwnClusterIT
ResultScanner rs = hIndexToolTable.getScanner(s);
int count =0;
for(Result r : rs) {
- Assert.assertTrue(r != null);
+ assertNotNull(r);
List<Cell> cells = r.getColumnCells(RESULT_TABLE_COLUMN_FAMILY, INDEX_TOOL_RUN_STATUS_BYTES);
Assert.assertEquals(cells.size(), 1);
Assert.assertTrue(Bytes.toString(CellUtil.cloneRow(cells.get(0))).startsWith(String.valueOf(scn)));
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolTimeRangeIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolTimeRangeIT.java
index 04a3b83..64efea7 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolTimeRangeIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolTimeRangeIT.java
@@ -24,6 +24,7 @@ import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.Map;
+import org.apache.phoenix.compat.hbase.HbaseCompatCapabilities;
import org.apache.phoenix.mapreduce.index.IndexTool;
import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.query.QueryServicesOptions;
@@ -33,6 +34,7 @@ import org.apache.phoenix.util.ReadOnlyProps;
import org.apache.phoenix.util.SchemaUtil;
import org.junit.AfterClass;
import org.junit.Assert;
+import org.junit.Assume;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Ignore;
@@ -132,6 +134,7 @@ public class IndexToolTimeRangeIT extends BaseUniqueNamesOwnClusterIT {
@Test
public void testValidTimeRange() throws Exception {
+ Assume.assumeTrue(HbaseCompatCapabilities.isRawFilterSupported());
String [] args = {"--delete-all-and-rebuild",
"--start-time", myClock.getRelativeTimeAsString(1),
"--end-time", myClock.getRelativeTimeAsString(9)};
@@ -142,6 +145,7 @@ public class IndexToolTimeRangeIT extends BaseUniqueNamesOwnClusterIT {
@Test
public void testValidTimeRange_startTimeInBetween() throws Exception {
+ Assume.assumeTrue(HbaseCompatCapabilities.isRawFilterSupported());
String [] args = {"--delete-all-and-rebuild",
"--start-time", myClock.getRelativeTimeAsString(6),
"--end-time", myClock.getRelativeTimeAsString(9)};
@@ -152,6 +156,7 @@ public class IndexToolTimeRangeIT extends BaseUniqueNamesOwnClusterIT {
@Test
public void testValidTimeRange_endTimeInBetween() throws Exception {
+ Assume.assumeTrue(HbaseCompatCapabilities.isRawFilterSupported());
String [] args = {"--delete-all-and-rebuild",
"--start-time", myClock.getRelativeTimeAsString(1),
"--end-time", myClock.getRelativeTimeAsString(6)};
@@ -170,6 +175,7 @@ public class IndexToolTimeRangeIT extends BaseUniqueNamesOwnClusterIT {
@Test
public void testValidTimeRange_onlyStartTimePassed() throws Exception {
+ Assume.assumeTrue(HbaseCompatCapabilities.isRawFilterSupported());
//starttime passed of last upsert
String [] args = {"--delete-all-and-rebuild",
"--start-time", myClock.getRelativeTimeAsString(8)};
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/MaxLookbackIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/MaxLookbackIT.java
index a1a5db0..6c9e311 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/MaxLookbackIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/MaxLookbackIT.java
@@ -35,6 +35,7 @@ import org.apache.phoenix.util.ReadOnlyProps;
import org.apache.phoenix.util.TestUtil;
import org.junit.After;
import org.junit.Assert;
+import org.junit.Assume;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -66,14 +67,12 @@ public class MaxLookbackIT extends BaseUniqueNamesOwnClusterIT {
private int ttl;
//max lookback isn't supported in HBase 2.1 and 2.2 because of missing coprocessor
// interfaces; see HBASE-24321
- private static boolean isMaxLookbackSupported =
+ private final static boolean isMaxLookbackSupported =
HbaseCompatCapabilities.isMaxLookbackTimeSupported();
@BeforeClass
public static synchronized void doSetup() throws Exception {
- if (!isMaxLookbackSupported) {
- return;
- }
+ Assume.assumeTrue(isMaxLookbackSupported);
Map<String, String> props = Maps.newHashMapWithExpectedSize(1);
props.put(QueryServices.GLOBAL_INDEX_ROW_AGE_THRESHOLD_TO_DELETE_MS_ATTRIB, Long.toString(0));
props.put(CompatBaseScannerRegionObserver.PHOENIX_MAX_LOOKBACK_AGE_CONF_KEY, Integer.toString(MAX_LOOKBACK_AGE));
@@ -82,9 +81,7 @@ public class MaxLookbackIT extends BaseUniqueNamesOwnClusterIT {
@Before
public void beforeTest(){
- if (!isMaxLookbackSupported) {
- return;
- }
+ Assume.assumeTrue(isMaxLookbackSupported);
EnvironmentEdgeManager.reset();
optionBuilder = new StringBuilder();
this.tableDDLOptions = optionBuilder.toString();
@@ -103,9 +100,7 @@ public class MaxLookbackIT extends BaseUniqueNamesOwnClusterIT {
@Test
public void testTooLowSCNWithMaxLookbackAge() throws Exception {
- if (!isMaxLookbackSupported) {
- return;
- }
+ Assume.assumeTrue(isMaxLookbackSupported);
String dataTableName = generateUniqueName();
createTable(dataTableName);
injectEdge.setValue(System.currentTimeMillis());
@@ -131,9 +126,7 @@ public class MaxLookbackIT extends BaseUniqueNamesOwnClusterIT {
@Test(timeout=120000L)
public void testRecentlyDeletedRowsNotCompactedAway() throws Exception {
- if (!isMaxLookbackSupported) {
- return;
- }
+ Assume.assumeTrue(isMaxLookbackSupported);
try (Connection conn = DriverManager.getConnection(getUrl())) {
String dataTableName = generateUniqueName();
String indexName = generateUniqueName();
@@ -201,9 +194,7 @@ public class MaxLookbackIT extends BaseUniqueNamesOwnClusterIT {
@Test(timeout=60000L)
public void testTTLAndMaxLookbackAge() throws Exception {
- if (!isMaxLookbackSupported) {
- return;
- }
+ Assume.assumeTrue(isMaxLookbackSupported);
ttl = 20;
optionBuilder.append("TTL=" + ttl);
tableDDLOptions = optionBuilder.toString();
@@ -276,9 +267,7 @@ public class MaxLookbackIT extends BaseUniqueNamesOwnClusterIT {
@Test(timeout=60000)
public void testRecentMaxVersionsNotCompactedAway() throws Exception {
- if (!isMaxLookbackSupported) {
- return;
- }
+ Assume.assumeTrue(isMaxLookbackSupported);
int versions = 2;
optionBuilder.append("VERSIONS=" + versions);
tableDDLOptions = optionBuilder.toString();
@@ -355,7 +344,7 @@ public class MaxLookbackIT extends BaseUniqueNamesOwnClusterIT {
}
private void flush(TableName table) throws IOException {
- Admin admin = getUtility().getHBaseAdmin();
+ Admin admin = getUtility().getAdmin();
admin.flush(table);
}
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/NonParameterizedIndexScrutinyToolIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/NonParameterizedIndexScrutinyToolIT.java
index 8b36518..a90e042 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/NonParameterizedIndexScrutinyToolIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/NonParameterizedIndexScrutinyToolIT.java
@@ -227,7 +227,7 @@ public class NonParameterizedIndexScrutinyToolIT extends IndexScrutinyToolBaseIT
}
}
- private void upsertRow(PreparedStatement stmt, int id, String name, byte[] val) throws
+ public static void upsertRow(PreparedStatement stmt, int id, String name, byte[] val) throws
SQLException {
int index = 1;
// insert row
@@ -237,13 +237,13 @@ public class NonParameterizedIndexScrutinyToolIT extends IndexScrutinyToolBaseIT
stmt.executeUpdate();
}
- private void upsertRow(PreparedStatement stmt, int id, String name, int zip)
+ public static void upsertRow(PreparedStatement stmt, int id, String name, int val)
throws SQLException {
int index = 1;
// insert row
stmt.setInt(index++, id);
stmt.setString(index++, name);
- stmt.setInt(index++, zip);
+ stmt.setInt(index++, val);
stmt.executeUpdate();
}
@@ -258,6 +258,6 @@ public class NonParameterizedIndexScrutinyToolIT extends IndexScrutinyToolBaseIT
cmdArgs =
getArgValues(schemaName, dataTableName, indexTableName, batchSize, sourceTable,
false, null, null, null, Long.MAX_VALUE);
- return runScrutiny(cmdArgs);
+ return runScrutiny(IndexScrutinyMapperForTest.class, cmdArgs);
}
}
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/DropColumnIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/DropColumnIT.java
index ac4896c..8c520cd 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/DropColumnIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/DropColumnIT.java
@@ -34,7 +34,7 @@ import java.util.Arrays;
import java.util.Collection;
import java.util.Properties;
-import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
@@ -50,7 +50,6 @@ import org.apache.phoenix.query.QueryConstants;
import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.schema.ColumnNotFoundException;
import org.apache.phoenix.schema.PColumn;
-import org.apache.phoenix.schema.PName;
import org.apache.phoenix.schema.PTable;
import org.apache.phoenix.schema.PTableImpl;
import org.apache.phoenix.schema.PTableKey;
@@ -203,7 +202,11 @@ public class DropColumnIT extends ParallelStatsDisabledIT {
Result result = results.next();
assertNotNull(result);
- assertEquals("data table column value should have been deleted", KeyValue.Type.DeleteColumn.getCode(), result.getColumnCells(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, dataCq).get(0).getTypeByte());
+ assertEquals("data table column value should have been deleted",
+ Cell.Type.DeleteColumn,
+ result.getColumnCells(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, dataCq).get
+ (0).getType());
+
assertNull(results.next());
// key value for v2 should have been deleted from the global index table
@@ -213,7 +216,10 @@ public class DropColumnIT extends ParallelStatsDisabledIT {
results = table.getScanner(scan);
result = results.next();
assertNotNull(result);
- assertEquals("data table column value should have been deleted", KeyValue.Type.DeleteColumn.getCode(), result.getColumnCells(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, globalIndexCq).get(0).getTypeByte());
+ assertEquals("data table column value should have been deleted",
+ Cell.Type.DeleteColumn,
+ result.getColumnCells(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES,
+ globalIndexCq).get(0).getType());
assertNull(results.next());
// key value for v2 should have been deleted from the local index table
@@ -224,8 +230,10 @@ public class DropColumnIT extends ParallelStatsDisabledIT {
results = table.getScanner(scan);
result = results.next();
assertNotNull(result);
- assertEquals("data table col"
- + "umn value should have been deleted", KeyValue.Type.DeleteColumn.getCode(), result.getColumnCells(QueryConstants.DEFAULT_LOCAL_INDEX_COLUMN_FAMILY_BYTES, localIndexCq).get(0).getTypeByte());
+ assertEquals("data table column value should have been deleted",
+ Cell.Type.DeleteColumn,
+ result.getColumnCells(QueryConstants.DEFAULT_LOCAL_INDEX_COLUMN_FAMILY_BYTES,
+ localIndexCq).get(0).getType());
assertNull(results.next());
}
else {
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/GlobalIndexCheckerIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/GlobalIndexCheckerIT.java
index 1453471..beda14b 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/GlobalIndexCheckerIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/GlobalIndexCheckerIT.java
@@ -18,6 +18,8 @@
package org.apache.phoenix.end2end.index;
import static org.apache.phoenix.mapreduce.PhoenixJobCounters.INPUT_RECORDS;
+import static org.apache.phoenix.mapreduce.index.PhoenixIndexToolJobCounters.BEFORE_REBUILD_BEYOND_MAXLOOKBACK_INVALID_INDEX_ROW_COUNT;
+import static org.apache.phoenix.mapreduce.index.PhoenixIndexToolJobCounters.BEFORE_REBUILD_BEYOND_MAXLOOKBACK_MISSING_INDEX_ROW_COUNT;
import static org.apache.phoenix.mapreduce.index.PhoenixIndexToolJobCounters.BEFORE_REBUILD_EXPIRED_INDEX_ROW_COUNT;
import static org.apache.phoenix.mapreduce.index.PhoenixIndexToolJobCounters.BEFORE_REBUILD_INVALID_INDEX_ROW_COUNT;
import static org.apache.phoenix.mapreduce.index.PhoenixIndexToolJobCounters.BEFORE_REBUILD_MISSING_INDEX_ROW_COUNT;
@@ -54,31 +56,26 @@ import org.apache.phoenix.util.QueryUtil;
import org.apache.phoenix.util.ReadOnlyProps;
import org.apache.phoenix.util.TestUtil;
import org.junit.After;
+import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameters;
+import com.google.common.collect.Lists;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.collect.Lists;
-
@RunWith(Parameterized.class)
public class GlobalIndexCheckerIT extends BaseUniqueNamesOwnClusterIT {
- private static final Logger LOG = LoggerFactory.getLogger(GlobalIndexCheckerIT.class);
+ private static final Logger LOGGER = LoggerFactory.getLogger(GlobalIndexCheckerIT.class);
private final boolean async;
+ private String tableDDLOptions;
+ private StringBuilder optionBuilder;
private final boolean encoded;
- private final String tableDDLOptions;
-
public GlobalIndexCheckerIT(boolean async, boolean encoded) {
this.async = async;
this.encoded = encoded;
- StringBuilder optionBuilder = new StringBuilder();
- if (!encoded) {
- optionBuilder.append(" COLUMN_ENCODED_BYTES=0 ");
- }
- this.tableDDLOptions = optionBuilder.toString();
}
@BeforeClass
@@ -88,6 +85,15 @@ public class GlobalIndexCheckerIT extends BaseUniqueNamesOwnClusterIT {
setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator()));
}
+ @Before
+ public void beforeTest(){
+ optionBuilder = new StringBuilder();
+ if (!encoded) {
+ optionBuilder.append(" COLUMN_ENCODED_BYTES=0");
+ }
+ this.tableDDLOptions = optionBuilder.toString();
+ }
+
@Parameters(
name = "async={0},encoded={1}")
public static synchronized Collection<Object[]> data() {
@@ -164,6 +170,7 @@ public class GlobalIndexCheckerIT extends BaseUniqueNamesOwnClusterIT {
String indexTableName = generateUniqueName();
conn.createStatement().execute("CREATE INDEX " + indexTableName + " on " +
dataTableName + " (val1) include (val2, val3)");
+
String dml = "DELETE from " + dataTableName + " WHERE id = 'a'";
conn.createStatement().executeUpdate(dml);
conn.commit();
@@ -218,6 +225,61 @@ public class GlobalIndexCheckerIT extends BaseUniqueNamesOwnClusterIT {
}
@Test
+ public void testLimitWithUnverifiedRows() throws Exception {
+ if (async) {
+ return;
+ }
+ String dataTableName = generateUniqueName();
+ populateTable(dataTableName); // with two rows ('a', 'ab', 'abc', 'abcd') and ('b', 'bc', 'bcd', 'bcde')
+ try (Connection conn = DriverManager.getConnection(getUrl())) {
+ String indexTableName = generateUniqueName();
+ conn.createStatement().execute("CREATE INDEX " + indexTableName + " on " +
+ dataTableName + " (val1) include (val2, val3)");
+ conn.commit();
+ // Read all index rows and rewrite them back directly. This will overwrite existing rows with newer
+ // timestamps and set the empty column to value "x". This will make them unverified
+ conn.createStatement().execute("UPSERT INTO " + indexTableName + " SELECT * FROM " +
+ indexTableName);
+ conn.commit();
+ // Verified that read repair will not reduce the number of rows returned for LIMIT queries
+ String selectSql = "SELECT * from " + indexTableName + " LIMIT 1";
+ ResultSet rs = conn.createStatement().executeQuery(selectSql);
+ assertTrue(rs.next());
+ assertEquals("ab", rs.getString(1));
+ assertEquals("a", rs.getString(2));
+ assertEquals("abc", rs.getString(3));
+ assertEquals("abcd", rs.getString(4));
+ assertFalse(rs.next());
+ selectSql = "SELECT val3 from " + dataTableName + " WHERE val1 = 'bc' LIMIT 1";
+ // Verify that we will read from the index table
+ assertExplainPlan(conn, selectSql, dataTableName, indexTableName);
+ rs = conn.createStatement().executeQuery(selectSql);
+ assertTrue(rs.next());
+ assertEquals("bcde", rs.getString(1));
+ assertFalse(rs.next());
+ // Configure IndexRegionObserver to fail the last write phase (i.e., the post index update phase) where the verify flag is set
+ // to true and/or index rows are deleted and check that this does not impact the correctness
+ IndexRegionObserver.setFailDataTableUpdatesForTesting(true);
+ // This is to cover the case where there is no data table row for an unverified index row
+ conn.createStatement().execute("upsert into " + dataTableName + " (id, val1, val2) values ('c', 'aa','cde')");
+ commitWithException(conn);
+ IndexRegionObserver.setFailDataTableUpdatesForTesting(false);
+ // Verified that read repair will not reduce the number of rows returned for LIMIT queries
+ selectSql = "SELECT * from " + indexTableName + " LIMIT 1";
+ rs = conn.createStatement().executeQuery(selectSql);
+ assertTrue(rs.next());
+ assertEquals("ab", rs.getString(1));
+ assertEquals("a", rs.getString(2));
+ assertEquals("abc", rs.getString(3));
+ assertEquals("abcd", rs.getString(4));
+ assertFalse(rs.next());
+ // Add rows and check everything is still okay
+ verifyTableHealth(conn, dataTableName, indexTableName);
+
+ }
+ }
+
+ @Test
public void testSimulateConcurrentUpdates() throws Exception {
try (Connection conn = DriverManager.getConnection(getUrl())) {
String dataTableName = generateUniqueName();
@@ -459,6 +521,8 @@ public class GlobalIndexCheckerIT extends BaseUniqueNamesOwnClusterIT {
assertEquals(0, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_EXPIRED_INDEX_ROW_COUNT).getValue());
assertEquals(0, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_INVALID_INDEX_ROW_COUNT).getValue());
assertEquals(0, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_MISSING_INDEX_ROW_COUNT).getValue());
+ assertEquals(0, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_BEYOND_MAXLOOKBACK_MISSING_INDEX_ROW_COUNT).getValue());
+ assertEquals(0, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_BEYOND_MAXLOOKBACK_INVALID_INDEX_ROW_COUNT).getValue());
assertEquals(2, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_UNVERIFIED_INDEX_ROW_COUNT).getValue());
assertEquals(0, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_OLD_INDEX_ROW_COUNT).getValue());
assertEquals(0, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_UNKNOWN_INDEX_ROW_COUNT).getValue());
@@ -619,17 +683,8 @@ public class GlobalIndexCheckerIT extends BaseUniqueNamesOwnClusterIT {
public void testOnePhaseOverwrite() throws Exception {
try (Connection conn = DriverManager.getConnection(getUrl())) {
String dataTableName = generateUniqueName();
- populateTable(dataTableName); // with two rows ('a', 'ab', 'abc', 'abcd') and ('b', 'bc', 'bcd', 'bcde')
String indexTableName = generateUniqueName();
- conn.createStatement().execute("CREATE INDEX " + indexTableName + "1 on " +
- dataTableName + " (val1) include (val2, val3)" + (async ? "ASYNC" : ""));
- conn.createStatement().execute("CREATE INDEX " + indexTableName + "2 on " +
- dataTableName + " (val2) include (val1, val3)" + (async ? "ASYNC" : ""));
- if (async) {
- // run the index MR job.
- IndexToolIT.runIndexTool(true, false, null, dataTableName, indexTableName + "1");
- IndexToolIT.runIndexTool(true, false, null, dataTableName, indexTableName + "2");
- }
+ createTableAndIndexes(conn, dataTableName, indexTableName);
// Configure IndexRegionObserver to skip the last two write phase (i.e., the data table update and post index
// update phase) and check that this does not impact the correctness (one overwrite)
IndexRegionObserver.setFailDataTableUpdatesForTesting(true);
@@ -699,21 +754,35 @@ public class GlobalIndexCheckerIT extends BaseUniqueNamesOwnClusterIT {
}
}
+ private void createTableAndIndexes(Connection conn, String dataTableName,
+ String indexTableName) throws Exception {
+ createTableAndIndexes(conn, dataTableName, indexTableName, 1);
+ }
+
+ private void createTableAndIndexes(Connection conn, String dataTableName,
+ String indexTableName, int indexVersions) throws Exception {
+ populateTable(dataTableName); // with two rows ('a', 'ab', 'abc', 'abcd') and ('b', 'bc', 'bcd', 'bcde')
+ conn.createStatement().execute("CREATE INDEX " + indexTableName + "1 on " +
+ dataTableName + " (val1) include (val2, val3)" + (async ? "ASYNC" : "") +
+ " VERSIONS=" + indexVersions);
+ conn.createStatement().execute("CREATE INDEX " + indexTableName + "2 on " +
+ dataTableName + " (val2) include (val1, val3)" + (async ? "ASYNC" : "")+
+ " VERSIONS=" + indexVersions);
+ conn.commit();
+ if (async) {
+ // run the index MR job.
+ IndexToolIT.runIndexTool(true, false, null, dataTableName, indexTableName + "1");
+ IndexToolIT.runIndexTool(true, false, null, dataTableName, indexTableName + "2");
+ }
+ }
+
@Test
public void testFailDataTableAndPostIndexRowUpdate() throws Exception {
- String dataTableName = generateUniqueName();
- populateTable(dataTableName); // with two rows ('a', 'ab', 'abc', 'abcd') and ('b', 'bc', 'bcd', 'bcde')
+
try (Connection conn = DriverManager.getConnection(getUrl())) {
+ String dataTableName = generateUniqueName();
String indexName = generateUniqueName();
- conn.createStatement().execute("CREATE INDEX " + indexName + "1 on " +
- dataTableName + " (val1) include (val2, val3)" + (async ? "ASYNC" : ""));
- conn.createStatement().execute("CREATE INDEX " + indexName + "2 on " +
- dataTableName + " (val2) include (val1, val3)" + (async ? "ASYNC" : ""));
- if (async) {
- // run the index MR job.
- IndexToolIT.runIndexTool(true, false, null, dataTableName, indexName + "1");
- IndexToolIT.runIndexTool(true, false, null, dataTableName, indexName + "2");
- }
+ createTableAndIndexes(conn, dataTableName, indexName);
// Configure IndexRegionObserver to fail the last two write phase (i.e., the data table update and post index update phase)
// and check that this does not impact the correctness
IndexRegionObserver.setFailDataTableUpdatesForTesting(true);
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexVerificationOutputRepositoryIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexVerificationOutputRepositoryIT.java
index 96386df..b76e408 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexVerificationOutputRepositoryIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexVerificationOutputRepositoryIT.java
@@ -19,8 +19,6 @@ package org.apache.phoenix.end2end.index;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
@@ -31,12 +29,15 @@ import org.apache.phoenix.end2end.ParallelStatsDisabledIT;
import org.apache.phoenix.jdbc.PhoenixConnection;
import org.apache.phoenix.mapreduce.index.IndexTool;
import org.apache.phoenix.mapreduce.index.IndexVerificationOutputRepository;
+import org.apache.phoenix.mapreduce.index.IndexVerificationOutputRepository.IndexVerificationErrorType;
import org.apache.phoenix.mapreduce.index.IndexVerificationOutputRow;
+import org.apache.phoenix.query.ConnectionQueryServices;
import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.util.EnvironmentEdgeManager;
import org.apache.phoenix.util.ManualEnvironmentEdge;
import org.apache.phoenix.util.ReadOnlyProps;
import org.apache.phoenix.util.TestUtil;
+import org.junit.After;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -51,6 +52,9 @@ import java.util.List;
import java.util.Map;
import static org.apache.phoenix.coprocessor.MetaDataProtocol.DEFAULT_LOG_TTL;
+import static org.apache.phoenix.mapreduce.index.IndexVerificationOutputRepository.IndexVerificationErrorType.BEYOND_MAX_LOOKBACK_INVALID;
+import static org.apache.phoenix.mapreduce.index.IndexVerificationOutputRepository.IndexVerificationErrorType.BEYOND_MAX_LOOKBACK_MISSING;
+import static org.apache.phoenix.mapreduce.index.IndexVerificationOutputRepository.IndexVerificationErrorType.INVALID_ROW;
import static org.apache.phoenix.mapreduce.index.IndexVerificationOutputRepository.OUTPUT_TABLE_NAME_BYTES;
import static org.apache.phoenix.mapreduce.index.IndexVerificationOutputRepository.PHASE_AFTER_VALUE;
import static org.apache.phoenix.mapreduce.index.IndexVerificationOutputRepository.PHASE_BEFORE_VALUE;
@@ -89,21 +93,23 @@ public class IndexVerificationOutputRepositoryIT extends ParallelStatsDisabledIT
long scanMaxTs = EnvironmentEdgeManager.currentTimeMillis();
outputRepository.logToIndexToolOutputTable(dataRowKey, indexRowKey, dataRowTs,
indexRowTs, expectedErrorMessage, expectedValue, actualValue,
- scanMaxTs, tableNameBytes, true);
+ scanMaxTs, tableNameBytes, true,
+ INVALID_ROW);
//now increment the scan time by 1 and do it again
outputRepository.logToIndexToolOutputTable(dataRowKey, indexRowKey, dataRowTs,
indexRowTs, expectedErrorMessage, expectedValue, actualValue,
- scanMaxTs +1, tableNameBytes, false);
+ scanMaxTs +1, tableNameBytes, false,
+ INVALID_ROW);
//make sure we only get the first row back
IndexVerificationOutputRow expectedRow = buildVerificationRow(dataRowKey, indexRowKey, dataRowTs,
indexRowTs, expectedErrorMessage, expectedValue, actualValue,
- scanMaxTs, tableNameBytes, indexNameBytes, PHASE_BEFORE_VALUE);
+ scanMaxTs, tableNameBytes, indexNameBytes, PHASE_BEFORE_VALUE, INVALID_ROW);
verifyOutputRow(outputRepository, scanMaxTs, indexNameBytes, expectedRow);
//make sure we get the second row back
IndexVerificationOutputRow secondExpectedRow = buildVerificationRow(dataRowKey,
indexRowKey, dataRowTs,
indexRowTs, expectedErrorMessage, expectedValue, actualValue,
- scanMaxTs + 1, tableNameBytes, indexNameBytes, PHASE_AFTER_VALUE);
+ scanMaxTs + 1, tableNameBytes, indexNameBytes, PHASE_AFTER_VALUE, INVALID_ROW);
verifyOutputRow(outputRepository, scanMaxTs+1, indexNameBytes, secondExpectedRow);
}
@@ -113,21 +119,25 @@ public class IndexVerificationOutputRepositoryIT extends ParallelStatsDisabledIT
public void testTTLOnOutputTable() throws SQLException, IOException {
String mockString = "mock_value";
byte[] mockStringBytes = Bytes.toBytes(mockString);
+
try (Connection conn = DriverManager.getConnection(getUrl())) {
+ Table hTable = conn.unwrap(PhoenixConnection.class).getQueryServices().getTable(
+ OUTPUT_TABLE_NAME_BYTES);
+
IndexVerificationOutputRepository
outputRepository =
new IndexVerificationOutputRepository(mockStringBytes, conn);
+
outputRepository.createOutputTable(conn);
+ TestUtil.assertTableHasTtl(conn, TableName.valueOf(OUTPUT_TABLE_NAME_BYTES), DEFAULT_LOG_TTL);
ManualEnvironmentEdge customClock = new ManualEnvironmentEdge();
- customClock.setValue(1);
+ customClock.setValue(EnvironmentEdgeManager.currentTimeMillis());
EnvironmentEdgeManager.injectEdge(customClock);
- TestUtil.assertTableHasTtl(conn, TableName.valueOf(OUTPUT_TABLE_NAME_BYTES), DEFAULT_LOG_TTL);
outputRepository.logToIndexToolOutputTable(mockStringBytes, mockStringBytes,
1, 2, mockString, mockStringBytes, mockStringBytes,
- EnvironmentEdgeManager.currentTimeMillis(), mockStringBytes, true);
- Table hTable =
- conn.unwrap(PhoenixConnection.class).getQueryServices().
- getTable(OUTPUT_TABLE_NAME_BYTES);
+ EnvironmentEdgeManager.currentTimeMillis(), mockStringBytes, true,
+ INVALID_ROW);
+ customClock.incrementValue(1L);
Assert.assertEquals(1, TestUtil.getRowCount(hTable, false));
customClock.incrementValue(1000*(DEFAULT_LOG_TTL+5));
@@ -145,7 +155,7 @@ public class IndexVerificationOutputRepositoryIT extends ParallelStatsDisabledIT
IndexTool.IndexDisableLoggingType disableLoggingVerifyType = IndexTool.IndexDisableLoggingType.BEFORE;
boolean expectedBefore = false;
boolean expectedAfter = true;
- verifyDisableLogging(disableLoggingVerifyType, expectedBefore, expectedAfter);
+ verifyDisableLogging(disableLoggingVerifyType, expectedBefore, expectedAfter, INVALID_ROW);
}
@Test
@@ -153,7 +163,7 @@ public class IndexVerificationOutputRepositoryIT extends ParallelStatsDisabledIT
IndexTool.IndexDisableLoggingType disableLoggingVerifyType = IndexTool.IndexDisableLoggingType.AFTER;
boolean expectedBefore = true;
boolean expectedAfter = false;
- verifyDisableLogging(disableLoggingVerifyType, expectedBefore, expectedAfter);
+ verifyDisableLogging(disableLoggingVerifyType, expectedBefore, expectedAfter, INVALID_ROW);
}
@Test
@@ -161,7 +171,7 @@ public class IndexVerificationOutputRepositoryIT extends ParallelStatsDisabledIT
IndexTool.IndexDisableLoggingType disableLoggingVerifyType = IndexTool.IndexDisableLoggingType.BOTH;
boolean expectedBefore = false;
boolean expectedAfter = false;
- verifyDisableLogging(disableLoggingVerifyType, expectedBefore, expectedAfter);
+ verifyDisableLogging(disableLoggingVerifyType, expectedBefore, expectedAfter, INVALID_ROW);
}
@Test
@@ -169,16 +179,46 @@ public class IndexVerificationOutputRepositoryIT extends ParallelStatsDisabledIT
IndexTool.IndexDisableLoggingType disableLoggingVerifyType = IndexTool.IndexDisableLoggingType.NONE;
boolean expectedBefore = true;
boolean expectedAfter = true;
- verifyDisableLogging(disableLoggingVerifyType, expectedBefore, expectedAfter);
+ verifyDisableLogging(disableLoggingVerifyType, expectedBefore, expectedAfter, INVALID_ROW);
+ }
+
+ @Test
+ public void testDisableLoggingBeyondMaxLookback() throws SQLException, IOException {
+ IndexTool.IndexDisableLoggingType disableLoggingVerifyType = IndexTool.IndexDisableLoggingType.NONE;
+ boolean expectedBefore = false;
+ boolean expectedAfter = false;
+ verifyDisableLogging(disableLoggingVerifyType, expectedBefore, expectedAfter,
+ BEYOND_MAX_LOOKBACK_INVALID, false);
+ verifyDisableLogging(disableLoggingVerifyType, expectedBefore, expectedAfter,
+ BEYOND_MAX_LOOKBACK_MISSING, false);
+
+ expectedBefore = true;
+ expectedAfter = true;
+ verifyDisableLogging(disableLoggingVerifyType, expectedBefore, expectedAfter,
+ BEYOND_MAX_LOOKBACK_INVALID, true);
+ verifyDisableLogging(disableLoggingVerifyType, expectedBefore, expectedAfter,
+ BEYOND_MAX_LOOKBACK_MISSING, true);
+ }
+
+ public void verifyDisableLogging(IndexTool.IndexDisableLoggingType disableLoggingVerifyType,
+ boolean expectedBefore, boolean expectedAfter,
+ IndexVerificationErrorType errorType) throws SQLException, IOException {
+ verifyDisableLogging(disableLoggingVerifyType, expectedBefore, expectedAfter, errorType,
+ true);
}
- public void verifyDisableLogging(IndexTool.IndexDisableLoggingType disableLoggingVerifyType, boolean expectedBefore, boolean expectedAfter) throws SQLException, IOException {
+ public void verifyDisableLogging(IndexTool.IndexDisableLoggingType disableLoggingVerifyType,
+ boolean expectedBefore, boolean expectedAfter,
+ IndexVerificationErrorType errorType,
+ boolean shouldLogBeyondMaxLookback) throws SQLException,
+ IOException {
Table mockOutputTable = Mockito.mock(Table.class);
Table mockIndexTable = Mockito.mock(Table.class);
when(mockIndexTable.getName()).thenReturn(TableName.valueOf("testDisableLoggingIndexName"));
IndexVerificationOutputRepository outputRepository =
new IndexVerificationOutputRepository(mockOutputTable,
mockIndexTable, disableLoggingVerifyType);
+ outputRepository.setShouldLogBeyondMaxLookback(shouldLogBeyondMaxLookback);
byte[] dataRowKey = Bytes.toBytes("dataRowKey");
byte[] indexRowKey = Bytes.toBytes("indexRowKey");
long dataRowTs = EnvironmentEdgeManager.currentTimeMillis();
@@ -190,9 +230,9 @@ public class IndexVerificationOutputRepositoryIT extends ParallelStatsDisabledIT
byte[] tableName = Bytes.toBytes("testDisableLoggingTableName");
outputRepository.logToIndexToolOutputTable(dataRowKey, indexRowKey, dataRowTs, indexRowTs
- , errorMsg, expectedValue, actualValue, scanMaxTs, tableName, true);
+ , errorMsg, expectedValue, actualValue, scanMaxTs, tableName, true, errorType);
outputRepository.logToIndexToolOutputTable(dataRowKey, indexRowKey, dataRowTs, indexRowTs
- , errorMsg, expectedValue, actualValue, scanMaxTs, tableName, false);
+ , errorMsg, expectedValue, actualValue, scanMaxTs, tableName, false, errorType);
int expectedRowsLogged = 0;
if (expectedBefore && expectedAfter) {
expectedRowsLogged = 2;
@@ -220,7 +260,8 @@ public class IndexVerificationOutputRepositoryIT extends ParallelStatsDisabledIT
long scanMaxTs,
byte[] tableNameBytes,
byte[] indexNameBytes,
- byte[] phaseBeforeValue) {
+ byte[] phaseBeforeValue,
+ IndexVerificationErrorType errorType) {
IndexVerificationOutputRow.IndexVerificationOutputRowBuilder builder =
new IndexVerificationOutputRow.IndexVerificationOutputRowBuilder();
return builder.setDataTableRowKey(dataRowKey).
@@ -237,6 +278,7 @@ public class IndexVerificationOutputRepositoryIT extends ParallelStatsDisabledIT
setDataTableName(Bytes.toString(tableNameBytes)).
setIndexTableName(Bytes.toString(indexNameBytes)).
setPhaseValue(phaseBeforeValue).
+ setErrorType(errorType).
build();
}
@@ -278,4 +320,18 @@ public class IndexVerificationOutputRepositoryIT extends ParallelStatsDisabledIT
dataTableName + " (val1) include (val2, val3)");
conn.commit();
}
+
+ @After
+ public void dropOutputTable() throws Exception {
+ try(Connection conn = DriverManager.getConnection(getUrl())) {
+ ConnectionQueryServices queryServices = conn.unwrap(PhoenixConnection.class).getQueryServices();
+ Admin admin = queryServices.getAdmin();
+ TableName outputTableName = TableName.valueOf(OUTPUT_TABLE_NAME_BYTES);
+ if (admin.tableExists(outputTableName)) {
+ admin.disableTable(outputTableName);
+ admin.deleteTable(outputTableName);
+ }
+ }
+ EnvironmentEdgeManager.reset();
+ }
}
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/ServerBuildIndexCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/ServerBuildIndexCompiler.java
index 2bbf5cb..0359bfa 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/ServerBuildIndexCompiler.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/ServerBuildIndexCompiler.java
@@ -91,7 +91,7 @@ public class ServerBuildIndexCompiler {
public MutationPlan compile(PTable index) throws SQLException {
try (final PhoenixStatement statement = new PhoenixStatement(connection)) {
- String query = "SELECT count(*) FROM " + tableName;
+ String query = "SELECT /*+ NO_INDEX */ count(*) FROM " + tableName;
this.plan = statement.compileQuery(query);
TableRef tableRef = plan.getTableRef();
Scan scan = plan.getContext().getScan();
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java
index 6fe5c95..712f719 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java
@@ -82,6 +82,8 @@ abstract public class BaseScannerRegionObserver extends CompatBaseScannerRegionO
public static final String INDEX_RETRY_VERIFY = "_IndexRetryVerify";
public static final String INDEX_REBUILD_DISABLE_LOGGING_VERIFY_TYPE =
"_IndexRebuildDisableLoggingVerifyType";
+ public static final String INDEX_REBUILD_DISABLE_LOGGING_BEYOND_MAXLOOKBACK_AGE =
+ "_IndexRebuildDisableLoggingBeyondMaxLookbackAge";
/*
* Attribute to denote that the index maintainer has been serialized using its proto-buf presentation.
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/GlobalIndexRegionScanner.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/GlobalIndexRegionScanner.java
index 9df3348..c5d2b56 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/GlobalIndexRegionScanner.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/GlobalIndexRegionScanner.java
@@ -19,6 +19,7 @@ package org.apache.phoenix.coprocessor;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Mutation;
@@ -31,43 +32,47 @@ import org.apache.hadoop.hbase.regionserver.Region;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Pair;
+import org.apache.phoenix.compat.hbase.coprocessor.CompatBaseScannerRegionObserver;
import org.apache.phoenix.hbase.index.ValueGetter;
import org.apache.phoenix.hbase.index.covered.update.ColumnReference;
-import org.apache.phoenix.hbase.index.parallel.TaskBatch;
import org.apache.phoenix.hbase.index.parallel.TaskRunner;
-import org.apache.phoenix.hbase.index.parallel.ThreadPoolBuilder;
-import org.apache.phoenix.hbase.index.parallel.ThreadPoolManager;
-import org.apache.phoenix.hbase.index.parallel.WaitForCompletionTaskRunner;
import org.apache.phoenix.hbase.index.table.HTableFactory;
import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
import org.apache.phoenix.hbase.index.write.IndexWriterUtils;
import org.apache.phoenix.index.IndexMaintainer;
import org.apache.phoenix.index.PhoenixIndexCodec;
+import org.apache.phoenix.mapreduce.index.IndexTool;
import org.apache.phoenix.mapreduce.index.IndexVerificationResultRepository;
import org.apache.phoenix.query.QueryServicesOptions;
-import org.apache.phoenix.util.ServerUtil;
import java.io.IOException;
import java.util.List;
+import java.util.Map;
+import java.util.NavigableSet;
-import static org.apache.phoenix.hbase.index.write.AbstractParallelWriterIndexCommitter.INDEX_WRITER_KEEP_ALIVE_TIME_CONF_KEY;
import static org.apache.phoenix.query.QueryServices.INDEX_REBUILD_PAGE_SIZE_IN_ROWS;
import static org.apache.phoenix.query.QueryServices.MUTATE_BATCH_SIZE_ATTRIB;
+import static org.apache.phoenix.query.QueryServices.MUTATE_BATCH_SIZE_BYTES_ATTRIB;
public abstract class GlobalIndexRegionScanner extends BaseRegionScanner {
public static final String NUM_CONCURRENT_INDEX_VERIFY_THREADS_CONF_KEY = "index.verify.threads.max";
- public static final int DEFAULT_CONCURRENT_INDEX_VERIFY_THREADS = 17;
- public static final String INDEX_VERIFY_ROW_COUNTS_PER_TASK_CONF_KEY = "index.verify.threads.max";
+ public static final int DEFAULT_CONCURRENT_INDEX_VERIFY_THREADS = 16;
+ public static final String INDEX_VERIFY_ROW_COUNTS_PER_TASK_CONF_KEY = "index.verify.row.count.per.task";
public static final int DEFAULT_INDEX_VERIFY_ROW_COUNTS_PER_TASK = 2048;
public static final String NO_EXPECTED_MUTATION = "No expected mutation";
public static final String ACTUAL_MUTATION_IS_NULL_OR_EMPTY = "actualMutationList is null or empty";
+ public static final String ERROR_MESSAGE_MISSING_INDEX_ROW_BEYOND_MAX_LOOKBACK = "Missing index row beyond maxLookBack";
public static final String ERROR_MESSAGE_MISSING_INDEX_ROW = "Missing index row";
protected long pageSizeInRows = Long.MAX_VALUE;
protected int rowCountPerTask;
protected boolean hasMore;
protected int maxBatchSize;
+ protected final long maxBatchSizeBytes;
+ protected final long blockingMemstoreSize;
+ protected final byte[] clientVersionBytes;
+ protected boolean useProto = true;
protected byte[] indexMetaData;
protected Scan scan;
protected RegionScanner innerScanner;
@@ -75,14 +80,18 @@ public abstract class GlobalIndexRegionScanner extends BaseRegionScanner {
protected IndexMaintainer indexMaintainer;
protected Table indexHTable;
protected TaskRunner pool;
- protected TaskBatch<Boolean> tasks;
protected String exceptionMessage;
protected HTableFactory hTableFactory;
protected int indexTableTTL;
- protected IndexToolVerificationResult verificationResult;
+ protected long maxLookBackInMills;
+ protected IndexToolVerificationResult verificationResult = null;
+ protected IndexVerificationResultRepository verificationResultRepository = null;
+ protected Map<byte[], NavigableSet<byte[]>> familyMap;
+ protected IndexTool.IndexVerifyType verifyType = IndexTool.IndexVerifyType.NONE;
+ protected boolean verify = false;
public GlobalIndexRegionScanner(RegionScanner innerScanner, final Region region, final Scan scan,
- final RegionCoprocessorEnvironment env) throws IOException {
+ final RegionCoprocessorEnvironment env) throws IOException {
super(innerScanner);
final Configuration config = env.getConfiguration();
if (scan.getAttribute(BaseScannerRegionObserver.INDEX_REBUILD_PAGING) != null) {
@@ -97,6 +106,14 @@ public abstract class GlobalIndexRegionScanner extends BaseRegionScanner {
}
}
maxBatchSize = config.getInt(MUTATE_BATCH_SIZE_ATTRIB, QueryServicesOptions.DEFAULT_MUTATE_BATCH_SIZE);
+ maxBatchSizeBytes = config.getLong(MUTATE_BATCH_SIZE_BYTES_ATTRIB,
+ QueryServicesOptions.DEFAULT_MUTATE_BATCH_SIZE_BYTES);
+ blockingMemstoreSize = UngroupedAggregateRegionObserver.getBlockingMemstoreSize(region, config);
+ clientVersionBytes = scan.getAttribute(BaseScannerRegionObserver.CLIENT_VERSION);
+ familyMap = scan.getFamilyMap();
+ if (familyMap.isEmpty()) {
+ familyMap = null;
+ }
indexMetaData = scan.getAttribute(PhoenixIndexCodec.INDEX_PROTO_MD);
if (indexMetaData == null) {
indexMetaData = scan.getAttribute(PhoenixIndexCodec.INDEX_MD);
@@ -106,19 +123,33 @@ public abstract class GlobalIndexRegionScanner extends BaseRegionScanner {
this.scan = scan;
this.innerScanner = innerScanner;
this.region = region;
+ byte[] valueBytes = scan.getAttribute(BaseScannerRegionObserver.INDEX_REBUILD_VERIFY_TYPE);
+ if (valueBytes != null) {
+ verifyType = IndexTool.IndexVerifyType.fromValue(valueBytes);
+ if (verifyType != IndexTool.IndexVerifyType.NONE) {
+ verify = true;
+ }
+ }
// Create the following objects only for rebuilds by IndexTool
hTableFactory = IndexWriterUtils.getDefaultDelegateHTableFactory(env);
indexHTable = hTableFactory.getTable(new ImmutableBytesPtr(indexMaintainer.getIndexTableName()));
- indexTableTTL = indexHTable.getTableDescriptor().getColumnFamilies()[0].getTimeToLive();
- pool = new WaitForCompletionTaskRunner(ThreadPoolManager.getExecutor(
- new ThreadPoolBuilder("IndexVerify",
- env.getConfiguration()).setMaxThread(NUM_CONCURRENT_INDEX_VERIFY_THREADS_CONF_KEY,
- DEFAULT_CONCURRENT_INDEX_VERIFY_THREADS).setCoreTimeout(
- INDEX_WRITER_KEEP_ALIVE_TIME_CONF_KEY), env));
+ indexTableTTL = indexHTable.getDescriptor().getColumnFamilies()[0].getTimeToLive();
+ maxLookBackInMills = CompatBaseScannerRegionObserver.getMaxLookbackInMillis(config);
rowCountPerTask = config.getInt(INDEX_VERIFY_ROW_COUNTS_PER_TASK_CONF_KEY,
DEFAULT_INDEX_VERIFY_ROW_COUNTS_PER_TASK);
}
+ @Override
+ public void close() throws IOException {
+ super.close();
+ if (indexHTable != null) {
+ indexHTable.close();
+ }
+ if (hTableFactory != null) {
+ this.hTableFactory.shutdown();
+ }
+ }
+
public static class SimpleValueGetter implements ValueGetter {
final ImmutableBytesWritable valuePtr = new ImmutableBytesWritable();
final Put put;
@@ -180,6 +211,14 @@ public abstract class GlobalIndexRegionScanner extends BaseRegionScanner {
return tsToCheck < (currentTime - tableTTL * 1000);
}
+ protected static boolean isTimestampBeyondMaxLookBack(long maxLookBackInMills,
+ long currentTime, long tsToCheck) {
+ if (!CompatBaseScannerRegionObserver.isMaxLookbackTimeEnabled(maxLookBackInMills)) {
+ return false;
+ }
+ return tsToCheck < (currentTime - maxLookBackInMills);
+ }
+
protected static long getMaxTimestamp(Pair<Put, Delete> pair) {
Put put = pair.getFirst();
long ts1 = 0;
@@ -191,6 +230,19 @@ public abstract class GlobalIndexRegionScanner extends BaseRegionScanner {
if (del != null) {
ts1 = getMaxTimestamp(del);
}
- return (ts1 > ts2) ? ts1 : ts2;
+ return Math.max(ts1, ts2);
+ }
+
+ protected boolean isColumnIncluded(Cell cell) {
+ byte[] family = CellUtil.cloneFamily(cell);
+ if (!familyMap.containsKey(family)) {
+ return false;
+ }
+ NavigableSet<byte[]> set = familyMap.get(family);
+ if (set == null || set.isEmpty()) {
+ return true;
+ }
+ byte[] qualifier = CellUtil.cloneQualifier(cell);
+ return set.contains(qualifier);
}
}
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/IndexRebuildRegionScanner.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/IndexRebuildRegionScanner.java
index b729a26..fdb10a4 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/IndexRebuildRegionScanner.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/IndexRebuildRegionScanner.java
@@ -21,33 +21,37 @@ import static org.apache.phoenix.hbase.index.IndexRegionObserver.UNVERIFIED_BYTE
import static org.apache.phoenix.hbase.index.IndexRegionObserver.VERIFIED_BYTES;
import static org.apache.phoenix.hbase.index.IndexRegionObserver.removeEmptyColumn;
import static org.apache.phoenix.hbase.index.write.AbstractParallelWriterIndexCommitter.INDEX_WRITER_KEEP_ALIVE_TIME_CONF_KEY;
+import static org.apache.phoenix.mapreduce.index.IndexVerificationOutputRepository.IndexVerificationErrorType.BEYOND_MAX_LOOKBACK_INVALID;
+import static org.apache.phoenix.mapreduce.index.IndexVerificationOutputRepository.IndexVerificationErrorType.BEYOND_MAX_LOOKBACK_MISSING;
+import static org.apache.phoenix.mapreduce.index.IndexVerificationOutputRepository.IndexVerificationErrorType.EXTRA_CELLS;
+import static org.apache.phoenix.mapreduce.index.IndexVerificationOutputRepository.IndexVerificationErrorType.INVALID_ROW;
+import static org.apache.phoenix.mapreduce.index.IndexVerificationOutputRepository.IndexVerificationErrorType.MISSING_ROW;
import static org.apache.phoenix.query.QueryConstants.AGG_TIMESTAMP;
import static org.apache.phoenix.query.QueryConstants.SINGLE_COLUMN;
import static org.apache.phoenix.query.QueryConstants.SINGLE_COLUMN_FAMILY;
import static org.apache.phoenix.query.QueryConstants.UNGROUPED_AGG_ROW_KEY;
-import static org.apache.phoenix.query.QueryServices.MUTATE_BATCH_SIZE_BYTES_ATTRIB;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.Iterator;
+import java.util.LinkedList;
import java.util.List;
import java.util.Map;
-import java.util.NavigableSet;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.TreeSet;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Lists;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.DoNotRetryIOException;
import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.client.Delete;
-import org.apache.hadoop.hbase.client.Durability;
import org.apache.hadoop.hbase.client.Mutation;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.RegionInfo;
@@ -60,31 +64,30 @@ import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.regionserver.Region;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.phoenix.compat.hbase.HbaseCompatCapabilities;
import org.apache.phoenix.filter.AllVersionsIndexRebuildFilter;
-import org.apache.phoenix.filter.SkipScanFilter;
+import org.apache.phoenix.hbase.index.parallel.ThreadPoolBuilder;
+import org.apache.phoenix.hbase.index.parallel.ThreadPoolManager;
+import org.apache.phoenix.hbase.index.parallel.WaitForCompletionTaskRunner;
+import org.apache.phoenix.mapreduce.index.IndexVerificationResultRepository;
+import org.apache.phoenix.query.HBaseFactoryProvider;
import org.apache.phoenix.util.ByteUtil;
import org.apache.phoenix.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.Pair;
-import org.apache.phoenix.cache.ServerCacheClient;
import org.apache.phoenix.compile.ScanRanges;
+import org.apache.phoenix.filter.SkipScanFilter;
import org.apache.phoenix.hbase.index.ValueGetter;
import org.apache.phoenix.hbase.index.parallel.EarlyExitFailure;
import org.apache.phoenix.hbase.index.parallel.Task;
import org.apache.phoenix.hbase.index.parallel.TaskBatch;
-import org.apache.phoenix.hbase.index.parallel.ThreadPoolBuilder;
-import org.apache.phoenix.hbase.index.parallel.ThreadPoolManager;
-import org.apache.phoenix.hbase.index.parallel.WaitForCompletionTaskRunner;
import org.apache.phoenix.hbase.index.util.GenericKeyValueBuilder;
import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
import org.apache.phoenix.hbase.index.util.IndexManagementUtil;
import org.apache.phoenix.index.GlobalIndexChecker;
import org.apache.phoenix.index.IndexMaintainer;
import org.apache.phoenix.mapreduce.index.IndexVerificationOutputRepository;
-import org.apache.phoenix.index.PhoenixIndexCodec;
import org.apache.phoenix.mapreduce.index.IndexTool;
-import org.apache.phoenix.mapreduce.index.IndexVerificationResultRepository;
import org.apache.phoenix.query.KeyRange;
-import org.apache.phoenix.query.QueryServicesOptions;
import org.apache.phoenix.schema.types.PLong;
import org.apache.phoenix.schema.types.PVarbinary;
import org.apache.phoenix.util.IndexUtil;
@@ -93,93 +96,83 @@ import org.apache.phoenix.util.ServerUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.base.Throwables;
import com.google.common.collect.Maps;
public class IndexRebuildRegionScanner extends GlobalIndexRegionScanner {
private static final Logger LOGGER = LoggerFactory.getLogger(IndexRebuildRegionScanner.class);
- private final long maxBatchSizeBytes;
- private final long blockingMemstoreSize;
- private final byte[] clientVersionBytes;
- private boolean useProto = true;
- private byte[] indexRowKey;
- private IndexTool.IndexVerifyType verifyType = IndexTool.IndexVerifyType.NONE;
+
+ public static final String PHOENIX_INDEX_MR_LOG_BEYOND_MAX_LOOKBACK_ERRORS =
+ "phoenix.index.mr.log.beyond.max.lookback.errors";
+ public static final boolean DEFAULT_PHOENIX_INDEX_MR_LOG_BEYOND_MAX_LOOKBACK_ERRORS = false;
+ private static boolean ignoreIndexRebuildForTesting = false;
+ private final UngroupedAggregateRegionObserver ungroupedAggregateRegionObserver;
+
+ public static void setIgnoreIndexRebuildForTesting(boolean ignore) { ignoreIndexRebuildForTesting = ignore; }
+ private byte[] indexRowKeyforReadRepair;
private IndexTool.IndexDisableLoggingType disableLoggingVerifyType = IndexTool.IndexDisableLoggingType.NONE;
- private boolean verify = false;
- private Map<byte[], List<Mutation>> indexKeyToMutationMap;
- private Map<byte[], Pair<Put, Delete>> dataKeyToMutationMap;
- private UngroupedAggregateRegionObserver ungroupedAggregateRegionObserver;
- protected UngroupedAggregateRegionObserver.MutationList mutations;
- private boolean isBeforeRebuilt = true;
- private boolean partialRebuild = false;
private int singleRowRebuildReturnCode;
- private Map<byte[], NavigableSet<byte[]>> familyMap;
private byte[][] viewConstants;
- private IndexVerificationOutputRepository verificationOutputRepository;
- protected IndexVerificationResultRepository verificationResultRepository;
+ private IndexVerificationOutputRepository verificationOutputRepository = null;
private boolean skipped = false;
private boolean shouldVerifyCheckDone = false;
+ final private RegionCoprocessorEnvironment env;
+ private byte[][] indexRegionEndKeys;
private byte[] nextStartKey;
private boolean hasMoreIncr = false;
private long minTimestamp = 0;
@VisibleForTesting
public IndexRebuildRegionScanner(final RegionScanner innerScanner, final Region region, final Scan scan,
- final RegionCoprocessorEnvironment env,
- UngroupedAggregateRegionObserver ungroupedAggregateRegionObserver) throws IOException {
+ final RegionCoprocessorEnvironment env,
+ final UngroupedAggregateRegionObserver ungroupedAggregateRegionObserver) throws IOException {
super(innerScanner, region, scan, env);
- final Configuration config = env.getConfiguration();
- if (scan.getAttribute(BaseScannerRegionObserver.INDEX_REBUILD_PAGING) == null) {
- partialRebuild = true;
- }
- maxBatchSizeBytes = config.getLong(MUTATE_BATCH_SIZE_BYTES_ATTRIB,
- QueryServicesOptions.DEFAULT_MUTATE_BATCH_SIZE_BYTES);
- mutations = new UngroupedAggregateRegionObserver.MutationList(maxBatchSize);
- blockingMemstoreSize = UngroupedAggregateRegionObserver.getBlockingMemstoreSize(region, config);
- clientVersionBytes = scan.getAttribute(BaseScannerRegionObserver.CLIENT_VERSION);
- indexMetaData = scan.getAttribute(PhoenixIndexCodec.INDEX_PROTO_MD);
- if (indexMetaData == null) {
- useProto = false;
- }
- familyMap = scan.getFamilyMap();
- if (familyMap.isEmpty()) {
- familyMap = null;
- }
+ this.env = env;
this.ungroupedAggregateRegionObserver = ungroupedAggregateRegionObserver;
- indexRowKey = scan.getAttribute(BaseScannerRegionObserver.INDEX_ROW_KEY);
- if (indexRowKey != null) {
+ indexRowKeyforReadRepair = scan.getAttribute(BaseScannerRegionObserver.INDEX_ROW_KEY);
+ if (indexRowKeyforReadRepair != null) {
setReturnCodeForSingleRowRebuild();
pageSizeInRows = 1;
+ return;
+ }
+ try(org.apache.hadoop.hbase.client.Connection connection =
+ HBaseFactoryProvider.getHConnectionFactory().createConnection(env.getConfiguration())) {
+ indexRegionEndKeys = connection.getRegionLocator(indexHTable.getName()).getEndKeys();
}
- byte[] valueBytes = scan.getAttribute(BaseScannerRegionObserver.INDEX_REBUILD_VERIFY_TYPE);
- if (valueBytes != null) {
- verifyType = IndexTool.IndexVerifyType.fromValue(valueBytes);
- if (verifyType != IndexTool.IndexVerifyType.NONE) {
- verify = true;
- viewConstants = IndexUtil.deserializeViewConstantsFromScan(scan);
- byte[] disableLoggingValueBytes =
+ pool = new WaitForCompletionTaskRunner(ThreadPoolManager.getExecutor(
+ new ThreadPoolBuilder("IndexVerify",
+ env.getConfiguration()).setMaxThread(NUM_CONCURRENT_INDEX_VERIFY_THREADS_CONF_KEY,
+ DEFAULT_CONCURRENT_INDEX_VERIFY_THREADS).setCoreTimeout(
+ INDEX_WRITER_KEEP_ALIVE_TIME_CONF_KEY), env));
+
+ if (verify) {
+ boolean shouldLogBeyondMaxLookbackInvalidRows;
+ byte[] scanParamShouldLogBeyondMaxLookbackInvalidRows =
+ scan.getAttribute(BaseScannerRegionObserver.INDEX_REBUILD_DISABLE_LOGGING_BEYOND_MAXLOOKBACK_AGE);
+ if (scanParamShouldLogBeyondMaxLookbackInvalidRows != null) {
+ shouldLogBeyondMaxLookbackInvalidRows =
+ Boolean.parseBoolean(Bytes.toString(scanParamShouldLogBeyondMaxLookbackInvalidRows));
+ } else {
+ shouldLogBeyondMaxLookbackInvalidRows =
+ env.getConfiguration().getBoolean(PHOENIX_INDEX_MR_LOG_BEYOND_MAX_LOOKBACK_ERRORS,
+ DEFAULT_PHOENIX_INDEX_MR_LOG_BEYOND_MAX_LOOKBACK_ERRORS);
+ }
+ viewConstants = IndexUtil.deserializeViewConstantsFromScan(scan);
+ byte[] disableLoggingValueBytes =
scan.getAttribute(BaseScannerRegionObserver.INDEX_REBUILD_DISABLE_LOGGING_VERIFY_TYPE);
- if (disableLoggingValueBytes != null) {
- disableLoggingVerifyType =
+ if (disableLoggingValueBytes != null) {
+ disableLoggingVerifyType =
IndexTool.IndexDisableLoggingType.fromValue(disableLoggingValueBytes);
- }
- verificationOutputRepository =
- new IndexVerificationOutputRepository(indexMaintainer.getIndexTableName()
- , hTableFactory, disableLoggingVerifyType);
- verificationResult = new IndexToolVerificationResult(scan);
- verificationResultRepository =
- new IndexVerificationResultRepository(indexMaintainer.getIndexTableName(), hTableFactory);
- indexKeyToMutationMap = Maps.newTreeMap(Bytes.BYTES_COMPARATOR);
- dataKeyToMutationMap = Maps.newTreeMap(Bytes.BYTES_COMPARATOR);
- pool = new WaitForCompletionTaskRunner(ThreadPoolManager.getExecutor(
- new ThreadPoolBuilder("IndexVerify",
- env.getConfiguration()).setMaxThread(NUM_CONCURRENT_INDEX_VERIFY_THREADS_CONF_KEY,
- DEFAULT_CONCURRENT_INDEX_VERIFY_THREADS).setCoreTimeout(
- INDEX_WRITER_KEEP_ALIVE_TIME_CONF_KEY), env));
- nextStartKey = null;
- minTimestamp = scan.getTimeRange().getMin();
}
+ verificationOutputRepository =
+ new IndexVerificationOutputRepository(indexMaintainer.getIndexTableName()
+ , hTableFactory, disableLoggingVerifyType);
+ verificationOutputRepository.setShouldLogBeyondMaxLookback(shouldLogBeyondMaxLookbackInvalidRows);
+ verificationResult = new IndexToolVerificationResult(scan);
+ verificationResultRepository =
+ new IndexVerificationResultRepository(indexMaintainer.getIndexTableName(), hTableFactory);
+ nextStartKey = null;
+ minTimestamp = scan.getTimeRange().getMin();
}
}
@@ -188,7 +181,7 @@ public class IndexRebuildRegionScanner extends GlobalIndexRegionScanner {
byte[] indexRowKey, Scan scan, Region region, IndexMaintainer indexMaintainer,
IndexVerificationResultRepository verificationResultRepository, boolean shouldVerifyCheckDone) throws IOException {
this.verifyType = verifyType;
- this.indexRowKey = indexRowKey;
+ this.indexRowKeyforReadRepair = indexRowKey;
this.scan = scan;
this.region = region;
this.indexMaintainer = indexMaintainer;
@@ -198,11 +191,11 @@ public class IndexRebuildRegionScanner extends GlobalIndexRegionScanner {
}
private boolean shouldVerify() throws IOException {
- //In case of read repair, proceed with rebuild
- //All other types of rebuilds/verification should be incrementally performed if appropriate param is passed
+ // In case of read repair, proceed with rebuild
+ // All other types of rebuilds/verification should be incrementally performed if appropriate param is passed
byte[] lastVerifyTimeValue = scan.getAttribute(UngroupedAggregateRegionObserver.INDEX_RETRY_VERIFY);
Long lastVerifyTime = lastVerifyTimeValue == null ? 0 : Bytes.toLong(lastVerifyTimeValue);
- if(indexRowKey != null || lastVerifyTime == 0 || shouldVerifyCheckDone) {
+ if(indexRowKeyforReadRepair != null || lastVerifyTime == 0 || shouldVerifyCheckDone) {
return true;
}
@@ -229,7 +222,7 @@ public class IndexRebuildRegionScanner extends GlobalIndexRegionScanner {
for (Cell cell : row) {
put.add(cell);
}
- if (checkIndexRow(indexRowKey, put)) {
+ if (checkIndexRow(indexRowKeyforReadRepair, put)) {
singleRowRebuildReturnCode = GlobalIndexChecker.RebuildReturnCode.INDEX_ROW_EXISTS.getValue();
} else {
singleRowRebuildReturnCode = GlobalIndexChecker.RebuildReturnCode.NO_INDEX_ROW.getValue();
@@ -251,44 +244,32 @@ public class IndexRebuildRegionScanner extends GlobalIndexRegionScanner {
@Override
public void close() throws IOException {
innerScanner.close();
- if (verify) {
- try {
- if (verificationResultRepository != null) {
- verificationResultRepository.logToIndexToolResultTable(verificationResult,
- verifyType, region.getRegionInfo().getRegionName(), skipped);
+ try {
+ if (indexRowKeyforReadRepair != null) {
+ return;
+ }
+ if (verify) {
+ try {
+ if (verificationResultRepository != null) {
+ verificationResultRepository.logToIndexToolResultTable(verificationResult,
+ verifyType, region.getRegionInfo().getRegionName(), skipped);
+ }
+ } finally {
+ this.pool.stop("IndexRebuildRegionScanner is closing");
+ if (verificationResultRepository != null) {
+ verificationResultRepository.close();
+ }
+ if (verificationOutputRepository != null) {
+ verificationOutputRepository.close();
+ }
}
- } finally {
+ } else {
this.pool.stop("IndexRebuildRegionScanner is closing");
- hTableFactory.shutdown();
- indexHTable.close();
- if (verificationResultRepository != null) {
- verificationResultRepository.close();
- }
- if (verificationOutputRepository != null) {
- verificationOutputRepository.close();
- }
}
+ } finally {
+ super.close();
}
- }
-
- private void setMutationAttributes(Mutation m, byte[] uuidValue) {
- m.setAttribute(useProto ? PhoenixIndexCodec.INDEX_PROTO_MD : PhoenixIndexCodec.INDEX_MD, indexMetaData);
- m.setAttribute(PhoenixIndexCodec.INDEX_UUID, uuidValue);
- m.setAttribute(BaseScannerRegionObserver.REPLAY_WRITES,
- BaseScannerRegionObserver.REPLAY_INDEX_REBUILD_WRITES);
- m.setAttribute(BaseScannerRegionObserver.CLIENT_VERSION, clientVersionBytes);
- // Since we're replaying existing mutations, it makes no sense to write them to the wal
- m.setDurability(Durability.SKIP_WAL);
- }
- private byte[] commitIfReady(byte[] uuidValue, UngroupedAggregateRegionObserver.MutationList mutationList) throws IOException {
- if (ServerUtil.readyToCommit(mutationList.size(), mutationList.byteSize(), maxBatchSize, maxBatchSizeBytes)) {
- ungroupedAggregateRegionObserver.checkForRegionClosingOrSplitting();
- ungroupedAggregateRegionObserver.commitBatchWithRetries(region, mutationList, blockingMemstoreSize);
- uuidValue = ServerCacheClient.generateId();
- mutationList.clear();
- }
- return uuidValue;
}
@VisibleForTesting
@@ -304,8 +285,8 @@ public class IndexRebuildRegionScanner extends GlobalIndexRegionScanner {
}
@VisibleForTesting
- public int setIndexKeyToMutationMap(Map<byte[], List<Mutation>> newTreeMap) {
- this.indexKeyToMutationMap = newTreeMap;
+ public long setMaxLookBackInMills(long maxLookBackInMills) {
+ this.maxLookBackInMills = maxLookBackInMills;
return 0;
}
@@ -319,17 +300,19 @@ public class IndexRebuildRegionScanner extends GlobalIndexRegionScanner {
}
public void logToIndexToolOutputTable(byte[] dataRowKey, byte[] indexRowKey, long dataRowTs, long indexRowTs,
- String errorMsg) throws IOException {
- logToIndexToolOutputTable(dataRowKey, indexRowKey, dataRowTs, indexRowTs, errorMsg, null, null);
+ String errorMsg, boolean isBeforeRebuild,
+ IndexVerificationOutputRepository.IndexVerificationErrorType errorType) throws IOException {
+ logToIndexToolOutputTable(dataRowKey, indexRowKey, dataRowTs, indexRowTs, errorMsg, null,
+ null, isBeforeRebuild, errorType);
}
@VisibleForTesting
public void logToIndexToolOutputTable(byte[] dataRowKey, byte[] indexRowKey, long dataRowTs, long indexRowTs,
- String errorMsg, byte[] expectedVaue, byte[] actualValue)
- throws IOException {
+ String errorMsg, byte[] expectedVaue, byte[] actualValue, boolean isBeforeRebuild,
+ IndexVerificationOutputRepository.IndexVerificationErrorType errorType) throws IOException {
verificationOutputRepository.logToIndexToolOutputTable(dataRowKey, indexRowKey, dataRowTs, indexRowTs,
errorMsg, expectedVaue, actualValue, scan.getTimeRange().getMax(),
- region.getRegionInfo().getTable().getName(), isBeforeRebuilt);
+ region.getRegionInfo().getTable().getName(), isBeforeRebuild, errorType);
}
private static Cell getCell(Mutation m, byte[] family, byte[] qualifier) {
@@ -345,13 +328,13 @@ public class IndexRebuildRegionScanner extends GlobalIndexRegionScanner {
return null;
}
- private boolean isMatchingMutation(Mutation expected, Mutation actual, int iteration, IndexToolVerificationResult.PhaseResult verificationPhaseResult) throws IOException {
+ private void logMismatch(Mutation expected, Mutation actual, int iteration, IndexToolVerificationResult.PhaseResult verificationPhaseResult, boolean isBeforeRebuild) throws IOException {
if (getTimestamp(expected) != getTimestamp(actual)) {
String errorMsg = "Not matching timestamp";
byte[] dataKey = indexMaintainer.buildDataRowKey(new ImmutableBytesWritable(expected.getRow()), viewConstants);
logToIndexToolOutputTable(dataKey, expected.getRow(), getTimestamp(expected), getTimestamp(actual),
- errorMsg, null, null);
- return false;
+ errorMsg, null, null, isBeforeRebuild, INVALID_ROW);
+ return;
}
int expectedCellCount = 0;
for (List<Cell> cells : expected.getFamilyCellMap().values()) {
@@ -364,19 +347,21 @@ public class IndexRebuildRegionScanner extends GlobalIndexRegionScanner {
byte[] qualifier = CellUtil.cloneQualifier(expectedCell);
Cell actualCell = getCell(actual, family, qualifier);
if (actualCell == null ||
- !CellUtil.matchingType(expectedCell, actualCell)) {
+ expectedCell.getType().equals(actualCell.getType())) {
byte[] dataKey = indexMaintainer.buildDataRowKey(new ImmutableBytesWritable(expected.getRow()), viewConstants);
String errorMsg = "Missing cell (in iteration " + iteration + ") " + Bytes.toString(family) + ":" + Bytes.toString(qualifier);
- logToIndexToolOutputTable(dataKey, expected.getRow(), getTimestamp(expected), getTimestamp(actual), errorMsg);
- verificationPhaseResult.setIndexHasMissingCellsCount(verificationPhaseResult.getIndexHasMissingCellsCount()+1);
- return false;
+ logToIndexToolOutputTable(dataKey, expected.getRow(), getTimestamp(expected),
+ getTimestamp(actual), errorMsg, isBeforeRebuild, INVALID_ROW);
+ verificationPhaseResult.setIndexHasMissingCellsCount(verificationPhaseResult.getIndexHasMissingCellsCount() + 1);
+ return;
}
if (!CellUtil.matchingValue(actualCell, expectedCell)) {
String errorMsg = "Not matching value (in iteration " + iteration + ") for " + Bytes.toString(family) + ":" + Bytes.toString(qualifier);
byte[] dataKey = indexMaintainer.buildDataRowKey(new ImmutableBytesWritable(expected.getRow()), viewConstants);
logToIndexToolOutputTable(dataKey, expected.getRow(), getTimestamp(expected), getTimestamp(actual),
- errorMsg, CellUtil.cloneValue(expectedCell), CellUtil.cloneValue(actualCell));
- return false;
+ errorMsg, CellUtil.cloneValue(expectedCell),
+ CellUtil.cloneValue(actualCell), isBeforeRebuild, INVALID_ROW);
+ return;
}
}
}
@@ -391,8 +376,42 @@ public class IndexRebuildRegionScanner extends GlobalIndexRegionScanner {
String errorMsg = "Index has extra cells (in iteration " + iteration + ")";
byte[] dataKey = indexMaintainer.buildDataRowKey(new ImmutableBytesWritable(expected.getRow()), viewConstants);
logToIndexToolOutputTable(dataKey, expected.getRow(), getTimestamp(expected), getTimestamp(actual),
- errorMsg);
- verificationPhaseResult.setIndexHasExtraCellsCount(verificationPhaseResult.getIndexHasExtraCellsCount()+1);
+ errorMsg, isBeforeRebuild, EXTRA_CELLS);
+ verificationPhaseResult.setIndexHasExtraCellsCount(verificationPhaseResult.getIndexHasExtraCellsCount() + 1);
+ }
+ }
+
+ private boolean isMatchingMutation(Mutation expected, Mutation actual) {
+ if (getTimestamp(expected) != getTimestamp(actual)) {
+ return false;
+ }
+ int expectedCellCount = 0;
+ for (List<Cell> cells : expected.getFamilyCellMap().values()) {
+ if (cells == null) {
+ continue;
+ }
+ for (Cell expectedCell : cells) {
+ expectedCellCount++;
+ byte[] family = CellUtil.cloneFamily(expectedCell);
+ byte[] qualifier = CellUtil.cloneQualifier(expectedCell);
+ Cell actualCell = getCell(actual, family, qualifier);
+ if (actualCell == null ||
+ !expectedCell.getType().equals(actualCell.getType())) {
+ return false;
+ }
+ if (!CellUtil.matchingValue(actualCell, expectedCell)) {
+ return false;
+ }
+ }
+ }
+ int actualCellCount = 0;
+ for (List<Cell> cells : actual.getFamilyCellMap().values()) {
+ if (cells == null) {
+ continue;
+ }
+ actualCellCount += cells.size();
+ }
+ if (expectedCellCount != actualCellCount) {
return false;
}
return true;
@@ -440,18 +459,7 @@ public class IndexRebuildRegionScanner extends GlobalIndexRegionScanner {
private boolean isDeleteFamily(Mutation mutation) {
for (List<Cell> cells : mutation.getFamilyCellMap().values()) {
for (Cell cell : cells) {
- if (KeyValue.Type.codeToType(cell.getTypeByte()) == KeyValue.Type.DeleteFamily) {
- return true;
- }
- }
- }
- return false;
- }
-
- private boolean isDeleteFamilyVersion(Mutation mutation) {
- for (List<Cell> cells : mutation.getFamilyCellMap().values()) {
- for (Cell cell : cells) {
- if (KeyValue.Type.codeToType(cell.getTypeByte()) == KeyValue.Type.DeleteFamilyVersion) {
+ if (cell.getType().equals(Cell.Type.DeleteFamily)) {
return true;
}
}
@@ -464,7 +472,7 @@ public class IndexRebuildRegionScanner extends GlobalIndexRegionScanner {
Put put = null;
Delete del = null;
for (Cell cell : indexRow.rawCells()) {
- if (KeyValue.Type.codeToType(cell.getTypeByte()) == KeyValue.Type.Put) {
+ if (cell.getType().equals(Cell.Type.Put)) {
if (put == null) {
put = new Put(CellUtil.cloneRow(cell));
}
@@ -473,37 +481,14 @@ public class IndexRebuildRegionScanner extends GlobalIndexRegionScanner {
if (del == null) {
del = new Delete(CellUtil.cloneRow(cell));
}
- del.addDeleteMarker(cell);
+ del.add(cell);
}
}
return getMutationsWithSameTS(put, del);
}
- private void updateUnverifiedIndexRowCounters(Put actual,
+ private void updateUnverifiedIndexRowCounters(Put actual, long expectedTs, List<Mutation> oldIndexRowsToBeDeletedList,
IndexToolVerificationResult.PhaseResult verificationPhaseResult) {
- // We are given an index row and need to check if this index row is unverified. However, before doing
- // that we need to check if this index row is the most recent version. To do that, we need to check its data
- // table row first.
- // Get the data row key from the index row key
- byte[] dataKey = indexMaintainer.buildDataRowKey(new ImmutableBytesWritable(actual.getRow()), viewConstants);
- // Get the data table row mutations using the data row key
- Pair<Put, Delete> putDeletePair = dataKeyToMutationMap.get(dataKey);
- Put put = putDeletePair.getFirst();
- if (put == null) {
- // The data table row does not exist, so there is nothing to do
- return;
- }
- Delete del = putDeletePair.getSecond();
- if (del != null && getMaxTimestamp(del) >= getTimestamp(put)) {
- // The data table row is deleted, i.e., the most recent mutation is delete. So there is nothing to do
- return;
- }
- // Get the index row key from the data table row and check if the given index row has the same row key
- byte[] indexRowKey = getIndexRowKey(indexMaintainer, put);
- if (Bytes.compareTo(actual.getRow(), 0, actual.getRow().length,
- indexRowKey, 0, indexRowKey.length) != 0) {
- return;
- }
// Get the empty column of the given index row
List<Cell> cellList = actual.get(indexMaintainer.getEmptyKeyValueFamily().copyBytesIfNecessary(),
indexMaintainer.getEmptyKeyValueQualifier());
@@ -525,6 +510,13 @@ public class IndexRebuildRegionScanner extends GlobalIndexRegionScanner {
}
// The empty column value is neither "verified" or "unverified". This must be a row from the old design
verificationPhaseResult.setOldIndexRowCount(verificationPhaseResult.getOldIndexRowCount() + 1);
+ if (verifyType == IndexTool.IndexVerifyType.BEFORE || verifyType == IndexTool.IndexVerifyType.BOTH) {
+ long actualTs = getTimestamp(actual);
+ if (actualTs > expectedTs) {
+ oldIndexRowsToBeDeletedList.add(indexMaintainer.buildRowDeleteMutation(actual.getRow(),
+ IndexMaintainer.DeleteType.SINGLE_VERSION, actualTs));
+ }
+ }
}
/**
@@ -578,12 +570,12 @@ public class IndexRebuildRegionScanner extends GlobalIndexRegionScanner {
while (iterator.hasNext()) {
Mutation mutation = iterator.next();
if ((mutation instanceof Put && !isVerified((Put) mutation)) ||
- (mutation instanceof Delete && isDeleteFamilyVersion(mutation))) {
+ (mutation instanceof Delete && !isDeleteFamily(mutation))) {
iterator.remove();
} else {
- if (previous != null && getTimestamp(previous) == getTimestamp(mutation) &&
- ((previous instanceof Put && mutation instanceof Put) ||
- previous instanceof Delete && mutation instanceof Delete)) {
+ if (((previous instanceof Put && mutation instanceof Put) ||
+ previous instanceof Delete && mutation instanceof Delete) &&
+ isMatchingMutation(previous, mutation)) {
iterator.remove();
} else {
previous = mutation;
@@ -660,9 +652,12 @@ public class IndexRebuildRegionScanner extends GlobalIndexRegionScanner {
*/
@VisibleForTesting
- public boolean verifySingleIndexRow(Result indexRow, IndexToolVerificationResult.PhaseResult verificationPhaseResult)
+ public boolean verifySingleIndexRow(Result indexRow, Map<byte[], List<Mutation>> perTaskIndexKeyToMutationMap,
+ Set<byte[]> mostRecentIndexRowKeys, List<Mutation> oldIndexRowsToBeDeletedList,
+ IndexToolVerificationResult.PhaseResult verificationPhaseResult,
+ boolean isBeforeRebuild)
throws IOException {
- List<Mutation> expectedMutationList = indexKeyToMutationMap.get(indexRow.getRow());
+ List<Mutation> expectedMutationList = perTaskIndexKeyToMutationMap.get(indexRow.getRow());
if (expectedMutationList == null) {
throw new DoNotRetryIOException(NO_EXPECTED_MUTATION);
}
@@ -672,24 +667,26 @@ public class IndexRebuildRegionScanner extends GlobalIndexRegionScanner {
}
Collections.sort(expectedMutationList, MUTATION_TS_DESC_COMPARATOR);
Collections.sort(actualMutationList, MUTATION_TS_DESC_COMPARATOR);
- if (verifyType == IndexTool.IndexVerifyType.ONLY) {
- if (actualMutationList.get(0) instanceof Put) {
+ if (isBeforeRebuild) {
+ Mutation m = actualMutationList.get(0);
+ if (m instanceof Put && mostRecentIndexRowKeys.contains(m.getRow())) {
// We do check here only the latest version as older versions will always be unverified before
// newer versions are inserted.
- updateUnverifiedIndexRowCounters((Put) actualMutationList.get(0), verificationPhaseResult);
+ updateUnverifiedIndexRowCounters((Put) m, getTimestamp(expectedMutationList.get(0)), oldIndexRowsToBeDeletedList, verificationPhaseResult);
}
+ }
+ if (verifyType == IndexTool.IndexVerifyType.ONLY) {
repairActualMutationList(actualMutationList, expectedMutationList);
}
cleanUpActualMutationList(actualMutationList);
long currentTime = EnvironmentEdgeManager.currentTimeMillis();
int actualIndex = 0;
int expectedIndex = 0;
- int matchingCount = 0;
int expectedSize = expectedMutationList.size();
int actualSize = actualMutationList.size();
Mutation expected = null;
Mutation previousExpected;
- Mutation actual;
+ Mutation actual = null;
while (expectedIndex < expectedSize && actualIndex <actualSize) {
previousExpected = expected;
expected = expectedMutationList.get(expectedIndex);
@@ -707,7 +704,8 @@ public class IndexRebuildRegionScanner extends GlobalIndexRegionScanner {
if (previousExpected instanceof Delete) {
// Between an expected delete and put, there can be one or more deletes due to
// concurrent mutations or data table write failures. Skip all of them if any
- while (getTimestamp(actual) > getTimestamp(expected) && (actual instanceof Delete)) {
+ // There cannot be any actual delete mutation between two expected put mutations.
+ while (getTimestamp(actual) >= getTimestamp(expected) && actual instanceof Delete) {
actualIndex++;
if (actualIndex == actualSize) {
break;
@@ -721,10 +719,9 @@ public class IndexRebuildRegionScanner extends GlobalIndexRegionScanner {
if (actual instanceof Delete) {
break;
}
- if (isMatchingMutation(expected, actual, expectedIndex, verificationPhaseResult)) {
+ if (isMatchingMutation(expected, actual)) {
expectedIndex++;
actualIndex++;
- matchingCount++;
continue;
}
} else { // expected instanceof Delete
@@ -744,122 +741,238 @@ public class IndexRebuildRegionScanner extends GlobalIndexRegionScanner {
(actual instanceof Delete && isDeleteFamily(actual))) {
expectedIndex++;
actualIndex++;
- matchingCount++;
continue;
}
}
- if (matchingCount > 0) {
- break;
+ break;
+ }
+
+ if (expectedIndex == expectedSize ){
+ // every expected mutation has its matching one in the actual list.
+ verificationPhaseResult.setValidIndexRowCount(verificationPhaseResult.getValidIndexRowCount() + 1);
+ return true;
+ }
+
+ if (isTimestampBeyondMaxLookBack(maxLookBackInMills, currentTime, getTimestamp(expectedMutationList.get(expectedIndex)))){
+ if (expectedIndex > 0) {
+ // if current expected index mutation is beyond max look back window, we only need to make sure its latest
+ // mutation is a matching one, as an SCN query is required.
+ verificationPhaseResult.
+ setValidIndexRowCount(verificationPhaseResult.getValidIndexRowCount() + 1);
+ return true;
}
- verificationPhaseResult.setInvalidIndexRowCount(verificationPhaseResult.getInvalidIndexRowCount() + 1);
+
+ // All expected mutations are beyond the maxLookBack window, none of them can find its matching one in actual list
+ // It may be caused by real bug or compaction on the index table.
+ // We report it as a failure, so "before" option can trigger the index rebuild for this row.
+ // This repair is required, when there is only one index row for a given data table row and the timestamp of that row
+ // can be beyond maxLookBack.
+ verificationPhaseResult.
+ setBeyondMaxLookBackInvalidIndexRowCount(verificationPhaseResult.getBeyondMaxLookBackInvalidIndexRowCount() + 1);
+ byte[] dataKey = indexMaintainer.buildDataRowKey(new ImmutableBytesWritable(indexRow.getRow()), viewConstants);
+ String errorMsg = String.format("Expect %1$s mutations but got %2$s (beyond maxLookBack)",
+ expectedSize,
+ actualSize);
+ logToIndexToolOutputTable(dataKey, indexRow.getRow(),
+ getTimestamp(expectedMutationList.get(expectedIndex)),
+ 0, errorMsg, isBeforeRebuild, BEYOND_MAX_LOOKBACK_INVALID);
return false;
}
- if ((expectedIndex != expectedSize) || actualIndex != actualSize) {
- if (matchingCount > 0) {
- if (verifyType != IndexTool.IndexVerifyType.ONLY) {
- // We do not consider this as a verification issue but log it for further information.
- // This may happen due to compaction
- byte[] dataKey = indexMaintainer.buildDataRowKey(new ImmutableBytesWritable(indexRow.getRow()), viewConstants);
- String errorMsg = "Expected to find " + expectedMutationList.size() + " mutations but got "
- + actualMutationList.size();
- logToIndexToolOutputTable(dataKey, indexRow.getRow(),
- getTimestamp(expectedMutationList.get(0)),
- getTimestamp(actualMutationList.get(0)), errorMsg);
- }
- } else {
+ else {
+ if (actualIndex < actualSize && actual instanceof Put && expected instanceof Put){
+ logMismatch(expected, actual, expectedIndex, verificationPhaseResult, isBeforeRebuild);
+ }
+ else {
byte[] dataKey = indexMaintainer.buildDataRowKey(new ImmutableBytesWritable(indexRow.getRow()), viewConstants);
String errorMsg = "Not matching index row";
logToIndexToolOutputTable(dataKey, indexRow.getRow(),
- getTimestamp(expectedMutationList.get(0)), 0L, errorMsg);
- verificationPhaseResult.setInvalidIndexRowCount(verificationPhaseResult.getInvalidIndexRowCount() + 1);
- return false;
+ getTimestamp(expectedMutationList.get(0)), 0L, errorMsg, isBeforeRebuild,
+ INVALID_ROW);
}
+ verificationPhaseResult.setInvalidIndexRowCount(verificationPhaseResult.getInvalidIndexRowCount() + 1);
+ return false;
}
- verificationPhaseResult.setValidIndexRowCount(verificationPhaseResult.getValidIndexRowCount() + 1);
- return true;
}
- private void verifyIndexRows(List<KeyRange> keys,
- IndexToolVerificationResult.PhaseResult verificationPhaseResult) throws IOException {
- List<KeyRange> invalidKeys = new ArrayList<>();
+ private void verifyIndexRows(Map<byte[], List<Mutation>> indexKeyToMutationMap,
+ Set<byte[]> mostRecentIndexRowKeys, List<Mutation> oldIndexRowsToBeDeletedList,
+ IndexToolVerificationResult.PhaseResult verificationPhaseResult,
+ boolean isBeforeRebuild) throws IOException {
+ List<KeyRange> keys = new LinkedList<KeyRange>();
+ for (byte[] indexKey: indexKeyToMutationMap.keySet()) {
+ keys.add(PVarbinary.INSTANCE.getKeyRange(indexKey));
+ }
+ Map<byte[], List<Mutation>> invalidIndexRows = Maps.newTreeMap(Bytes.BYTES_COMPARATOR);
ScanRanges scanRanges = ScanRanges.createPointLookup(keys);
Scan indexScan = new Scan();
indexScan.setTimeRange(scan.getTimeRange().getMin(), scan.getTimeRange().getMax());
scanRanges.initializeScan(indexScan);
- /*
- SkipScanFilter skipScanFilter = scanRanges.getSkipScanFilter();
- indexScan.setFilter(new SkipScanFilter(skipScanFilter, true));
- */
+ boolean useSkipScanFilter = HbaseCompatCapabilities.isRawFilterSupported();
+ if (useSkipScanFilter) {
+ SkipScanFilter skipScanFilter = scanRanges.getSkipScanFilter();
+ indexScan.setFilter(new SkipScanFilter(skipScanFilter, true));
+ }
+
indexScan.setRaw(true);
- indexScan.setMaxVersions();
+ indexScan.readAllVersions();
indexScan.setCacheBlocks(false);
try (ResultScanner resultScanner = indexHTable.getScanner(indexScan)) {
for (Result result = resultScanner.next(); (result != null); result = resultScanner.next()) {
ungroupedAggregateRegionObserver.checkForRegionClosingOrSplitting();
- KeyRange keyRange = PVarbinary.INSTANCE.getKeyRange(result.getRow());
- if (!keys.contains(keyRange)) {
- continue;
+ //if we couldn't use skip scan filter, we're scanning through a mix of rows we
+ //want to verify and those we don't, so we have to check first
+ if (!useSkipScanFilter) {
+ KeyRange keyRange = PVarbinary.INSTANCE.getKeyRange(result.getRow());
+ if (!keys.remove(keyRange)) {
+ continue;
+ }
}
- if (!verifySingleIndexRow(result, verificationPhaseResult)) {
- invalidKeys.add(keyRange);
+ if (!verifySingleIndexRow(result, indexKeyToMutationMap, mostRecentIndexRowKeys, oldIndexRowsToBeDeletedList,
+ verificationPhaseResult, isBeforeRebuild)) {
+ invalidIndexRows.put(result.getRow(), indexKeyToMutationMap.get(result.getRow()));
}
- keys.remove(keyRange);
+ indexKeyToMutationMap.remove(result.getRow());
}
} catch (Throwable t) {
ServerUtil.throwIOException(indexHTable.getName().toString(), t);
}
+ List<byte[]> expiredIndexRows = new ArrayList<>();
// Check if any expected rows from index(which we didn't get) are already expired due to TTL
// TODO: metrics for expired rows
- if (!keys.isEmpty()) {
- Iterator<KeyRange> itr = keys.iterator();
- long currentTime = EnvironmentEdgeManager.currentTimeMillis();
- while(itr.hasNext()) {
- KeyRange keyRange = itr.next();
- byte[] key = keyRange.getLowerRange();
- List<Mutation> mutationList = indexKeyToMutationMap.get(key);
- if (isTimestampBeforeTTL(indexTableTTL, currentTime, getTimestamp(mutationList.get(mutationList.size() - 1)))) {
- itr.remove();
- verificationPhaseResult.setExpiredIndexRowCount(verificationPhaseResult.getExpiredIndexRowCount() + 1);
- }
+ long currentTime = EnvironmentEdgeManager.currentTimeMillis();
+ for (Map.Entry<byte[], List<Mutation>> entry: indexKeyToMutationMap.entrySet()) {
+ List<Mutation> mutationList = entry.getValue();
+ if (isTimestampBeforeTTL(indexTableTTL, currentTime, getTimestamp(mutationList.get(mutationList.size() - 1)))) {
+ verificationPhaseResult.setExpiredIndexRowCount(verificationPhaseResult.getExpiredIndexRowCount() + 1);
+ expiredIndexRows.add(entry.getKey());
}
}
- if (keys.size() > 0) {
- for (KeyRange keyRange : keys) {
- String errorMsg = "Missing index row";
- byte[] key = keyRange.getLowerRange();
- List<Mutation> mutationList = indexKeyToMutationMap.get(key);
- if (mutationList.get(mutationList.size() - 1) instanceof Delete) {
- continue;
- }
- byte[] dataKey = indexMaintainer.buildDataRowKey(new ImmutableBytesWritable(keyRange.getLowerRange()), viewConstants);
- logToIndexToolOutputTable(dataKey,
- keyRange.getLowerRange(),
- getMaxTimestamp(dataKeyToMutationMap.get(dataKey)),
- getTimestamp(mutationList.get(mutationList.size() - 1)), errorMsg);
+ // Remove the expired rows from indexKeyToMutationMap
+ for (byte[] indexKey : expiredIndexRows) {
+ indexKeyToMutationMap.remove(indexKey);
+ }
+ // Count and log missing rows
+ for (Map.Entry<byte[], List<Mutation>> entry: indexKeyToMutationMap.entrySet()) {
+ byte[] indexKey = entry.getKey();
+ List<Mutation> mutationList = entry.getValue();
+ Mutation mutation = mutationList.get(mutationList.size() - 1);
+ if (mutation instanceof Delete) {
+ continue;
+ }
+ currentTime = EnvironmentEdgeManager.currentTimeMillis();
+ String errorMsg;
+ IndexVerificationOutputRepository.IndexVerificationErrorType errorType;
+ if (isTimestampBeyondMaxLookBack(maxLookBackInMills, currentTime, getTimestamp(mutation))){
+ errorMsg = ERROR_MESSAGE_MISSING_INDEX_ROW_BEYOND_MAX_LOOKBACK;
+ errorType = BEYOND_MAX_LOOKBACK_MISSING;
+ verificationPhaseResult.
+ setBeyondMaxLookBackMissingIndexRowCount(verificationPhaseResult.getBeyondMaxLookBackMissingIndexRowCount() + 1);
+ }
+ else {
+ errorMsg = ERROR_MESSAGE_MISSING_INDEX_ROW;
+ errorType = MISSING_ROW;
verificationPhaseResult.setMissingIndexRowCount(verificationPhaseResult.getMissingIndexRowCount() + 1);
}
+ byte[] dataKey = indexMaintainer.buildDataRowKey(new ImmutableBytesWritable(indexKey), viewConstants);
+ logToIndexToolOutputTable(dataKey, indexKey, getTimestamp(mutation), 0, errorMsg,
+ isBeforeRebuild, errorType);
}
- keys.addAll(invalidKeys);
+ // Leave the invalid and missing rows in indexKeyToMutationMap
+ indexKeyToMutationMap.putAll(invalidIndexRows);
}
- private boolean isTimestampBeforeTTL(long currentTime, long tsToCheck) {
- if (indexTableTTL == HConstants.FOREVER) {
- return false;
+ private void rebuildIndexRows(Map<byte[], List<Mutation>> indexKeyToMutationMap,
+ List<Mutation> oldIndexRowsToBeDeletedList,
+ IndexToolVerificationResult verificationResult) throws IOException {
+ if (ignoreIndexRebuildForTesting) {
+ return;
+ }
+ try {
+ ungroupedAggregateRegionObserver.checkForRegionClosingOrSplitting();
+ int batchSize = 0;
+ List<Mutation> indexUpdates = new ArrayList<>(maxBatchSize);
+ for (List<Mutation> mutationList : indexKeyToMutationMap.values()) {
+ indexUpdates.addAll(mutationList);
+ batchSize += mutationList.size();
+ if (batchSize >= maxBatchSize) {
+ Object[] results = new Object[batchSize];
+ indexHTable.batch(indexUpdates, results);
+ batchSize = 0;
+ indexUpdates = new ArrayList<>(maxBatchSize);
+ }
+ }
+ if (batchSize > 0) {
+ Object[] results = new Object[batchSize];
+ indexHTable.batch(indexUpdates, results);
+ }
+ batchSize = 0;
+ indexUpdates = new ArrayList<>(maxBatchSize);
+ for (Mutation mutation : oldIndexRowsToBeDeletedList) {
+ indexUpdates.add(mutation);
+ batchSize ++;
+ if (batchSize >= maxBatchSize) {
+ Object[] results = new Object[batchSize];
+ indexHTable.batch(indexUpdates, results);
+ batchSize = 0;
+ indexUpdates = new ArrayList<>(maxBatchSize);
+ }
+ }
+ if (batchSize > 0) {
+ Object[] results = new Object[batchSize];
+ ungroupedAggregateRegionObserver.checkForRegionClosingOrSplitting();
+ indexHTable.batch(indexUpdates, results);
+ }
+ if (verify) {
+ verificationResult.setRebuiltIndexRowCount(verificationResult.getRebuiltIndexRowCount() + indexKeyToMutationMap.size());
+ }
+ ungroupedAggregateRegionObserver.checkForRegionClosingOrSplitting();
+ } catch (Throwable t) {
+ ServerUtil.throwIOException(indexHTable.getName().toString(), t);
}
- return tsToCheck < (currentTime - (long) indexTableTTL * 1000);
}
- private void addVerifyTask(final List<KeyRange> keys,
- final IndexToolVerificationResult.PhaseResult verificationPhaseResult) {
+ private void rebuildAndOrVerifyIndexRows(Map<byte[], List<Mutation>> indexKeyToMutationMap,
+ Set<byte[]> mostRecentIndexRowKeys,
+ IndexToolVerificationResult verificationResult) throws IOException {
+ List<Mutation> oldIndexRowsToBeDeletedList = new ArrayList<>();
+ if (verifyType == IndexTool.IndexVerifyType.NONE) {
+ rebuildIndexRows(indexKeyToMutationMap, oldIndexRowsToBeDeletedList, verificationResult);
+ } else if (verifyType == IndexTool.IndexVerifyType.ONLY) {
+ verifyIndexRows(indexKeyToMutationMap, mostRecentIndexRowKeys, Collections.EMPTY_LIST, verificationResult.getBefore(), true);
+ } else if (verifyType == IndexTool.IndexVerifyType.BEFORE) {
+ verifyIndexRows(indexKeyToMutationMap, mostRecentIndexRowKeys, oldIndexRowsToBeDeletedList, verificationResult.getBefore(), true);
+ if (!indexKeyToMutationMap.isEmpty() || !oldIndexRowsToBeDeletedList.isEmpty()) {
+ rebuildIndexRows(indexKeyToMutationMap, oldIndexRowsToBeDeletedList, verificationResult);
+ }
+ } else if (verifyType == IndexTool.IndexVerifyType.AFTER) {
+ rebuildIndexRows(indexKeyToMutationMap, Collections.EMPTY_LIST, verificationResult);
+ verifyIndexRows(indexKeyToMutationMap, mostRecentIndexRowKeys, Collections.EMPTY_LIST, verificationResult.getAfter(), false);
+ } else { // verifyType == IndexTool.IndexVerifyType.BOTH
+ verifyIndexRows(indexKeyToMutationMap, mostRecentIndexRowKeys, oldIndexRowsToBeDeletedList, verificationResult.getBefore(), true);
+ if (!indexKeyToMutationMap.isEmpty() || !oldIndexRowsToBeDeletedList.isEmpty()) {
+ rebuildIndexRows(indexKeyToMutationMap, oldIndexRowsToBeDeletedList, verificationResult);
+ }
+ if (!indexKeyToMutationMap.isEmpty()) {
+ verifyIndexRows(indexKeyToMutationMap, mostRecentIndexRowKeys, Collections.EMPTY_LIST, verificationResult.getAfter(), false);
+ }
+ }
+ }
+
+ private void addRebuildAndOrVerifyTask(TaskBatch<Boolean> tasks,
+ final Map<byte[], List<Mutation>> indexKeyToMutationMap,
+ final Set<byte[]> mostRecentIndexRowKeys,
+ final IndexToolVerificationResult verificationResult) {
tasks.add(new Task<Boolean>() {
@Override
public Boolean call() throws Exception {
try {
+ //in HBase 1.x we could check if the coproc environment was closed or aborted,
+ //but in HBase 2.x the coproc environment can't check region server services
if (Thread.currentThread().isInterrupted()) {
- exceptionMessage = "Pool closed, not attempting to verify index rows! " + indexHTable.getName();
+ exceptionMessage = "Pool closed, not attempting to rebuild and/or verify index rows! " + indexHTable.getName();
throw new IOException(exceptionMessage);
}
- verifyIndexRows(keys, verificationPhaseResult);
+ rebuildAndOrVerifyIndexRows(indexKeyToMutationMap, mostRecentIndexRowKeys, verificationResult);
} catch (Exception e) {
throw e;
}
@@ -868,31 +981,10 @@ public class IndexRebuildRegionScanner extends GlobalIndexRegionScanner {
});
}
- private void parallelizeIndexVerify(IndexToolVerificationResult.PhaseResult verificationPhaseResult) throws IOException {
- int taskCount = (indexKeyToMutationMap.size() + rowCountPerTask - 1) / rowCountPerTask;
- tasks = new TaskBatch<>(taskCount);
- List<List<KeyRange>> listOfKeyRangeList = new ArrayList<>(taskCount);
- List<IndexToolVerificationResult.PhaseResult> verificationPhaseResultList = new ArrayList<>(taskCount);
- List<KeyRange> keys = new ArrayList<>(rowCountPerTask);
- listOfKeyRangeList.add(keys);
- IndexToolVerificationResult.PhaseResult perTaskVerificationPhaseResult = new IndexToolVerificationResult.PhaseResult();
- verificationPhaseResultList.add(perTaskVerificationPhaseResult);
- for (byte[] indexKey: indexKeyToMutationMap.keySet()) {
- keys.add(PVarbinary.INSTANCE.getKeyRange(indexKey));
- if (keys.size() == rowCountPerTask) {
- addVerifyTask(keys, perTaskVerificationPhaseResult);
- keys = new ArrayList<>(rowCountPerTask);
- listOfKeyRangeList.add(keys);
- perTaskVerificationPhaseResult = new IndexToolVerificationResult.PhaseResult();
- verificationPhaseResultList.add(perTaskVerificationPhaseResult);
- }
- }
- if (keys.size() > 0) {
- addVerifyTask(keys, perTaskVerificationPhaseResult);
- }
+ private void submitTasks(TaskBatch<Boolean> tasks) throws IOException{
Pair<List<Boolean>, List<Future<Boolean>>> resultsAndFutures = null;
try {
- LOGGER.debug("Waiting on index verify tasks to complete...");
+ LOGGER.debug("Waiting on index verify and/or rebuild tasks to complete...");
resultsAndFutures = this.pool.submitUninterruptible(tasks);
} catch (ExecutionException e) {
throw new RuntimeException("Should not fail on the results while using a WaitForCompletionTaskRunner", e);
@@ -908,111 +1000,58 @@ public class IndexRebuildRegionScanner extends GlobalIndexRegionScanner {
}
index++;
}
- for (IndexToolVerificationResult.PhaseResult result : verificationPhaseResultList) {
- verificationPhaseResult.add(result);
- }
- if (verifyType == IndexTool.IndexVerifyType.BEFORE || verifyType == IndexTool.IndexVerifyType.BOTH) {
- Map<byte[], Pair<Put, Delete>> newDataKeyToMutationMap = Maps.newTreeMap(Bytes.BYTES_COMPARATOR);
- for (List<KeyRange> keyRangeList : listOfKeyRangeList) {
- for (KeyRange keyRange : keyRangeList) {
- byte[] dataKey = indexMaintainer.buildDataRowKey(new ImmutableBytesWritable(keyRange.getLowerRange()), viewConstants);
- newDataKeyToMutationMap.put(dataKey, dataKeyToMutationMap.get(dataKey));
- }
- }
- dataKeyToMutationMap.clear();
- dataKeyToMutationMap = newDataKeyToMutationMap;
- }
}
- private void rebuildIndexRows(UngroupedAggregateRegionObserver.MutationList mutationList) throws IOException {
- byte[] uuidValue = ServerCacheClient.generateId();
- UngroupedAggregateRegionObserver.MutationList currentMutationList =
- new UngroupedAggregateRegionObserver.MutationList(maxBatchSize);
- Put put = null;
- for (Mutation mutation : mutationList) {
- if (mutation instanceof Put) {
- if (put != null) {
- // back to back put, i.e., no delete in between. we can commit the previous put
- uuidValue = commitIfReady(uuidValue, currentMutationList);
+ public static List<Map<byte[], List<Mutation>>> getPerTaskIndexKeyToMutationMaps(
+ TreeMap<byte[], List<Mutation>> indexKeyToMutationMap, byte[][] endKeys, int maxMapSize) {
+ List<Map<byte[], List<Mutation>>> mapList = new ArrayList<>();
+ int regionCount = endKeys.length;
+ int regionIndex = 0;
+ byte[] indexKey = indexKeyToMutationMap.firstKey();
+ Map<byte[], List<Mutation>> perTaskIndexKeyToMutationMap = Maps.newTreeMap(Bytes.BYTES_COMPARATOR);
+ mapList.add(perTaskIndexKeyToMutationMap);
+ // Find the region including the first index key
+ while (regionIndex < regionCount - 1 && Bytes.BYTES_COMPARATOR.compare(indexKey, endKeys[regionIndex]) > 0) {
+ regionIndex++;
+ }
+ for (Map.Entry<byte[], List<Mutation>> entry: indexKeyToMutationMap.entrySet()) {
+ indexKey = entry.getKey();
+ if (perTaskIndexKeyToMutationMap.size() == maxMapSize ||
+ (regionIndex < regionCount - 1 && Bytes.BYTES_COMPARATOR.compare(indexKey, endKeys[regionIndex]) > 0)) {
+ perTaskIndexKeyToMutationMap = Maps.newTreeMap(Bytes.BYTES_COMPARATOR);
+ mapList.add(perTaskIndexKeyToMutationMap);
+ // Find the region including indexKey
+ while (regionIndex < regionCount - 1 && Bytes.BYTES_COMPARATOR.compare(indexKey, endKeys[regionIndex]) > 0) {
+ regionIndex++;
}
- currentMutationList.add(mutation);
- setMutationAttributes(mutation, uuidValue);
- put = (Put)mutation;
- } else {
- currentMutationList.add(mutation);
- setMutationAttributes(mutation, uuidValue);
- uuidValue = commitIfReady(uuidValue, currentMutationList);
- put = null;
}
+ perTaskIndexKeyToMutationMap.put(indexKey, entry.getValue());
}
- if (!currentMutationList.isEmpty()) {
- ungroupedAggregateRegionObserver.checkForRegionClosingOrSplitting();
- ungroupedAggregateRegionObserver.commitBatchWithRetries(region, currentMutationList, blockingMemstoreSize);
- }
+ return mapList;
}
- private void verifyAndOrRebuildIndex() throws IOException {
- isBeforeRebuilt = true;
- IndexToolVerificationResult nextVerificationResult = new IndexToolVerificationResult(scan);
- nextVerificationResult.setScannedDataRowCount(dataKeyToMutationMap.size());
- if (verifyType == IndexTool.IndexVerifyType.AFTER || verifyType == IndexTool.IndexVerifyType.NONE) {
- // For these options we start with rebuilding index rows
- rebuildIndexRows(mutations);
- nextVerificationResult.setRebuiltIndexRowCount(dataKeyToMutationMap.size());
- isBeforeRebuilt = false;
- }
- if (verifyType == IndexTool.IndexVerifyType.NONE) {
+ private void verifyAndOrRebuildIndex(Map<byte[], List<Mutation>> indexKeyToMutationMap,
+ Set<byte[]> mostRecentIndexRowKeys) throws IOException {
+ if (indexKeyToMutationMap.size() == 0) {
return;
}
- if (verifyType == IndexTool.IndexVerifyType.BEFORE || verifyType == IndexTool.IndexVerifyType.BOTH ||
- verifyType == IndexTool.IndexVerifyType.ONLY) {
- IndexToolVerificationResult.PhaseResult verificationPhaseResult = new IndexToolVerificationResult.PhaseResult();
- // For these options we start with verifying index rows
- parallelizeIndexVerify(verificationPhaseResult);
- nextVerificationResult.getBefore().add(verificationPhaseResult);
- }
- if (verifyType == IndexTool.IndexVerifyType.BEFORE || verifyType == IndexTool.IndexVerifyType.BOTH) {
- // For these options, we have identified the rows to be rebuilt and now need to rebuild them
- // At this point, dataKeyToDataPutMap includes mapping only for the rows to be rebuilt
- mutations.clear();
-
- for (Map.Entry<byte[], Pair<Put, Delete>> entry: dataKeyToMutationMap.entrySet()) {
- if (entry.getValue().getFirst() != null) {
- mutations.add(entry.getValue().getFirst());
- }
- if (entry.getValue().getSecond() != null) {
- mutations.add(entry.getValue().getSecond());
- }
- }
- rebuildIndexRows(mutations);
- nextVerificationResult.setRebuiltIndexRowCount(nextVerificationResult.getRebuiltIndexRowCount() + dataKeyToMutationMap.size());
- isBeforeRebuilt = false;
- }
-
- if (verifyType == IndexTool.IndexVerifyType.AFTER || verifyType == IndexTool.IndexVerifyType.BOTH) {
- // We have rebuilt index row and now we need to verify them
- IndexToolVerificationResult.PhaseResult verificationPhaseResult = new IndexToolVerificationResult.PhaseResult();
- indexKeyToMutationMap.clear();
- for (Map.Entry<byte[], Pair<Put, Delete>> entry: dataKeyToMutationMap.entrySet()) {
- prepareIndexMutations(entry.getValue().getFirst(), entry.getValue().getSecond());
+ List<Map<byte[], List<Mutation>>> mapList =
+ getPerTaskIndexKeyToMutationMaps((TreeMap<byte[], List<Mutation>>)indexKeyToMutationMap,
+ indexRegionEndKeys, rowCountPerTask);
+ int taskCount = mapList.size();
+ TaskBatch<Boolean> tasks = new TaskBatch<>(taskCount);
+ List<IndexToolVerificationResult> verificationResultList = new ArrayList<>(taskCount);
+ for (int i = 0; i < taskCount; i++) {
+ IndexToolVerificationResult perTaskVerificationResult = new IndexToolVerificationResult(scan);
+ verificationResultList.add(perTaskVerificationResult);
+ addRebuildAndOrVerifyTask(tasks, mapList.get(i), mostRecentIndexRowKeys, perTaskVerificationResult);
+ }
+ submitTasks(tasks);
+ if (verify) {
+ for (IndexToolVerificationResult result : verificationResultList) {
+ verificationResult.add(result);
}
- parallelizeIndexVerify(verificationPhaseResult);
- nextVerificationResult.getAfter().add(verificationPhaseResult);
}
- verificationResult.add(nextVerificationResult);
- }
-
- private boolean isColumnIncluded(Cell cell) {
- byte[] family = CellUtil.cloneFamily(cell);
- if (!familyMap.containsKey(family)) {
- return false;
- }
- NavigableSet<byte[]> set = familyMap.get(family);
- if (set == null || set.isEmpty()) {
- return true;
- }
- byte[] qualifier = CellUtil.cloneQualifier(cell);
- return set.contains(qualifier);
}
/**
@@ -1056,7 +1095,7 @@ public class IndexRebuildRegionScanner extends GlobalIndexRegionScanner {
}
private static Put prepareIndexPutForRebuid(IndexMaintainer indexMaintainer, ImmutableBytesPtr rowKeyPtr,
- ValueGetter mergedRowVG, long ts)
+ ValueGetter mergedRowVG, long ts)
throws IOException {
Put indexPut = indexMaintainer.buildUpdateMutation(GenericKeyValueBuilder.INSTANCE,
mergedRowVG, rowKeyPtr, ts, null, null);
@@ -1112,17 +1151,17 @@ public class IndexRebuildRegionScanner extends GlobalIndexRegionScanner {
private static void applyDeleteOnPut(Delete del, Put put) throws IOException {
for (List<Cell> cells : del.getFamilyCellMap().values()) {
for (Cell cell : cells) {
- switch ((KeyValue.Type.codeToType(cell.getTypeByte()))) {
- case DeleteFamily:
- put.getFamilyCellMap().remove(CellUtil.cloneFamily(cell));
- break;
- case DeleteColumn:
- removeColumn(put, cell);
- break;
- default:
- // We do not expect this can happen
- throw new DoNotRetryIOException("Single version delete marker in data mutation " +
- del);
+ switch (cell.getType()) {
+ case DeleteFamily:
+ put.getFamilyCellMap().remove(CellUtil.cloneFamily(cell));
+ break;
+ case DeleteColumn:
+ removeColumn(put, cell);
+ break;
+ default:
+ // We do not expect this can happen
+ throw new DoNotRetryIOException("Single version delete marker in data mutation " +
+ del);
}
}
}
@@ -1136,7 +1175,7 @@ public class IndexRebuildRegionScanner extends GlobalIndexRegionScanner {
* after put.
*/
public static List<Mutation> prepareIndexMutationsForRebuild(IndexMaintainer indexMaintainer,
- Put dataPut, Delete dataDel) throws IOException {
+ Put dataPut, Delete dataDel) throws IOException {
List<Mutation> dataMutations = getMutationsWithSameTS(dataPut, dataDel);
List<Mutation> indexMutations = Lists.newArrayListWithExpectedSize(dataMutations.size());
// The row key ptr of the data table row for which we will build index rows here
@@ -1169,7 +1208,7 @@ public class IndexRebuildRegionScanner extends GlobalIndexRegionScanner {
}
}
} else {
- /**
+ /*
* When the mutation is empty after the delete mutation is applied, we should
* reuse the logical of applying a delete mutation on currentDataRowState.
*/
@@ -1237,12 +1276,24 @@ public class IndexRebuildRegionScanner extends GlobalIndexRegionScanner {
}
@VisibleForTesting
- public int prepareIndexMutations(Put put, Delete del) throws IOException {
+ public int prepareIndexMutations(Put put, Delete del, Map<byte[], List<Mutation>> indexKeyToMutationMap,
+ Set<byte[]> mostRecentIndexRowKeys) throws IOException {
List<Mutation> indexMutations = prepareIndexMutationsForRebuild(indexMaintainer, put, del);
+ boolean mostRecentDone = false;
+ // Do not populate mostRecentIndexRowKeys when verifyType is NONE or AFTER
+ if (verifyType == IndexTool.IndexVerifyType.NONE || verifyType == IndexTool.IndexVerifyType.AFTER) {
+ mostRecentDone = true;
+ }
for (Mutation mutation : indexMutations) {
byte[] indexRowKey = mutation.getRow();
List<Mutation> mutationList = indexKeyToMutationMap.get(indexRowKey);
if (mutationList == null) {
+ if (!mostRecentDone) {
+ if (mutation instanceof Put) {
+ mostRecentIndexRowKeys.add(indexRowKey);
+ mostRecentDone = true;
+ }
+ }
mutationList = new ArrayList<>();
mutationList.add(mutation);
indexKeyToMutationMap.put(indexRowKey, mutationList);
@@ -1250,26 +1301,29 @@ public class IndexRebuildRegionScanner extends GlobalIndexRegionScanner {
mutationList.add(mutation);
}
}
- return 0;
+ return indexMutations.size();
}
@Override
public boolean next(List<Cell> results) throws IOException {
- if (indexRowKey != null &&
+ if (indexRowKeyforReadRepair != null &&
singleRowRebuildReturnCode == GlobalIndexChecker.RebuildReturnCode.NO_DATA_ROW.getValue()) {
byte[] rowCountBytes =
PLong.INSTANCE.toBytes(Long.valueOf(singleRowRebuildReturnCode));
- final Cell aggKeyValue = PhoenixKeyValueUtil.newKeyValue(UNGROUPED_AGG_ROW_KEY, SINGLE_COLUMN_FAMILY,
+ final Cell aggKeyValue = PhoenixKeyValueUtil.newKeyValue(UNGROUPED_AGG_ROW_KEY,
+ SINGLE_COLUMN_FAMILY,
SINGLE_COLUMN, AGG_TIMESTAMP, rowCountBytes, 0, rowCountBytes.length);
results.add(aggKeyValue);
return false;
}
+ Map<byte[], List<Mutation>> indexKeyToMutationMap = Maps.newTreeMap(Bytes.BYTES_COMPARATOR);
+ Set<byte[]> mostRecentIndexRowKeys = new TreeSet<>(Bytes.BYTES_COMPARATOR);
Cell lastCell = null;
- int rowCount = 0;
+ int dataRowCount = 0;
+ int indexMutationCount = 0;
region.startRegionOperation();
RegionScanner localScanner = null;
try {
- byte[] uuidValue = ServerCacheClient.generateId();
localScanner = getLocalScanner();
if (localScanner == null) {
return false;
@@ -1280,21 +1334,21 @@ public class IndexRebuildRegionScanner extends GlobalIndexRegionScanner {
return false;
}
do {
- /**
+ /*
If region is closing and there are large number of rows being verified/rebuilt with IndexTool,
not having this check will impact/delay the region closing -- affecting the availability
as this method holds the read lock on the region.
* */
ungroupedAggregateRegionObserver.checkForRegionClosingOrSplitting();
- List<Cell> row = new ArrayList<Cell>();
+ List<Cell> row = new ArrayList<>();
hasMore = localScanner.nextRaw(row);
if (!row.isEmpty()) {
lastCell = row.get(0); // lastCell is any cell from the last visited row
Put put = null;
Delete del = null;
for (Cell cell : row) {
- if (KeyValue.Type.codeToType(cell.getTypeByte()) == KeyValue.Type.Put) {
- if (!partialRebuild && familyMap != null && !isColumnIncluded(cell)) {
+ if (cell.getType().equals(Cell.Type.Put)) {
+ if (familyMap != null && !isColumnIncluded(cell)) {
continue;
}
if (put == null) {
@@ -1305,44 +1359,27 @@ public class IndexRebuildRegionScanner extends GlobalIndexRegionScanner {
if (del == null) {
del = new Delete(CellUtil.cloneRow(cell));
}
- del.addDeleteMarker(cell);
+ del.add(cell);
}
}
if (put == null && del == null) {
continue;
}
- // Always add the put first and then delete for a given row. This simplifies the logic in
- // IndexRegionObserver
- if (put != null) {
- mutations.add(put);
- }
- if (del != null) {
- mutations.add(del);
- }
- if (!verify) {
- if (put != null) {
- setMutationAttributes(put, uuidValue);
- }
- if (del != null) {
- setMutationAttributes(del, uuidValue);
- }
- uuidValue = commitIfReady(uuidValue, mutations);
- } else {
- byte[] dataKey = (put != null) ? put.getRow() : del.getRow();
- prepareIndexMutations(put, del);
- dataKeyToMutationMap.put(dataKey, new Pair<Put, Delete>(put, del));
- }
- rowCount++;
+ indexMutationCount += prepareIndexMutations(put, del, indexKeyToMutationMap, mostRecentIndexRowKeys);
+ dataRowCount++;
}
- } while (hasMore && rowCount < pageSizeInRows);
- if (!mutations.isEmpty()) {
- if (verify) {
- verifyAndOrRebuildIndex();
+ } while (hasMore && indexMutationCount < pageSizeInRows
+ && dataRowCount < pageSizeInRows);
+ if (!indexKeyToMutationMap.isEmpty()) {
+ if (indexRowKeyforReadRepair != null) {
+ rebuildIndexRows(indexKeyToMutationMap, Collections.EMPTY_LIST, verificationResult);
} else {
- ungroupedAggregateRegionObserver.checkForRegionClosingOrSplitting();
- ungroupedAggregateRegionObserver.commitBatchWithRetries(region, mutations, blockingMemstoreSize);
+ verifyAndOrRebuildIndex(indexKeyToMutationMap, mostRecentIndexRowKeys);
}
}
+ if (verify) {
+ verificationResult.setScannedDataRowCount(verificationResult.getScannedDataRowCount() + dataRowCount);
+ }
}
} catch (Throwable e) {
LOGGER.error("Exception in IndexRebuildRegionScanner for region "
@@ -1350,28 +1387,25 @@ public class IndexRebuildRegionScanner extends GlobalIndexRegionScanner {
throw e;
} finally {
region.closeRegionOperation();
- mutations.clear();
- if (verify) {
- dataKeyToMutationMap.clear();
- indexKeyToMutationMap.clear();
- }
if (localScanner!=null && localScanner!=innerScanner) {
localScanner.close();
}
}
- if (indexRowKey != null) {
- rowCount = singleRowRebuildReturnCode;
+ if (indexRowKeyforReadRepair != null) {
+ dataRowCount = singleRowRebuildReturnCode;
}
if (minTimestamp != 0) {
nextStartKey = ByteUtil.calculateTheClosestNextRowKeyForPrefix(CellUtil.cloneRow(lastCell));
}
- byte[] rowCountBytes = PLong.INSTANCE.toBytes(Long.valueOf(rowCount));
+ byte[] rowCountBytes = PLong.INSTANCE.toBytes(Long.valueOf(dataRowCount));
final Cell aggKeyValue;
if (lastCell == null) {
- aggKeyValue = PhoenixKeyValueUtil.newKeyValue(UNGROUPED_AGG_ROW_KEY, SINGLE_COLUMN_FAMILY,
+ aggKeyValue = PhoenixKeyValueUtil.newKeyValue(UNGROUPED_AGG_ROW_KEY,
+ SINGLE_COLUMN_FAMILY,
SINGLE_COLUMN, AGG_TIMESTAMP, rowCountBytes, 0, rowCountBytes.length);
} else {
- aggKeyValue = PhoenixKeyValueUtil.newKeyValue(CellUtil.cloneRow(lastCell), SINGLE_COLUMN_FAMILY,
+ aggKeyValue = PhoenixKeyValueUtil.newKeyValue(CellUtil.cloneRow(lastCell),
+ SINGLE_COLUMN_FAMILY,
SINGLE_COLUMN, AGG_TIMESTAMP, rowCountBytes, 0, rowCountBytes.length);
}
results.add(aggKeyValue);
@@ -1382,11 +1416,11 @@ public class IndexRebuildRegionScanner extends GlobalIndexRegionScanner {
// override the filter to skip scan and open new scanner
// when lower bound of timerange is passed or newStartKey was populated
// from previous call to next()
- if (minTimestamp != 0) {
+ if(minTimestamp!= 0) {
Scan incrScan = new Scan(scan);
incrScan.setTimeRange(minTimestamp, scan.getTimeRange().getMax());
incrScan.setRaw(true);
- incrScan.setMaxVersions();
+ incrScan.readAllVersions();
incrScan.getFamilyMap().clear();
incrScan.setCacheBlocks(false);
for (byte[] family : scan.getFamilyMap().keySet()) {
@@ -1401,11 +1435,11 @@ public class IndexRebuildRegionScanner extends GlobalIndexRegionScanner {
// Override the filter so that we get all versions
incrScan.setFilter(new AllVersionsIndexRebuildFilter(scan.getFilter()));
}
- if (nextStartKey != null) {
- incrScan.setStartRow(nextStartKey);
+ if(nextStartKey != null) {
+ incrScan.withStartRow(nextStartKey);
}
List<KeyRange> keys = new ArrayList<>();
- try (RegionScanner scanner = region.getScanner(incrScan)) {
+ try(RegionScanner scanner = region.getScanner(incrScan)) {
List<Cell> row = new ArrayList<>();
int rowCount = 0;
// collect row keys that have been modified in the given time-range
@@ -1442,4 +1476,4 @@ public class IndexRebuildRegionScanner extends GlobalIndexRegionScanner {
public long getMaxResultSize() {
return scan.getMaxResultSize();
}
-}
\ No newline at end of file
+}
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/IndexToolVerificationResult.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/IndexToolVerificationResult.java
index 1581678..2e02263 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/IndexToolVerificationResult.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/IndexToolVerificationResult.java
@@ -22,11 +22,16 @@ import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.util.Bytes;
+import static org.apache.phoenix.mapreduce.index.IndexVerificationResultRepository.AFTER_REBUILD_BEYOND_MAXLOOKBACK_INVALID_INDEX_ROW_COUNT_BYTES;
+import static org.apache.phoenix.mapreduce.index.IndexVerificationResultRepository.AFTER_REBUILD_BEYOND_MAXLOOKBACK_MISSING_INDEX_ROW_COUNT_BYTES;
import static org.apache.phoenix.mapreduce.index.IndexVerificationResultRepository.AFTER_REBUILD_EXPIRED_INDEX_ROW_COUNT_BYTES;
import static org.apache.phoenix.mapreduce.index.IndexVerificationResultRepository.AFTER_REBUILD_INVALID_INDEX_ROW_COUNT_BYTES;
+import static org.apache.phoenix.mapreduce.index.IndexVerificationResultRepository.AFTER_REBUILD_INVALID_INDEX_ROW_COUNT_COZ_EXTRA_CELLS_BYTES;
import static org.apache.phoenix.mapreduce.index.IndexVerificationResultRepository.AFTER_REBUILD_INVALID_INDEX_ROW_COUNT_COZ_MISSING_CELLS_BYTES;
import static org.apache.phoenix.mapreduce.index.IndexVerificationResultRepository.AFTER_REBUILD_MISSING_INDEX_ROW_COUNT_BYTES;
import static org.apache.phoenix.mapreduce.index.IndexVerificationResultRepository.AFTER_REBUILD_VALID_INDEX_ROW_COUNT_BYTES;
+import static org.apache.phoenix.mapreduce.index.IndexVerificationResultRepository.BEFORE_REBUILD_BEYOND_MAXLOOKBACK_INVALID_INDEX_ROW_COUNT_BYTES;
+import static org.apache.phoenix.mapreduce.index.IndexVerificationResultRepository.BEFORE_REBUILD_BEYOND_MAXLOOKBACK_MISSING_INDEX_ROW_COUNT_BYTES;
import static org.apache.phoenix.mapreduce.index.IndexVerificationResultRepository.BEFORE_REBUILD_EXPIRED_INDEX_ROW_COUNT_BYTES;
import static org.apache.phoenix.mapreduce.index.IndexVerificationResultRepository.BEFORE_REBUILD_INVALID_INDEX_ROW_COUNT_BYTES;
import static org.apache.phoenix.mapreduce.index.IndexVerificationResultRepository.BEFORE_REBUILD_INVALID_INDEX_ROW_COUNT_COZ_EXTRA_CELLS_BYTES;
@@ -39,7 +44,6 @@ import static org.apache.phoenix.mapreduce.index.IndexVerificationResultReposito
import static org.apache.phoenix.mapreduce.index.IndexVerificationResultRepository.REBUILT_INDEX_ROW_COUNT_BYTES;
import static org.apache.phoenix.mapreduce.index.IndexVerificationResultRepository.RESULT_TABLE_COLUMN_FAMILY;
import static org.apache.phoenix.mapreduce.index.IndexVerificationResultRepository.SCANNED_DATA_ROW_COUNT_BYTES;
-import static org.apache.phoenix.mapreduce.index.IndexVerificationResultRepository.AFTER_REBUILD_INVALID_INDEX_ROW_COUNT_COZ_EXTRA_CELLS_BYTES;
public class IndexToolVerificationResult {
@@ -83,13 +87,13 @@ public class IndexToolVerificationResult {
this.scanMaxTs = scanMaxTs;
}
- public IndexToolVerificationResult(byte[] startRow, byte[] stopRow, long scanMaxTs){
+ public IndexToolVerificationResult(byte[] startRow, byte[] stopRow, long scanMaxTs) {
this.setStartRow(startRow);
this.setStopRow(stopRow);
this.scanMaxTs = scanMaxTs;
}
- public IndexToolVerificationResult(Scan scan){
+ public IndexToolVerificationResult(Scan scan) {
this.setStartRow(scan.getStartRow());
this.setStopRow(scan.getStopRow());
this.scanMaxTs = scan.getTimeRange().getMax();
@@ -112,6 +116,8 @@ public class IndexToolVerificationResult {
private long expiredIndexRowCount = 0;
private long missingIndexRowCount = 0;
private long invalidIndexRowCount = 0;
+ private long beyondMaxLookBackMissingIndexRowCount = 0;
+ private long beyondMaxLookBackInvalidIndexRowCount = 0;
private long indexHasExtraCellsCount = 0;
private long indexHasMissingCellsCount = 0;
private long unverifiedIndexRowCount = 0;
@@ -119,6 +125,10 @@ public class IndexToolVerificationResult {
private long unknownIndexRowCount = 0;
public void add(PhaseResult phaseResult) {
+ setBeyondMaxLookBackMissingIndexRowCount(getBeyondMaxLookBackMissingIndexRowCount() +
+ phaseResult.getBeyondMaxLookBackMissingIndexRowCount());
+ setBeyondMaxLookBackInvalidIndexRowCount(getBeyondMaxLookBackInvalidIndexRowCount() +
+ phaseResult.getBeyondMaxLookBackInvalidIndexRowCount());
setValidIndexRowCount(getValidIndexRowCount() + phaseResult.getValidIndexRowCount());
setExpiredIndexRowCount(getExpiredIndexRowCount() + phaseResult.getExpiredIndexRowCount());
setMissingIndexRowCount(getMissingIndexRowCount() + phaseResult.getMissingIndexRowCount());
@@ -130,35 +140,50 @@ public class IndexToolVerificationResult {
setOldIndexRowCount(getOldIndexRowCount() + phaseResult.getOldIndexRowCount());
}
- public PhaseResult(){}
+ public PhaseResult() {
+ }
public PhaseResult(long validIndexRowCount, long expiredIndexRowCount,
- long missingIndexRowCount, long invalidIndexRowCount, long indexHasExtraCellsCount, long indexHasMissingCellsCount) {
+ long missingIndexRowCount, long invalidIndexRowCount,
+ long beyondMaxLookBackMissingIndexRowCount,
+ long beyondMaxLookBackInvalidIndexRowCount,
+ long indexHasExtraCellsCount, long indexHasMissingCellsCount) {
this.setValidIndexRowCount(validIndexRowCount);
this.setExpiredIndexRowCount(expiredIndexRowCount);
this.setMissingIndexRowCount(missingIndexRowCount);
this.setInvalidIndexRowCount(invalidIndexRowCount);
+ this.setBeyondMaxLookBackInvalidIndexRowCount(beyondMaxLookBackInvalidIndexRowCount);
+ this.setBeyondMaxLookBackMissingIndexRowCount(beyondMaxLookBackMissingIndexRowCount);
this.setIndexHasExtraCellsCount(indexHasExtraCellsCount);
this.setIndexHasMissingCellsCount(indexHasMissingCellsCount);
}
+
public long getTotalCount() {
- return getValidIndexRowCount() + getExpiredIndexRowCount() + getMissingIndexRowCount() + getInvalidIndexRowCount();
+ return getValidIndexRowCount() + getExpiredIndexRowCount() + getMissingIndexRowCount() + getInvalidIndexRowCount()
+ + getBeyondMaxLookBackMissingIndexRowCount() + getBeyondMaxLookBackInvalidIndexRowCount();
}
public long getIndexHasExtraCellsCount() {
return indexHasExtraCellsCount;
}
+ public long getIndexHasMissingCellsCount() {
+ return indexHasMissingCellsCount;
+ }
+
@Override
public String toString() {
return "PhaseResult{" +
- "validIndexRowCount=" + getValidIndexRowCount() +
- ", expiredIndexRowCount=" + getExpiredIndexRowCount() +
- ", missingIndexRowCount=" + getMissingIndexRowCount() +
- ", invalidIndexRowCount=" + getInvalidIndexRowCount() +
- ", extraCellsOnIndexCount=" + getIndexHasExtraCellsCount() +
- ", missingCellsOnIndexCount=" + getIndexHasMissingCellsCount() +
+
+ "validIndexRowCount=" + validIndexRowCount +
+ ", expiredIndexRowCount=" + expiredIndexRowCount +
+ ", missingIndexRowCount=" + missingIndexRowCount +
+ ", invalidIndexRowCount=" + invalidIndexRowCount +
+ ", beyondMaxLookBackMissingIndexRowCount=" + getBeyondMaxLookBackMissingIndexRowCount() +
+ ", beyondMaxLookBackInvalidIndexRowCount=" + getBeyondMaxLookBackInvalidIndexRowCount() +
+ ", extraCellsOnIndexCount=" + indexHasExtraCellsCount +
+ ", missingCellsOnIndexCount=" + indexHasMissingCellsCount +
'}';
}
@@ -171,12 +196,15 @@ public class IndexToolVerificationResult {
return false;
}
PhaseResult pr = (PhaseResult) o;
- return this.getExpiredIndexRowCount() == pr.getExpiredIndexRowCount()
- && this.getValidIndexRowCount() == pr.getValidIndexRowCount()
- && this.getInvalidIndexRowCount() == pr.getInvalidIndexRowCount()
- && this.getMissingIndexRowCount() == pr.getMissingIndexRowCount()
- && this.getIndexHasMissingCellsCount() == pr.getIndexHasMissingCellsCount()
- && this.getIndexHasExtraCellsCount() == pr.getIndexHasExtraCellsCount();
+
+ return this.expiredIndexRowCount == pr.expiredIndexRowCount
+ && this.validIndexRowCount == pr.validIndexRowCount
+ && this.invalidIndexRowCount == pr.invalidIndexRowCount
+ && this.missingIndexRowCount == pr.missingIndexRowCount
+ && this.getBeyondMaxLookBackInvalidIndexRowCount() == pr.getBeyondMaxLookBackInvalidIndexRowCount()
+ && this.getBeyondMaxLookBackMissingIndexRowCount() == pr.getBeyondMaxLookBackMissingIndexRowCount()
+ && this.indexHasMissingCellsCount == pr.indexHasMissingCellsCount
+ && this.indexHasExtraCellsCount == pr.indexHasExtraCellsCount;
}
@Override
@@ -186,6 +214,8 @@ public class IndexToolVerificationResult {
result = 31 * result + getValidIndexRowCount();
result = 31 * result + getMissingIndexRowCount();
result = 31 * result + getInvalidIndexRowCount();
+ result = 31 * result + getBeyondMaxLookBackMissingIndexRowCount();
+ result = 31 * result + getBeyondMaxLookBackInvalidIndexRowCount();
result = 31 * result + getIndexHasMissingCellsCount();
result = 31 * result + getIndexHasExtraCellsCount();
result = 31 * result + getUnverifiedIndexRowCount();
@@ -226,18 +256,30 @@ public class IndexToolVerificationResult {
this.invalidIndexRowCount = invalidIndexRowCount;
}
- public void setIndexHasExtraCellsCount(long indexHasExtraCellsCount) {
- this.indexHasExtraCellsCount = indexHasExtraCellsCount;
+ public long getBeyondMaxLookBackMissingIndexRowCount() {
+ return beyondMaxLookBackMissingIndexRowCount;
}
- public long getIndexHasMissingCellsCount() {
- return indexHasMissingCellsCount;
+ public void setBeyondMaxLookBackMissingIndexRowCount(long beyondMaxLookBackMissingIndexRowCount) {
+ this.beyondMaxLookBackMissingIndexRowCount = beyondMaxLookBackMissingIndexRowCount;
+ }
+
+ public long getBeyondMaxLookBackInvalidIndexRowCount() {
+ return beyondMaxLookBackInvalidIndexRowCount;
+ }
+
+ public void setBeyondMaxLookBackInvalidIndexRowCount(long beyondMaxLookBackInvalidIndexRowCount) {
+ this.beyondMaxLookBackInvalidIndexRowCount = beyondMaxLookBackInvalidIndexRowCount;
}
public void setIndexHasMissingCellsCount(long indexHasMissingCellsCount) {
this.indexHasMissingCellsCount = indexHasMissingCellsCount;
}
+ public void setIndexHasExtraCellsCount(long indexHasExtraCellsCount) {
+ this.indexHasExtraCellsCount = indexHasExtraCellsCount;
+ }
+
public long getUnverifiedIndexRowCount() {
return unverifiedIndexRowCount;
}
@@ -302,7 +344,6 @@ public class IndexToolVerificationResult {
return getBefore().getInvalidIndexRowCount();
}
-
public long getBeforeRebuildUnverifiedIndexRowCount() {
return getBefore().getUnverifiedIndexRowCount();
}
@@ -315,14 +356,21 @@ public class IndexToolVerificationResult {
return getBefore().getUnknownIndexRowCount();
}
+ public long getBeforeRebuildBeyondMaxLookBackMissingIndexRowCount() {
+ return before.getBeyondMaxLookBackMissingIndexRowCount();
+ }
+
+ public long getBeforeRebuildBeyondMaxLookBackInvalidIndexRowCount() {
+ return before.getBeyondMaxLookBackInvalidIndexRowCount();
+ }
public long getBeforeRebuildMissingIndexRowCount() {
return getBefore().getMissingIndexRowCount();
}
- public long getBeforeIndexHasMissingCellsCount() {return before.indexHasMissingCellsCount; }
+ public long getBeforeIndexHasMissingCellsCount() {return getBefore().getIndexHasMissingCellsCount(); }
- public long getBeforeIndexHasExtraCellsCount() {return before.indexHasExtraCellsCount; }
+ public long getBeforeIndexHasExtraCellsCount() {return getBefore().getIndexHasExtraCellsCount(); }
public long getAfterRebuildValidIndexRowCount() {
return getAfter().getValidIndexRowCount();
@@ -340,9 +388,17 @@ public class IndexToolVerificationResult {
return getAfter().getMissingIndexRowCount();
}
- public long getAfterIndexHasMissingCellsCount() { return after.indexHasMissingCellsCount; }
+ public long getAfterRebuildBeyondMaxLookBackMissingIndexRowCount() {
+ return after.getBeyondMaxLookBackMissingIndexRowCount();
+ }
+
+ public long getAfterRebuildBeyondMaxLookBackInvalidIndexRowCount() {
+ return after.getBeyondMaxLookBackInvalidIndexRowCount();
+ }
+
+ public long getAfterIndexHasMissingCellsCount() { return getAfter().getIndexHasMissingCellsCount(); }
- public long getAfterIndexHasExtraCellsCount() { return after.indexHasExtraCellsCount; }
+ public long getAfterIndexHasExtraCellsCount() { return getAfter().getIndexHasExtraCellsCount(); }
private void addScannedDataRowCount(long count) {
this.setScannedDataRowCount(this.getScannedDataRowCount() + count);
@@ -368,12 +424,19 @@ public class IndexToolVerificationResult {
getBefore().setInvalidIndexRowCount(getBefore().getInvalidIndexRowCount() + count);
}
+ private void addBeforeRebuildBeyondMaxLookBackMissingIndexRowCount(long count) {
+ before.setBeyondMaxLookBackMissingIndexRowCount(before.getBeyondMaxLookBackMissingIndexRowCount() + count);
+ }
+
+ private void addBeforeRebuildBeyondMaxLookBackInvalidIndexRowCount(long count) {
+ before.setBeyondMaxLookBackInvalidIndexRowCount(before.getBeyondMaxLookBackInvalidIndexRowCount() + count);
+ }
public void addBeforeIndexHasMissingCellsCount(long count) {
- before.indexHasMissingCellsCount += count;
+ getBefore().setIndexHasMissingCellsCount(getBefore().getIndexHasMissingCellsCount() + count);
}
public void addBeforeIndexHasExtraCellsCount(long count) {
- before.indexHasExtraCellsCount += count;
+ getBefore().setIndexHasExtraCellsCount(getBefore().getIndexHasExtraCellsCount() + count);
}
public void addBeforeUnverifiedIndexRowCount(long count) {
@@ -404,12 +467,20 @@ public class IndexToolVerificationResult {
getAfter().setInvalidIndexRowCount(getAfter().getInvalidIndexRowCount() + count);
}
+ private void addAfterRebuildBeyondMaxLookBackMissingIndexRowCount(long count) {
+ after.setBeyondMaxLookBackMissingIndexRowCount(after.getBeyondMaxLookBackMissingIndexRowCount() + count);
+ }
+
+ private void addAfterRebuildBeyondMaxLookBackInvalidIndexRowCount(long count) {
+ after.setBeyondMaxLookBackInvalidIndexRowCount(after.getBeyondMaxLookBackInvalidIndexRowCount() + count);
+ }
+
public void addAfterIndexHasMissingCellsCount(long count) {
- after.indexHasMissingCellsCount += count;
+ getAfter().setIndexHasMissingCellsCount(getAfter().getIndexHasMissingCellsCount() + count);
}
public void addAfterIndexHasExtraCellsCount(long count) {
- after.indexHasExtraCellsCount += count;
+ getAfter().setIndexHasExtraCellsCount(getAfter().getIndexHasExtraCellsCount() + count);
}
private static boolean isAfterRebuildInvalidIndexRowCount(Cell cell) {
@@ -440,6 +511,10 @@ public class IndexToolVerificationResult {
addBeforeRebuildMissingIndexRowCount(getValue(cell));
} else if (CellUtil.matchingColumn(cell, RESULT_TABLE_COLUMN_FAMILY, BEFORE_REBUILD_INVALID_INDEX_ROW_COUNT_BYTES)) {
addBeforeRebuildInvalidIndexRowCount(getValue(cell));
+ } else if (CellUtil.matchingColumn(cell, RESULT_TABLE_COLUMN_FAMILY, BEFORE_REBUILD_BEYOND_MAXLOOKBACK_MISSING_INDEX_ROW_COUNT_BYTES)) {
+ addBeforeRebuildBeyondMaxLookBackMissingIndexRowCount(getValue(cell));
+ } else if (CellUtil.matchingColumn(cell, RESULT_TABLE_COLUMN_FAMILY, BEFORE_REBUILD_BEYOND_MAXLOOKBACK_INVALID_INDEX_ROW_COUNT_BYTES)) {
+ addBeforeRebuildBeyondMaxLookBackInvalidIndexRowCount(getValue(cell));
} else if (CellUtil.matchingColumn(cell, RESULT_TABLE_COLUMN_FAMILY, BEFORE_REBUILD_INVALID_INDEX_ROW_COUNT_COZ_EXTRA_CELLS_BYTES)) {
addBeforeIndexHasExtraCellsCount(getValue(cell));
} else if (CellUtil.matchingColumn(cell, RESULT_TABLE_COLUMN_FAMILY, BEFORE_REBUILD_INVALID_INDEX_ROW_COUNT_COZ_MISSING_CELLS_BYTES)) {
@@ -458,6 +533,10 @@ public class IndexToolVerificationResult {
addAfterRebuildMissingIndexRowCount(getValue(cell));
} else if (CellUtil.matchingColumn(cell, RESULT_TABLE_COLUMN_FAMILY, AFTER_REBUILD_INVALID_INDEX_ROW_COUNT_BYTES)) {
addAfterRebuildInvalidIndexRowCount(getValue(cell));
+ } else if (CellUtil.matchingColumn(cell, RESULT_TABLE_COLUMN_FAMILY, AFTER_REBUILD_BEYOND_MAXLOOKBACK_MISSING_INDEX_ROW_COUNT_BYTES)) {
+ addAfterRebuildBeyondMaxLookBackMissingIndexRowCount(getValue(cell));
+ } else if (CellUtil.matchingColumn(cell, RESULT_TABLE_COLUMN_FAMILY, AFTER_REBUILD_BEYOND_MAXLOOKBACK_INVALID_INDEX_ROW_COUNT_BYTES)) {
+ addAfterRebuildBeyondMaxLookBackInvalidIndexRowCount(getValue(cell));
} else if (CellUtil.matchingColumn(cell, RESULT_TABLE_COLUMN_FAMILY, AFTER_REBUILD_INVALID_INDEX_ROW_COUNT_COZ_EXTRA_CELLS_BYTES)) {
addAfterIndexHasExtraCellsCount(getValue(cell));
} else if (CellUtil.matchingColumn(cell, RESULT_TABLE_COLUMN_FAMILY, AFTER_REBUILD_INVALID_INDEX_ROW_COUNT_COZ_MISSING_CELLS_BYTES)) {
@@ -466,6 +545,8 @@ public class IndexToolVerificationResult {
}
public boolean isVerificationFailed() {
+ //we don't want to count max look back failures alone as failing an index rebuild job
+ //so we omit them from the below calculation.
if (getAfter().getInvalidIndexRowCount() + getAfter().getMissingIndexRowCount() > 0) {
return true;
}
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/IndexerRegionScanner.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/IndexerRegionScanner.java
index fabda3b..2c55133 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/IndexerRegionScanner.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/IndexerRegionScanner.java
@@ -17,6 +17,7 @@
*/
package org.apache.phoenix.coprocessor;
+import static org.apache.phoenix.hbase.index.write.AbstractParallelWriterIndexCommitter.INDEX_WRITER_KEEP_ALIVE_TIME_CONF_KEY;
import static org.apache.phoenix.query.QueryConstants.AGG_TIMESTAMP;
import static org.apache.phoenix.query.QueryConstants.EMPTY_COLUMN_VALUE_BYTES;
import static org.apache.phoenix.query.QueryConstants.SINGLE_COLUMN;
@@ -35,11 +36,9 @@ import java.util.concurrent.Future;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
-import org.apache.hadoop.hbase.DoNotRetryIOException;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.KeyValue;
-
-import org.apache.hadoop.hbase.KeyValueUtil;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Durability;
+import org.apache.hadoop.hbase.client.Mutation;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.RegionInfo;
import org.apache.hadoop.hbase.client.Result;
@@ -51,16 +50,20 @@ import org.apache.hadoop.hbase.regionserver.Region;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
-
import org.apache.hadoop.hbase.util.Pair;
+import org.apache.phoenix.cache.ServerCacheClient;
import org.apache.phoenix.compile.ScanRanges;
import org.apache.phoenix.filter.SkipScanFilter;
import org.apache.phoenix.hbase.index.ValueGetter;
import org.apache.phoenix.hbase.index.parallel.EarlyExitFailure;
import org.apache.phoenix.hbase.index.parallel.Task;
import org.apache.phoenix.hbase.index.parallel.TaskBatch;
+import org.apache.phoenix.hbase.index.parallel.ThreadPoolBuilder;
+import org.apache.phoenix.hbase.index.parallel.ThreadPoolManager;
+import org.apache.phoenix.hbase.index.parallel.WaitForCompletionTaskRunner;
import org.apache.phoenix.hbase.index.util.GenericKeyValueBuilder;
+import org.apache.phoenix.index.PhoenixIndexCodec;
import org.apache.phoenix.mapreduce.index.IndexTool;
import org.apache.phoenix.mapreduce.index.IndexVerificationResultRepository;
import org.apache.phoenix.query.KeyRange;
@@ -78,15 +81,31 @@ public class IndexerRegionScanner extends GlobalIndexRegionScanner {
private static final Logger LOGGER = LoggerFactory.getLogger(IndexerRegionScanner.class);
protected Map<byte[], Put> indexKeyToDataPutMap;
- protected IndexVerificationResultRepository verificationResultRepository;
+ protected UngroupedAggregateRegionObserver.MutationList mutations;
+ private boolean partialRebuild = false;
+ private final UngroupedAggregateRegionObserver ungroupedAggregateRegionObserver;
IndexerRegionScanner (final RegionScanner innerScanner, final Region region, final Scan scan,
- final RegionCoprocessorEnvironment env) throws IOException {
+ final RegionCoprocessorEnvironment env,
+ UngroupedAggregateRegionObserver ungroupedAggregateRegionObserver) throws IOException {
super(innerScanner, region, scan, env);
- indexKeyToDataPutMap = Maps.newTreeMap(Bytes.BYTES_COMPARATOR);
- verificationResult = new IndexToolVerificationResult(scan);
- verificationResultRepository =
- new IndexVerificationResultRepository(indexMaintainer.getIndexTableName(), hTableFactory);
+ pool = new WaitForCompletionTaskRunner(ThreadPoolManager.getExecutor(
+ new ThreadPoolBuilder("IndexVerify",
+ env.getConfiguration()).setMaxThread(NUM_CONCURRENT_INDEX_VERIFY_THREADS_CONF_KEY,
+ DEFAULT_CONCURRENT_INDEX_VERIFY_THREADS).setCoreTimeout(
+ INDEX_WRITER_KEEP_ALIVE_TIME_CONF_KEY), env));
+ this.ungroupedAggregateRegionObserver = ungroupedAggregateRegionObserver;
+ if (scan.getAttribute(BaseScannerRegionObserver.INDEX_REBUILD_PAGING) == null) {
+ partialRebuild = true;
+ }
+ if (verify) {
+ indexKeyToDataPutMap = Maps.newTreeMap(Bytes.BYTES_COMPARATOR);
+ verificationResult = new IndexToolVerificationResult(scan);
+ verificationResultRepository =
+ new IndexVerificationResultRepository(indexMaintainer.getIndexTableName(), hTableFactory);
+ } else {
+ mutations = new UngroupedAggregateRegionObserver.MutationList(maxBatchSize);
+ }
}
@Override
@@ -101,13 +120,16 @@ public class IndexerRegionScanner extends GlobalIndexRegionScanner {
public void close() throws IOException {
innerScanner.close();
try {
- verificationResultRepository.logToIndexToolResultTable(verificationResult,
- IndexTool.IndexVerifyType.ONLY, region.getRegionInfo().getRegionName());
+ if (verify) {
+ verificationResultRepository.logToIndexToolResultTable(verificationResult,
+ IndexTool.IndexVerifyType.ONLY, region.getRegionInfo().getRegionName());
+ }
} finally {
this.pool.stop("IndexerRegionScanner is closing");
- hTableFactory.shutdown();
- indexHTable.close();
- verificationResultRepository.close();
+ if (verify) {
+ verificationResultRepository.close();
+ }
+ super.close();
}
}
@@ -142,6 +164,7 @@ public class IndexerRegionScanner extends GlobalIndexRegionScanner {
// get a value back from index if it has already expired between our rebuild and
// verify
+ // or if cell timestamp is beyond maxlookback
if (isTimestampBeforeTTL(indexTableTTL, currentTime, expectedCell.getTimestamp())) {
continue;
}
@@ -154,6 +177,12 @@ public class IndexerRegionScanner extends GlobalIndexRegionScanner {
}
// Check all columns
if (!CellUtil.matchingValue(actualCell, expectedCell) || actualCell.getTimestamp() != ts) {
+ if(isTimestampBeyondMaxLookBack(maxLookBackInMills, currentTime, actualCell.getTimestamp())) {
+ verificationPhaseResult
+ .setBeyondMaxLookBackInvalidIndexRowCount(verificationPhaseResult
+ .getBeyondMaxLookBackInvalidIndexRowCount()+1);
+ continue;
+ }
return false;
}
cellCount++;
@@ -204,12 +233,22 @@ public class IndexerRegionScanner extends GlobalIndexRegionScanner {
}
// Check if any expected rows from index(which we didn't get) are beyond max look back and have been compacted away
if (!perTaskDataKeyToDataPutMap.isEmpty()) {
- verificationPhaseResult.setMissingIndexRowCount(
- verificationPhaseResult.getMissingIndexRowCount() + perTaskDataKeyToDataPutMap.size());
+ for (Entry<byte[], Put> entry : perTaskDataKeyToDataPutMap.entrySet()) {
+ Put put = entry.getValue();
+ long ts = getMaxTimestamp(put);
+ long currentTime = EnvironmentEdgeManager.currentTime();
+ if (isTimestampBeyondMaxLookBack(maxLookBackInMills, currentTime, ts)) {
+ verificationPhaseResult.
+ setBeyondMaxLookBackMissingIndexRowCount(verificationPhaseResult.getBeyondMaxLookBackMissingIndexRowCount() + 1);
+ } else {
+ verificationPhaseResult.setMissingIndexRowCount(
+ verificationPhaseResult.getMissingIndexRowCount() + 1);
+ }
+ }
}
}
- private void addVerifyTask(final List<KeyRange> keys, final Map<byte[], Put> perTaskDataKeyToDataPutMap,
+ private void addVerifyTask(TaskBatch<Boolean> tasks, final List<KeyRange> keys, final Map<byte[], Put> perTaskDataKeyToDataPutMap,
final IndexToolVerificationResult.PhaseResult verificationPhaseResult) {
tasks.add(new Task<Boolean>() {
@Override
@@ -230,7 +269,7 @@ public class IndexerRegionScanner extends GlobalIndexRegionScanner {
private void parallelizeIndexVerify(IndexToolVerificationResult.PhaseResult verificationPhaseResult) throws IOException {
int taskCount = (indexKeyToDataPutMap.size() + rowCountPerTask - 1) / rowCountPerTask;
- tasks = new TaskBatch<>(taskCount);
+ TaskBatch<Boolean> tasks = new TaskBatch<>(taskCount);
List<Map<byte[], Put>> dataPutMapList = new ArrayList<>(taskCount);
List<IndexToolVerificationResult.PhaseResult> verificationPhaseResultList = new ArrayList<>(taskCount);
@@ -247,7 +286,7 @@ public class IndexerRegionScanner extends GlobalIndexRegionScanner {
keys.add(PVarbinary.INSTANCE.getKeyRange(entry.getKey()));
perTaskDataKeyToDataPutMap.put(entry.getValue().getRow(), entry.getValue());
if (keys.size() == rowCountPerTask) {
- addVerifyTask(keys, perTaskDataKeyToDataPutMap, perTaskVerificationPhaseResult);
+ addVerifyTask(tasks, keys, perTaskDataKeyToDataPutMap, perTaskVerificationPhaseResult);
keys = new ArrayList<>(rowCountPerTask);
perTaskDataKeyToDataPutMap = Maps.newTreeMap(Bytes.BYTES_COMPARATOR);
dataPutMapList.add(perTaskDataKeyToDataPutMap);
@@ -256,7 +295,7 @@ public class IndexerRegionScanner extends GlobalIndexRegionScanner {
}
}
if (keys.size() > 0) {
- addVerifyTask(keys, perTaskDataKeyToDataPutMap, perTaskVerificationPhaseResult);
+ addVerifyTask(tasks, keys, perTaskDataKeyToDataPutMap, perTaskVerificationPhaseResult);
}
Pair<List<Boolean>, List<Future<Boolean>>> resultsAndFutures = null;
try {
@@ -292,6 +331,26 @@ public class IndexerRegionScanner extends GlobalIndexRegionScanner {
verificationResult.add(nextVerificationResult);
}
+ private void setMutationAttributes(Mutation m, byte[] uuidValue) {
+ m.setAttribute(useProto ? PhoenixIndexCodec.INDEX_PROTO_MD : PhoenixIndexCodec.INDEX_MD, indexMetaData);
+ m.setAttribute(PhoenixIndexCodec.INDEX_UUID, uuidValue);
+ m.setAttribute(BaseScannerRegionObserver.REPLAY_WRITES,
+ BaseScannerRegionObserver.REPLAY_INDEX_REBUILD_WRITES);
+ m.setAttribute(BaseScannerRegionObserver.CLIENT_VERSION, clientVersionBytes);
+ // Since we're replaying existing mutations, it makes no sense to write them to the wal
+ m.setDurability(Durability.SKIP_WAL);
+ }
+
+ private byte[] commitIfReady(byte[] uuidValue, UngroupedAggregateRegionObserver.MutationList mutationList) throws IOException {
+ if (ServerUtil.readyToCommit(mutationList.size(), mutationList.byteSize(), maxBatchSize, maxBatchSizeBytes)) {
+ ungroupedAggregateRegionObserver.checkForRegionClosingOrSplitting();
+ ungroupedAggregateRegionObserver.commitBatchWithRetries(region, mutationList, blockingMemstoreSize);
+ uuidValue = ServerCacheClient.generateId();
+ mutationList.clear();
+ }
+ return uuidValue;
+ }
+
@Override
public boolean next(List<Cell> results) throws IOException {
Cell lastCell = null;
@@ -299,43 +358,79 @@ public class IndexerRegionScanner extends GlobalIndexRegionScanner {
region.startRegionOperation();
try {
synchronized (innerScanner) {
+ byte[] uuidValue = ServerCacheClient.generateId();
do {
List<Cell> row = new ArrayList<>();
hasMore = innerScanner.nextRaw(row);
if (!row.isEmpty()) {
- lastCell = row.get(0);
+ lastCell = row.get(0); // lastCell is any cell from the last visited row
Put put = null;
+ Delete del = null;
for (Cell cell : row) {
- if (KeyValue.Type.codeToType(cell.getTypeByte()) == KeyValue.Type.Put) {
+ if (cell.getType().equals(Cell.Type.Put)) {
+ if (!partialRebuild && familyMap != null && !isColumnIncluded(cell)) {
+ continue;
+ }
if (put == null) {
put = new Put(CellUtil.cloneRow(cell));
}
put.add(cell);
} else {
- throw new DoNotRetryIOException("Scan without raw found a deleted cell");
+ if (del == null) {
+ del = new Delete(CellUtil.cloneRow(cell));
+ }
+ del.add(cell);
}
}
+ if (put == null && del == null) {
+ continue;
+ }
+ if (!verify) {
+ if (put != null) {
+ setMutationAttributes(put, uuidValue);
+ mutations.add(put);
+ }
+ if (del != null) {
+ setMutationAttributes(del, uuidValue);
+ mutations.add(del);
+ }
+ uuidValue = commitIfReady(uuidValue, mutations);
+ } else {
+ indexKeyToDataPutMap
+ .put(getIndexRowKey(indexMaintainer, put), put);
+ }
rowCount++;
- indexKeyToDataPutMap
- .put(getIndexRowKey(indexMaintainer, put), put);
+
}
} while (hasMore && rowCount < pageSizeInRows);
- verifyIndex();
+ if (verify) {
+ verifyIndex();
+ } else if (!mutations.isEmpty()) {
+ ungroupedAggregateRegionObserver.checkForRegionClosingOrSplitting();
+ ungroupedAggregateRegionObserver.commitBatchWithRetries(region, mutations, blockingMemstoreSize);
+ mutations.clear();
+ }
}
} catch (IOException e) {
LOGGER.error(String.format("IOException during rebuilding: %s", Throwables.getStackTraceAsString(e)));
throw e;
} finally {
region.closeRegionOperation();
- indexKeyToDataPutMap.clear();
+ if (verify) {
+ indexKeyToDataPutMap.clear();
+ } else {
+ mutations.clear();
+ }
}
byte[] rowCountBytes = PLong.INSTANCE.toBytes(Long.valueOf(rowCount));
final Cell aggKeyValue;
if (lastCell == null) {
- aggKeyValue = PhoenixKeyValueUtil.newKeyValue(UNGROUPED_AGG_ROW_KEY, SINGLE_COLUMN_FAMILY,
+ aggKeyValue = PhoenixKeyValueUtil.newKeyValue(UNGROUPED_AGG_ROW_KEY,
+ SINGLE_COLUMN_FAMILY,
SINGLE_COLUMN, AGG_TIMESTAMP, rowCountBytes,0, rowCountBytes.length);
} else {
- aggKeyValue = PhoenixKeyValueUtil.newKeyValue(CellUtil.cloneRow(lastCell), SINGLE_COLUMN_FAMILY,
+ aggKeyValue = PhoenixKeyValueUtil.newKeyValue(CellUtil.cloneRow(lastCell),
+ SINGLE_COLUMN_FAMILY,
SINGLE_COLUMN, AGG_TIMESTAMP, rowCountBytes, 0, rowCountBytes.length);
}
results.add(aggKeyValue);
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
index fc1902b..453172b 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
@@ -17,9 +17,6 @@
*/
package org.apache.phoenix.coprocessor;
-import static org.apache.phoenix.hbase.index.IndexRegionObserver.VERIFIED_BYTES;
-import static org.apache.phoenix.hbase.index.IndexRegionObserver.removeEmptyColumn;
-import static org.apache.phoenix.hbase.index.write.AbstractParallelWriterIndexCommitter.INDEX_WRITER_KEEP_ALIVE_TIME_CONF_KEY;
import static org.apache.phoenix.query.QueryConstants.AGG_TIMESTAMP;
import static org.apache.phoenix.query.QueryConstants.SINGLE_COLUMN;
import static org.apache.phoenix.query.QueryConstants.SINGLE_COLUMN_FAMILY;
@@ -32,7 +29,6 @@ import static org.apache.phoenix.schema.stats.StatisticsCollectionRunTracker.CON
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
-import java.io.Closeable;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
@@ -41,34 +37,29 @@ import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
-import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutionException;
import javax.annotation.concurrent.GuardedBy;
-import com.google.common.collect.Maps;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellBuilderFactory;
+import org.apache.hadoop.hbase.CellBuilderType;
import org.apache.hadoop.hbase.CoprocessorEnvironment;
import org.apache.hadoop.hbase.DoNotRetryIOException;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.KeepDeletedCells;
-import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.NamespaceDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.Delete;
-import org.apache.hadoop.hbase.client.Durability;
import org.apache.hadoop.hbase.client.Mutation;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.RegionInfo;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.TableDescriptor;
@@ -95,7 +86,6 @@ import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.io.WritableUtils;
import org.apache.phoenix.cache.GlobalCache;
import org.apache.phoenix.cache.TenantCache;
-import org.apache.phoenix.compile.ScanRanges;
import org.apache.phoenix.coprocessor.generated.PTableProtos;
import org.apache.phoenix.exception.DataExceedsCapacityException;
import org.apache.phoenix.exception.SQLExceptionCode;
@@ -106,19 +96,10 @@ import org.apache.phoenix.expression.aggregator.Aggregator;
import org.apache.phoenix.expression.aggregator.Aggregators;
import org.apache.phoenix.expression.aggregator.ServerAggregators;
import org.apache.phoenix.filter.AllVersionsIndexRebuildFilter;
-import org.apache.phoenix.filter.SkipScanFilter;
import org.apache.phoenix.hbase.index.Indexer;
import org.apache.phoenix.hbase.index.ValueGetter;
import org.apache.phoenix.hbase.index.covered.update.ColumnReference;
import org.apache.phoenix.hbase.index.exception.IndexWriteException;
-import org.apache.phoenix.hbase.index.parallel.EarlyExitFailure;
-import org.apache.phoenix.hbase.index.parallel.Task;
-import org.apache.phoenix.hbase.index.parallel.TaskBatch;
-import org.apache.phoenix.hbase.index.parallel.TaskRunner;
-import org.apache.phoenix.hbase.index.parallel.ThreadPoolBuilder;
-import org.apache.phoenix.hbase.index.parallel.ThreadPoolManager;
-import org.apache.phoenix.hbase.index.parallel.WaitForCompletionTaskRunner;
-import org.apache.phoenix.hbase.index.table.HTableFactory;
import org.apache.phoenix.hbase.index.util.GenericKeyValueBuilder;
import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
import org.apache.phoenix.hbase.index.util.KeyValueBuilder;
@@ -132,7 +113,6 @@ import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
import org.apache.phoenix.join.HashJoinInfo;
import org.apache.phoenix.mapreduce.index.IndexTool;
import org.apache.phoenix.memory.MemoryManager.MemoryChunk;
-import org.apache.phoenix.query.KeyRange;
import org.apache.phoenix.query.QueryConstants;
import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.query.QueryServicesOptions;
@@ -151,6 +131,7 @@ import org.apache.phoenix.schema.stats.NoOpStatisticsCollector;
import org.apache.phoenix.schema.stats.StatisticsCollectionRunTracker;
import org.apache.phoenix.schema.stats.StatisticsCollector;
import org.apache.phoenix.schema.stats.StatisticsCollectorFactory;
+
import org.apache.phoenix.schema.stats.StatsCollectionDisabledOnServerException;
import org.apache.phoenix.schema.tuple.EncodedColumnQualiferCellsList;
import org.apache.phoenix.schema.tuple.MultiKeyValueTuple;
@@ -162,7 +143,6 @@ import org.apache.phoenix.schema.types.PDataType;
import org.apache.phoenix.schema.types.PDouble;
import org.apache.phoenix.schema.types.PFloat;
import org.apache.phoenix.schema.types.PLong;
-import org.apache.phoenix.schema.types.PVarbinary;
import org.apache.phoenix.transaction.PhoenixTransactionContext;
import org.apache.phoenix.transaction.PhoenixTransactionProvider;
import org.apache.phoenix.transaction.TransactionFactory;
@@ -171,8 +151,8 @@ import org.apache.phoenix.util.EncodedColumnsUtil;
import org.apache.phoenix.util.EnvironmentEdgeManager;
import org.apache.phoenix.util.ExpressionUtil;
import org.apache.phoenix.util.IndexUtil;
-import org.apache.phoenix.util.LogUtil;
import org.apache.phoenix.util.PhoenixKeyValueUtil;
+import org.apache.phoenix.util.LogUtil;
import org.apache.phoenix.util.PhoenixRuntime;
import org.apache.phoenix.util.PropertiesUtil;
import org.apache.phoenix.util.QueryUtil;
@@ -244,10 +224,9 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver
@Override
public Optional<RegionObserver> getRegionObserver() {
- return Optional.of(this);
+ return Optional.of(this);
}
-
@Override
public void start(CoprocessorEnvironment e) throws IOException {
// Can't use ClientKeyValueBuilder on server-side because the memstore expects to
@@ -300,11 +279,10 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver
return;
}
- Mutation[] mutationArray = new Mutation[mutations.size()];
+ Mutation[] mutationArray = new Mutation[mutations.size()];
// When memstore size reaches blockingMemstoreSize we are waiting 3 seconds for the
// flush happen which decrease the memstore size and then writes allowed on the region.
- for (int i = 0; blockingMemstoreSize > 0 && (region.getMemStoreHeapSize() + region.getMemStoreOffHeapSize()) > blockingMemstoreSize
- && i < 30; i++) {
+ for (int i = 0; blockingMemstoreSize > 0 && region.getMemStoreHeapSize() > blockingMemstoreSize && i < 30; i++) {
try {
checkForRegionClosingOrSplitting();
Thread.sleep(100);
@@ -319,8 +297,8 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver
}
private void setIndexAndTransactionProperties(List<Mutation> mutations, byte[] indexUUID,
- byte[] indexMaintainersPtr, byte[] txState, byte[] clientVersionBytes,
- boolean useIndexProto) {
+ byte[] indexMaintainersPtr, byte[] txState,
+ byte[] clientVersionBytes, boolean useIndexProto) {
for (Mutation m : mutations) {
if (indexMaintainersPtr != null) {
m.setAttribute(useIndexProto ? PhoenixIndexCodec.INDEX_PROTO_MD : PhoenixIndexCodec.INDEX_MD, indexMaintainersPtr);
@@ -337,14 +315,15 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver
}
}
- private void commitBatchWithHTable(Table table, List<Mutation> mutations) throws IOException {
+ private void commitBatchWithTable(Table table, List<Mutation> mutations) throws IOException {
if (mutations.isEmpty()) {
return;
}
LOGGER.debug("Committing batch of " + mutations.size() + " mutations for " + table);
try {
- table.batch(mutations, null);
+ Object[] results = new Object[mutations.size()];
+ table.batch(mutations, results);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw new RuntimeException(e);
@@ -371,9 +350,9 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver
}
@Override
- public void preScannerOpen(org.apache.hadoop.hbase.coprocessor.ObserverContext<RegionCoprocessorEnvironment> c,
- Scan scan) throws IOException {
- super.preScannerOpen(c, scan);
+ public void preScannerOpen(ObserverContext<RegionCoprocessorEnvironment> e, Scan scan)
+ throws IOException {
+ super.preScannerOpen(e, scan);
if (ScanUtil.isAnalyzeTable(scan)) {
// We are setting the start row and stop row such that it covers the entire region. As part
// of Phonenix-1263 we are storing the guideposts against the physical table rather than
@@ -384,8 +363,8 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver
}
}
- public static class MutationList extends ArrayList<Mutation> {
- private long byteSize = 0l;
+ public static class MutationList extends ArrayList<Mutation> {
+ private long byteSize = 0L;
public MutationList() {
super();
}
@@ -409,22 +388,22 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver
@Override
public void clear() {
- byteSize = 0l;
+ byteSize = 0L;
super.clear();
}
}
public static long getBlockingMemstoreSize(Region region, Configuration conf) {
- long flushSize = region.getTableDescriptor().getMemStoreFlushSize();
-
- if (flushSize <= 0) {
- flushSize = conf.getLong(HConstants.HREGION_MEMSTORE_FLUSH_SIZE,
- TableDescriptorBuilder.DEFAULT_MEMSTORE_FLUSH_SIZE);
- }
- return flushSize * (conf.getLong(HConstants.HREGION_MEMSTORE_BLOCK_MULTIPLIER,
- HConstants.DEFAULT_HREGION_MEMSTORE_BLOCK_MULTIPLIER)-1) ;
- }
-
+ long flushSize = region.getTableDescriptor().getMemStoreFlushSize();
+
+ if (flushSize <= 0) {
+ flushSize = conf.getLong(HConstants.HREGION_MEMSTORE_FLUSH_SIZE,
+ TableDescriptorBuilder.DEFAULT_MEMSTORE_FLUSH_SIZE);
+ }
+ return flushSize * (conf.getLong(HConstants.HREGION_MEMSTORE_BLOCK_MULTIPLIER,
+ HConstants.DEFAULT_HREGION_MEMSTORE_BLOCK_MULTIPLIER)-1);
+ }
+
@Override
protected RegionScanner doPostScannerOpen(final ObserverContext<RegionCoprocessorEnvironment> c, final Scan scan, final RegionScanner s) throws IOException, SQLException {
final RegionCoprocessorEnvironment env = c.getEnvironment();
@@ -513,8 +492,8 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver
byte[] deleteCQ = null;
byte[] deleteCF = null;
byte[] emptyCF = null;
- Table targetHTable = null;
Connection targetHConn = null;
+ Table targetHTable = null;
boolean isPKChanging = false;
ImmutableBytesWritable ptr = new ImmutableBytesWritable();
if (upsertSelectTable != null) {
@@ -522,7 +501,7 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver
projectedTable = deserializeTable(upsertSelectTable);
targetHConn = ConnectionFactory.createConnection(upsertSelectConfig);
targetHTable = targetHConn.getTable(
- TableName.valueOf(projectedTable.getPhysicalName().getBytes()));
+ TableName.valueOf(projectedTable.getPhysicalName().getBytes()));
selectExpressions = deserializeExpressions(scan.getAttribute(BaseScannerRegionObserver.UPSERT_SELECT_EXPRS));
values = new byte[projectedTable.getPKColumns().size()][];
isPKChanging = ExpressionUtil.isPkPositionChanging(new TableRef(projectedTable), selectExpressions);
@@ -568,17 +547,17 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver
* bytes. This avoids flush storm to hdfs for cases like index building where reads and
* write happen to all the table regions in the server.
*/
- final long blockingMemStoreSize = getBlockingMemstoreSize(region, conf);
+ final long blockingMemStoreSize = getBlockingMemstoreSize(region, conf) ;
boolean buildLocalIndex = indexMaintainers != null && dataColumns==null && !localIndexScan;
if(buildLocalIndex) {
checkForLocalIndexColumnFamilies(region, indexMaintainers);
}
- if (isDescRowKeyOrderUpgrade || isDelete || isUpsert ||
- (deleteCQ != null && deleteCF != null) || emptyCF != null || buildLocalIndex) {
+ if (isDescRowKeyOrderUpgrade || isDelete || isUpsert
+ || (deleteCQ != null && deleteCF != null) || emptyCF != null || buildLocalIndex) {
needToWrite = true;
- if(isUpsert && (targetHTable == null ||
- !targetHTable.getName().equals(region.getTableDescriptor().getTableName()))) {
+ if((isUpsert && (targetHTable == null ||
+ !targetHTable.getName().equals(region.getTableDescriptor().getTableName())))) {
needToWrite = false;
}
maxBatchSize = conf.getInt(MUTATE_BATCH_SIZE_ATTRIB, QueryServicesOptions.DEFAULT_MUTATE_BATCH_SIZE);
@@ -685,27 +664,38 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver
byte[] newRow = ByteUtil.copyKeyBytesIfNecessary(ptr);
if (offset > 0) { // for local indexes (prepend region start key)
byte[] newRowWithOffset = new byte[offset + newRow.length];
- System.arraycopy(firstKV.getRowArray(), firstKV.getRowOffset(), newRowWithOffset, 0, offset);;
+ System.arraycopy(firstKV.getRowArray(), firstKV.getRowOffset(), newRowWithOffset, 0, offset);
System.arraycopy(newRow, 0, newRowWithOffset, offset, newRow.length);
newRow = newRowWithOffset;
}
byte[] oldRow = Bytes.copy(firstKV.getRowArray(), firstKV.getRowOffset(), firstKV.getRowLength());
for (Cell cell : results) {
// Copy existing cell but with new row key
- Cell newCell = new KeyValue(newRow, 0, newRow.length,
- cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength(),
- cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength(),
- cell.getTimestamp(), KeyValue.Type.codeToType(cell.getTypeByte()),
- cell.getValueArray(), cell.getValueOffset(), cell.getValueLength());
- switch (KeyValue.Type.codeToType(cell.getTypeByte())) {
+ Cell newCell =
+ CellBuilderFactory.create(CellBuilderType.DEEP_COPY).
+ setRow(newRow).
+ setFamily(cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength()).
+ setQualifier(cell.getQualifierArray(),
+ cell.getQualifierOffset(), cell.getQualifierLength()).
+ setTimestamp(cell.getTimestamp()).
+ setType(cell.getType()).setValue(cell.getValueArray(),
+ cell.getValueOffset(), cell.getValueLength()).build();
+ switch (cell.getType()) {
case Put:
// If Put, point delete old Put
Delete del = new Delete(oldRow);
- del.addDeleteMarker(new KeyValue(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength(),
- cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength(),
- cell.getQualifierArray(), cell.getQualifierOffset(),
- cell.getQualifierLength(), cell.getTimestamp(), KeyValue.Type.Delete,
- ByteUtil.EMPTY_BYTE_ARRAY, 0, 0));
+ Cell newDelCell =
+ CellBuilderFactory.create(CellBuilderType.DEEP_COPY).
+ setRow(newRow).
+ setFamily(cell.getFamilyArray(), cell.getFamilyOffset(),
+ cell.getFamilyLength()).
+ setQualifier(cell.getQualifierArray(),
+ cell.getQualifierOffset(), cell.getQualifierLength()).
+ setTimestamp(cell.getTimestamp()).
+ setType(Cell.Type.Delete).
+ setValue(ByteUtil.EMPTY_BYTE_ARRAY,
+ 0, 0).build();
+ del.add(newDelCell);
mutations.add(del);
Put put = new Put(newRow);
@@ -717,7 +707,7 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver
case DeleteFamily:
case DeleteFamilyVersion:
Delete delete = new Delete(newRow);
- delete.addDeleteMarker(newCell);
+ delete.add(newCell);
mutations.add(delete);
break;
}
@@ -889,8 +879,20 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver
}
}
try {
- tryClosingResourceSilently(targetHTable);
- tryClosingResourceSilently(targetHConn);
+ if (targetHTable != null) {
+ try {
+ targetHTable.close();
+ } catch (IOException e) {
+ LOGGER.error("Closing table: " + targetHTable + " failed: ", e);
+ }
+ }
+ if (targetHConn != null) {
+ try {
+ targetHConn.close();
+ } catch (IOException e) {
+ LOGGER.error("Closing connection: " + targetHConn + " failed: ", e);
+ }
+ }
} finally {
try {
innerScanner.close();
@@ -907,7 +909,8 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver
Cell keyValue = null;
if (hadAny) {
byte[] value = aggregators.toBytes(rowAggregators);
- keyValue = PhoenixKeyValueUtil.newKeyValue(UNGROUPED_AGG_ROW_KEY, SINGLE_COLUMN_FAMILY, SINGLE_COLUMN, AGG_TIMESTAMP, value, 0, value.length);
+ keyValue = PhoenixKeyValueUtil.newKeyValue(UNGROUPED_AGG_ROW_KEY, SINGLE_COLUMN_FAMILY,
+ SINGLE_COLUMN, AGG_TIMESTAMP, value, 0, value.length);
}
final Cell aggKeyValue = keyValue;
@@ -936,16 +939,6 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver
}
- private static void tryClosingResourceSilently(Closeable res) {
- if (res != null) {
- try {
- res.close();
- } catch (IOException e) {
- LOGGER.error("Closing resource: " + res + " failed: ", e);
- }
- }
- }
-
private void checkForLocalIndexColumnFamilies(Region region,
List<IndexMaintainer> indexMaintainers) throws IOException {
TableDescriptor tableDesc = region.getTableDescriptor();
@@ -977,67 +970,66 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver
}
}
- private void commit(Region region, List<Mutation> mutations, byte[] indexUUID,
- long blockingMemStoreSize, byte[] indexMaintainersPtr, byte[] txState,
- Table targetHTable, boolean useIndexProto, boolean isPKChanging,
- byte[] clientVersionBytes) throws IOException {
- List<Mutation> localRegionMutations = Lists.newArrayList();
- List<Mutation> remoteRegionMutations = Lists.newArrayList();
- setIndexAndTransactionProperties(mutations, indexUUID, indexMaintainersPtr, txState,
- clientVersionBytes, useIndexProto);
+ private void commit(final Region region, List<Mutation> mutations, byte[] indexUUID, final long blockingMemStoreSize,
+ byte[] indexMaintainersPtr, byte[] txState, final Table targetHTable, boolean useIndexProto,
+ boolean isPKChanging, byte[] clientVersionBytes)
+ throws IOException {
+ final List<Mutation> localRegionMutations = Lists.newArrayList();
+ final List<Mutation> remoteRegionMutations = Lists.newArrayList();
+ setIndexAndTransactionProperties(mutations, indexUUID, indexMaintainersPtr, txState, clientVersionBytes, useIndexProto);
separateLocalAndRemoteMutations(targetHTable, region, mutations, localRegionMutations, remoteRegionMutations,
isPKChanging);
commitBatchWithRetries(region, localRegionMutations, blockingMemStoreSize);
- try {
- commitBatchWithHTable(targetHTable, remoteRegionMutations);
- } catch (IOException e) {
- handleIndexWriteException(remoteRegionMutations, e, new MutateCommand() {
- @Override
- public void doMutation() throws IOException {
- commitBatchWithHTable(targetHTable, remoteRegionMutations);
- }
+ try {
+ commitBatchWithTable(targetHTable, remoteRegionMutations);
+ } catch (IOException e) {
+ handleIndexWriteException(remoteRegionMutations, e, new MutateCommand() {
+ @Override
+ public void doMutation() throws IOException {
+ commitBatchWithTable(targetHTable, remoteRegionMutations);
+ }
@Override
public List<Mutation> getMutationList() {
return remoteRegionMutations;
}
- });
- }
+ });
+ }
localRegionMutations.clear();
remoteRegionMutations.clear();
}
private void handleIndexWriteException(final List<Mutation> localRegionMutations, IOException origIOE,
- MutateCommand mutateCommand) throws IOException {
- long serverTimestamp = ServerUtil.parseTimestampFromRemoteException(origIOE);
- SQLException inferredE = ServerUtil.parseLocalOrRemoteServerException(origIOE);
- if (inferredE != null && inferredE.getErrorCode() == SQLExceptionCode.INDEX_WRITE_FAILURE.getErrorCode()) {
- // For an index write failure, the data table write succeeded,
- // so when we retry we need to set REPLAY_WRITES
- for (Mutation mutation : localRegionMutations) {
- if (PhoenixIndexMetaData.isIndexRebuild(mutation.getAttributesMap())) {
- mutation.setAttribute(BaseScannerRegionObserver.REPLAY_WRITES,
- BaseScannerRegionObserver.REPLAY_INDEX_REBUILD_WRITES);
- } else {
- mutation.setAttribute(BaseScannerRegionObserver.REPLAY_WRITES,
- BaseScannerRegionObserver.REPLAY_ONLY_INDEX_WRITES);
- }
- // use the server timestamp for index write retries
- PhoenixKeyValueUtil.setTimestamp(mutation, serverTimestamp);
- }
- IndexWriteException iwe = PhoenixIndexFailurePolicy.getIndexWriteException(inferredE);
- try (PhoenixConnection conn =
- QueryUtil.getConnectionOnServer(indexWriteConfig)
- .unwrap(PhoenixConnection.class)) {
- PhoenixIndexFailurePolicy.doBatchWithRetries(mutateCommand, iwe, conn,
- indexWriteProps);
- } catch (Exception e) {
- throw new DoNotRetryIOException(e);
- }
- } else {
- throw origIOE;
- }
- }
+ MutateCommand mutateCommand) throws IOException {
+ long serverTimestamp = ServerUtil.parseTimestampFromRemoteException(origIOE);
+ SQLException inferredE = ServerUtil.parseLocalOrRemoteServerException(origIOE);
+ if (inferredE != null && inferredE.getErrorCode() == SQLExceptionCode.INDEX_WRITE_FAILURE.getErrorCode()) {
+ // For an index write failure, the data table write succeeded,
+ // so when we retry we need to set REPLAY_WRITES
+ for (Mutation mutation : localRegionMutations) {
+ if (PhoenixIndexMetaData.isIndexRebuild(mutation.getAttributesMap())) {
+ mutation.setAttribute(BaseScannerRegionObserver.REPLAY_WRITES,
+ BaseScannerRegionObserver.REPLAY_INDEX_REBUILD_WRITES);
+ } else {
+ mutation.setAttribute(BaseScannerRegionObserver.REPLAY_WRITES,
+ BaseScannerRegionObserver.REPLAY_ONLY_INDEX_WRITES);
+ }
+ // use the server timestamp for index write retrys
+ PhoenixKeyValueUtil.setTimestamp(mutation, serverTimestamp);
+ }
+ IndexWriteException iwe = PhoenixIndexFailurePolicy.getIndexWriteException(inferredE);
+ try (PhoenixConnection conn =
+ QueryUtil.getConnectionOnServer(indexWriteConfig)
+ .unwrap(PhoenixConnection.class)) {
+ PhoenixIndexFailurePolicy.doBatchWithRetries(mutateCommand, iwe, conn,
+ indexWriteProps);
+ } catch (Exception e) {
+ throw new DoNotRetryIOException(e);
+ }
+ } else {
+ throw origIOE;
+ }
+ }
private void separateLocalAndRemoteMutations(Table targetHTable, Region region, List<Mutation> mutations,
List<Mutation> localRegionMutations, List<Mutation> remoteRegionMutations,
@@ -1067,45 +1059,39 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver
}
@Override
- public InternalScanner preCompact(
- org.apache.hadoop.hbase.coprocessor.ObserverContext<RegionCoprocessorEnvironment> c, Store store,
- InternalScanner scanner, ScanType scanType,
- org.apache.hadoop.hbase.regionserver.compactions.CompactionLifeCycleTracker tracker,
- CompactionRequest request) throws IOException {
+ public InternalScanner preCompact(ObserverContext<RegionCoprocessorEnvironment> c, Store store,
+ InternalScanner scanner, ScanType scanType, CompactionLifeCycleTracker tracker,
+ CompactionRequest request)
+ throws IOException {
if (scanType.equals(ScanType.COMPACT_DROP_DELETES)) {
final TableName table = c.getEnvironment().getRegion().getRegionInfo().getTable();
- // Compaction and split upcalls run with the effective user context of the requesting user.
- // This will lead to failure of cross cluster RPC if the effective user is not
- // the login user. Switch to the login user context to ensure we have the expected
- // security context.
- return User.runAsLoginUser(
- new PrivilegedExceptionAction<InternalScanner>() {
- @Override
- public InternalScanner run() throws Exception {
- InternalScanner internalScanner = scanner;
- try {
- long clientTimeStamp = EnvironmentEdgeManager.currentTimeMillis();
- DelegateRegionCoprocessorEnvironment compactionConfEnv =
- new DelegateRegionCoprocessorEnvironment(
- c.getEnvironment(), ConnectionType.COMPACTION_CONNECTION);
- StatisticsCollector statisticsCollector =
- StatisticsCollectorFactory.createStatisticsCollector(
- compactionConfEnv,
- table.getNameAsString(),
- clientTimeStamp,
- store.getColumnFamilyDescriptor().getName());
- statisticsCollector.init();
- internalScanner = statisticsCollector.createCompactionScanner(compactionConfEnv, store, scanner);
- } catch (Exception e) {
- // If we can't reach the stats table, don't interrupt the normal
- // compaction operation, just log a warning.
- if (LOGGER.isWarnEnabled()) {
- LOGGER.warn("Unable to collect stats for " + table, e);
+ // Compaction and split upcalls run with the effective user context of the requesting user.
+ // This will lead to failure of cross cluster RPC if the effective user is not
+ // the login user. Switch to the login user context to ensure we have the expected
+ // security context.
+ return User.runAsLoginUser(new PrivilegedExceptionAction<InternalScanner>() {
+ @Override public InternalScanner run() throws Exception {
+ InternalScanner internalScanner = scanner;
+ try {
+ long clientTimeStamp = EnvironmentEdgeManager.currentTimeMillis();
+ DelegateRegionCoprocessorEnvironment compactionConfEnv =
+ new DelegateRegionCoprocessorEnvironment(c.getEnvironment(),
+ ConnectionType.COMPACTION_CONNECTION);
+ StatisticsCollector statisticsCollector = StatisticsCollectorFactory.createStatisticsCollector(
+ compactionConfEnv, table.getNameAsString(), clientTimeStamp,
+ store.getColumnFamilyDescriptor().getName());
+ statisticsCollector.init();
+ internalScanner = statisticsCollector.createCompactionScanner(compactionConfEnv, store, internalScanner);
+ } catch (Exception e) {
+ // If we can't reach the stats table, don't interrupt the normal
+ // compaction operation, just log a warning.
+ if (LOGGER.isWarnEnabled()) {
+ LOGGER.warn("Unable to collect stats for " + table, e);
+ }
+ }
+ return internalScanner;
}
- }
- return internalScanner;
- }
- });
+ });
}
return scanner;
}
@@ -1124,14 +1110,14 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver
boolean oldCoproc = region.getTableDescriptor().hasCoprocessor(Indexer.class.getCanonicalName());
byte[] valueBytes = scan.getAttribute(BaseScannerRegionObserver.INDEX_REBUILD_VERIFY_TYPE);
IndexTool.IndexVerifyType verifyType = (valueBytes != null) ?
- IndexTool.IndexVerifyType.fromValue(valueBytes):IndexTool.IndexVerifyType.NONE;
- if(oldCoproc && verifyType == IndexTool.IndexVerifyType.ONLY) {
- return new IndexerRegionScanner(innerScanner, region, scan, env);
+ IndexTool.IndexVerifyType.fromValue(valueBytes) : IndexTool.IndexVerifyType.NONE;
+ if (oldCoproc && verifyType == IndexTool.IndexVerifyType.ONLY) {
+ return new IndexerRegionScanner(innerScanner, region, scan, env, this);
}
if (!scan.isRaw()) {
Scan rawScan = new Scan(scan);
rawScan.setRaw(true);
- rawScan.setMaxVersions();
+ rawScan.readAllVersions();
rawScan.getFamilyMap().clear();
// For rebuilds we use count (*) as query for regular tables which ends up setting the FKOF on scan
// This filter doesn't give us all columns and skips to the next row as soon as it finds 1 col
@@ -1148,9 +1134,17 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver
}
innerScanner.close();
RegionScanner scanner = region.getScanner(rawScan);
- return new IndexRebuildRegionScanner(scanner, region, scan, env, this);
+ if (oldCoproc) {
+ return new IndexerRegionScanner(scanner, region, scan, env, this);
+ } else {
+ return new IndexRebuildRegionScanner(scanner, region, scan, env, this);
+ }
+ }
+ if (oldCoproc) {
+ return new IndexerRegionScanner(innerScanner, region, scan, env, this);
+ } else {
+ return new IndexRebuildRegionScanner(innerScanner, region, scan, env, this);
}
- return new IndexRebuildRegionScanner(innerScanner, region, scan, env, this);
}
private RegionScanner collectStats(final RegionScanner innerScanner, StatisticsCollector stats,
@@ -1260,7 +1254,7 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver
statsCollector.init();
synchronized (innerScanner) {
do {
- List<Cell> results = new ArrayList<Cell>();
+ List<Cell> results = new ArrayList<>();
hasMore = innerScanner.nextRaw(results);
statsCollector.collectStatistics(results);
rowCount++;
@@ -1287,7 +1281,7 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver
}
} finally {
try {
- StatisticsCollectionRunTracker.getInstance(config).removeUpdateStatsCommandRegion(region.getRegionInfo(),scan.getFamilyMap().keySet());
+ StatisticsCollectionRunTracker.getInstance(config).removeUpdateStatsCommandRegion(region.getRegionInfo(), scan.getFamilyMap().keySet());
statsCollector.close();
} finally {
try {
@@ -1352,6 +1346,32 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver
}
}
+ // Don't allow splitting/closing if operations need read and write to same region are going on in the
+ // the coprocessors to avoid dead lock scenario. See PHOENIX-3111.
+ private void waitForScansToFinish(ObserverContext<RegionCoprocessorEnvironment> c) throws IOException {
+ int maxWaitTime = c.getEnvironment().getConfiguration().getInt(HConstants.HBASE_CLIENT_OPERATION_TIMEOUT,
+ HConstants.DEFAULT_HBASE_CLIENT_OPERATION_TIMEOUT);
+ long start = EnvironmentEdgeManager.currentTimeMillis();
+ synchronized (lock) {
+ isRegionClosingOrSplitting = true;
+ while (scansReferenceCount > 0) {
+ try {
+ lock.wait(1000);
+ if (EnvironmentEdgeManager.currentTimeMillis() - start >= maxWaitTime) {
+ isRegionClosingOrSplitting = false; // must reset in case split is not retried
+ throw new IOException(String.format(
+ "Operations like local index building/delete/upsert select"
+ + " might be going on so not allowing to split/close. scansReferenceCount=%s region=%s",
+ scansReferenceCount,
+ c.getEnvironment().getRegionInfo().getRegionNameAsString()));
+ }
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ }
+ }
+ }
+
@Override
public void preBulkLoadHFile(ObserverContext<RegionCoprocessorEnvironment> c,
List<Pair<byte[], String>> familyPaths) throws IOException {
@@ -1368,16 +1388,7 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver
@Override
public void preClose(ObserverContext<RegionCoprocessorEnvironment> c, boolean abortRequested)
throws IOException {
- synchronized (lock) {
- isRegionClosingOrSplitting = true;
- while (scansReferenceCount > 0) {
- try {
- lock.wait(1000);
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- }
- }
- }
+ waitForScansToFinish(c);
}
@Override
@@ -1387,8 +1398,8 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver
@Override
public void preCompactScannerOpen(final ObserverContext<RegionCoprocessorEnvironment> c, final Store store,
- ScanType scanType, ScanOptions options, CompactionLifeCycleTracker tracker,
- final CompactionRequest request) throws IOException {
+ ScanType scanType, ScanOptions options, CompactionLifeCycleTracker tracker,
+ final CompactionRequest request) throws IOException {
// Compaction and split upcalls run with the effective user context of the requesting user.
// This will lead to failure of cross cluster RPC if the effective user is not
// the login user. Switch to the login user context to ensure we have the expected
@@ -1401,7 +1412,7 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver
public Void run() throws Exception {
// If the index is disabled, keep the deleted cells so the rebuild doesn't corrupt the index
try (PhoenixConnection conn =
- QueryUtil.getConnectionOnServer(compactionConfig).unwrap(PhoenixConnection.class)) {
+ QueryUtil.getConnectionOnServer(compactionConfig).unwrap(PhoenixConnection.class)) {
PTable table = PhoenixRuntime.getTableNoCache(conn, fullTableName);
List<PTable> indexes = PTableType.INDEX.equals(table.getType()) ? Lists.newArrayList(table) : table.getIndexes();
// FIXME need to handle views and indexes on views as well
@@ -1409,7 +1420,7 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver
if (index.getIndexDisableTimestamp() != 0) {
LOGGER.info(
"Modifying major compaction scanner to retain deleted cells for a table with disabled index: "
- + fullTableName);
+ + fullTableName);
options.setKeepDeletedCells(KeepDeletedCells.TRUE);
options.readAllVersions();
options.setTTL(Long.MAX_VALUE);
@@ -1422,7 +1433,7 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver
} else {
LOGGER.error("Unable to modify compaction scanner to retain deleted cells for a table with disabled Index; "
+ fullTableName,
- e);
+ e);
}
}
return null;
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/index/GlobalIndexChecker.java b/phoenix-core/src/main/java/org/apache/phoenix/index/GlobalIndexChecker.java
index 6c4a054..2645ca3 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/index/GlobalIndexChecker.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/index/GlobalIndexChecker.java
@@ -42,10 +42,14 @@ import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
import org.apache.hadoop.hbase.coprocessor.ObserverContext;
import org.apache.hadoop.hbase.coprocessor.RegionCoprocessor;
import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
import org.apache.hadoop.hbase.coprocessor.RegionObserver;
+import org.apache.hadoop.hbase.filter.Filter;
+import org.apache.hadoop.hbase.filter.FilterList;
+import org.apache.hadoop.hbase.filter.PageFilter;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.regionserver.Region;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
@@ -88,9 +92,12 @@ import org.slf4j.LoggerFactory;
* the verified version that is masked by the unverified version(s).
*
*/
-public class GlobalIndexChecker implements RegionCoprocessor, RegionObserver {
- private static final Logger LOG = LoggerFactory.getLogger(GlobalIndexChecker.class);
+public class GlobalIndexChecker extends BaseRegionObserver implements RegionCoprocessor, RegionObserver {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(GlobalIndexChecker.class);
private GlobalIndexCheckerSource metricsSource;
+ private CoprocessorEnvironment env;
+
public enum RebuildReturnCode {
NO_DATA_ROW(0),
NO_INDEX_ROW(1),
@@ -130,7 +137,10 @@ public class GlobalIndexChecker implements RegionCoprocessor, RegionObserver {
private long minTimestamp;
private long maxTimestamp;
private GlobalIndexCheckerSource metricsSource;
-
+ private long rowCount = 0;
+ private long pageSize = Long.MAX_VALUE;
+ private boolean restartScanDueToPageFilterRemoval = false;
+ private boolean hasMore;
public GlobalIndexScanner(RegionCoprocessorEnvironment env,
Scan scan,
RegionScanner scanner,
@@ -169,12 +179,14 @@ public class GlobalIndexChecker implements RegionCoprocessor, RegionObserver {
return scanner.getMaxResultSize();
}
- @Override
- public boolean next(List<Cell> result) throws IOException {
+ public boolean next(List<Cell> result, boolean raw) throws IOException {
try {
- boolean hasMore;
do {
- hasMore = scanner.next(result);
+ if (raw) {
+ hasMore = scanner.nextRaw(result);
+ } else {
+ hasMore = scanner.next(result);
+ }
if (result.isEmpty()) {
break;
}
@@ -184,6 +196,10 @@ public class GlobalIndexChecker implements RegionCoprocessor, RegionObserver {
// skip this row as it is invalid
// if there is no more row, then result will be an empty list
} while (hasMore);
+ rowCount++;
+ if (rowCount == pageSize) {
+ return false;
+ }
return hasMore;
} catch (Throwable t) {
ServerUtil.throwIOException(region.getRegionInfo().getRegionNameAsString(), t);
@@ -192,6 +208,16 @@ public class GlobalIndexChecker implements RegionCoprocessor, RegionObserver {
}
@Override
+ public boolean next(List<Cell> result) throws IOException {
+ return next(result, false);
+ }
+
+ @Override
+ public boolean nextRaw(List<Cell> result) throws IOException {
+ return next(result, true);
+ }
+
+ @Override
public boolean next(List<Cell> result, ScannerContext scannerContext) throws IOException {
throw new IOException("next with scannerContext should not be called in Phoenix environment");
}
@@ -229,27 +255,6 @@ public class GlobalIndexChecker implements RegionCoprocessor, RegionObserver {
return scanner.getMvccReadPoint();
}
- @Override
- public boolean nextRaw(List<Cell> result) throws IOException {
- try {
- boolean hasMore;
- do {
- hasMore = scanner.nextRaw(result);
- if (result.isEmpty()) {
- break;
- }
- if (verifyRowAndRepairIfNecessary(result)) {
- break;
- }
- // skip this row as it is invalid
- // if there is no more row, then result will be an empty list
- } while (hasMore);
- return hasMore;
- } catch (Throwable t) {
- ServerUtil.throwIOException(region.getRegionInfo().getRegionNameAsString(), t);
- return false; // impossible
- }
- }
private void deleteRowIfAgedEnough(byte[] indexRowKey, List<Cell> row, long ts, boolean specific) throws IOException {
if ((EnvironmentEdgeManager.currentTimeMillis() - ts) > ageThreshold) {
@@ -266,16 +271,54 @@ public class GlobalIndexChecker implements RegionCoprocessor, RegionObserver {
}
}
+ private PageFilter removePageFilterFromFilterList(FilterList filterList) {
+ Iterator<Filter> filterIterator = filterList.getFilters().iterator();
+ while (filterIterator.hasNext()) {
+ Filter filter = filterIterator.next();
+ if (filter instanceof PageFilter) {
+ filterIterator.remove();
+ return (PageFilter) filter;
+ } else if (filter instanceof FilterList) {
+ PageFilter pageFilter = removePageFilterFromFilterList((FilterList) filter);
+ if (pageFilter != null) {
+ return pageFilter;
+ }
+ }
+ }
+ return null;
+ }
+
+ // This method assumes that there is at most one instance of PageFilter in a scan
+ private PageFilter removePageFilter(Scan scan) {
+ Filter filter = scan.getFilter();
+ if (filter != null) {
+ if (filter instanceof PageFilter) {
+ scan.setFilter(null);
+ return (PageFilter) filter;
+ } else if (filter instanceof FilterList) {
+ return removePageFilterFromFilterList((FilterList) filter);
+ }
+ }
+ return null;
+ }
+
private void repairIndexRows(byte[] indexRowKey, long ts, List<Cell> row) throws IOException {
- // Build the data table row key from the index table row key
if (buildIndexScan == null) {
+ PageFilter pageFilter = removePageFilter(scan);
+ if (pageFilter != null) {
+ pageSize = pageFilter.getPageSize();
+ restartScanDueToPageFilterRemoval = true;
+ }
buildIndexScan = new Scan();
indexScan = new Scan(scan);
deleteRowScan = new Scan();
singleRowIndexScan = new Scan(scan);
byte[] dataTableName = scan.getAttribute(PHYSICAL_DATA_TABLE_NAME);
- dataHTable = ServerUtil.ConnectionFactory.getConnection(ServerUtil.ConnectionType.INDEX_WRITER_CONNECTION,
- env).getTable(TableName.valueOf(dataTableName));
+ dataHTable =
+ ServerUtil.ConnectionFactory.
+ getConnection(ServerUtil.ConnectionType.INDEX_WRITER_CONNECTION, env).
+ getTable(TableName.valueOf(dataTableName));
+
viewConstants = IndexUtil.deserializeViewConstantsFromScan(scan);
// The following attributes are set to instruct UngroupedAggregateRegionObserver to do partial index rebuild
// i.e., rebuild a subset of index rows.
@@ -315,10 +358,20 @@ public class GlobalIndexChecker implements RegionCoprocessor, RegionObserver {
// Skip this unverified row (i.e., do not return it to the client). Just retuning empty row is
// sufficient to do that
row.clear();
+ if (restartScanDueToPageFilterRemoval) {
+ scanner.close();
+ indexScan.withStartRow(indexRowKey, false);
+ scanner = region.getScanner(indexScan);
+ hasMore = true;
+ // Set restartScanDueToPageFilterRemoval to false as we do not restart the scan unnecessarily next time
+ restartScanDueToPageFilterRemoval = false;
+ }
return;
}
// An index row has been built. Close the current scanner as the newly built row will not be visible to it
scanner.close();
+ // Set restartScanDueToPageFilterRemoval to false as we do not restart the scan unnecessarily next time
+ restartScanDueToPageFilterRemoval = false;
if (code == RebuildReturnCode.NO_INDEX_ROW.getValue()) {
// This means there exists a data table row for the data row key derived from this unverified index row
// but the data table row does not point back to the index row.
@@ -327,6 +380,7 @@ public class GlobalIndexChecker implements RegionCoprocessor, RegionObserver {
// Open a new scanner starting from the row after the current row
indexScan.withStartRow(indexRowKey, false);
scanner = region.getScanner(indexScan);
+ hasMore = true;
// Skip this unverified row (i.e., do not return it to the client). Just retuning empty row is
// sufficient to do that
row.clear();
@@ -336,7 +390,7 @@ public class GlobalIndexChecker implements RegionCoprocessor, RegionObserver {
// Open a new scanner starting from the current row
indexScan.withStartRow(indexRowKey, true);
scanner = region.getScanner(indexScan);
- scanner.next(row);
+ hasMore = scanner.next(row);
if (row.isEmpty()) {
// This means the index row has been deleted before opening the new scanner.
return;
@@ -353,6 +407,7 @@ public class GlobalIndexChecker implements RegionCoprocessor, RegionObserver {
// so that it can be repaired
scanner.close();
scanner = region.getScanner(indexScan);
+ hasMore = true;
row.clear();
return;
}
@@ -529,11 +584,6 @@ public class GlobalIndexChecker implements RegionCoprocessor, RegionObserver {
}
@Override
- public Optional<RegionObserver> getRegionObserver() {
- return Optional.of(this);
- }
-
- @Override
public RegionScanner postScannerOpen(ObserverContext<RegionCoprocessorEnvironment> c,
Scan scan, RegionScanner s) throws IOException {
if (scan.getAttribute(CHECK_VERIFY_COLUMN) == null) {
@@ -543,7 +593,14 @@ public class GlobalIndexChecker implements RegionCoprocessor, RegionObserver {
}
@Override
+ public Optional<RegionObserver> getRegionObserver() {
+ return Optional.of(this);
+ }
+
+ @Override
public void start(CoprocessorEnvironment e) throws IOException {
+ this.env = e;
this.metricsSource = MetricsIndexerSourceFactory.getInstance().getGlobalIndexCheckerSource();
}
+
}
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/PhoenixServerBuildIndexInputFormat.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/PhoenixServerBuildIndexInputFormat.java
index 27287e5..056daf6 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/PhoenixServerBuildIndexInputFormat.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/PhoenixServerBuildIndexInputFormat.java
@@ -30,6 +30,7 @@ import org.apache.hadoop.mapreduce.JobContext;
import org.apache.hadoop.mapreduce.lib.db.DBWritable;
import org.apache.phoenix.compile.*;
import org.apache.phoenix.coprocessor.BaseScannerRegionObserver;
+import org.apache.phoenix.coprocessor.IndexRebuildRegionScanner;
import org.apache.phoenix.jdbc.PhoenixConnection;
import org.apache.phoenix.mapreduce.util.ConnectionUtil;
import org.apache.phoenix.mapreduce.util.PhoenixConfigurationUtil;
@@ -116,6 +117,12 @@ public class PhoenixServerBuildIndexInputFormat<T extends DBWritable> extends Ph
scan.setAttribute(BaseScannerRegionObserver.INDEX_RETRY_VERIFY, Bytes.toBytes(lastVerifyTimeValue));
scan.setAttribute(BaseScannerRegionObserver.INDEX_REBUILD_DISABLE_LOGGING_VERIFY_TYPE,
getDisableLoggingVerifyType(configuration).toBytes());
+ String shouldLogMaxLookbackOutput =
+ configuration.get(IndexRebuildRegionScanner.PHOENIX_INDEX_MR_LOG_BEYOND_MAX_LOOKBACK_ERRORS);
+ if (shouldLogMaxLookbackOutput != null) {
+ scan.setAttribute(BaseScannerRegionObserver.INDEX_REBUILD_DISABLE_LOGGING_BEYOND_MAXLOOKBACK_AGE,
+ Bytes.toBytes(shouldLogMaxLookbackOutput));
+ }
} catch (IOException e) {
throw new SQLException(e);
}
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexScrutinyMapper.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexScrutinyMapper.java
index 43628c9..5ed3178 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexScrutinyMapper.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexScrutinyMapper.java
@@ -36,13 +36,14 @@ import java.util.Set;
import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.codec.binary.Hex;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.TableDescriptor;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.phoenix.compat.hbase.coprocessor.CompatBaseScannerRegionObserver;
import org.apache.phoenix.jdbc.PhoenixConnection;
import org.apache.phoenix.jdbc.PhoenixResultSet;
import org.apache.phoenix.mapreduce.PhoenixJobCounters;
@@ -52,7 +53,6 @@ import org.apache.phoenix.mapreduce.util.ConnectionUtil;
import org.apache.phoenix.mapreduce.util.PhoenixConfigurationUtil;
import org.apache.phoenix.parse.HintNode.Hint;
import org.apache.phoenix.query.ConnectionQueryServices;
-import org.apache.phoenix.query.ConnectionQueryServicesImpl;
import org.apache.phoenix.schema.PTable;
import org.apache.phoenix.schema.PTableType;
import org.apache.phoenix.util.ColumnInfo;
@@ -66,7 +66,6 @@ import org.slf4j.LoggerFactory;
import com.google.common.base.Joiner;
-
/**
* Mapper that reads from the data table and checks the rows against the index table
*/
@@ -96,6 +95,12 @@ public class IndexScrutinyMapper extends Mapper<NullWritable, PhoenixIndexDBWrit
private long outputMaxRows;
private MessageDigest md5;
private long ttl;
+ private long scnTimestamp;
+ private long maxLookbackAgeMillis;
+
+ protected long getScrutinyTs(){
+ return scnTimestamp;
+ }
@Override
protected void setup(final Context context) throws IOException, InterruptedException {
@@ -107,7 +112,7 @@ public class IndexScrutinyMapper extends Mapper<NullWritable, PhoenixIndexDBWrit
final Properties overrideProps = new Properties();
String scn = configuration.get(PhoenixConfigurationUtil.CURRENT_SCN_VALUE);
overrideProps.put(PhoenixRuntime.CURRENT_SCN_ATTRIB, scn);
-
+ scnTimestamp = new Long(scn);
connection = ConnectionUtil.getOutputConnection(configuration, overrideProps);
connection.setAutoCommit(false);
batchSize = PhoenixConfigurationUtil.getScrutinyBatchSize(configuration);
@@ -162,12 +167,18 @@ public class IndexScrutinyMapper extends Mapper<NullWritable, PhoenixIndexDBWrit
LOGGER.info("Target table base query: " + targetTableQuery);
md5 = MessageDigest.getInstance("MD5");
ttl = getTableTtl();
+ maxLookbackAgeMillis = CompatBaseScannerRegionObserver.getMaxLookbackInMillis(configuration);
} catch (SQLException | NoSuchAlgorithmException e) {
tryClosingResourceSilently(this.outputUpsertStmt);
tryClosingResourceSilently(this.connection);
tryClosingResourceSilently(this.outputConn);
throw new RuntimeException(e);
}
+ postSetup();
+ }
+
+ protected void postSetup() {
+
}
private static void tryClosingResourceSilently(AutoCloseable res) {
@@ -245,14 +256,10 @@ public class IndexScrutinyMapper extends Mapper<NullWritable, PhoenixIndexDBWrit
// fetch results from the target table and output invalid rows
queryTargetTable(context, targetStatement, targetPkToSourceValues);
- //check if there are any invalid rows that have been expired, report them
- //with EXPIRED_ROW_COUNT
- checkIfInvalidRowsExpired(context, targetPkToSourceValues);
+ //check if any invalid rows are just temporary side effects of ttl or compaction,
+ //and if so remove them from the list and count them as separate metrics
+ categorizeInvalidRows(context, targetPkToSourceValues);
- // any source values we have left over are invalid (e.g. data table rows without
- // corresponding index row)
- context.getCounter(PhoenixScrutinyJobCounters.INVALID_ROW_COUNT)
- .increment(targetPkToSourceValues.size());
if (outputInvalidRows) {
for (Pair<Long, List<Object>> sourceRowWithoutTargetRow : targetPkToSourceValues.values()) {
List<Object> valuesWithoutTarget = sourceRowWithoutTargetRow.getSecond();
@@ -275,8 +282,8 @@ public class IndexScrutinyMapper extends Mapper<NullWritable, PhoenixIndexDBWrit
protected void preQueryTargetTable() { }
- protected void checkIfInvalidRowsExpired(Context context,
- Map<String, Pair<Long,
+ protected void categorizeInvalidRows(Context context,
+ Map<String, Pair<Long,
List<Object>>> targetPkToSourceValues) {
Set<Map.Entry<String, Pair<Long, List<Object>>>>
entrySet = targetPkToSourceValues.entrySet();
@@ -289,8 +296,15 @@ public class IndexScrutinyMapper extends Mapper<NullWritable, PhoenixIndexDBWrit
Pair<Long, List<Object>> sourceValues = entry.getValue();
Long sourceTS = sourceValues.getFirst();
if (hasRowExpiredOnSource(sourceTS, ttl)) {
- context.getCounter(PhoenixScrutinyJobCounters.EXPIRED_ROW_COUNT).increment(1);
- itr.remove();
+ context.getCounter(PhoenixScrutinyJobCounters.EXPIRED_ROW_COUNT).increment(1L);
+ itr.remove(); //don't output to the scrutiny table
+ } else if (isRowOlderThanMaxLookback(sourceTS)) {
+ context.getCounter(PhoenixScrutinyJobCounters.BEYOND_MAX_LOOKBACK_COUNT).increment(1L);
+ //still output to the scrutiny table just in case it's useful
+ } else {
+ // otherwise it's invalid (e.g. data table rows without corresponding index row)
+ context.getCounter(PhoenixScrutinyJobCounters.INVALID_ROW_COUNT)
+ .increment(1L);
}
}
}
@@ -300,7 +314,16 @@ public class IndexScrutinyMapper extends Mapper<NullWritable, PhoenixIndexDBWrit
return ttl != Integer.MAX_VALUE && sourceTS + ttl*1000 < currentTS;
}
- private long getTableTtl() throws SQLException, IOException {
+ protected boolean isRowOlderThanMaxLookback(Long sourceTS){
+ if (maxLookbackAgeMillis == CompatBaseScannerRegionObserver.DEFAULT_PHOENIX_MAX_LOOKBACK_AGE * 1000){
+ return false;
+ }
+ long now = EnvironmentEdgeManager.currentTimeMillis();
+ long maxLookBackTimeMillis = now - maxLookbackAgeMillis;
+ return sourceTS <= maxLookBackTimeMillis;
+ }
+
+ private int getTableTtl() throws SQLException, IOException {
PTable pSourceTable = PhoenixRuntime.getTable(connection, qSourceTable);
if (pSourceTable.getType() == PTableType.INDEX
&& pSourceTable.getIndexType() == PTable.IndexType.LOCAL) {
@@ -311,8 +334,8 @@ public class IndexScrutinyMapper extends Mapper<NullWritable, PhoenixIndexDBWrit
Admin admin = cqsi.getAdmin();
String physicalTable = getSourceTableName(pSourceTable,
SchemaUtil.isNamespaceMappingEnabled(null, cqsi.getProps()));
- HTableDescriptor tableDesc = admin.getTableDescriptor(TableName.valueOf(physicalTable));
- return tableDesc.getFamily(SchemaUtil.getEmptyColumnFamily(pSourceTable)).getTimeToLive();
+ TableDescriptor tableDesc = admin.getDescriptor(TableName.valueOf(physicalTable));
+ return tableDesc.getColumnFamily(SchemaUtil.getEmptyColumnFamily(pSourceTable)).getTimeToLive();
}
@VisibleForTesting
@@ -418,6 +441,7 @@ public class IndexScrutinyMapper extends Mapper<NullWritable, PhoenixIndexDBWrit
outputUpsertStmt.setLong(index++, sourceTS); // SOURCE_TS
outputUpsertStmt.setLong(index++, targetTS); // TARGET_TS
outputUpsertStmt.setBoolean(index++, targetValues != null); // HAS_TARGET_ROW
+ outputUpsertStmt.setBoolean(index++, isRowOlderThanMaxLookback(sourceTS));
index = setStatementObjects(sourceValues, index, sourceTblColumnMetadata);
if (targetValues != null) {
index = setStatementObjects(targetValues, index, targetTblColumnMetadata);
@@ -499,4 +523,5 @@ public class IndexScrutinyMapper extends Mapper<NullWritable, PhoenixIndexDBWrit
md5.reset();
}
}
+
}
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexScrutinyTableOutput.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexScrutinyTableOutput.java
index ac69bd8..986b435 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexScrutinyTableOutput.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexScrutinyTableOutput.java
@@ -26,7 +26,6 @@ import java.sql.SQLException;
import java.util.Arrays;
import java.util.List;
-import com.google.common.base.Strings;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.Counters;
import org.apache.hadoop.mapreduce.Job;
@@ -69,6 +68,7 @@ public class IndexScrutinyTableOutput {
" SOURCE_TS BIGINT,\n" +
" TARGET_TS BIGINT,\n" +
" HAS_TARGET_ROW BOOLEAN,\n" +
+ " BEYOND_MAX_LOOKBACK BOOLEAN,\n" +
" CONSTRAINT PK PRIMARY KEY\n" +
" (\n" +
" " + SOURCE_TABLE_COL_NAME + ",\n" +
@@ -76,7 +76,10 @@ public class IndexScrutinyTableOutput {
" " + SCRUTINY_EXECUTE_TIME_COL_NAME + ",\n" + // time at which the scrutiny ran
" SOURCE_ROW_PK_HASH\n" + // this hash makes the PK unique
" )\n" + // dynamic columns consisting of the source and target columns will follow
- ")";
+ ") COLUMN_ENCODED_BYTES = 0 "; //column encoding not supported with dyn columns (PHOENIX-5107)
+ public static final String OUTPUT_TABLE_BEYOND_LOOKBACK_DDL = "" +
+ "ALTER TABLE " + OUTPUT_TABLE_NAME + "\n" +
+ " ADD IF NOT EXISTS BEYOND_MAX_LOOKBACK BOOLEAN";
/**
* This table holds metadata about a scrutiny job - result counters and queries to fetch invalid
@@ -103,6 +106,8 @@ public class IndexScrutinyTableOutput {
" INVALID_ROWS_QUERY_ALL VARCHAR,\n" + // stored sql query to fetch all the invalid rows from the output table
" INVALID_ROWS_QUERY_MISSING_TARGET VARCHAR,\n" + // stored sql query to fetch all the invalid rows which are missing a target row
" INVALID_ROWS_QUERY_BAD_COVERED_COL_VAL VARCHAR,\n" + // stored sql query to fetch all the invalid rows which have bad covered column values
+ " INVALID_ROWS_QUERY_BEYOND_MAX_LOOKBACK VARCHAR,\n" + // stored sql query to fetch all the potentially invalid rows which are before max lookback age
+ " BEYOND_MAX_LOOKBACK_COUNT BIGINT,\n" +
" CONSTRAINT PK PRIMARY KEY\n" +
" (\n" +
" " + SOURCE_TABLE_COL_NAME + ",\n" +
@@ -110,8 +115,12 @@ public class IndexScrutinyTableOutput {
" " + SCRUTINY_EXECUTE_TIME_COL_NAME + "\n" +
" )\n" +
")\n";
+ public static final String OUTPUT_METADATA_BEYOND_LOOKBACK_COUNTER_DDL = "" +
+ "ALTER TABLE " + OUTPUT_METADATA_TABLE_NAME + "\n" +
+ " ADD IF NOT EXISTS INVALID_ROWS_QUERY_BEYOND_MAX_LOOKBACK VARCHAR, \n" +
+ " BEYOND_MAX_LOOKBACK_COUNT BIGINT";
- public static final String UPSERT_METADATA_SQL = "UPSERT INTO " + OUTPUT_METADATA_TABLE_NAME + " VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)";
+ public static final String UPSERT_METADATA_SQL = "UPSERT INTO " + OUTPUT_METADATA_TABLE_NAME + " VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)";
/**
* Gets the parameterized upsert sql to the output table Used by the scrutiny MR job to write
@@ -178,6 +187,21 @@ public class IndexScrutinyTableOutput {
return paramQuery.replaceFirst("\\?", "true");
}
+ public static String getSqlQueryBeyondMaxLookback(Connection conn,
+ SourceTargetColumnNames columnNames, long scrutinyTimeMillis) throws SQLException {
+ String whereQuery =
+ constructOutputTableQuery(conn, columnNames,
+ getPksCsv() + ", " + SchemaUtil.getEscapedFullColumnName("HAS_TARGET_ROW")
+ + ", " + SchemaUtil.getEscapedFullColumnName("BEYOND_MAX_LOOKBACK"));
+ String inClause =
+ " IN " + QueryUtil.constructParameterizedInClause(getPkCols().size() + 2, 1);
+ String paramQuery = whereQuery + inClause;
+ paramQuery = bindPkCols(columnNames, scrutinyTimeMillis, paramQuery);
+ paramQuery = paramQuery.replaceFirst("\\?", "false"); //has_target_row false
+ paramQuery = paramQuery.replaceFirst("\\?", "true"); //beyond_max_lookback true
+ return paramQuery;
+ }
+
/**
* Query the metadata table for the given columns
* @param conn connection to use
@@ -198,6 +222,22 @@ public class IndexScrutinyTableOutput {
return ps.executeQuery();
}
+ public static ResultSet queryAllLatestMetadata(Connection conn, String qSourceTableName,
+ String qTargetTableName) throws SQLException {
+ String sql = "SELECT MAX(" + SCRUTINY_EXECUTE_TIME_COL_NAME + ") " +
+ "FROM " + OUTPUT_METADATA_TABLE_NAME +
+ " WHERE " + SOURCE_TABLE_COL_NAME + " = ?" + " AND " + TARGET_TABLE_COL_NAME + "= ?";
+ PreparedStatement stmt = conn.prepareStatement(sql);
+ stmt.setString(1, qSourceTableName);
+ stmt.setString(2, qTargetTableName);
+ ResultSet rs = stmt.executeQuery();
+ long scrutinyTimeMillis = 0L;
+ if (rs.next()){
+ scrutinyTimeMillis = rs.getLong(1);
+ } //even if we didn't find one, still need to do a query to return the right columns
+ return queryAllMetadata(conn, qSourceTableName, qTargetTableName, scrutinyTimeMillis);
+ }
+
/**
* Query the metadata table for all columns
* @param conn connection to use
@@ -259,6 +299,9 @@ public class IndexScrutinyTableOutput {
pStmt.setString(index++, getSqlQueryAllInvalidRows(conn, columnNames, scrutinyExecuteTime));
pStmt.setString(index++, getSqlQueryMissingTargetRows(conn, columnNames, scrutinyExecuteTime));
pStmt.setString(index++, getSqlQueryBadCoveredColVal(conn, columnNames, scrutinyExecuteTime));
+ pStmt.setString(index++, getSqlQueryBeyondMaxLookback(conn, columnNames, scrutinyExecuteTime));
+ pStmt.setLong(index++,
+ counters.findCounter(PhoenixScrutinyJobCounters.BEYOND_MAX_LOOKBACK_COUNT).getValue());
pStmt.addBatch();
}
pStmt.executeBatch();
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexScrutinyTool.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexScrutinyTool.java
index fd856fb..5402303 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexScrutinyTool.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexScrutinyTool.java
@@ -19,8 +19,6 @@ package org.apache.phoenix.mapreduce.index;
import java.io.IOException;
import java.sql.Connection;
-import java.sql.DatabaseMetaData;
-import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.List;
@@ -33,7 +31,7 @@ import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
-import org.apache.commons.lang3.exception.ExceptionUtils;
+import org.apache.commons.lang.exception.ExceptionUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FileSystem;
@@ -43,7 +41,6 @@ import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
import org.apache.hadoop.hbase.util.CommonFSUtils;
-import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
@@ -51,16 +48,14 @@ import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
+import org.apache.phoenix.compat.hbase.coprocessor.CompatBaseScannerRegionObserver;
import org.apache.phoenix.jdbc.PhoenixConnection;
import org.apache.phoenix.mapreduce.CsvBulkImportUtil;
import org.apache.phoenix.mapreduce.util.ConnectionUtil;
import org.apache.phoenix.mapreduce.util.PhoenixConfigurationUtil;
import org.apache.phoenix.mapreduce.util.PhoenixMapReduceUtil;
import org.apache.phoenix.parse.HintNode.Hint;
-import org.apache.phoenix.schema.PColumn;
import org.apache.phoenix.schema.PTable;
-import org.apache.phoenix.schema.PTableType;
-import org.apache.phoenix.schema.types.PDataType;
import org.apache.phoenix.util.EnvironmentEdgeManager;
import org.apache.phoenix.util.PhoenixRuntime;
import org.apache.phoenix.util.QueryUtil;
@@ -72,8 +67,6 @@ import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
-import static org.apache.phoenix.util.MetaDataUtil.VIEW_INDEX_TABLE_PREFIX;
-
/**
* An MR job to verify that the index table is in sync with the data table.
*
@@ -124,9 +117,9 @@ public class IndexScrutinyTool extends Configured implements Tool {
public static final String INDEX_JOB_NAME_TEMPLATE = "PHOENIX_SCRUTINY_[%s]_[%s]";
@Inject
- Class<IndexScrutinyMapperForTest> mapperClass = null;
+ Class<? extends IndexScrutinyMapper> mapperClass = null;
- public IndexScrutinyTool(Class<IndexScrutinyMapperForTest> indexScrutinyMapperForTestClass) {
+ public IndexScrutinyTool(Class<? extends IndexScrutinyMapper> indexScrutinyMapperForTestClass) {
this.mapperClass = indexScrutinyMapperForTestClass;
}
@@ -140,7 +133,7 @@ public class IndexScrutinyTool extends Configured implements Tool {
/**
* Runs two separate jobs to iterate over both tables
*/
- BOTH;
+ BOTH
}
public enum OutputFormat {
@@ -215,19 +208,20 @@ public class IndexScrutinyTool extends Configured implements Tool {
class JobFactory {
Connection connection;
Configuration configuration;
- private boolean useSnapshot;
- private long ts;
- private boolean outputInvalidRows;
- private OutputFormat outputFormat;
- private String basePath;
- private long scrutinyExecuteTime;
- private long outputMaxRows; // per mapper
- private String tenantId;
- Class<IndexScrutinyMapperForTest> mapperClass;
+ private final boolean useSnapshot;
+ private final long ts;
+ private final boolean outputInvalidRows;
+ private final OutputFormat outputFormat;
+ private final String basePath;
+ private final long scrutinyExecuteTime;
+ private final long outputMaxRows; // per mapper
+ private final String tenantId;
+ Class<? extends IndexScrutinyMapper> mapperClass;
public JobFactory(Connection connection, Configuration configuration, long batchSize,
boolean useSnapshot, long ts, boolean outputInvalidRows, OutputFormat outputFormat,
- String basePath, long outputMaxRows, String tenantId, Class<IndexScrutinyMapperForTest> mapperClass) {
+ String basePath, long outputMaxRows, String tenantId,
+ Class<? extends IndexScrutinyMapper> mapperClass) {
this.outputInvalidRows = outputInvalidRows;
this.outputFormat = outputFormat;
this.basePath = basePath;
@@ -250,7 +244,7 @@ public class IndexScrutinyTool extends Configured implements Tool {
}
public Job createSubmittableJob(String schemaName, String indexTable, String dataTable,
- SourceTable sourceTable, Class<IndexScrutinyMapperForTest> mapperClass) throws Exception {
+ SourceTable sourceTable, Class<? extends IndexScrutinyMapper> mapperClass) throws Exception {
Preconditions.checkArgument(SourceTable.DATA_TABLE_SOURCE.equals(sourceTable)
|| SourceTable.INDEX_TABLE_SOURCE.equals(sourceTable));
@@ -329,6 +323,7 @@ public class IndexScrutinyTool extends Configured implements Tool {
// root dir not a subdirectory of hbase dir
Path rootDir = new Path("hdfs:///index-snapshot-dir");
CommonFSUtils.setRootDir(configuration, rootDir);
+ Path restoreDir = new Path(CommonFSUtils.getRootDir(configuration), "restore-dir");
// set input for map reduce job using hbase snapshots
//PhoenixMapReduceUtil.setInput(job, PhoenixIndexDBWritable.class, snapshotName,
@@ -343,7 +338,7 @@ public class IndexScrutinyTool extends Configured implements Tool {
return configureSubmittableJob(job, outputPath, mapperClass);
}
- private Job configureSubmittableJob(Job job, Path outputPath, Class<IndexScrutinyMapperForTest> mapperClass) throws Exception {
+ private Job configureSubmittableJob(Job job, Path outputPath, Class<? extends IndexScrutinyMapper> mapperClass) throws Exception {
Configuration conf = job.getConfiguration();
conf.setBoolean("mapreduce.job.user.classpath.first", true);
HBaseConfiguration.merge(conf, HBaseConfiguration.create(conf));
@@ -421,6 +416,8 @@ public class IndexScrutinyTool extends Configured implements Tool {
? Long.parseLong(cmdLine.getOptionValue(TIMESTAMP.getOpt()))
: EnvironmentEdgeManager.currentTimeMillis() - 60000;
+ validateTimestamp(configuration, ts);
+
if (indexTable != null) {
if (!IndexTool.isValidIndexTable(connection, qDataTable, indexTable, tenantId)) {
throw new IllegalArgumentException(String
@@ -445,8 +442,12 @@ public class IndexScrutinyTool extends Configured implements Tool {
outputConfiguration.unset(PhoenixRuntime.TENANT_ID_ATTRIB);
try (Connection outputConn = ConnectionUtil.getOutputConnection(outputConfiguration)) {
outputConn.createStatement().execute(IndexScrutinyTableOutput.OUTPUT_TABLE_DDL);
+ outputConn.createStatement().
+ execute(IndexScrutinyTableOutput.OUTPUT_TABLE_BEYOND_LOOKBACK_DDL);
outputConn.createStatement()
.execute(IndexScrutinyTableOutput.OUTPUT_METADATA_DDL);
+ outputConn.createStatement().
+ execute(IndexScrutinyTableOutput.OUTPUT_METADATA_BEYOND_LOOKBACK_COUNTER_DDL);
}
}
@@ -512,6 +513,18 @@ public class IndexScrutinyTool extends Configured implements Tool {
}
}
+ private void validateTimestamp(Configuration configuration, long ts) {
+ long maxLookBackAge = CompatBaseScannerRegionObserver.getMaxLookbackInMillis(configuration);
+ if (maxLookBackAge != CompatBaseScannerRegionObserver.DEFAULT_PHOENIX_MAX_LOOKBACK_AGE * 1000L) {
+ long minTimestamp = EnvironmentEdgeManager.currentTimeMillis() - maxLookBackAge;
+ if (ts < minTimestamp){
+ throw new IllegalArgumentException("Index scrutiny can't look back past the configured" +
+ "max lookback age: " + maxLookBackAge / 1000 + " seconds");
+ }
+ }
+
+ }
+
@VisibleForTesting
public List<Job> getJobs() {
return jobs;
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexTool.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexTool.java
index ee7b17e..f7ef6a3 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexTool.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexTool.java
@@ -74,6 +74,7 @@ import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
+import org.apache.phoenix.compat.hbase.HbaseCompatCapabilities;
import org.apache.phoenix.compile.PostIndexDDLCompiler;
import org.apache.phoenix.coprocessor.BaseScannerRegionObserver;
import org.apache.phoenix.hbase.index.ValueGetter;
@@ -383,6 +384,11 @@ public class IndexTool extends Configured implements Tool {
"DisableLoggingType: ["
+ cmdLine.getOptionValue(DISABLE_LOGGING_OPTION.getOpt()) + "]");
}
+ if ((cmdLine.hasOption(START_TIME_OPTION.getOpt()) || cmdLine.hasOption(RETRY_VERIFY_OPTION.getOpt()))
+ && !HbaseCompatCapabilities.isRawFilterSupported()) {
+ throw new IllegalStateException("Can't do incremental index verification on this " +
+ "version of HBase because raw skip scan filters are not supported.");
+ }
return cmdLine;
}
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexVerificationOutputRepository.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexVerificationOutputRepository.java
index 0a1dc5f..da1b245 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexVerificationOutputRepository.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexVerificationOutputRepository.java
@@ -47,11 +47,13 @@ import java.util.List;
public class IndexVerificationOutputRepository implements AutoCloseable {
public static final byte[] ROW_KEY_SEPARATOR_BYTE = Bytes.toBytes("|");
+
private Table indexTable;
private byte[] indexName;
private Table outputTable;
private IndexTool.IndexDisableLoggingType disableLoggingVerifyType =
IndexTool.IndexDisableLoggingType.NONE;
+ private boolean shouldLogBeyondMaxLookback = true;
public final static String OUTPUT_TABLE_NAME = "PHOENIX_INDEX_TOOL";
public final static byte[] OUTPUT_TABLE_NAME_BYTES = Bytes.toBytes(OUTPUT_TABLE_NAME);
@@ -71,6 +73,8 @@ public class IndexVerificationOutputRepository implements AutoCloseable {
public final static byte[] INDEX_TABLE_TS_BYTES = Bytes.toBytes(INDEX_TABLE_TS);
public final static String ERROR_MESSAGE = "Error";
public final static byte[] ERROR_MESSAGE_BYTES = Bytes.toBytes(ERROR_MESSAGE);
+ public static final String ERROR_TYPE = "ErrorType";
+ public static final byte[] ERROR_TYPE_BYTES = Bytes.toBytes(ERROR_TYPE);
public static String VERIFICATION_PHASE = "Phase";
public final static byte[] VERIFICATION_PHASE_BYTES = Bytes.toBytes(VERIFICATION_PHASE);
@@ -85,6 +89,15 @@ public class IndexVerificationOutputRepository implements AutoCloseable {
public static final byte[] PHASE_BEFORE_VALUE = Bytes.toBytes("BEFORE");
public static final byte[] PHASE_AFTER_VALUE = Bytes.toBytes("AFTER");
+
+ public enum IndexVerificationErrorType {
+ INVALID_ROW,
+ MISSING_ROW,
+ EXTRA_CELLS,
+ BEYOND_MAX_LOOKBACK_INVALID,
+ BEYOND_MAX_LOOKBACK_MISSING,
+ UNKNOWN
+ }
/**
* Only usable for the create table / read path or for testing. Use setOutputTable and
* setIndexTable first to write.
@@ -117,6 +130,10 @@ public class IndexVerificationOutputRepository implements AutoCloseable {
this.disableLoggingVerifyType = disableLoggingVerifyType;
}
+ public void setShouldLogBeyondMaxLookback(boolean shouldLogBeyondMaxLookback) {
+ this.shouldLogBeyondMaxLookback = shouldLogBeyondMaxLookback;
+ }
+
public static byte[] generateOutputTableRowKey(long ts, byte[] indexTableName, byte[] dataRowKey ) {
byte[] keyPrefix = Bytes.toBytes(Long.toString(ts));
byte[] rowKey;
@@ -162,7 +179,7 @@ public class IndexVerificationOutputRepository implements AutoCloseable {
ColumnFamilyDescriptorBuilder.newBuilder(OUTPUT_TABLE_COLUMN_FAMILY).
setTimeToLive(MetaDataProtocol.DEFAULT_LOG_TTL).build();
TableDescriptor tableDescriptor =
- TableDescriptorBuilder.newBuilder(outputTableName).
+ TableDescriptorBuilder.newBuilder(TableName.valueOf(OUTPUT_TABLE_NAME)).
setColumnFamily(columnDescriptor).build();
admin.createTable(tableDescriptor);
outputTable = admin.getConnection().getTable(outputTableName);
@@ -173,9 +190,11 @@ public class IndexVerificationOutputRepository implements AutoCloseable {
public void logToIndexToolOutputTable(byte[] dataRowKey, byte[] indexRowKey, long dataRowTs,
long indexRowTs,
String errorMsg, byte[] expectedValue, byte[] actualValue,
- long scanMaxTs, byte[] tableName, boolean isBeforeRebuild)
+ long scanMaxTs, byte[] tableName,
+ boolean isBeforeRebuild,
+ IndexVerificationErrorType errorType)
throws IOException {
- if (shouldLogOutput(isBeforeRebuild)) {
+ if (shouldLogOutput(isBeforeRebuild, errorType)) {
byte[] rowKey = generateOutputTableRowKey(scanMaxTs, indexTable.getName().toBytes(), dataRowKey);
Put put = new Put(rowKey);
put.addColumn(OUTPUT_TABLE_COLUMN_FAMILY, DATA_TABLE_NAME_BYTES, tableName);
@@ -193,6 +212,8 @@ public class IndexVerificationOutputRepository implements AutoCloseable {
errorMessageBytes = Bytes.toBytes(errorMsg);
}
put.addColumn(OUTPUT_TABLE_COLUMN_FAMILY, ERROR_MESSAGE_BYTES, errorMessageBytes);
+ put.addColumn(OUTPUT_TABLE_COLUMN_FAMILY, ERROR_TYPE_BYTES,
+ Bytes.toBytes(errorType.toString()));
if (isBeforeRebuild) {
put.addColumn(OUTPUT_TABLE_COLUMN_FAMILY, VERIFICATION_PHASE_BYTES, PHASE_BEFORE_VALUE);
} else {
@@ -202,7 +223,12 @@ public class IndexVerificationOutputRepository implements AutoCloseable {
}
}
- public boolean shouldLogOutput(boolean isBeforeRebuild) {
+ public boolean shouldLogOutput(boolean isBeforeRebuild, IndexVerificationErrorType errorType) {
+ return shouldLogOutputForVerifyType(isBeforeRebuild) &&
+ shouldLogOutputForErrorType(errorType);
+ }
+
+ private boolean shouldLogOutputForVerifyType(boolean isBeforeRebuild) {
if (disableLoggingVerifyType.equals(IndexTool.IndexDisableLoggingType.BOTH)) {
return false;
}
@@ -219,6 +245,15 @@ public class IndexVerificationOutputRepository implements AutoCloseable {
return false;
}
+ private boolean shouldLogOutputForErrorType(IndexVerificationErrorType errorType) {
+ if (errorType != null &&
+ (errorType.equals(IndexVerificationErrorType.BEYOND_MAX_LOOKBACK_INVALID) ||
+ errorType.equals(IndexVerificationErrorType.BEYOND_MAX_LOOKBACK_MISSING))){
+ return shouldLogBeyondMaxLookback;
+ }
+ return true;
+ }
+
public static byte[] getErrorMessageBytes(String errorMsg, byte[] expectedValue, byte[] actualValue) {
byte[] errorMessageBytes;
errorMessageBytes = new byte[errorMsg.length() + expectedValue.length + actualValue.length +
@@ -255,20 +290,20 @@ public class IndexVerificationOutputRepository implements AutoCloseable {
return outputRowList;
}
- @VisibleForTesting
- public Iterator<IndexVerificationOutputRow> getOutputRowIteratorForAllRows()
+ public Iterator<IndexVerificationOutputRow> getOutputRowIterator(long ts, byte[] indexName)
throws IOException {
Scan scan = new Scan();
+ byte[] partialKey = generatePartialOutputTableRowKey(ts, indexName);
+ scan.setStartRow(partialKey);
+ scan.setStopRow(ByteUtil.calculateTheClosestNextRowKeyForPrefix(partialKey));
ResultScanner scanner = outputTable.getScanner(scan);
return new IndexVerificationOutputRowIterator(scanner.iterator());
}
- public Iterator<IndexVerificationOutputRow> getOutputRowIterator(long ts, byte[] indexName)
+ @VisibleForTesting
+ public Iterator<IndexVerificationOutputRow> getOutputRowIteratorForAllRows()
throws IOException {
Scan scan = new Scan();
- byte[] partialKey = generatePartialOutputTableRowKey(ts, indexName);
- scan.withStartRow(partialKey);
- scan.withStopRow(ByteUtil.calculateTheClosestNextRowKeyForPrefix(partialKey));
ResultScanner scanner = outputTable.getScanner(scan);
return new IndexVerificationOutputRowIterator(scanner.iterator());
}
@@ -297,6 +332,18 @@ public class IndexVerificationOutputRepository implements AutoCloseable {
builder.setExpectedValue(result.getValue(OUTPUT_TABLE_COLUMN_FAMILY, EXPECTED_VALUE_BYTES));
builder.setActualValue(result.getValue(OUTPUT_TABLE_COLUMN_FAMILY, ACTUAL_VALUE_BYTES));
builder.setPhaseValue(result.getValue(OUTPUT_TABLE_COLUMN_FAMILY, VERIFICATION_PHASE_BYTES));
+ IndexVerificationErrorType errorType;
+ try {
+ errorType =
+ IndexVerificationErrorType.valueOf(
+ Bytes.toString(result.getValue(OUTPUT_TABLE_COLUMN_FAMILY, ERROR_TYPE_BYTES)));
+ } catch (Throwable e) {
+ //in case we have a cast exception because an incompatible version of the enum produced
+ //the row, or an earlier version that didn't record error types, it's better to mark
+ // the error type unknown and move on rather than fail
+ errorType = IndexVerificationErrorType.UNKNOWN;
+ }
+ builder.setErrorType(errorType);
return builder.build();
}
@@ -309,7 +356,7 @@ public class IndexVerificationOutputRepository implements AutoCloseable {
}
}
- public class IndexVerificationOutputRowIterator implements Iterator<IndexVerificationOutputRow> {
+ public static class IndexVerificationOutputRowIterator implements Iterator<IndexVerificationOutputRow> {
Iterator<Result> delegate;
public IndexVerificationOutputRowIterator(Iterator<Result> delegate){
this.delegate = delegate;
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexVerificationOutputRow.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexVerificationOutputRow.java
index 1ecac08..4dad9b6 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexVerificationOutputRow.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexVerificationOutputRow.java
@@ -18,11 +18,13 @@
package org.apache.phoenix.mapreduce.index;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.phoenix.mapreduce.index.IndexVerificationOutputRepository.IndexVerificationErrorType;
import java.util.Arrays;
import java.util.Objects;
public class IndexVerificationOutputRow {
+ public static final String SCAN_MAX_TIMESTAMP = "ScanMaxTimestamp: ";
private String dataTableName;
private String indexTableName;
private Long scanMaxTimestamp;
@@ -34,13 +36,14 @@ public class IndexVerificationOutputRow {
private byte[] expectedValue;
private byte[] actualValue;
private byte[] phaseValue;
+ private IndexVerificationErrorType errorType;
private IndexVerificationOutputRow(String dataTableName, String indexTableName,
byte[] dataTableRowKey, Long scanMaxTimestamp,
byte[] indexTableRowKey,
long dataTableRowTimestamp, long indexTableRowTimestamp,
String errorMessage, byte[] expectedValue, byte[] actualValue,
- byte[] phaseValue) {
+ byte[] phaseValue, IndexVerificationErrorType errorType) {
this.dataTableName = dataTableName;
this.indexTableName = indexTableName;
this.scanMaxTimestamp = scanMaxTimestamp;
@@ -52,6 +55,7 @@ public class IndexVerificationOutputRow {
this.expectedValue = expectedValue;
this.actualValue = actualValue;
this.phaseValue = phaseValue;
+ this.errorType = errorType;
}
public String getDataTableName() {
@@ -118,7 +122,8 @@ public class IndexVerificationOutputRow {
Objects.equals(errorMessage, otherRow.getErrorMessage()) &&
Arrays.equals(expectedValue, otherRow.getExpectedValue()) &&
Arrays.equals(actualValue, otherRow.getActualValue()) &&
- Arrays.equals(phaseValue, otherRow.getPhaseValue());
+ Arrays.equals(phaseValue, otherRow.getPhaseValue()) &&
+ Objects.equals(errorType, otherRow.getErrorType());
}
@Override
@@ -130,20 +135,26 @@ public class IndexVerificationOutputRow {
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
- sb.append("DataTableName: ").append(dataTableName).append(",");
- sb.append("IndexTableName: ").append(indexTableName).append(",");
- sb.append("ScanMaxTimestamp: ").append(scanMaxTimestamp).append(",");
- sb.append("DataTableRowKey: ").append(Bytes.toString(dataTableRowKey)).append(",");
- sb.append("IndexTableRowKey: ").append(Bytes.toString(indexTableRowKey)).append(",");
- sb.append("DataTableRowTimestamp: ").append(dataTableRowTimestamp).append(",");
- sb.append("IndexTableRowTimestamp: ").append(indexTableRowTimestamp).append(",");
- sb.append("ErrorMessage: ").append(errorMessage).append(",");
- sb.append("ExpectedValue: ").append(Bytes.toString(expectedValue)).append(",");
- sb.append("ActualValue: ").append(Bytes.toString(actualValue)).append(",");
- sb.append("PhaseValue: ").append(Bytes.toString(phaseValue));
+ sb.append(IndexVerificationOutputRepository.DATA_TABLE_NAME + ": ").append(dataTableName).append(",");
+ sb.append(IndexVerificationOutputRepository.INDEX_TABLE_NAME + ": ").append(indexTableName).append(",");
+ sb.append(SCAN_MAX_TIMESTAMP).append(": ").append(scanMaxTimestamp).append(",");
+ sb.append(IndexVerificationOutputRepository.DATA_TABLE_ROW_KEY + ": ").append(Bytes.toString(dataTableRowKey)).append(",");
+ sb.append(IndexVerificationOutputRepository.INDEX_TABLE_ROW_KEY + ": ").append(Bytes.toString(indexTableRowKey)).append(",");
+ sb.append(IndexVerificationOutputRepository.DATA_TABLE_TS + ": ").append(dataTableRowTimestamp).append(",");
+ sb.append(IndexVerificationOutputRepository.INDEX_TABLE_TS + ": ").append(indexTableRowTimestamp).append(",");
+ sb.append(IndexVerificationOutputRepository.ERROR_MESSAGE + ": ").append(errorMessage).append(",");
+ sb.append(IndexVerificationOutputRepository.EXPECTED_VALUE + ": ").append(Bytes.toString(expectedValue)).append(",");
+ sb.append(IndexVerificationOutputRepository.ACTUAL_VALUE + ": ").append(Bytes.toString(actualValue)).append(
+ ",");
+ sb.append(IndexVerificationOutputRepository.VERIFICATION_PHASE + ": ").append(Bytes.toString(phaseValue));
+ sb.append(IndexVerificationOutputRepository.ERROR_TYPE + ": " ).append(Objects.toString(errorType));
return sb.toString();
}
+ public IndexVerificationErrorType getErrorType() {
+ return errorType;
+ }
+
public static class IndexVerificationOutputRowBuilder {
private String dataTableName;
private String indexTableName;
@@ -156,6 +167,7 @@ public class IndexVerificationOutputRow {
private byte[] expectedValue;
private byte[] actualValue;
private byte[] phaseValue;
+ private IndexVerificationErrorType errorType;
public IndexVerificationOutputRowBuilder setDataTableName(String dataTableName) {
this.dataTableName = dataTableName;
@@ -212,10 +224,15 @@ public class IndexVerificationOutputRow {
return this;
}
+ public IndexVerificationOutputRowBuilder setErrorType(IndexVerificationErrorType errorType) {
+ this.errorType = errorType;
+ return this;
+ }
+
public IndexVerificationOutputRow build() {
return new IndexVerificationOutputRow(dataTableName, indexTableName, dataTableRowKey,
scanMaxTimestamp, indexTableRowKey, dataTableRowTimestamp, indexTableRowTimestamp,
- errorMessage, expectedValue, actualValue, phaseValue);
+ errorMessage, expectedValue, actualValue, phaseValue, errorType);
}
}
}
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexVerificationResultRepository.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexVerificationResultRepository.java
index 2cb7a53..d424dbe 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexVerificationResultRepository.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexVerificationResultRepository.java
@@ -49,8 +49,8 @@ public class IndexVerificationResultRepository implements AutoCloseable {
public static final String RUN_STATUS_SKIPPED = "Skipped";
public static final String RUN_STATUS_EXECUTED = "Executed";
- private Table resultHTable;
- private Table indexHTable;
+ private Table resultTable;
+ private Table indexTable;
public static final String ROW_KEY_SEPARATOR = "|";
public static final byte[] ROW_KEY_SEPARATOR_BYTE = Bytes.toBytes(ROW_KEY_SEPARATOR);
public final static String RESULT_TABLE_NAME = "PHOENIX_INDEX_TOOL_RESULT";
@@ -95,14 +95,33 @@ public class IndexVerificationResultRepository implements AutoCloseable {
public final static String AFTER_REBUILD_INVALID_INDEX_ROW_COUNT =
"AfterRebuildInvalidIndexRowCount";
public final static byte[] AFTER_REBUILD_INVALID_INDEX_ROW_COUNT_BYTES = Bytes.toBytes(AFTER_REBUILD_INVALID_INDEX_ROW_COUNT);
+ public final static String BEFORE_REBUILD_BEYOND_MAXLOOKBACK_MISSING_INDEX_ROW_COUNT =
+ "BeforeRebuildBeyondMaxLookBackMissingIndexRowCount";
+ public final static byte[] BEFORE_REBUILD_BEYOND_MAXLOOKBACK_MISSING_INDEX_ROW_COUNT_BYTES =
+ Bytes.toBytes(BEFORE_REBUILD_BEYOND_MAXLOOKBACK_MISSING_INDEX_ROW_COUNT);
+ public final static String BEFORE_REBUILD_BEYOND_MAXLOOKBACK_INVALID_INDEX_ROW_COUNT =
+ "BeforeRebuildBeyondMaxLookBackInvalidIndexRowCount";
+ public final static byte[] BEFORE_REBUILD_BEYOND_MAXLOOKBACK_INVALID_INDEX_ROW_COUNT_BYTES =
+ Bytes.toBytes(BEFORE_REBUILD_BEYOND_MAXLOOKBACK_INVALID_INDEX_ROW_COUNT);
+
+ public final static String AFTER_REBUILD_BEYOND_MAXLOOKBACK_MISSING_INDEX_ROW_COUNT =
+ "AfterRebuildBeyondMaxLookBackMissingIndexRowCount";
+ public final static byte[] AFTER_REBUILD_BEYOND_MAXLOOKBACK_MISSING_INDEX_ROW_COUNT_BYTES =
+ Bytes.toBytes(AFTER_REBUILD_BEYOND_MAXLOOKBACK_MISSING_INDEX_ROW_COUNT);
+ public final static String AFTER_REBUILD_BEYOND_MAXLOOKBACK_INVALID_INDEX_ROW_COUNT =
+ "AfterRebuildBeyondMaxLookBackInvalidIndexRowCount";
+ public final static byte[] AFTER_REBUILD_BEYOND_MAXLOOKBACK_INVALID_INDEX_ROW_COUNT_BYTES =
+ Bytes.toBytes(AFTER_REBUILD_BEYOND_MAXLOOKBACK_INVALID_INDEX_ROW_COUNT);
+
+ public static String BEFORE_REBUILD_INVALID_INDEX_ROW_COUNT_COZ_EXTRA_CELLS = "BeforeRebuildInvalidIndexRowCountCozExtraCells";
+ public final static byte[] BEFORE_REBUILD_INVALID_INDEX_ROW_COUNT_COZ_EXTRA_CELLS_BYTES = Bytes.toBytes(BEFORE_REBUILD_INVALID_INDEX_ROW_COUNT_COZ_EXTRA_CELLS);
+ public static String BEFORE_REBUILD_INVALID_INDEX_ROW_COUNT_COZ_MISSING_CELLS = "BeforeRebuildInvalidIndexRowCountCozMissingCells";
+ public final static byte[] BEFORE_REBUILD_INVALID_INDEX_ROW_COUNT_COZ_MISSING_CELLS_BYTES = Bytes.toBytes(BEFORE_REBUILD_INVALID_INDEX_ROW_COUNT_COZ_MISSING_CELLS);
+
public static String AFTER_REBUILD_INVALID_INDEX_ROW_COUNT_COZ_EXTRA_CELLS = "AfterRebuildInvalidIndexRowCountCozExtraCells";
public final static byte[] AFTER_REBUILD_INVALID_INDEX_ROW_COUNT_COZ_EXTRA_CELLS_BYTES = Bytes.toBytes(AFTER_REBUILD_INVALID_INDEX_ROW_COUNT_COZ_EXTRA_CELLS);
public static String AFTER_REBUILD_INVALID_INDEX_ROW_COUNT_COZ_MISSING_CELLS = "AfterRebuildInvalidIndexRowCountCozMissingCells";
public final static byte[] AFTER_REBUILD_INVALID_INDEX_ROW_COUNT_COZ_MISSING_CELLS_BYTES = Bytes.toBytes(AFTER_REBUILD_INVALID_INDEX_ROW_COUNT_COZ_MISSING_CELLS);
- public static String BEFORE_REBUILD_INVALID_INDEX_ROW_COUNT_COZ_EXTRA_CELLS = "BeforeRebuildInvalidIndexRowCountCozExtraCells";
- public final static byte[] BEFORE_REBUILD_INVALID_INDEX_ROW_COUNT_COZ_EXTRA_CELLS_BYTES = Bytes.toBytes(BEFORE_REBUILD_INVALID_INDEX_ROW_COUNT_COZ_EXTRA_CELLS);
- public static String BEFORE_REBUILD_INVALID_INDEX_ROW_COUNT_COZ_MISSING_CELLS = "BeforeRebuildInvalidIndexRowCountCozMissingCells";
- public final static byte[] BEFORE_REBUILD_INVALID_INDEX_ROW_COUNT_COZ_MISSING_CELLS_BYTES = Bytes.toBytes(BEFORE_REBUILD_INVALID_INDEX_ROW_COUNT_COZ_MISSING_CELLS);
/***
* Only usable for read / create methods. To write use setResultTable and setIndexTable first
@@ -112,14 +131,14 @@ public class IndexVerificationResultRepository implements AutoCloseable {
}
public IndexVerificationResultRepository(Connection conn, byte[] indexNameBytes) throws SQLException {
- resultHTable = getTable(conn, RESULT_TABLE_NAME_BYTES);
- indexHTable = getTable(conn, indexNameBytes);
+ resultTable = getTable(conn, RESULT_TABLE_NAME_BYTES);
+ indexTable = getTable(conn, indexNameBytes);
}
public IndexVerificationResultRepository(byte[] indexName,
HTableFactory hTableFactory) throws IOException {
- resultHTable = hTableFactory.getTable(new ImmutableBytesPtr(RESULT_TABLE_NAME_BYTES));
- indexHTable = hTableFactory.getTable(new ImmutableBytesPtr(indexName));
+ resultTable = hTableFactory.getTable(new ImmutableBytesPtr(RESULT_TABLE_NAME_BYTES));
+ indexTable = hTableFactory.getTable(new ImmutableBytesPtr(indexName));
}
public void createResultTable(Connection connection) throws IOException, SQLException {
@@ -134,7 +153,7 @@ public class IndexVerificationResultRepository implements AutoCloseable {
TableDescriptorBuilder.newBuilder(resultTableName).
setColumnFamily(columnDescriptor).build();
admin.createTable(tableDescriptor);
- resultHTable = admin.getConnection().getTable(resultTableName);
+ resultTable = admin.getConnection().getTable(resultTableName);
}
}
private static byte[] generatePartialResultTableRowKey(long ts, byte[] indexTableName) {
@@ -189,7 +208,7 @@ public class IndexVerificationResultRepository implements AutoCloseable {
public void logToIndexToolResultTable(IndexToolVerificationResult verificationResult,
IndexTool.IndexVerifyType verifyType, byte[] region, boolean skipped) throws IOException {
long scanMaxTs = verificationResult.getScanMaxTs();
- byte[] rowKey = generateResultTableRowKey(scanMaxTs, indexHTable.getName().toBytes(),
+ byte[] rowKey = generateResultTableRowKey(scanMaxTs, indexTable.getName().toBytes(),
region, verificationResult.getStartRow(),
verificationResult.getStopRow());
Put put = new Put(rowKey);
@@ -209,6 +228,10 @@ public class IndexVerificationResultRepository implements AutoCloseable {
Bytes.toBytes(Long.toString(verificationResult.getBeforeRebuildMissingIndexRowCount())));
put.addColumn(RESULT_TABLE_COLUMN_FAMILY, BEFORE_REBUILD_INVALID_INDEX_ROW_COUNT_BYTES,
Bytes.toBytes(Long.toString(verificationResult.getBeforeRebuildInvalidIndexRowCount())));
+ put.addColumn(RESULT_TABLE_COLUMN_FAMILY, BEFORE_REBUILD_BEYOND_MAXLOOKBACK_MISSING_INDEX_ROW_COUNT_BYTES,
+ Bytes.toBytes(Long.toString(verificationResult.getBefore().getBeyondMaxLookBackMissingIndexRowCount())));
+ put.addColumn(RESULT_TABLE_COLUMN_FAMILY, BEFORE_REBUILD_BEYOND_MAXLOOKBACK_INVALID_INDEX_ROW_COUNT_BYTES,
+ Bytes.toBytes(Long.toString(verificationResult.getBefore().getBeyondMaxLookBackInvalidIndexRowCount())));
put.addColumn(RESULT_TABLE_COLUMN_FAMILY, BEFORE_REBUILD_INVALID_INDEX_ROW_COUNT_COZ_EXTRA_CELLS_BYTES,
Bytes.toBytes(Long.toString(verificationResult.getBeforeIndexHasExtraCellsCount())));
put.addColumn(RESULT_TABLE_COLUMN_FAMILY, BEFORE_REBUILD_INVALID_INDEX_ROW_COUNT_COZ_MISSING_CELLS_BYTES,
@@ -229,12 +252,16 @@ public class IndexVerificationResultRepository implements AutoCloseable {
Bytes.toBytes(Long.toString(verificationResult.getAfterRebuildMissingIndexRowCount())));
put.addColumn(RESULT_TABLE_COLUMN_FAMILY, AFTER_REBUILD_INVALID_INDEX_ROW_COUNT_BYTES,
Bytes.toBytes(Long.toString(verificationResult.getAfterRebuildInvalidIndexRowCount())));
+ put.addColumn(RESULT_TABLE_COLUMN_FAMILY, AFTER_REBUILD_BEYOND_MAXLOOKBACK_MISSING_INDEX_ROW_COUNT_BYTES,
+ Bytes.toBytes(Long.toString(verificationResult.getAfter().getBeyondMaxLookBackMissingIndexRowCount())));
+ put.addColumn(RESULT_TABLE_COLUMN_FAMILY, AFTER_REBUILD_BEYOND_MAXLOOKBACK_INVALID_INDEX_ROW_COUNT_BYTES,
+ Bytes.toBytes(Long.toString(verificationResult.getAfter().getBeyondMaxLookBackInvalidIndexRowCount())));
put.addColumn(RESULT_TABLE_COLUMN_FAMILY, AFTER_REBUILD_INVALID_INDEX_ROW_COUNT_COZ_EXTRA_CELLS_BYTES,
Bytes.toBytes(Long.toString(verificationResult.getAfterIndexHasExtraCellsCount())));
put.addColumn(RESULT_TABLE_COLUMN_FAMILY, AFTER_REBUILD_INVALID_INDEX_ROW_COUNT_COZ_MISSING_CELLS_BYTES,
Bytes.toBytes(Long.toString(verificationResult.getAfterIndexHasMissingCellsCount())));
}
- resultHTable.put(put);
+ resultTable.put(put);
}
public Table getTable(Connection conn, byte[] tableName) throws SQLException {
@@ -248,8 +275,8 @@ public class IndexVerificationResultRepository implements AutoCloseable {
byte[] stopRowKey = ByteUtil.calculateTheClosestNextRowKeyForPrefix(startRowKey);
IndexToolVerificationResult verificationResult = new IndexToolVerificationResult(ts);
Scan scan = new Scan();
- scan.withStartRow(startRowKey);
- scan.withStopRow(stopRowKey);
+ scan.setStartRow(startRowKey);
+ scan.setStopRow(stopRowKey);
return aggregateVerificationResult(htable, verificationResult, scan);
}
@@ -305,27 +332,28 @@ public class IndexVerificationResultRepository implements AutoCloseable {
}
public void close() throws IOException {
- if (resultHTable != null) {
- resultHTable.close();
+ if (resultTable != null) {
+ resultTable.close();
}
- if (indexHTable != null) {
- indexHTable.close();
+ if (indexTable != null) {
+ indexTable.close();
}
}
public void setResultTable(Table resultTable) {
- this.resultHTable = resultTable;
+ this.resultTable = resultTable;
}
public void setIndexTable(Table indexTable) {
- this.indexHTable = indexTable;
+ this.indexTable = indexTable;
}
public IndexToolVerificationResult getVerificationResult(Long ts, Scan scan, Region region, byte[] indexTableName) throws IOException {
byte [] rowKey = generateResultTableRowKey(ts,
indexTableName, region.getRegionInfo().getRegionName(),
scan.getStartRow(), scan.getStopRow());
- return getVerificationResult(resultHTable, rowKey, scan);
+ return getVerificationResult(resultTable, rowKey, scan);
+
}
}
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/PhoenixIndexImportDirectReducer.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/PhoenixIndexImportDirectReducer.java
index b8cb34b..8ec3603 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/PhoenixIndexImportDirectReducer.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/PhoenixIndexImportDirectReducer.java
@@ -82,6 +82,10 @@ public class PhoenixIndexImportDirectReducer extends
setValue(verificationResult.getBeforeRebuildInvalidIndexRowCount());
context.getCounter(PhoenixIndexToolJobCounters.BEFORE_REBUILD_INVALID_INDEX_ROW_COUNT_COZ_EXTRA_CELLS).
setValue(verificationResult.getBeforeIndexHasExtraCellsCount());
+ context.getCounter(PhoenixIndexToolJobCounters.BEFORE_REBUILD_BEYOND_MAXLOOKBACK_MISSING_INDEX_ROW_COUNT).
+ setValue(verificationResult.getBeforeRebuildBeyondMaxLookBackMissingIndexRowCount());
+ context.getCounter(PhoenixIndexToolJobCounters.BEFORE_REBUILD_BEYOND_MAXLOOKBACK_INVALID_INDEX_ROW_COUNT).
+ setValue(verificationResult.getBeforeRebuildBeyondMaxLookBackInvalidIndexRowCount());
context.getCounter(PhoenixIndexToolJobCounters.BEFORE_REBUILD_INVALID_INDEX_ROW_COUNT_COZ_MISSING_CELLS).
setValue(verificationResult.getBeforeIndexHasMissingCellsCount());
context.getCounter(PhoenixIndexToolJobCounters.BEFORE_REBUILD_UNVERIFIED_INDEX_ROW_COUNT).
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/PhoenixIndexToolJobCounters.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/PhoenixIndexToolJobCounters.java
index 1d5eac3..df8f0a8 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/PhoenixIndexToolJobCounters.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/PhoenixIndexToolJobCounters.java
@@ -1,5 +1,5 @@
/**
-/*
+ /*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -28,6 +28,8 @@ public enum PhoenixIndexToolJobCounters {
BEFORE_REBUILD_EXPIRED_INDEX_ROW_COUNT,
BEFORE_REBUILD_MISSING_INDEX_ROW_COUNT,
BEFORE_REBUILD_INVALID_INDEX_ROW_COUNT,
+ BEFORE_REBUILD_BEYOND_MAXLOOKBACK_MISSING_INDEX_ROW_COUNT,
+ BEFORE_REBUILD_BEYOND_MAXLOOKBACK_INVALID_INDEX_ROW_COUNT,
BEFORE_REBUILD_INVALID_INDEX_ROW_COUNT_COZ_EXTRA_CELLS,
BEFORE_REBUILD_INVALID_INDEX_ROW_COUNT_COZ_MISSING_CELLS,
BEFORE_REBUILD_UNVERIFIED_INDEX_ROW_COUNT,
@@ -37,6 +39,8 @@ public enum PhoenixIndexToolJobCounters {
AFTER_REBUILD_EXPIRED_INDEX_ROW_COUNT,
AFTER_REBUILD_MISSING_INDEX_ROW_COUNT,
AFTER_REBUILD_INVALID_INDEX_ROW_COUNT,
+ AFTER_REBUILD_BEYOND_MAXLOOKBACK_MISSING_INDEX_ROW_COUNT,
+ AFTER_REBUILD_BEYOND_MAXLOOKBACK_INVALID_INDEX_ROW_COUNT,
AFTER_REBUILD_INVALID_INDEX_ROW_COUNT_COZ_EXTRA_CELLS,
- AFTER_REBUILD_INVALID_INDEX_ROW_COUNT_COZ_MISSING_CELLS,;
+ AFTER_REBUILD_INVALID_INDEX_ROW_COUNT_COZ_MISSING_CELLS
}
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/PhoenixScrutinyJobCounters.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/PhoenixScrutinyJobCounters.java
index 43965b7..54e8a94 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/PhoenixScrutinyJobCounters.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/PhoenixScrutinyJobCounters.java
@@ -41,5 +41,11 @@ public enum PhoenixScrutinyJobCounters {
/**
* Number of batches processed
*/
- BATCHES_PROCESSED_COUNT;
+ BATCHES_PROCESSED_COUNT,
+ /**
+ * Number of rows in source that became older than the max lookback age while scrutiny
+ * was comparing them with the target, and didn't match. We break these out separately because
+ * they could be due to extra versions being compacted, and are harmless.
+ */
+ BEYOND_MAX_LOOKBACK_COUNT;
}
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java
index a61fdf7..17ae907 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java
@@ -60,6 +60,7 @@ import org.apache.hadoop.hbase.client.TableDescriptor;
import org.apache.hadoop.hbase.client.coprocessor.Batch;
import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
import org.apache.hadoop.hbase.filter.Filter;
+import org.apache.hadoop.hbase.filter.FilterList;
import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.ipc.CoprocessorRpcUtils.BlockingRpcCallback;
@@ -144,7 +145,8 @@ public class IndexUtil {
public static final String INDEX_COLUMN_NAME_SEP = ":";
public static final byte[] INDEX_COLUMN_NAME_SEP_BYTES = Bytes.toBytes(INDEX_COLUMN_NAME_SEP);
- private static Cache<String, Boolean> indexNameGlobalIndexCheckerEnabledMap = CacheBuilder.newBuilder()
+ private final static Cache<String, Boolean> indexNameGlobalIndexCheckerEnabledMap =
+ CacheBuilder.newBuilder()
.expireAfterWrite(QueryServicesOptions.GLOBAL_INDEX_CHECKER_ENABLED_MAP_EXPIRATION_MIN,
TimeUnit.MINUTES)
.build();
@@ -577,8 +579,8 @@ public class IndexUtil {
joinResult = dataRegion.get(get);
} else {
TableName dataTable =
- TableName.valueOf(MetaDataUtil.getLocalIndexUserTableName(
- environment.getRegion().getTableDescriptor().getTableName().getNameAsString()));
+ TableName.valueOf(MetaDataUtil.getLocalIndexUserTableName(environment.getRegion().
+ getTableDescriptor().getTableName().getNameAsString()));
Table table = null;
try {
table = environment.getConnection().getTable(dataTable);
@@ -596,7 +598,8 @@ public class IndexUtil {
tupleProjector.getSchema().toBytes(joinTuple, tupleProjector.getExpressions(),
tupleProjector.getValueBitSet(), ptr);
Cell keyValue =
- PhoenixKeyValueUtil.newKeyValue(firstCell.getRowArray(),firstCell.getRowOffset(),firstCell.getRowLength(), VALUE_COLUMN_FAMILY,
+ PhoenixKeyValueUtil.newKeyValue(firstCell.getRowArray(),
+ firstCell.getRowOffset(),firstCell.getRowLength(), VALUE_COLUMN_FAMILY,
VALUE_COLUMN_QUALIFIER, firstCell.getTimestamp(), value, 0, value.length);
result.add(keyValue);
}
@@ -641,7 +644,8 @@ public class IndexUtil {
m.setDurability(Durability.SKIP_WAL);
}
}
- region.batchMutate(mutations.toArray(new Mutation[mutations.size()]));
+ region.batchMutate(
+ mutations.toArray(new Mutation[mutations.size()]));
}
public static MetaDataMutationResult updateIndexState(String indexTableName, long minTimeStamp,
@@ -667,7 +671,7 @@ public class IndexUtil {
@Override
public MetaDataResponse call(MetaDataService instance) throws IOException {
ServerRpcController controller = new ServerRpcController();
- BlockingRpcCallback<MetaDataResponse> rpcCallback = new BlockingRpcCallback<MetaDataResponse>();
+ BlockingRpcCallback<MetaDataResponse> rpcCallback = new BlockingRpcCallback<>();
UpdateIndexStateRequest.Builder builder = UpdateIndexStateRequest.newBuilder();
for (Mutation m : tableMetadata) {
MutationProto mp = ProtobufUtil.toProto(m);
@@ -817,30 +821,61 @@ public class IndexUtil {
return false;
}
+ private static boolean addEmptyColumnToFilter(Scan scan, byte[] emptyCF, byte[] emptyCQ, Filter filter, boolean addedEmptyColumn) {
+ if (filter instanceof EncodedQualifiersColumnProjectionFilter) {
+ ((EncodedQualifiersColumnProjectionFilter) filter).addTrackedColumn(ENCODED_EMPTY_COLUMN_NAME);
+ if (!addedEmptyColumn && containsOneOrMoreColumn(scan)) {
+ scan.addColumn(emptyCF, emptyCQ);
+ return true;
+ }
+ }
+ else if (filter instanceof ColumnProjectionFilter) {
+ ((ColumnProjectionFilter) filter).addTrackedColumn(new ImmutableBytesPtr(emptyCF), new ImmutableBytesPtr(emptyCQ));
+ if (!addedEmptyColumn && containsOneOrMoreColumn(scan)) {
+ scan.addColumn(emptyCF, emptyCQ);
+ return true;
+ }
+ }
+ else if (filter instanceof MultiEncodedCQKeyValueComparisonFilter) {
+ ((MultiEncodedCQKeyValueComparisonFilter) filter).setMinQualifier(ENCODED_EMPTY_COLUMN_NAME);
+ }
+ else if (!addedEmptyColumn && filter instanceof FirstKeyOnlyFilter) {
+ scan.addColumn(emptyCF, emptyCQ);
+ return true;
+ }
+ return addedEmptyColumn;
+ }
+
+ private static boolean addEmptyColumnToFilterList(Scan scan, byte[] emptyCF, byte[] emptyCQ, FilterList filterList, boolean addedEmptyColumn) {
+ Iterator<Filter> filterIterator = filterList.getFilters().iterator();
+ while (filterIterator.hasNext()) {
+ Filter filter = filterIterator.next();
+ if (filter instanceof FilterList) {
+ if (addEmptyColumnToFilterList(scan, emptyCF, emptyCQ, (FilterList) filter, addedEmptyColumn)) {
+ addedEmptyColumn = true;
+ }
+ } else {
+ if (addEmptyColumnToFilter(scan, emptyCF, emptyCQ, filter, addedEmptyColumn)) {
+ addedEmptyColumn = true;
+ }
+ }
+ }
+ return addedEmptyColumn;
+ }
+
public static void addEmptyColumnToScan(Scan scan, byte[] emptyCF, byte[] emptyCQ) {
boolean addedEmptyColumn = false;
- Iterator<Filter> iterator = ScanUtil.getFilterIterator(scan);
- while (iterator.hasNext()) {
- Filter filter = iterator.next();
- if (filter instanceof EncodedQualifiersColumnProjectionFilter) {
- ((EncodedQualifiersColumnProjectionFilter) filter).addTrackedColumn(ENCODED_EMPTY_COLUMN_NAME);
- if (!addedEmptyColumn && containsOneOrMoreColumn(scan)) {
- scan.addColumn(emptyCF, emptyCQ);
+ Filter filter = scan.getFilter();
+ if (filter != null) {
+ if (filter instanceof FilterList) {
+ if (addEmptyColumnToFilterList(scan, emptyCF, emptyCQ, (FilterList) filter, addedEmptyColumn)) {
+ addedEmptyColumn = true;
}
- }
- else if (filter instanceof ColumnProjectionFilter) {
- ((ColumnProjectionFilter) filter).addTrackedColumn(new ImmutableBytesPtr(emptyCF), new ImmutableBytesPtr(emptyCQ));
- if (!addedEmptyColumn && containsOneOrMoreColumn(scan)) {
- scan.addColumn(emptyCF, emptyCQ);
+ } else {
+ if (addEmptyColumnToFilter(scan, emptyCF, emptyCQ, filter, addedEmptyColumn)) {
+ addedEmptyColumn = true;
}
}
- else if (filter instanceof MultiEncodedCQKeyValueComparisonFilter) {
- ((MultiEncodedCQKeyValueComparisonFilter) filter).setMinQualifier(ENCODED_EMPTY_COLUMN_NAME);
- }
- else if (!addedEmptyColumn && filter instanceof FirstKeyOnlyFilter) {
- scan.addColumn(emptyCF, emptyCQ);
- addedEmptyColumn = true;
- }
}
if (!addedEmptyColumn && containsOneOrMoreColumn(scan)) {
scan.addColumn(emptyCF, emptyCQ);
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/index/IndexToolTest.java b/phoenix-core/src/test/java/org/apache/phoenix/index/IndexToolTest.java
index 87215d0..6f88cb0 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/index/IndexToolTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/index/IndexToolTest.java
@@ -18,12 +18,14 @@
package org.apache.phoenix.index;
import org.apache.commons.cli.CommandLine;
+import org.apache.phoenix.compat.hbase.HbaseCompatCapabilities;
import org.apache.phoenix.end2end.IndexToolIT;
import org.apache.phoenix.mapreduce.index.IndexTool;
import org.apache.phoenix.query.BaseTest;
import org.apache.phoenix.schema.PTable;
import org.apache.phoenix.util.EnvironmentEdgeManager;
import org.junit.Assert;
+import org.junit.Assume;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
@@ -65,6 +67,7 @@ public class IndexToolTest extends BaseTest {
@Test
public void testParseOptions_timeRange_timeRangeNotNull() throws Exception {
+ Assume.assumeTrue(HbaseCompatCapabilities.isRawFilterSupported());
Long startTime = 10L;
Long endTime = 15L;
String [] args =
@@ -90,6 +93,7 @@ public class IndexToolTest extends BaseTest {
@Test
public void testParseOptions_timeRange_startTimeNotNull() throws Exception {
+ Assume.assumeTrue(HbaseCompatCapabilities.isRawFilterSupported());
Long startTime = 10L;
String [] args =
IndexToolIT.getArgValues(true, true, schema,
@@ -129,6 +133,7 @@ public class IndexToolTest extends BaseTest {
@Test
public void testParseOptions_timeRange_endTimeNullStartTimeInFuture() throws Exception {
+ Assume.assumeTrue(HbaseCompatCapabilities.isRawFilterSupported());
Long startTime = EnvironmentEdgeManager.currentTimeMillis() + 100000;
String [] args =
IndexToolIT.getArgValues(true, true, schema,
@@ -142,6 +147,7 @@ public class IndexToolTest extends BaseTest {
@Test(timeout = 10000 /* 10 secs */)
public void testParseOptions_timeRange_startTimeInFuture() throws Exception {
+ Assume.assumeTrue(HbaseCompatCapabilities.isRawFilterSupported());
Long startTime = EnvironmentEdgeManager.currentTimeMillis() + 100000;
Long endTime = EnvironmentEdgeManager.currentTimeMillis() + 200000;
String [] args =
@@ -156,6 +162,7 @@ public class IndexToolTest extends BaseTest {
@Test(timeout = 10000 /* 10 secs */)
public void testParseOptions_timeRange_endTimeInFuture() throws Exception {
+ Assume.assumeTrue(HbaseCompatCapabilities.isRawFilterSupported());
Long startTime = EnvironmentEdgeManager.currentTimeMillis();
Long endTime = EnvironmentEdgeManager.currentTimeMillis() + 100000;
String [] args =
@@ -170,6 +177,7 @@ public class IndexToolTest extends BaseTest {
@Test
public void testParseOptions_timeRange_startTimeEqEndTime() throws Exception {
+ Assume.assumeTrue(HbaseCompatCapabilities.isRawFilterSupported());
Long startTime = 10L;
Long endTime = 10L;
String [] args =
@@ -184,6 +192,7 @@ public class IndexToolTest extends BaseTest {
@Test
public void testParseOptions_timeRange_startTimeGtEndTime() throws Exception {
+ Assume.assumeTrue(HbaseCompatCapabilities.isRawFilterSupported());
Long startTime = 10L;
Long endTime = 1L;
String [] args =
@@ -206,6 +215,7 @@ public class IndexToolTest extends BaseTest {
@Test
public void testIncrcementalVerifyOption() throws Exception {
+ Assume.assumeTrue(HbaseCompatCapabilities.isRawFilterSupported());
IndexTool mockTool = Mockito.mock(IndexTool.class);
when(mockTool.getLastVerifyTime()).thenCallRealMethod();
Long lastVerifyTime = 10L;
@@ -231,6 +241,7 @@ public class IndexToolTest extends BaseTest {
@Test
public void testIncrcementalVerifyOption_notApplicable() throws Exception {
+ Assume.assumeTrue(HbaseCompatCapabilities.isRawFilterSupported());
IndexTool mockTool = Mockito.mock(IndexTool.class);
when(mockTool.getLastVerifyTime()).thenCallRealMethod();
Long lastVerifyTime = 10L;
@@ -253,8 +264,41 @@ public class IndexToolTest extends BaseTest {
}
@Test
+ public void testIncrementalVerifyNotSupportedWithoutRawSkipScanFilters() {
+ //We should give an exception if we try to use incremental verification on HBase 2.1
+ // which lacks HBASE-22710 enabling raw skip scan filters. For 2.2 we assume 2.2.5+
+ Assume.assumeFalse(HbaseCompatCapabilities.isRawFilterSupported());
+ try {
+ IndexTool it = new IndexTool();
+ Long lastVerifyTime = 10L;
+ String[] args =
+ IndexToolIT.getArgValues(true, true, schema,
+ dataTable, indexTable, tenantId, IndexTool.IndexVerifyType.AFTER,
+ lastVerifyTime, null, IndexTool.IndexDisableLoggingType.NONE,
+ lastVerifyTime);
+ it.parseOptions(args);
+ Assert.fail("Should have thrown an IllegalStateException");
+ } catch (IllegalStateException ise) {
+ //eat exception
+ }
+ //now check retry-verify
+ try {
+ IndexTool it = new IndexTool();
+ Long lastVerifyTime = 10L;
+ String[] args =
+ IndexToolIT.getArgValues(true, true, schema,
+ dataTable, indexTable, tenantId, IndexTool.IndexVerifyType.AFTER,
+ null, null, IndexTool.IndexDisableLoggingType.NONE,
+ lastVerifyTime);
+ it.parseOptions(args);
+ Assert.fail("Should have thrown an IllegalStateException");
+ } catch (IllegalStateException ise) {
+ //eat exception
+ }
+ }
+ @Test
public void testCheckVerifyAndDisableLogging_defaultsNone() throws Exception {
- Long startTime = 1L;
+ Long startTime = null;
Long endTime = 10L;
String [] args =
IndexToolIT.getArgValues(true, true, schema,
@@ -312,7 +356,7 @@ public class IndexToolTest extends BaseTest {
public void verifyDisableLogging(IndexTool.IndexDisableLoggingType disableType,
IndexTool.IndexVerifyType verifyType) throws Exception {
- Long startTime = 1L;
+ Long startTime = null;
Long endTime = 10L;
String[] args =
IndexToolIT.getArgValues(true, true, schema,
@@ -325,7 +369,7 @@ public class IndexToolTest extends BaseTest {
public void verifyDisableLoggingException(IndexTool.IndexDisableLoggingType disableType,
IndexTool.IndexVerifyType verifyType) {
- Long startTime = 1L;
+ Long startTime = null;
Long endTime = 10L;
String[] args =
IndexToolIT.getArgValues(true, true, schema,
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/index/IndexUpgradeToolTest.java b/phoenix-core/src/test/java/org/apache/phoenix/index/IndexUpgradeToolTest.java
index 0ce54fb..c3e717d 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/index/IndexUpgradeToolTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/index/IndexUpgradeToolTest.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HConstants;
+import org.apache.phoenix.compat.hbase.HbaseCompatCapabilities;
import org.apache.phoenix.mapreduce.index.IndexTool;
import org.apache.phoenix.mapreduce.index.IndexUpgradeTool;
import org.apache.phoenix.query.QueryServices;
@@ -38,6 +39,7 @@ import org.apache.phoenix.query.QueryServicesOptions;
import org.apache.phoenix.util.PhoenixRuntime;
import org.junit.Assert;
+import org.junit.Assume;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
@@ -98,6 +100,7 @@ public class IndexUpgradeToolTest {
@Test
public void testIfOptionsArePassedToIndexTool() throws Exception {
+ Assume.assumeTrue(HbaseCompatCapabilities.isRawFilterSupported());
if (!upgrade) {
return;
}
@@ -127,6 +130,7 @@ public class IndexUpgradeToolTest {
@Test
public void testMalformedSpacingOptionsArePassedToIndexTool() throws Exception {
+ Assume.assumeTrue(HbaseCompatCapabilities.isRawFilterSupported());
if (!upgrade) {
return;
}
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/index/VerifySingleIndexRowTest.java b/phoenix-core/src/test/java/org/apache/phoenix/index/VerifySingleIndexRowTest.java
index 1797e48..f1ada0b 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/index/VerifySingleIndexRowTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/index/VerifySingleIndexRowTest.java
@@ -20,29 +20,31 @@ package org.apache.phoenix.index;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellBuilderFactory;
+import org.apache.hadoop.hbase.CellBuilderType;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.DoNotRetryIOException;
import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Mutation;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Pair;
+import org.apache.phoenix.compat.hbase.HbaseCompatCapabilities;
import org.apache.phoenix.coprocessor.GlobalIndexRegionScanner;
import org.apache.phoenix.coprocessor.IndexRebuildRegionScanner;
import org.apache.phoenix.coprocessor.IndexToolVerificationResult;
+import org.apache.phoenix.hbase.index.IndexRegionObserver;
import org.apache.phoenix.jdbc.PhoenixConnection;
+import org.apache.phoenix.mapreduce.index.IndexVerificationOutputRepository;
import org.apache.phoenix.query.BaseConnectionlessQueryTest;
+import org.apache.phoenix.query.QueryConstants;
import org.apache.phoenix.schema.PTable;
import org.apache.phoenix.schema.PTableKey;
-import org.apache.phoenix.util.EnvironmentEdge;
-import org.apache.phoenix.util.EnvironmentEdgeManager;
-import org.apache.phoenix.util.PhoenixRuntime;
-import org.apache.phoenix.util.SchemaUtil;
+import org.apache.phoenix.util.*;
+import org.junit.Assume;
import org.junit.Before;
-import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
@@ -56,18 +58,13 @@ import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.SQLException;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.NavigableMap;
-import java.util.Properties;
+import java.util.*;
import static org.apache.phoenix.hbase.index.IndexRegionObserver.UNVERIFIED_BYTES;
import static org.apache.phoenix.hbase.index.IndexRegionObserver.VERIFIED_BYTES;
import static org.apache.phoenix.query.QueryConstants.EMPTY_COLUMN_BYTES;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.*;
+import static org.mockito.Matchers.any;
import static org.mockito.Mockito.doNothing;
import static org.mockito.Mockito.when;
@@ -81,9 +78,9 @@ public class VerifySingleIndexRowTest extends BaseConnectionlessQueryTest {
public static final String SECOND_VALUE = "SECOND_VALUE";
public static final String
CREATE_TABLE_DDL = "CREATE TABLE IF NOT EXISTS %s (FIRST_ID BIGINT NOT NULL, "
- + "SECOND_ID BIGINT NOT NULL, FIRST_VALUE VARCHAR(20), "
- + "SECOND_VALUE INTEGER "
- + "CONSTRAINT PK PRIMARY KEY(FIRST_ID, SECOND_ID)) COLUMN_ENCODED_BYTES=0";
+ + "SECOND_ID BIGINT NOT NULL, FIRST_VALUE VARCHAR(20), "
+ + "SECOND_VALUE INTEGER "
+ + "CONSTRAINT PK PRIMARY KEY(FIRST_ID, SECOND_ID)) COLUMN_ENCODED_BYTES=0";
public static final String
CREATE_INDEX_DDL = "CREATE INDEX %s ON %s (SECOND_VALUE) INCLUDE (FIRST_VALUE)";
@@ -139,7 +136,8 @@ public class VerifySingleIndexRowTest extends BaseConnectionlessQueryTest {
Delete delete = null;
PhoenixConnection pconn;
IndexToolVerificationResult.PhaseResult actualPR;
- public Map<byte[], List<Mutation>> indexKeyToMutationMapLocal;
+ public Map<byte[], List<Mutation>> indexKeyToMutationMap = null;
+ Set<byte[]> mostRecentIndexRowKeys;
private IndexMaintainer indexMaintainer;
@Before
@@ -227,7 +225,7 @@ public class VerifySingleIndexRowTest extends BaseConnectionlessQueryTest {
}
private void upsertCompleteRow(int key1, int key2, String val1
- , int val2) throws SQLException, IOException {
+ , int val2) throws SQLException, IOException {
try(Connection conn = DriverManager.getConnection(getUrl(), new Properties())) {
PreparedStatement
ps = conn.prepareStatement(String.format(COMPLETE_ROW_UPSERT, dataTableFullName));
@@ -249,9 +247,13 @@ public class VerifySingleIndexRowTest extends BaseConnectionlessQueryTest {
for (Cell kv : elem.getSecond()) {
Cell cell =
- CellUtil.createCell(CellUtil.cloneRow(kv), CellUtil.cloneFamily(kv), CellUtil.cloneQualifier(kv),
- mutationTS, kv.getTypeByte(), CellUtil.cloneValue(kv));
- if (KeyValue.Type.codeToType(cell.getTypeByte()) == KeyValue.Type.Put) {
+ CellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(CellUtil.cloneRow(kv)).
+ setFamily(CellUtil.cloneFamily(kv)).
+ setQualifier(CellUtil.cloneQualifier(kv)).
+ setTimestamp(mutationTS).
+ setType(kv.getType()).
+ setValue(CellUtil.cloneValue(kv)).build();
+ if (cell.getType().equals(Cell.Type.Put)) {
if (put == null ) {
put = new Put(key);
}
@@ -260,7 +262,7 @@ public class VerifySingleIndexRowTest extends BaseConnectionlessQueryTest {
if (delete == null) {
delete = new Delete(key);
}
- delete.addDeleteMarker(cell);
+ delete.add(cell);
}
}
}
@@ -268,51 +270,52 @@ public class VerifySingleIndexRowTest extends BaseConnectionlessQueryTest {
private void initializeRebuildScannerAttributes() {
when(rebuildScanner.setIndexTableTTL(Matchers.anyInt())).thenCallRealMethod();
when(rebuildScanner.setIndexMaintainer(Matchers.<IndexMaintainer>any())).thenCallRealMethod();
- when(rebuildScanner.setIndexKeyToMutationMap(Matchers.<Map>any())).thenCallRealMethod();
+ when(rebuildScanner.setMaxLookBackInMills(Matchers.anyLong())).thenCallRealMethod();
rebuildScanner.setIndexTableTTL(HConstants.FOREVER);
indexMaintainer = pIndexTable.getIndexMaintainer(pDataTable, pconn);
rebuildScanner.setIndexMaintainer(indexMaintainer);
+ // set the maxLookBack to infinite to avoid the compaction
+ rebuildScanner.setMaxLookBackInMills(Long.MAX_VALUE);
}
private void initializeGlobalMockitoSetup() throws IOException {
//setup
when(GlobalIndexRegionScanner.getIndexRowKey(indexMaintainer, put)).thenCallRealMethod();
- when(rebuildScanner.prepareIndexMutations(put, delete)).thenCallRealMethod();
- when(rebuildScanner.verifySingleIndexRow(Matchers.<Result>any(),
- Matchers.<IndexToolVerificationResult.PhaseResult>any())).thenCallRealMethod();
+ when(rebuildScanner.prepareIndexMutations(put, delete, indexKeyToMutationMap, mostRecentIndexRowKeys)).thenCallRealMethod();
+ when(rebuildScanner.verifySingleIndexRow(Matchers.<Result>any(), Matchers.<Map>any(),
+ Matchers.<Set>any(), Matchers.<List>any(),
+ Matchers.<IndexToolVerificationResult.PhaseResult>any(), Matchers.anyBoolean())).thenCallRealMethod();
doNothing().when(rebuildScanner)
.logToIndexToolOutputTable(Matchers.<byte[]>any(),Matchers.<byte[]>any(),
- Mockito.anyLong(),Mockito.anyLong(), Mockito.anyString(),
- Matchers.<byte[]>any(), Matchers.<byte[]>any());
+ Mockito.anyLong(),Mockito.anyLong(), Mockito.anyString(),
+ Matchers.<byte[]>any(), Matchers.<byte[]>any(), Matchers.anyBoolean(),
+ Mockito.<IndexVerificationOutputRepository.IndexVerificationErrorType>any());
doNothing().when(rebuildScanner)
.logToIndexToolOutputTable(Matchers.<byte[]>any(),Matchers.<byte[]>any(),
- Mockito.anyLong(),Mockito.anyLong(), Mockito.anyString());
+ Mockito.anyLong(),Mockito.anyLong(), Mockito.anyString(),
+ Matchers.anyBoolean(),
+ Mockito.<IndexVerificationOutputRepository.IndexVerificationErrorType>any());
//populate the local map to use to create actual mutations
- indexKeyToMutationMapLocal = Maps.newTreeMap(Bytes.BYTES_COMPARATOR);
- rebuildScanner.setIndexKeyToMutationMap(indexKeyToMutationMapLocal);
- rebuildScanner.prepareIndexMutations(put, delete);
-
- //populate map to use in test code
- Map<byte[], List<Mutation>> indexKeyToMutationMap = Maps.newTreeMap((Bytes.BYTES_COMPARATOR));
- rebuildScanner.setIndexKeyToMutationMap(indexKeyToMutationMap);
- rebuildScanner.prepareIndexMutations(put, delete);
+ indexKeyToMutationMap = Maps.newTreeMap(Bytes.BYTES_COMPARATOR);
+ rebuildScanner.prepareIndexMutations(put, delete, indexKeyToMutationMap, mostRecentIndexRowKeys);
}
private byte[] getValidRowKey() {
- return indexKeyToMutationMapLocal.entrySet().iterator().next().getKey();
+ return indexKeyToMutationMap.entrySet().iterator().next().getKey();
}
@Test
public void testVerifySingleIndexRow_validIndexRowCount_nonZero() throws IOException {
IndexToolVerificationResult.PhaseResult expectedPR = getValidPhaseResult();
for (Map.Entry<byte[], List<Mutation>>
- entry : indexKeyToMutationMapLocal.entrySet()) {
+ entry : indexKeyToMutationMap.entrySet()) {
initializeLocalMockitoSetup(entry, TestType.VALID_EXACT_MATCH);
//test code
- rebuildScanner.verifySingleIndexRow(indexRow, actualPR);
+ rebuildScanner.verifySingleIndexRow(indexRow, indexKeyToMutationMap,
+ mostRecentIndexRowKeys, Collections.emptyList(), actualPR, true);
- assertTrue(actualPR.equals(expectedPR));
+ assertEquals(actualPR, expectedPR);
}
}
@@ -320,12 +323,13 @@ public class VerifySingleIndexRowTest extends BaseConnectionlessQueryTest {
public void testVerifySingleIndexRow_validIndexRowCount_moreActual() throws IOException {
IndexToolVerificationResult.PhaseResult expectedPR = getValidPhaseResult();
for (Map.Entry<byte[], List<Mutation>>
- entry : indexKeyToMutationMapLocal.entrySet()) {
+ entry : indexKeyToMutationMap.entrySet()) {
initializeLocalMockitoSetup(entry, TestType.VALID_MORE_MUTATIONS);
//test code
- rebuildScanner.verifySingleIndexRow(indexRow, actualPR);
+ rebuildScanner.verifySingleIndexRow(indexRow, indexKeyToMutationMap,
+ mostRecentIndexRowKeys, Collections.emptyList(), actualPR, true);
- assertTrue(actualPR.equals(expectedPR));
+ assertEquals(actualPR, expectedPR);
}
}
@@ -333,12 +337,13 @@ public class VerifySingleIndexRowTest extends BaseConnectionlessQueryTest {
public void testVerifySingleIndexRow_allMix() throws IOException {
IndexToolVerificationResult.PhaseResult expectedPR = getValidPhaseResult();
for (Map.Entry<byte[], List<Mutation>>
- entry : indexKeyToMutationMapLocal.entrySet()) {
+ entry : indexKeyToMutationMap.entrySet()) {
initializeLocalMockitoSetup(entry, TestType.VALID_MIX_MUTATIONS);
//test code
- rebuildScanner.verifySingleIndexRow(indexRow, actualPR);
+ rebuildScanner.verifySingleIndexRow(indexRow, indexKeyToMutationMap,
+ mostRecentIndexRowKeys, Collections.emptyList(), actualPR, true);
- assertTrue(actualPR.equals(expectedPR));
+ assertEquals(actualPR, expectedPR);
}
}
@@ -346,85 +351,91 @@ public class VerifySingleIndexRowTest extends BaseConnectionlessQueryTest {
public void testVerifySingleIndexRow_allUnverified() throws IOException {
IndexToolVerificationResult.PhaseResult expectedPR = getValidPhaseResult();
for (Map.Entry<byte[], List<Mutation>>
- entry : indexKeyToMutationMapLocal.entrySet()) {
+ entry : indexKeyToMutationMap.entrySet()) {
initializeLocalMockitoSetup(entry, TestType.VALID_NEW_UNVERIFIED_MUTATIONS);
//test code
- rebuildScanner.verifySingleIndexRow(indexRow, actualPR);
+ rebuildScanner.verifySingleIndexRow(indexRow, indexKeyToMutationMap,
+ mostRecentIndexRowKeys, Collections.emptyList(), actualPR, true);
- assertTrue(actualPR.equals(expectedPR));
+ assertEquals(actualPR, expectedPR);
}
}
@Test
public void testVerifySingleIndexRow_expiredIndexRowCount_nonZero() throws IOException {
IndexToolVerificationResult.PhaseResult
- expectedPR = new IndexToolVerificationResult.PhaseResult(0, 1, 0, 0, 0, 0);
+ expectedPR = new IndexToolVerificationResult.PhaseResult(0, 1, 0, 0, 0, 0,0,0);
try {
for (Map.Entry<byte[], List<Mutation>>
- entry : indexKeyToMutationMapLocal.entrySet()) {
+ entry : indexKeyToMutationMap.entrySet()) {
initializeLocalMockitoSetup(entry, TestType.EXPIRED);
expireThisRow();
//test code
- rebuildScanner.verifySingleIndexRow(indexRow, actualPR);
+ rebuildScanner.verifySingleIndexRow(indexRow, indexKeyToMutationMap,
+ mostRecentIndexRowKeys, Collections.emptyList(), actualPR, true);
- assertTrue(actualPR.equals(expectedPR));
+ assertEquals(actualPR, expectedPR);
}
} finally {
EnvironmentEdgeManager.reset();
}
}
- @Ignore
+
@Test
public void testVerifySingleIndexRow_invalidIndexRowCount_cellValue() throws IOException {
IndexToolVerificationResult.PhaseResult expectedPR = getInvalidPhaseResult();
+ expectedPR.setIndexHasExtraCellsCount(1);
for (Map.Entry<byte[], List<Mutation>>
- entry : indexKeyToMutationMapLocal.entrySet()) {
+ entry : indexKeyToMutationMap.entrySet()) {
initializeLocalMockitoSetup(entry, TestType.INVALID_CELL_VALUE);
//test code
- rebuildScanner.verifySingleIndexRow(indexRow, actualPR);
+ rebuildScanner.verifySingleIndexRow(indexRow, indexKeyToMutationMap,
+ mostRecentIndexRowKeys, Collections.emptyList(), actualPR, true);
- assertTrue(actualPR.equals(expectedPR));
+ assertEquals(actualPR, expectedPR);
}
}
- @Ignore
@Test
public void testVerifySingleIndexRow_invalidIndexRowCount_emptyCell() throws IOException {
IndexToolVerificationResult.PhaseResult expectedPR = getInvalidPhaseResult();
for (Map.Entry<byte[], List<Mutation>>
- entry : indexKeyToMutationMapLocal.entrySet()) {
+ entry : indexKeyToMutationMap.entrySet()) {
initializeLocalMockitoSetup(entry, TestType.INVALID_EMPTY_CELL);
//test code
- rebuildScanner.verifySingleIndexRow(indexRow, actualPR);
+ rebuildScanner.verifySingleIndexRow(indexRow, indexKeyToMutationMap,
+ mostRecentIndexRowKeys, Collections.emptyList(), actualPR, true);
- assertTrue(actualPR.equals(expectedPR));
+ assertEquals(actualPR, expectedPR);
}
}
@Test
public void testVerifySingleIndexRow_invalidIndexRowCount_diffColumn() throws IOException {
IndexToolVerificationResult.PhaseResult expectedPR = getInvalidPhaseResult();
+ expectedPR.setIndexHasExtraCellsCount(1);
for (Map.Entry<byte[], List<Mutation>>
- entry : indexKeyToMutationMapLocal.entrySet()) {
+ entry : indexKeyToMutationMap.entrySet()) {
initializeLocalMockitoSetup(entry, TestType.INVALID_COLUMN);
//test code
- rebuildScanner.verifySingleIndexRow(indexRow, actualPR);
+ rebuildScanner.verifySingleIndexRow(indexRow, indexKeyToMutationMap,
+ mostRecentIndexRowKeys, Collections.emptyList(), actualPR, true);
- assertTrue(actualPR.equals(expectedPR));
+ assertEquals(actualPR, expectedPR);
}
}
- @Ignore
@Test
public void testVerifySingleIndexRow_invalidIndexRowCount_extraCell() throws IOException {
IndexToolVerificationResult.PhaseResult expectedPR = getInvalidPhaseResult();
for (Map.Entry<byte[], List<Mutation>>
- entry : indexKeyToMutationMapLocal.entrySet()) {
+ entry : indexKeyToMutationMap.entrySet()) {
initializeLocalMockitoSetup(entry, TestType.INVALID_EXTRA_CELL);
//test code
- rebuildScanner.verifySingleIndexRow(indexRow, actualPR);
+ rebuildScanner.verifySingleIndexRow(indexRow, indexKeyToMutationMap,
+ mostRecentIndexRowKeys, Collections.emptyList(), actualPR, true);
- assertTrue(actualPR.equals(expectedPR));
+ assertEquals(actualPR, expectedPR);
}
}
@@ -433,48 +444,154 @@ public class VerifySingleIndexRowTest extends BaseConnectionlessQueryTest {
when(indexRow.getRow()).thenReturn(Bytes.toBytes(1));
exceptionRule.expect(DoNotRetryIOException.class);
exceptionRule.expectMessage(IndexRebuildRegionScanner.NO_EXPECTED_MUTATION);
- rebuildScanner.verifySingleIndexRow(indexRow, actualPR);
- }
-
- @Test
- public void testVerifySingleIndexRow_actualMutations_null() throws IOException {
- byte [] validRowKey = getValidRowKey();
- when(indexRow.getRow()).thenReturn(validRowKey);
- when(rebuildScanner.prepareActualIndexMutations(indexRow)).thenReturn(null);
- exceptionRule.expect(DoNotRetryIOException.class);
- exceptionRule.expectMessage(IndexRebuildRegionScanner.ACTUAL_MUTATION_IS_NULL_OR_EMPTY);
- rebuildScanner.verifySingleIndexRow(indexRow, actualPR);
+ rebuildScanner.verifySingleIndexRow(indexRow, indexKeyToMutationMap,
+ mostRecentIndexRowKeys, Collections.emptyList(), actualPR, true);
}
@Test
public void testVerifySingleIndexRow_validIndexRowCount_extraCell() throws IOException {
for (Map.Entry<byte[], List<Mutation>>
- entry : indexKeyToMutationMapLocal.entrySet()) {
+ entry : indexKeyToMutationMap.entrySet()) {
initializeLocalMockitoSetup(entry, TestType.VALID_EXTRA_CELL);
//test code
- rebuildScanner.verifySingleIndexRow(indexRow, actualPR);
+ rebuildScanner.verifySingleIndexRow(indexRow, indexKeyToMutationMap,
+ mostRecentIndexRowKeys, Collections.emptyList(), actualPR, true);
assertEquals(1, actualPR.getIndexHasExtraCellsCount());
}
}
+ // Test the major compaction on index table only.
+ // There is at least one expected mutation within maxLookBack that has its matching one in the actual list.
+ // However there are some expected mutations outside of maxLookBack, which matching ones in actual list may be compacted away.
+ // We will report such row as a valid row.
@Test
- public void testVerifySingleIndexRow_actualMutations_empty() throws IOException {
- byte [] validRowKey = getValidRowKey();
- when(indexRow.getRow()).thenReturn(validRowKey);
- actualMutationList = new ArrayList<>();
- when(rebuildScanner.prepareActualIndexMutations(indexRow)).thenReturn(actualMutationList);
- exceptionRule.expect(DoNotRetryIOException.class);
- exceptionRule.expectMessage(IndexRebuildRegionScanner.ACTUAL_MUTATION_IS_NULL_OR_EMPTY);
- rebuildScanner.verifySingleIndexRow(indexRow, actualPR);
+ public void testVerifySingleIndexRow_compactionOnIndexTable_atLeastOneExpectedMutationWithinMaxLookBack() throws Exception {
+ Assume.assumeTrue(HbaseCompatCapabilities.isMaxLookbackTimeSupported());
+ String dataRowKey = "k1";
+ byte[] indexRowKey1Bytes = generateIndexRowKey(dataRowKey, "val1");
+ ManualEnvironmentEdge injectEdge = new ManualEnvironmentEdge();
+ injectEdge.setValue(1);
+ EnvironmentEdgeManager.injectEdge(injectEdge);
+
+ List<Mutation> expectedMutations = new ArrayList<>();
+ List<Mutation> actualMutations = new ArrayList<>();
+ // change the maxLookBack from infinite to some interval, which allows to simulate the mutation beyond the maxLookBack window.
+ long maxLookbackInMills = 10 * 1000;
+ rebuildScanner.setMaxLookBackInMills(maxLookbackInMills);
+
+ Put put = new Put(indexRowKey1Bytes);
+ Cell cell = getNewCell(indexRowKey1Bytes,
+ QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES,
+ QueryConstants.EMPTY_COLUMN_BYTES,
+ EnvironmentEdgeManager.currentTimeMillis(),
+ Cell.Type.Put,
+ IndexRegionObserver.VERIFIED_BYTES);
+ put.add(cell);
+ // This mutation is beyond maxLookBack, so add it to expectedMutations only.
+ expectedMutations.add(put);
+
+ // advance the time of maxLookBack, so last mutation will be outside of maxLookBack,
+ // next mutation will be within maxLookBack
+ injectEdge.incrementValue(maxLookbackInMills);
+ put = new Put(indexRowKey1Bytes);
+ cell = getNewCell(indexRowKey1Bytes,
+ QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES,
+ QueryConstants.EMPTY_COLUMN_BYTES,
+ EnvironmentEdgeManager.currentTimeMillis(),
+ Cell.Type.Put,
+ IndexRegionObserver.VERIFIED_BYTES);
+ put.add(cell);
+ // This mutation is in both expectedMutations and actualMutations, as it is within the maxLookBack, so it will not get chance to be compacted away
+ expectedMutations.add(put);
+ actualMutations.add(put);
+ Result actualMutationsScanResult = Result.create(Arrays.asList(cell));
+
+ Map<byte[], List<Mutation>> indexKeyToMutationMap = Maps.newTreeMap((Bytes.BYTES_COMPARATOR));
+ indexKeyToMutationMap.put(indexRowKey1Bytes, expectedMutations);
+ when(rebuildScanner.prepareActualIndexMutations(any(Result.class))).thenReturn(actualMutations);
+ when(indexRow.getRow()).thenReturn(indexRowKey1Bytes);
+ injectEdge.incrementValue(1);
+ IndexToolVerificationResult.PhaseResult actualPR = new IndexToolVerificationResult.PhaseResult();
+ // Report this validation as a success
+ assertTrue(rebuildScanner.verifySingleIndexRow(indexRow, indexKeyToMutationMap, mostRecentIndexRowKeys, Collections.EMPTY_LIST, actualPR, false));
+ // validIndexRowCount = 1
+ IndexToolVerificationResult.PhaseResult expectedPR = new IndexToolVerificationResult.PhaseResult(1, 0, 0, 0, 0, 0, 0, 0);
+ assertEquals(actualPR, expectedPR);
+ }
+
+ // Test the major compaction on index table only.
+ // All expected mutations are beyond the maxLookBack, and there are no matching ones in the actual list because of major compaction.
+ // We will report such row as an invalid beyond maxLookBack row.
+ @Test
+ public void testVerifySingleIndexRow_compactionOnIndexTable_noExpectedMutationWithinMaxLookBack() throws Exception {
+ Assume.assumeTrue(HbaseCompatCapabilities.isMaxLookbackTimeSupported());
+ String dataRowKey = "k1";
+ byte[] indexRowKey1Bytes = generateIndexRowKey(dataRowKey, "val1");
+ List<Mutation> expectedMutations = new ArrayList<>();
+ List<Mutation> actualMutations = new ArrayList<>();
+ // change the maxLookBack from infinite to some interval, which allows to simulate the mutation beyond the maxLookBack window.
+ long maxLookbackInMills = 10 * 1000;
+ rebuildScanner.setMaxLookBackInMills(maxLookbackInMills);
+
+ ManualEnvironmentEdge injectEdge = new ManualEnvironmentEdge();
+ injectEdge.setValue(1);
+ EnvironmentEdgeManager.injectEdge(injectEdge);
+
+ Put put = new Put(indexRowKey1Bytes);
+ Cell cell = getNewCell(indexRowKey1Bytes,
+ QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES,
+ QueryConstants.EMPTY_COLUMN_BYTES,
+ EnvironmentEdgeManager.currentTimeMillis(),
+ Cell.Type.Put,
+ VERIFIED_BYTES);
+ put.add(cell);
+ // This mutation is beyond maxLookBack, so add it to expectedMutations only.
+ expectedMutations.add(put);
+
+ injectEdge.incrementValue(maxLookbackInMills);
+ put = new Put(indexRowKey1Bytes);
+ cell = getNewCell(indexRowKey1Bytes,
+ QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES,
+ QueryConstants.EMPTY_COLUMN_BYTES,
+ EnvironmentEdgeManager.currentTimeMillis(),
+ Cell.Type.Put,
+ UNVERIFIED_BYTES);
+ put.add(cell);
+ // This mutation is actualMutations only, as it is an unverified put
+ actualMutations.add(put);
+ Result actualMutationsScanResult = Result.create(Arrays.asList(cell));
+
+ Map<byte[], List<Mutation>> indexKeyToMutationMap = Maps.newTreeMap((Bytes.BYTES_COMPARATOR));
+ indexKeyToMutationMap.put(indexRowKey1Bytes, expectedMutations);
+ mostRecentIndexRowKeys = new TreeSet<>(Bytes.BYTES_COMPARATOR);
+ when(rebuildScanner.prepareActualIndexMutations(any(Result.class))).thenReturn(actualMutations);
+ when(indexRow.getRow()).thenReturn(indexRowKey1Bytes);
+
+ injectEdge.incrementValue(1);
+ IndexToolVerificationResult.PhaseResult actualPR = new IndexToolVerificationResult.PhaseResult();
+ // Report this validation as a failure
+ assertFalse(rebuildScanner.verifySingleIndexRow(indexRow, indexKeyToMutationMap, mostRecentIndexRowKeys, new ArrayList<Mutation>(), actualPR, true));
+ // beyondMaxLookBackInvalidIndexRowCount = 1
+ IndexToolVerificationResult.PhaseResult expectedPR = new IndexToolVerificationResult.PhaseResult(0, 0, 0, 0, 0, 1, 0, 0);
+ assertEquals(actualPR, expectedPR);
+ }
+
+ private static byte[] generateIndexRowKey(String dataRowKey, String dataVal){
+ List<Byte> idxKey = new ArrayList<>();
+ if (dataVal != null && !dataVal.isEmpty())
+ idxKey.addAll(com.google.common.primitives.Bytes.asList(Bytes.toBytes(dataVal)));
+ idxKey.add(QueryConstants.SEPARATOR_BYTE);
+ idxKey.addAll(com.google.common.primitives.Bytes.asList(Bytes.toBytes(dataRowKey)));
+ return com.google.common.primitives.Bytes.toArray(idxKey);
}
private IndexToolVerificationResult.PhaseResult getValidPhaseResult() {
- return new IndexToolVerificationResult.PhaseResult(1,0,0,0,0, 0);
+ return new IndexToolVerificationResult.PhaseResult(1, 0, 0, 0, 0, 0, 0, 0);
}
private IndexToolVerificationResult.PhaseResult getInvalidPhaseResult() {
- return new IndexToolVerificationResult.PhaseResult(0, 0, 0, 1, 0, 0);
+ return new IndexToolVerificationResult.PhaseResult(0, 0, 0, 1, 0, 0, 0, 0);
}
private void initializeLocalMockitoSetup(Map.Entry<byte[], List<Mutation>> entry,
@@ -489,7 +606,7 @@ public class VerifySingleIndexRowTest extends BaseConnectionlessQueryTest {
private List<Mutation> buildActualIndexMutationsList(TestType testType) {
List<Mutation> actualMutations = new ArrayList<>();
- actualMutations.addAll(indexKeyToMutationMapLocal.get(indexRow.getRow()));
+ actualMutations.addAll(indexKeyToMutationMap.get(indexRow.getRow()));
if(testType.equals(TestType.EXPIRED)) {
return actualMutations;
}
@@ -530,9 +647,8 @@ public class VerifySingleIndexRowTest extends BaseConnectionlessQueryTest {
for (Mutation m : newActualMutations) {
if (m instanceof Put) {
List<Cell> origList = m.getFamilyCellMap().firstEntry().getValue();
- Cell newCell =
- CellUtil.createCell(m.getRow(), CellUtil.cloneFamily(origList.get(0)),
- Bytes.toBytes("EXTRACOL"), m.getTimestamp(), KeyValue.Type.Put.getCode(),
+ Cell newCell = getNewCell(m.getRow(), CellUtil.cloneFamily(origList.get(0)),
+ Bytes.toBytes("EXTRACOL"), m.getTimestamp(), Cell.Type.Put,
Bytes.toBytes("asdfg"));
byte[] fam = CellUtil.cloneFamily(origList.get(0));
m.getFamilyCellMap().get(fam).add(newCell);
@@ -557,8 +673,13 @@ public class VerifySingleIndexRowTest extends BaseConnectionlessQueryTest {
infiltrateCell(c, newCellList, testType);
}
familyCellMap.put(fam, newCellList);
- m.setFamilyCellMap(familyCellMap);
- newActualMutations.add(m);
+ Mutation newM;
+ if (m instanceof Put) {
+ newM = new Put(m.getRow(), m.getTimestamp(), familyCellMap);
+ } else {
+ newM = new Delete(m.getRow(), m.getTimestamp(), familyCellMap);
+ }
+ newActualMutations.add(newM);
}
return newActualMutations;
}
@@ -568,11 +689,10 @@ public class VerifySingleIndexRowTest extends BaseConnectionlessQueryTest {
Cell emptyCell;
switch(e) {
case INVALID_COLUMN:
- newCell =
- CellUtil.createCell(CellUtil.cloneRow(c), CellUtil.cloneFamily(c),
+ newCell = getNewCell(CellUtil.cloneRow(c), CellUtil.cloneFamily(c),
Bytes.toBytes(UNEXPECTED_COLUMN),
- EnvironmentEdgeManager.currentTimeMillis(),
- KeyValue.Type.Put.getCode(), Bytes.toBytes("zxcv"));
+ c.getTimestamp(),
+ Cell.Type.Put, Bytes.toBytes("zxcv"));
newCellList.add(newCell);
newCellList.add(c);
break;
@@ -589,9 +709,9 @@ public class VerifySingleIndexRowTest extends BaseConnectionlessQueryTest {
case INVALID_EMPTY_CELL:
if (CellUtil.matchingQualifier(c, EMPTY_COLUMN_BYTES)) {
newCell =
- CellUtil.createCell(CellUtil.cloneRow(c), CellUtil.cloneFamily(c),
+ getNewCell(CellUtil.cloneRow(c), CellUtil.cloneFamily(c),
CellUtil.cloneQualifier(c), c.getTimestamp(),
- KeyValue.Type.Delete.getCode(), VERIFIED_BYTES);
+ Cell.Type.Delete, VERIFIED_BYTES);
newCellList.add(newCell);
} else {
newCellList.add(c);
@@ -607,16 +727,16 @@ public class VerifySingleIndexRowTest extends BaseConnectionlessQueryTest {
}
private Cell getVerifiedEmptyCell(Cell c) {
- return CellUtil.createCell(CellUtil.cloneRow(c), CellUtil.cloneFamily(c),
+ return getNewCell(CellUtil.cloneRow(c), CellUtil.cloneFamily(c),
indexMaintainer.getEmptyKeyValueQualifier(),
- EnvironmentEdgeManager.currentTimeMillis(),
- KeyValue.Type.Put.getCode(), VERIFIED_BYTES);
+ c.getTimestamp(),
+ Cell.Type.Put, VERIFIED_BYTES);
}
private Cell getCellWithPut(Cell c) {
- return CellUtil.createCell(CellUtil.cloneRow(c),
+ return getNewCell(CellUtil.cloneRow(c),
CellUtil.cloneFamily(c), Bytes.toBytes(INCLUDED_COLUMN),
- c.getTimestamp(), KeyValue.Type.Put.getCode(),
+ c.getTimestamp(), Cell.Type.Put,
Bytes.toBytes("zxcv"));
}
@@ -630,8 +750,8 @@ public class VerifySingleIndexRowTest extends BaseConnectionlessQueryTest {
Mutation m = new Delete(orig.getRow());
List<Cell> origList = orig.getFamilyCellMap().firstEntry().getValue();
ts = ts == null ? EnvironmentEdgeManager.currentTimeMillis() : ts;
- Cell c = getNewPutCell(orig, origList, ts, KeyValue.Type.DeleteFamilyVersion);
- Cell empty = getEmptyCell(orig, origList, ts, KeyValue.Type.Put, true);
+ Cell c = getNewPutCell(orig, origList, ts, Cell.Type.DeleteFamilyVersion);
+ Cell empty = getEmptyCell(orig, origList, ts, Cell.Type.Put, true);
byte[] fam = CellUtil.cloneFamily(origList.get(0));
List<Cell> famCells = Lists.newArrayList();
m.getFamilyCellMap().put(fam, famCells);
@@ -649,8 +769,8 @@ public class VerifySingleIndexRowTest extends BaseConnectionlessQueryTest {
}
List<Cell> origList = orig.getFamilyCellMap().firstEntry().getValue();
ts = ts == null ? EnvironmentEdgeManager.currentTimeMillis() : ts;
- Cell c = getNewPutCell(orig, origList, ts, KeyValue.Type.Put);
- Cell empty = getEmptyCell(orig, origList, ts, KeyValue.Type.Put, false);
+ Cell c = getNewPutCell(orig, origList, ts, Cell.Type.Put);
+ Cell empty = getEmptyCell(orig, origList, ts, Cell.Type.Put, false);
byte[] fam = CellUtil.cloneFamily(origList.get(0));
List<Cell> famCells = Lists.newArrayList();
m.getFamilyCellMap().put(fam, famCells);
@@ -659,16 +779,24 @@ public class VerifySingleIndexRowTest extends BaseConnectionlessQueryTest {
return m;
}
- private Cell getEmptyCell(Mutation orig, List<Cell> origList, Long ts, KeyValue.Type type,
+ private Cell getEmptyCell(Mutation orig, List<Cell> origList, Long ts, Cell.Type type,
boolean verified) {
- return CellUtil.createCell(orig.getRow(), CellUtil.cloneFamily(origList.get(0)),
+ return getNewCell(orig.getRow(), CellUtil.cloneFamily(origList.get(0)),
indexMaintainer.getEmptyKeyValueQualifier(),
- ts, type.getCode(), verified ? VERIFIED_BYTES : UNVERIFIED_BYTES);
+ ts, type, verified ? VERIFIED_BYTES : UNVERIFIED_BYTES);
}
- private Cell getNewPutCell(Mutation orig, List<Cell> origList, Long ts, KeyValue.Type type) {
- return CellUtil.createCell(orig.getRow(),
+ private Cell getNewPutCell(Mutation orig, List<Cell> origList, Long ts, Cell.Type type) {
+ return getNewCell(orig.getRow(),
CellUtil.cloneFamily(origList.get(0)), Bytes.toBytes(INCLUDED_COLUMN),
- ts, type.getCode(), Bytes.toBytes("asdfg"));
+ ts, type, Bytes.toBytes("asdfg"));
+ }
+
+ private Cell getNewCell(byte[] row, byte[] family, byte[] qualifier,
+ long timestamp, Cell.Type type, byte[] value) {
+ return CellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(row).
+ setFamily(family).setQualifier(qualifier).
+ setTimestamp(timestamp).setType(type).
+ setValue(value).build();
}
}
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/mapreduce/index/IndexScrutinyTableOutputTest.java b/phoenix-core/src/test/java/org/apache/phoenix/mapreduce/index/IndexScrutinyTableOutputTest.java
index a904bca..9955fec 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/mapreduce/index/IndexScrutinyTableOutputTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/mapreduce/index/IndexScrutinyTableOutputTest.java
@@ -33,8 +33,8 @@ public class IndexScrutinyTableOutputTest extends BaseIndexTest {
@Test
public void testConstructMetadataParamQuery() {
String metadataParamQuery =
- IndexScrutinyTableOutput
- .constructMetadataParamQuery(Arrays.asList("INVALID_ROWS_QUERY_ALL"));
+ IndexScrutinyTableOutput
+ .constructMetadataParamQuery(Arrays.asList("INVALID_ROWS_QUERY_ALL"));
assertEquals(
"SELECT \"INVALID_ROWS_QUERY_ALL\" FROM PHOENIX_INDEX_SCRUTINY_METADATA WHERE (\"SOURCE_TABLE\",\"TARGET_TABLE\",\"SCRUTINY_EXECUTE_TIME\") IN ((?,?,?))",
metadataParamQuery);
@@ -43,33 +43,44 @@ public class IndexScrutinyTableOutputTest extends BaseIndexTest {
@Test
public void testGetSqlQueryAllInvalidRows() throws SQLException {
SourceTargetColumnNames columnNames =
- new SourceTargetColumnNames.DataSourceColNames(pDataTable, pIndexTable);
+ new SourceTargetColumnNames.DataSourceColNames(pDataTable, pIndexTable);
String sqlStr =
- IndexScrutinyTableOutput.getSqlQueryAllInvalidRows(conn, columnNames,
- SCRUTINY_TIME_MILLIS);
- assertEquals("SELECT \"SOURCE_TABLE\" , \"TARGET_TABLE\" , \"SCRUTINY_EXECUTE_TIME\" , \"SOURCE_ROW_PK_HASH\" , \"SOURCE_TS\" , \"TARGET_TS\" , \"HAS_TARGET_ROW\" , \"ID\" , \"PK_PART2\" , \"NAME\" , \"ZIP\" , \":ID\" , \":PK_PART2\" , \"0:NAME\" , \"0:ZIP\" FROM PHOENIX_INDEX_SCRUTINY(\"ID\" INTEGER,\"PK_PART2\" TINYINT,\"NAME\" VARCHAR,\"ZIP\" BIGINT,\":ID\" INTEGER,\":PK_PART2\" TINYINT,\"0:NAME\" VARCHAR,\"0:ZIP\" BIGINT) WHERE (\"SOURCE_TABLE\",\"TARGET_TABLE\",\"SCRUTINY_EX [...]
+ IndexScrutinyTableOutput.getSqlQueryAllInvalidRows(conn, columnNames,
+ SCRUTINY_TIME_MILLIS);
+ assertEquals("SELECT \"SOURCE_TABLE\" , \"TARGET_TABLE\" , \"SCRUTINY_EXECUTE_TIME\" , \"SOURCE_ROW_PK_HASH\" , \"SOURCE_TS\" , \"TARGET_TS\" , \"HAS_TARGET_ROW\" , \"BEYOND_MAX_LOOKBACK\" , \"ID\" , \"PK_PART2\" , \"NAME\" , \"ZIP\" , \":ID\" , \":PK_PART2\" , \"0:NAME\" , \"0:ZIP\" FROM PHOENIX_INDEX_SCRUTINY(\"ID\" INTEGER,\"PK_PART2\" TINYINT,\"NAME\" VARCHAR,\"ZIP\" BIGINT,\":ID\" INTEGER,\":PK_PART2\" TINYINT,\"0:NAME\" VARCHAR,\"0:ZIP\" BIGINT) WHERE (\"SOURCE_TABLE\",\"TA [...]
sqlStr);
}
@Test
public void testGetSqlQueryMissingTargetRows() throws SQLException {
SourceTargetColumnNames columnNames =
- new SourceTargetColumnNames.DataSourceColNames(pDataTable, pIndexTable);
+ new SourceTargetColumnNames.DataSourceColNames(pDataTable, pIndexTable);
String query =
- IndexScrutinyTableOutput.getSqlQueryMissingTargetRows(conn, columnNames,
- SCRUTINY_TIME_MILLIS);
- assertEquals("SELECT \"SOURCE_TABLE\" , \"TARGET_TABLE\" , \"SCRUTINY_EXECUTE_TIME\" , \"SOURCE_ROW_PK_HASH\" , \"SOURCE_TS\" , \"TARGET_TS\" , \"HAS_TARGET_ROW\" , \"ID\" , \"PK_PART2\" , \"NAME\" , \"ZIP\" , \":ID\" , \":PK_PART2\" , \"0:NAME\" , \"0:ZIP\" FROM PHOENIX_INDEX_SCRUTINY(\"ID\" INTEGER,\"PK_PART2\" TINYINT,\"NAME\" VARCHAR,\"ZIP\" BIGINT,\":ID\" INTEGER,\":PK_PART2\" TINYINT,\"0:NAME\" VARCHAR,\"0:ZIP\" BIGINT) WHERE (\"SOURCE_TABLE\",\"TARGET_TABLE\",\"SCRUTINY_EX [...]
+ IndexScrutinyTableOutput.getSqlQueryMissingTargetRows(conn, columnNames,
+ SCRUTINY_TIME_MILLIS);
+ assertEquals("SELECT \"SOURCE_TABLE\" , \"TARGET_TABLE\" , \"SCRUTINY_EXECUTE_TIME\" , \"SOURCE_ROW_PK_HASH\" , \"SOURCE_TS\" , \"TARGET_TS\" , \"HAS_TARGET_ROW\" , \"BEYOND_MAX_LOOKBACK\" , \"ID\" , \"PK_PART2\" , \"NAME\" , \"ZIP\" , \":ID\" , \":PK_PART2\" , \"0:NAME\" , \"0:ZIP\" FROM PHOENIX_INDEX_SCRUTINY(\"ID\" INTEGER,\"PK_PART2\" TINYINT,\"NAME\" VARCHAR,\"ZIP\" BIGINT,\":ID\" INTEGER,\":PK_PART2\" TINYINT,\"0:NAME\" VARCHAR,\"0:ZIP\" BIGINT) WHERE (\"SOURCE_TABLE\",\"TA [...]
query);
}
@Test
public void testGetSqlQueryBadCoveredColVal() throws SQLException {
SourceTargetColumnNames columnNames =
- new SourceTargetColumnNames.DataSourceColNames(pDataTable, pIndexTable);
+ new SourceTargetColumnNames.DataSourceColNames(pDataTable, pIndexTable);
String query =
- IndexScrutinyTableOutput.getSqlQueryBadCoveredColVal(conn, columnNames,
- SCRUTINY_TIME_MILLIS);
- assertEquals("SELECT \"SOURCE_TABLE\" , \"TARGET_TABLE\" , \"SCRUTINY_EXECUTE_TIME\" , \"SOURCE_ROW_PK_HASH\" , \"SOURCE_TS\" , \"TARGET_TS\" , \"HAS_TARGET_ROW\" , \"ID\" , \"PK_PART2\" , \"NAME\" , \"ZIP\" , \":ID\" , \":PK_PART2\" , \"0:NAME\" , \"0:ZIP\" FROM PHOENIX_INDEX_SCRUTINY(\"ID\" INTEGER,\"PK_PART2\" TINYINT,\"NAME\" VARCHAR,\"ZIP\" BIGINT,\":ID\" INTEGER,\":PK_PART2\" TINYINT,\"0:NAME\" VARCHAR,\"0:ZIP\" BIGINT) WHERE (\"SOURCE_TABLE\",\"TARGET_TABLE\",\"SCRUTINY_EX [...]
+ IndexScrutinyTableOutput.getSqlQueryBadCoveredColVal(conn, columnNames,
+ SCRUTINY_TIME_MILLIS);
+ assertEquals("SELECT \"SOURCE_TABLE\" , \"TARGET_TABLE\" , \"SCRUTINY_EXECUTE_TIME\" , \"SOURCE_ROW_PK_HASH\" , \"SOURCE_TS\" , \"TARGET_TS\" , \"HAS_TARGET_ROW\" , \"BEYOND_MAX_LOOKBACK\" , \"ID\" , \"PK_PART2\" , \"NAME\" , \"ZIP\" , \":ID\" , \":PK_PART2\" , \"0:NAME\" , \"0:ZIP\" FROM PHOENIX_INDEX_SCRUTINY(\"ID\" INTEGER,\"PK_PART2\" TINYINT,\"NAME\" VARCHAR,\"ZIP\" BIGINT,\":ID\" INTEGER,\":PK_PART2\" TINYINT,\"0:NAME\" VARCHAR,\"0:ZIP\" BIGINT) WHERE (\"SOURCE_TABLE\",\"TA [...]
+ query);
+ }
+
+ @Test
+ public void testGetSqlQueryBeyondMaxLookback() throws SQLException {
+ SourceTargetColumnNames columnNames =
+ new SourceTargetColumnNames.DataSourceColNames(pDataTable, pIndexTable);
+ String query =
+ IndexScrutinyTableOutput.getSqlQueryBeyondMaxLookback(conn, columnNames,
+ SCRUTINY_TIME_MILLIS);
+ assertEquals("SELECT \"SOURCE_TABLE\" , \"TARGET_TABLE\" , \"SCRUTINY_EXECUTE_TIME\" , \"SOURCE_ROW_PK_HASH\" , \"SOURCE_TS\" , \"TARGET_TS\" , \"HAS_TARGET_ROW\" , \"BEYOND_MAX_LOOKBACK\" , \"ID\" , \"PK_PART2\" , \"NAME\" , \"ZIP\" , \":ID\" , \":PK_PART2\" , \"0:NAME\" , \"0:ZIP\" FROM PHOENIX_INDEX_SCRUTINY(\"ID\" INTEGER,\"PK_PART2\" TINYINT,\"NAME\" VARCHAR,\"ZIP\" BIGINT,\":ID\" INTEGER,\":PK_PART2\" TINYINT,\"0:NAME\" VARCHAR,\"0:ZIP\" BIGINT) WHERE (\"SOURCE_TABLE\",\"TA [...]
query);
}
@@ -77,10 +88,10 @@ public class IndexScrutinyTableOutputTest extends BaseIndexTest {
public void testGetOutputTableUpsert() throws Exception {
IndexColumnNames columnNames = new IndexColumnNames(pDataTable, pIndexTable);
String outputTableUpsert =
- IndexScrutinyTableOutput.constructOutputTableUpsert(
- columnNames.getDynamicDataCols(), columnNames.getDynamicIndexCols(), conn);
+ IndexScrutinyTableOutput.constructOutputTableUpsert(
+ columnNames.getDynamicDataCols(), columnNames.getDynamicIndexCols(), conn);
conn.prepareStatement(outputTableUpsert); // shouldn't throw
- assertEquals("UPSERT INTO PHOENIX_INDEX_SCRUTINY (\"SOURCE_TABLE\", \"TARGET_TABLE\", \"SCRUTINY_EXECUTE_TIME\", \"SOURCE_ROW_PK_HASH\", \"SOURCE_TS\", \"TARGET_TS\", \"HAS_TARGET_ROW\", \"ID\" INTEGER, \"PK_PART2\" TINYINT, \"NAME\" VARCHAR, \"ZIP\" BIGINT, \":ID\" INTEGER, \":PK_PART2\" TINYINT, \"0:NAME\" VARCHAR, \"0:ZIP\" BIGINT) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
+ assertEquals("UPSERT INTO PHOENIX_INDEX_SCRUTINY (\"SOURCE_TABLE\", \"TARGET_TABLE\", \"SCRUTINY_EXECUTE_TIME\", \"SOURCE_ROW_PK_HASH\", \"SOURCE_TS\", \"TARGET_TS\", \"HAS_TARGET_ROW\", \"BEYOND_MAX_LOOKBACK\", \"ID\" INTEGER, \"PK_PART2\" TINYINT, \"NAME\" VARCHAR, \"ZIP\" BIGINT, \":ID\" INTEGER, \":PK_PART2\" TINYINT, \"0:NAME\" VARCHAR, \"0:ZIP\" BIGINT) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
outputTableUpsert);
}
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java b/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java
index d04f6f6..9b9c5ef 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java
@@ -157,7 +157,7 @@ import com.google.common.collect.Lists;
public class TestUtil {
private static final Logger LOGGER = LoggerFactory.getLogger(TestUtil.class);
-
+
private static final Long ZERO = new Long(0);
public static final String DEFAULT_SCHEMA_NAME = "S";
public static final String DEFAULT_DATA_TABLE_NAME = "T";
@@ -902,7 +902,7 @@ public class TestUtil {
System.out.println("************ dumping " + table + " **************");
Scan s = new Scan();
s.setRaw(true);;
- s.setMaxVersions();
+ s.readAllVersions();
try (ResultScanner scanner = table.getScanner(s)) {
Result result = null;
while ((result = scanner.next()) != null) {
@@ -910,7 +910,9 @@ public class TestUtil {
Cell current = null;
while (cellScanner.advance()) {
current = cellScanner.current();
- System.out.println(current);
+ System.out.println(current + " column= " +
+ Bytes.toString(CellUtil.cloneQualifier(current)) +
+ " val=" + Bytes.toStringBinary(CellUtil.cloneValue(current)));
}
}
}
diff --git a/phoenix-hbase-compat-2.1.6/src/main/java/org/apache/phoenix/compat/hbase/HbaseCompatCapabilities.java b/phoenix-hbase-compat-2.1.6/src/main/java/org/apache/phoenix/compat/hbase/HbaseCompatCapabilities.java
index 629946b..8e1284d 100644
--- a/phoenix-hbase-compat-2.1.6/src/main/java/org/apache/phoenix/compat/hbase/HbaseCompatCapabilities.java
+++ b/phoenix-hbase-compat-2.1.6/src/main/java/org/apache/phoenix/compat/hbase/HbaseCompatCapabilities.java
@@ -29,4 +29,9 @@ public class HbaseCompatCapabilities {
//In HBase 2.1 and 2.2, a lookback query won't return any results if covered by a future delete
public static boolean isLookbackBeyondDeletesSupported() { return false; }
+ //HBase 2.1 does not have HBASE-22710, which is necessary for raw scan skip scan and
+ // AllVersionsIndexRebuild filters to
+ // show all versions properly. HBase 2.2.5+ and HBase 2.3.0+ have this fix.
+ public static boolean isRawFilterSupported() { return false; }
+
}
diff --git a/phoenix-hbase-compat-2.2.1/src/main/java/org/apache/phoenix/compat/hbase/HbaseCompatCapabilities.java b/phoenix-hbase-compat-2.2.1/src/main/java/org/apache/phoenix/compat/hbase/HbaseCompatCapabilities.java
index 629946b..a9158b4 100644
--- a/phoenix-hbase-compat-2.2.1/src/main/java/org/apache/phoenix/compat/hbase/HbaseCompatCapabilities.java
+++ b/phoenix-hbase-compat-2.2.1/src/main/java/org/apache/phoenix/compat/hbase/HbaseCompatCapabilities.java
@@ -29,4 +29,9 @@ public class HbaseCompatCapabilities {
//In HBase 2.1 and 2.2, a lookback query won't return any results if covered by a future delete
public static boolean isLookbackBeyondDeletesSupported() { return false; }
+ //HBase 2.1 does not have HBASE-22710, which is necessary for raw scan skip scan and
+ // AllVersionsIndexRebuild filters to
+ // show all versions properly. HBase 2.2.5+ and HBase 2.3.0+ have this fix.
+ public static boolean isRawFilterSupported() { return true; }
+
}
diff --git a/phoenix-hbase-compat-2.3.0/src/main/java/org/apache/phoenix/compat/hbase/HbaseCompatCapabilities.java b/phoenix-hbase-compat-2.3.0/src/main/java/org/apache/phoenix/compat/hbase/HbaseCompatCapabilities.java
index 9b83e6d..b86466f 100644
--- a/phoenix-hbase-compat-2.3.0/src/main/java/org/apache/phoenix/compat/hbase/HbaseCompatCapabilities.java
+++ b/phoenix-hbase-compat-2.3.0/src/main/java/org/apache/phoenix/compat/hbase/HbaseCompatCapabilities.java
@@ -28,4 +28,9 @@ public class HbaseCompatCapabilities {
//but in 2.3 and later we have the preSoreScannerOpen hook that overrides that behavior
public static boolean isLookbackBeyondDeletesSupported() { return true; }
+ //HBase 2.1 does not have HBASE-22710, which is necessary for raw scan skip scan and
+ // AllVersionsIndexRebuild filters to
+ // show all versions properly. HBase 2.2.5+ and HBase 2.3.0+ have this fix.
+ public static boolean isRawFilterSupported() { return true; }
+
}