You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by gj...@apache.org on 2020/05/22 18:29:06 UTC
[phoenix] branch 4.x updated: PHOENIX-5875 - Optional logging for
IndexTool verification
This is an automated email from the ASF dual-hosted git repository.
gjacoby pushed a commit to branch 4.x
in repository https://gitbox.apache.org/repos/asf/phoenix.git
The following commit(s) were added to refs/heads/4.x by this push:
new 5e4044f PHOENIX-5875 - Optional logging for IndexTool verification
5e4044f is described below
commit 5e4044fdf11f7d695dd2cd06110f485752a96ba3
Author: Geoffrey Jacoby <gj...@apache.org>
AuthorDate: Mon May 18 17:07:05 2020 -0700
PHOENIX-5875 - Optional logging for IndexTool verification
---
.../end2end/IndexToolForNonTxGlobalIndexIT.java | 149 +++++++++++++++++++--
.../org/apache/phoenix/end2end/IndexToolIT.java | 59 +++++---
.../index/IndexVerificationOutputRepositoryIT.java | 67 +++++++++
.../coprocessor/BaseScannerRegionObserver.java | 2 +
.../coprocessor/IndexRebuildRegionScanner.java | 27 +++-
.../PhoenixServerBuildIndexInputFormat.java | 3 +
.../apache/phoenix/mapreduce/index/IndexTool.java | 97 ++++++++++++++
.../index/IndexVerificationOutputRepository.java | 74 +++++++---
.../mapreduce/util/PhoenixConfigurationUtil.java | 15 +++
.../org/apache/phoenix/index/IndexToolTest.java | 106 +++++++++++++--
10 files changed, 537 insertions(+), 62 deletions(-)
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolForNonTxGlobalIndexIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolForNonTxGlobalIndexIT.java
index 16813ee..aca88b3 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolForNonTxGlobalIndexIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolForNonTxGlobalIndexIT.java
@@ -23,6 +23,9 @@ import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HConnection;
+import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
@@ -34,6 +37,8 @@ import org.apache.phoenix.hbase.index.IndexRegionObserver;
import org.apache.phoenix.jdbc.PhoenixConnection;
import org.apache.phoenix.mapreduce.index.IndexTool;
import org.apache.phoenix.mapreduce.index.IndexVerificationOutputRepository;
+import org.apache.phoenix.mapreduce.index.IndexVerificationOutputRow;
+import org.apache.phoenix.mapreduce.index.IndexVerificationResultRepository;
import org.apache.phoenix.query.ConnectionQueryServices;
import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.query.QueryServicesOptions;
@@ -44,6 +49,7 @@ import org.apache.phoenix.util.PropertiesUtil;
import org.apache.phoenix.util.ReadOnlyProps;
import org.apache.phoenix.util.SchemaUtil;
import org.apache.phoenix.util.TestUtil;
+import org.junit.After;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Rule;
@@ -89,7 +95,9 @@ import static org.apache.phoenix.mapreduce.index.PhoenixIndexToolJobCounters.REB
import static org.apache.phoenix.mapreduce.index.PhoenixIndexToolJobCounters.SCANNED_DATA_ROW_COUNT;
import static org.apache.phoenix.mapreduce.util.PhoenixConfigurationUtil.CURRENT_SCN_VALUE;
import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
+import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
@RunWith(Parameterized.class)
@@ -136,6 +144,17 @@ public class IndexToolForNonTxGlobalIndexIT extends BaseUniqueNamesOwnClusterIT
new ReadOnlyProps(clientProps.entrySet().iterator()));
}
+ @After
+ public void cleanup() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ deleteAllRows(conn,
+ TableName.valueOf(IndexVerificationOutputRepository.OUTPUT_TABLE_NAME_BYTES));
+ deleteAllRows(conn,
+ TableName.valueOf(IndexVerificationResultRepository.RESULT_TABLE_NAME));
+ }
+ }
+
@Test
public void testWithSetNull() throws Exception {
// This tests the cases where a column having a null value is overwritten with a not null value and vice versa;
@@ -166,11 +185,11 @@ public class IndexToolForNonTxGlobalIndexIT extends BaseUniqueNamesOwnClusterIT
IndexTool
indexTool = IndexToolIT.runIndexTool(directApi, useSnapshot, schemaName, dataTableName, indexTableName, null, 0, new String[0]);
assertEquals(NROWS, indexTool.getJob().getCounters().findCounter(INPUT_RECORDS).getValue());
+ assertTrue("Index rebuild failed!", indexTool.getJob().isSuccessful());
+ TestUtil.assertIndexState(conn, indexTableFullName, PIndexState.ACTIVE, null);
long actualRowCount = IndexScrutiny
.scrutinizeIndex(conn, dataTableFullName, indexTableFullName);
assertEquals(NROWS, actualRowCount);
- actualRowCount = IndexScrutiny.scrutinizeIndex(conn, dataTableFullName, indexTableFullName);
- assertEquals(NROWS, actualRowCount);
IndexToolIT.setEveryNthRowWithNull(NROWS, 5, stmt);
conn.commit();
actualRowCount = IndexScrutiny.scrutinizeIndex(conn, dataTableFullName, indexTableFullName);
@@ -192,7 +211,6 @@ public class IndexToolForNonTxGlobalIndexIT extends BaseUniqueNamesOwnClusterIT
assertEquals(0, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_MISSING_INDEX_ROW_COUNT).getValue());
assertEquals(0, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_BEYOND_MAXLOOKBACK_INVALID_INDEX_ROW_COUNT).getValue());
assertEquals(0, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_BEYOND_MAXLOOKBACK_MISSING_INDEX_ROW_COUNT).getValue());
- IndexToolIT.dropIndexToolTables(conn);
}
}
@@ -259,7 +277,6 @@ public class IndexToolForNonTxGlobalIndexIT extends BaseUniqueNamesOwnClusterIT
.getTable(indexToolOutputTable.getName());
Result r = hIndexToolTable.getScanner(scan).next();
assertTrue(r == null);
- IndexToolIT.dropIndexToolTables(conn);
}
}
@@ -376,7 +393,6 @@ public class IndexToolForNonTxGlobalIndexIT extends BaseUniqueNamesOwnClusterIT
assertEquals(0, indexTool.getJob().getCounters().findCounter(AFTER_REBUILD_BEYOND_MAXLOOKBACK_INVALID_INDEX_ROW_COUNT).getValue());
actualRowCount = IndexScrutiny.scrutinizeIndex(conn, dataTableFullName, indexTableFullName);
assertEquals(2 * NROWS, actualRowCount);
- IndexToolIT.dropIndexToolTables(conn);
}
}
@@ -416,7 +432,6 @@ public class IndexToolForNonTxGlobalIndexIT extends BaseUniqueNamesOwnClusterIT
IndexToolIT.runIndexTool(directApi, useSnapshot, schemaName, viewName, indexTableName,
null, 0, IndexTool.IndexVerifyType.BOTH);
assertEquals(0, IndexToolIT.MutationCountingRegionObserver.getMutationCount());
- IndexToolIT.dropIndexToolTables(conn);
}
}
@@ -457,7 +472,6 @@ public class IndexToolForNonTxGlobalIndexIT extends BaseUniqueNamesOwnClusterIT
Assert.fail("Fail to parsing the error message from IndexToolOutputTable");
}
IndexRegionObserver.setIgnoreIndexRebuildForTesting(false);
- IndexToolIT.dropIndexToolTables(conn);
}
}
@@ -501,7 +515,6 @@ public class IndexToolForNonTxGlobalIndexIT extends BaseUniqueNamesOwnClusterIT
null, 0, IndexTool.IndexVerifyType.AFTER);
IndexToolIT.runIndexTool(directApi, useSnapshot, schemaName, dataTableName, indexTableName,
null, 0, IndexTool.IndexVerifyType.ONLY);
- IndexToolIT.dropIndexToolTables(conn);
}
}
@@ -566,9 +579,127 @@ public class IndexToolForNonTxGlobalIndexIT extends BaseUniqueNamesOwnClusterIT
Assert.assertFalse(it.isValidLastVerifyTime(10L));
Assert.assertFalse(it.isValidLastVerifyTime(EnvironmentEdgeManager.currentTimeMillis() - 1000L));
Assert.assertTrue(it.isValidLastVerifyTime(scn));
+ }
+ }
- IndexToolIT.dropIndexToolTables(conn);
+ @Test
+ public void testDisableOutputLogging() throws Exception {
+ if (!mutable || useSnapshot) {
+ return;
+ }
+
+ String schemaName = generateUniqueName();
+ String dataTableName = generateUniqueName();
+ String dataTableFullName = SchemaUtil.getTableName(schemaName, dataTableName);
+ String indexTableName = generateUniqueName();
+ String indexTableFullName = SchemaUtil.getTableName(schemaName, indexTableName);
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+
+ try(Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ String stmString1 =
+ "CREATE TABLE " + dataTableFullName
+ + " (ID INTEGER NOT NULL PRIMARY KEY, NAME VARCHAR, ZIP INTEGER) "
+ + tableDDLOptions;
+ conn.createStatement().execute(stmString1);
+ String upsertQuery = String.format("UPSERT INTO %s VALUES(?, ?, ?)", dataTableFullName);
+ PreparedStatement stmt1 = conn.prepareStatement(upsertQuery);
+
+ // insert two rows
+ IndexToolIT.upsertRow(stmt1, 1);
+ IndexToolIT.upsertRow(stmt1, 2);
+ conn.commit();
+
+ //create ASYNC
+ String stmtString2 =
+ String.format(
+ "CREATE INDEX %s ON %s (LPAD(UPPER(NAME, 'en_US'),8,'x')||'_xyz') ASYNC ",
+ indexTableName, dataTableFullName);
+ conn.createStatement().execute(stmtString2);
+ conn.commit();
+
+ // run the index MR job as ONLY so the index doesn't get rebuilt. Should be 2 missing
+ //rows. We pass in --disable-logging BEFORE to silence the output logging to
+ // PHOENIX_INDEX_TOOL, since ONLY logs BEFORE the (non-existent in this case)
+ // rebuild
+ assertDisableLogging(conn, 0, IndexTool.IndexVerifyType.ONLY,
+ IndexTool.IndexDisableLoggingType.BEFORE, null, schemaName, dataTableName, indexTableName,
+ indexTableFullName, 0);
+
+ //now check that disabling logging AFTER leaves only the BEFORE logs on a BOTH run
+ assertDisableLogging(conn, 2, IndexTool.IndexVerifyType.BOTH,
+ IndexTool.IndexDisableLoggingType.AFTER,
+ IndexVerificationOutputRepository.PHASE_BEFORE_VALUE, schemaName,
+ dataTableName, indexTableName,
+ indexTableFullName, 0);
+
+ deleteAllRows(conn,
+ TableName.valueOf(IndexVerificationOutputRepository.OUTPUT_TABLE_NAME));
+ deleteAllRows(conn, TableName.valueOf(indexTableFullName));
+
+ //now check that disabling logging BEFORE creates only the AFTER logs on a BOTH run
+ //the index tool run fails validation at the end because we suppressed the BEFORE logs
+ //which prevented the rebuild from working properly, but that's ok for this test.
+ assertDisableLogging(conn, 2, IndexTool.IndexVerifyType.BOTH,
+ IndexTool.IndexDisableLoggingType.BEFORE,
+ IndexVerificationOutputRepository.PHASE_AFTER_VALUE, schemaName,
+ dataTableName, indexTableName,
+ indexTableFullName, -1);
+
+ deleteAllRows(conn, TableName.valueOf(IndexVerificationOutputRepository.OUTPUT_TABLE_NAME));
+ deleteAllRows(conn, TableName.valueOf(indexTableFullName));
+
+ //now check that disabling logging BOTH creates no logs on a BOTH run
+ assertDisableLogging(conn, 0, IndexTool.IndexVerifyType.BOTH,
+ IndexTool.IndexDisableLoggingType.BOTH,
+ IndexVerificationOutputRepository.PHASE_AFTER_VALUE, schemaName,
+ dataTableName, indexTableName,
+ indexTableFullName, -1);
+
+ }
+ }
+
+ public void deleteAllRows(Connection conn, TableName tableName) throws SQLException,
+ IOException {
+ Scan scan = new Scan();
+ HBaseAdmin admin = conn.unwrap(PhoenixConnection.class).getQueryServices().
+ getAdmin();
+ HConnection hbaseConn = admin.getConnection();
+ HTableInterface table = hbaseConn.getTable(tableName);
+ try (ResultScanner scanner = table.getScanner(scan)) {
+ for (Result r : scanner) {
+ Delete del = new Delete(r.getRow());
+ table.delete(del);
+ }
+ }
+ }
+
+ private void assertDisableLogging(Connection conn, int expectedRows,
+ IndexTool.IndexVerifyType verifyType,
+ IndexTool.IndexDisableLoggingType disableLoggingType,
+ byte[] expectedPhase,
+ String schemaName, String dataTableName,
+ String indexTableName, String indexTableFullName,
+ int expectedStatus) throws Exception {
+ IndexTool tool = IndexToolIT.runIndexTool(true, false, schemaName, dataTableName,
+ indexTableName,
+ null,
+ expectedStatus, verifyType, "-et",
+ Long.toString(EnvironmentEdgeManager.currentTimeMillis()),"-dl", disableLoggingType.toString());
+ assertNotNull(tool);
+ assertNotNull(tool.getEndTime());
+ byte[] indexTableFullNameBytes = Bytes.toBytes(indexTableFullName);
+
+ IndexVerificationOutputRepository outputRepository =
+ new IndexVerificationOutputRepository(indexTableFullNameBytes, conn);
+ List<IndexVerificationOutputRow> rows =
+ outputRepository.getOutputRows(tool.getEndTime(),
+ indexTableFullNameBytes);
+ assertEquals(expectedRows, rows.size());
+ if (expectedRows > 0) {
+ assertArrayEquals(expectedPhase, rows.get(0).getPhaseValue());
}
+ TestUtil.dumpTable(conn,
+ TableName.valueOf(IndexVerificationOutputRepository.OUTPUT_TABLE_NAME));
}
private void deleteOneRowFromResultTable(Connection conn, Long scn, String indexTable)
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolIT.java
index a2bd788..5310618 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolIT.java
@@ -23,11 +23,13 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.HBaseIOException;
-import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HConnection;
+import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Mutation;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
@@ -40,11 +42,9 @@ import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.MiniBatchOperationInProgress;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.mapreduce.Job;
-import org.apache.phoenix.coprocessor.IndexRebuildRegionScanner;
-import org.apache.phoenix.hbase.index.IndexRegionObserver;
import org.apache.phoenix.mapreduce.index.IndexVerificationOutputRepository;
+import org.apache.phoenix.mapreduce.index.IndexVerificationOutputRow;
import org.apache.phoenix.mapreduce.index.IndexVerificationResultRepository;
import org.apache.phoenix.jdbc.PhoenixConnection;
import org.apache.phoenix.mapreduce.index.IndexTool;
@@ -53,24 +53,23 @@ import org.apache.phoenix.mapreduce.index.PhoenixServerBuildIndexMapper;
import org.apache.phoenix.query.ConnectionQueryServices;
import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.query.QueryServicesOptions;
-import org.apache.phoenix.schema.PIndexState;
import org.apache.phoenix.schema.PTable;
import org.apache.phoenix.transaction.PhoenixTransactionProvider.Feature;
import org.apache.phoenix.transaction.TransactionFactory;
-import org.apache.phoenix.util.IndexScrutiny;
+import org.apache.phoenix.util.EnvironmentEdgeManager;
import org.apache.phoenix.util.PhoenixRuntime;
import org.apache.phoenix.util.PropertiesUtil;
import org.apache.phoenix.util.QueryUtil;
import org.apache.phoenix.util.ReadOnlyProps;
import org.apache.phoenix.util.SchemaUtil;
import org.apache.phoenix.util.TestUtil;
-import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameters;
+import java.io.IOException;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
@@ -106,8 +105,10 @@ import static org.apache.phoenix.mapreduce.index.PhoenixIndexToolJobCounters.BEF
import static org.apache.phoenix.mapreduce.index.PhoenixIndexToolJobCounters.REBUILT_INDEX_ROW_COUNT;
import static org.apache.phoenix.mapreduce.index.PhoenixIndexToolJobCounters.SCANNED_DATA_ROW_COUNT;
import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
+import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -161,7 +162,8 @@ public class IndexToolIT extends BaseUniqueNamesOwnClusterIT {
}
@Parameters(
- name = "transactionProvider={0},mutable={1},localIndex={2},directApi={3},useSnapshot={4}")
+ name = "transactionProvider={0},mutable={1},localIndex={2},directApi={3}," +
+ "useSnapshot={4},useTenant={5}")
public static synchronized Collection<Object[]> data() {
List<Object[]> list = Lists.newArrayListWithExpectedSize(48);
boolean[] Booleans = new boolean[] { false, true };
@@ -382,7 +384,10 @@ public class IndexToolIT extends BaseUniqueNamesOwnClusterIT {
if (Bytes.compareTo(cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength(),
IndexVerificationOutputRepository.DATA_TABLE_NAME_BYTES, 0, IndexVerificationOutputRepository.DATA_TABLE_NAME_BYTES.length) == 0) {
dataTableNameCheck = true;
- assertTrue(Bytes.compareTo(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength(),
+ assertTrue("Value was different! Expected: " + Bytes.toString(dataTableFullNameBytes)
+ + " Actual: " + Bytes.toString(cell.getValueArray(),
+ cell.getValueOffset(), cell.getValueLength()),
+ Bytes.compareTo(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength(),
dataTableFullNameBytes, 0, dataTableFullNameBytes.length) == 0);
} else if (Bytes.compareTo(cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength(),
IndexVerificationOutputRepository.INDEX_TABLE_NAME_BYTES, 0, IndexVerificationOutputRepository.INDEX_TABLE_NAME_BYTES.length) == 0) {
@@ -404,8 +409,10 @@ public class IndexToolIT extends BaseUniqueNamesOwnClusterIT {
scan = new Scan();
scanner = hIndexTable.getScanner(scan);
Result result = scanner.next();
- assert(result != null);
- verifyIndexTableRowKey(CellUtil.cloneRow(result.rawCells()[0]), indexTableFullName);
+ if (result != null) {
+ verifyIndexTableRowKey(CellUtil.cloneRow(result.rawCells()[0]), indexTableFullName);
+ }
+
return errorMessageCell;
}
@@ -630,8 +637,18 @@ public class IndexToolIT extends BaseUniqueNamesOwnClusterIT {
}
private static List<String> getArgList (boolean directApi, boolean useSnapshot, String schemaName,
+ String dataTable, String indxTable, String tenantId,
+ IndexTool.IndexVerifyType verifyType, Long startTime,
+ Long endTime, Long incrementalVerify) {
+ return getArgList(directApi, useSnapshot, schemaName, dataTable, indxTable, tenantId,
+ verifyType, startTime, endTime, IndexTool.IndexDisableLoggingType.NONE, incrementalVerify);
+ }
+
+ private static List<String> getArgList (boolean directApi, boolean useSnapshot, String schemaName,
String dataTable, String indxTable, String tenantId,
- IndexTool.IndexVerifyType verifyType, Long startTime, Long endTime, Long incrementalVerify) {
+ IndexTool.IndexVerifyType verifyType, Long startTime, Long endTime,
+ IndexTool.IndexDisableLoggingType disableLoggingType,
+ Long incrementalVerify) {
List<String> args = Lists.newArrayList();
if (schemaName != null) {
args.add("-s");
@@ -644,7 +661,8 @@ public class IndexToolIT extends BaseUniqueNamesOwnClusterIT {
if (directApi) {
args.add("-direct");
}
- args.add("-v" + verifyType.getValue()); // verify index rows inline
+ args.add("-v");
+ args.add(verifyType.getValue()); // verify index rows inline
// Need to run this job in foreground for the test to be deterministic
args.add("-runfg");
@@ -664,6 +682,12 @@ public class IndexToolIT extends BaseUniqueNamesOwnClusterIT {
args.add("-et");
args.add(String.valueOf(endTime));
}
+
+ if (disableLoggingType != IndexTool.IndexDisableLoggingType.NONE) {
+ args.add("-dl");
+ args.add(disableLoggingType.getValue());
+ }
+
if(incrementalVerify!=null) {
args.add("-rv");
args.add(String.valueOf(incrementalVerify));
@@ -689,10 +713,13 @@ public class IndexToolIT extends BaseUniqueNamesOwnClusterIT {
}
public static String [] getArgValues(boolean directApi, boolean useSnapshot, String schemaName,
- String dataTable, String indexTable, String tenantId,
- IndexTool.IndexVerifyType verifyType, Long incrementalVerify) {
+ String dataTable, String indexTable, String tenantId,
+ IndexTool.IndexVerifyType verifyType, Long startTime,
+ Long endTime,
+ IndexTool.IndexDisableLoggingType disableLoggingType,
+ Long incrementalVerify ) {
List<String> args = getArgList(directApi, useSnapshot, schemaName, dataTable, indexTable,
- tenantId, verifyType, null, null, incrementalVerify);
+ tenantId, verifyType, startTime, endTime, disableLoggingType, incrementalVerify);
return args.toArray(new String[0]);
}
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexVerificationOutputRepositoryIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexVerificationOutputRepositoryIT.java
index 1464e80..e772fa7 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexVerificationOutputRepositoryIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexVerificationOutputRepositoryIT.java
@@ -21,6 +21,7 @@ import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
@@ -28,6 +29,7 @@ import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.phoenix.end2end.ParallelStatsDisabledIT;
import org.apache.phoenix.jdbc.PhoenixConnection;
+import org.apache.phoenix.mapreduce.index.IndexTool;
import org.apache.phoenix.mapreduce.index.IndexVerificationOutputRepository;
import org.apache.phoenix.mapreduce.index.IndexVerificationOutputRow;
import org.apache.phoenix.query.ConnectionQueryServices;
@@ -40,6 +42,7 @@ import org.junit.After;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
+import org.mockito.Mockito;
import java.io.IOException;
import java.sql.Connection;
@@ -55,6 +58,7 @@ import static org.apache.phoenix.mapreduce.index.IndexVerificationOutputReposito
import static org.apache.phoenix.mapreduce.index.IndexVerificationOutputRepository.PHASE_BEFORE_VALUE;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
+import static org.mockito.Mockito.when;
public class IndexVerificationOutputRepositoryIT extends ParallelStatsDisabledIT {
@@ -136,6 +140,69 @@ public class IndexVerificationOutputRepositoryIT extends ParallelStatsDisabledIT
}
}
+ @Test
+ public void testDisableLoggingBefore() throws SQLException, IOException {
+ IndexTool.IndexDisableLoggingType disableLoggingVerifyType = IndexTool.IndexDisableLoggingType.BEFORE;
+ boolean expectedBefore = false;
+ boolean expectedAfter = true;
+ verifyDisableLogging(disableLoggingVerifyType, expectedBefore, expectedAfter);
+ }
+
+ @Test
+ public void testDisableLoggingAfter() throws SQLException, IOException {
+ IndexTool.IndexDisableLoggingType disableLoggingVerifyType = IndexTool.IndexDisableLoggingType.AFTER;
+ boolean expectedBefore = true;
+ boolean expectedAfter = false;
+ verifyDisableLogging(disableLoggingVerifyType, expectedBefore, expectedAfter);
+ }
+
+ @Test
+ public void testDisableLoggingBoth() throws SQLException, IOException {
+ IndexTool.IndexDisableLoggingType disableLoggingVerifyType = IndexTool.IndexDisableLoggingType.BOTH;
+ boolean expectedBefore = false;
+ boolean expectedAfter = false;
+ verifyDisableLogging(disableLoggingVerifyType, expectedBefore, expectedAfter);
+ }
+
+ @Test
+ public void testDisableLoggingNone() throws SQLException, IOException {
+ IndexTool.IndexDisableLoggingType disableLoggingVerifyType = IndexTool.IndexDisableLoggingType.NONE;
+ boolean expectedBefore = true;
+ boolean expectedAfter = true;
+ verifyDisableLogging(disableLoggingVerifyType, expectedBefore, expectedAfter);
+ }
+
+ public void verifyDisableLogging(IndexTool.IndexDisableLoggingType disableLoggingVerifyType, boolean expectedBefore, boolean expectedAfter) throws SQLException, IOException {
+ Table mockOutputTable = Mockito.mock(Table.class);
+ Table mockIndexTable = Mockito.mock(Table.class);
+ when(mockIndexTable.getName()).thenReturn(TableName.valueOf("testDisableLoggingIndexName"));
+ IndexVerificationOutputRepository outputRepository =
+ new IndexVerificationOutputRepository(mockOutputTable,
+ mockIndexTable, disableLoggingVerifyType);
+ byte[] dataRowKey = Bytes.toBytes("dataRowKey");
+ byte[] indexRowKey = Bytes.toBytes("indexRowKey");
+ long dataRowTs = EnvironmentEdgeManager.currentTimeMillis();
+ long indexRowTs = EnvironmentEdgeManager.currentTimeMillis();
+ String errorMsg = "";
+ byte[] expectedValue = Bytes.toBytes("expectedValue");
+ byte[] actualValue = Bytes.toBytes("actualValue");
+ long scanMaxTs = EnvironmentEdgeManager.currentTimeMillis();
+ byte[] tableName = Bytes.toBytes("testDisableLoggingTableName");
+
+ outputRepository.logToIndexToolOutputTable(dataRowKey, indexRowKey, dataRowTs, indexRowTs
+ , errorMsg, expectedValue, actualValue, scanMaxTs, tableName, true);
+ outputRepository.logToIndexToolOutputTable(dataRowKey, indexRowKey, dataRowTs, indexRowTs
+ , errorMsg, expectedValue, actualValue, scanMaxTs, tableName, false);
+ int expectedRowsLogged = 0;
+ if (expectedBefore && expectedAfter) {
+ expectedRowsLogged = 2;
+ } else if (expectedBefore || expectedAfter) {
+ expectedRowsLogged = 1;
+ }
+ Mockito.verify(mockOutputTable, Mockito.times(expectedRowsLogged)).
+ put(Mockito.any(Put.class));
+ }
+
public void verifyOutputRow(IndexVerificationOutputRepository outputRepository, long scanMaxTs,
byte[] indexNameBytes, IndexVerificationOutputRow expectedRow)
throws IOException {
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java
index 4897741..4d61295 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java
@@ -89,6 +89,8 @@ abstract public class BaseScannerRegionObserver extends BaseRegionObserver {
// Index verification type done by the index tool
public static final String INDEX_REBUILD_VERIFY_TYPE = "_IndexRebuildVerifyType";
public static final String INDEX_RETRY_VERIFY = "_IndexRetryVerify";
+ public static final String INDEX_REBUILD_DISABLE_LOGGING_VERIFY_TYPE =
+ "_IndexRebuildDisableLoggingVerifyType";
/*
* Attribute to denote that the index maintainer has been serialized using its proto-buf presentation.
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/IndexRebuildRegionScanner.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/IndexRebuildRegionScanner.java
index 76ae453..a5a4ec1 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/IndexRebuildRegionScanner.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/IndexRebuildRegionScanner.java
@@ -39,7 +39,6 @@ import java.util.concurrent.Future;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Lists;
-import com.sun.org.apache.xpath.internal.operations.Bool;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
@@ -104,6 +103,7 @@ public class IndexRebuildRegionScanner extends GlobalIndexRegionScanner {
private boolean useProto = true;
private byte[] indexRowKey;
private IndexTool.IndexVerifyType verifyType = IndexTool.IndexVerifyType.NONE;
+ private IndexTool.IndexDisableLoggingType disableLoggingVerifyType = IndexTool.IndexDisableLoggingType.NONE;
private boolean verify = false;
private Map<byte[], List<Mutation>> indexKeyToMutationMap;
private Map<byte[], Pair<Put, Delete>> dataKeyToMutationMap;
@@ -152,11 +152,20 @@ public class IndexRebuildRegionScanner extends GlobalIndexRegionScanner {
if (verifyType != IndexTool.IndexVerifyType.NONE) {
verify = true;
viewConstants = IndexUtil.deserializeViewConstantsFromScan(scan);
+ byte[] disableLoggingValueBytes =
+ scan.getAttribute(BaseScannerRegionObserver.INDEX_REBUILD_DISABLE_LOGGING_VERIFY_TYPE);
+ if (disableLoggingValueBytes != null) {
+ disableLoggingVerifyType =
+ IndexTool.IndexDisableLoggingType.fromValue(disableLoggingValueBytes);
+ }
verificationOutputRepository =
- new IndexVerificationOutputRepository(indexMaintainer.getIndexTableName(), hTableFactory);
+ new IndexVerificationOutputRepository(indexMaintainer.getIndexTableName()
+ , hTableFactory, disableLoggingVerifyType);
verificationResult = new IndexToolVerificationResult(scan);
+ new IndexVerificationOutputRepository(indexMaintainer.getIndexTableName()
+ , hTableFactory, disableLoggingVerifyType);
verificationResultRepository =
- new IndexVerificationResultRepository(indexMaintainer.getIndexTableName(), hTableFactory);
+ new IndexVerificationResultRepository(indexMaintainer.getIndexTableName(), hTableFactory);
indexKeyToMutationMap = Maps.newTreeMap(Bytes.BYTES_COMPARATOR);
dataKeyToMutationMap = Maps.newTreeMap(Bytes.BYTES_COMPARATOR);
pool = new WaitForCompletionTaskRunner(ThreadPoolManager.getExecutor(
@@ -240,14 +249,20 @@ public class IndexRebuildRegionScanner extends GlobalIndexRegionScanner {
innerScanner.close();
if (verify) {
try {
- verificationResultRepository.logToIndexToolResultTable(verificationResult,
+ if (verificationResultRepository != null) {
+ verificationResultRepository.logToIndexToolResultTable(verificationResult,
verifyType, region.getRegionInfo().getRegionName(), skipped);
+ }
} finally {
this.pool.stop("IndexRebuildRegionScanner is closing");
hTableFactory.shutdown();
indexHTable.close();
- verificationResultRepository.close();
- verificationOutputRepository.close();
+ if (verificationResultRepository != null) {
+ verificationResultRepository.close();
+ }
+ if (verificationOutputRepository != null) {
+ verificationOutputRepository.close();
+ }
}
}
}
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/PhoenixServerBuildIndexInputFormat.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/PhoenixServerBuildIndexInputFormat.java
index 9408369..696659b 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/PhoenixServerBuildIndexInputFormat.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/PhoenixServerBuildIndexInputFormat.java
@@ -41,6 +41,7 @@ import org.slf4j.LoggerFactory;
import com.google.common.base.Preconditions;
import static org.apache.phoenix.mapreduce.util.PhoenixConfigurationUtil.getCurrentScnValue;
+import static org.apache.phoenix.mapreduce.util.PhoenixConfigurationUtil.getDisableLoggingVerifyType;
import static org.apache.phoenix.mapreduce.util.PhoenixConfigurationUtil.getIndexToolDataTableName;
import static org.apache.phoenix.mapreduce.util.PhoenixConfigurationUtil.getIndexToolIndexTableName;
import static org.apache.phoenix.mapreduce.util.PhoenixConfigurationUtil.getIndexToolLastVerifyTime;
@@ -106,6 +107,8 @@ public class PhoenixServerBuildIndexInputFormat<T extends DBWritable> extends Ph
scan.setAttribute(BaseScannerRegionObserver.INDEX_REBUILD_PAGING, TRUE_BYTES);
scan.setAttribute(BaseScannerRegionObserver.INDEX_REBUILD_VERIFY_TYPE, getIndexVerifyType(configuration).toBytes());
scan.setAttribute(BaseScannerRegionObserver.INDEX_RETRY_VERIFY, Bytes.toBytes(lastVerifyTimeValue));
+ scan.setAttribute(BaseScannerRegionObserver.INDEX_REBUILD_DISABLE_LOGGING_VERIFY_TYPE,
+ getDisableLoggingVerifyType(configuration).toBytes());
} catch (IOException e) {
throw new SQLException(e);
}
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexTool.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexTool.java
index 4c0c2d2..3477049 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexTool.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexTool.java
@@ -156,6 +156,42 @@ public class IndexTool extends Configured implements Tool {
}
}
+ public enum IndexDisableLoggingType {
+ NONE("NONE"),
+ BEFORE("BEFORE"),
+ AFTER("AFTER"),
+ BOTH("BOTH");
+
+ private String value;
+ private byte[] valueBytes;
+
+ IndexDisableLoggingType(String value) {
+ this.value = value;
+ this.valueBytes = PVarchar.INSTANCE.toBytes(value);
+ }
+
+ public String getValue() {
+ return this.value;
+ }
+
+ public byte[] toBytes() {
+ return this.valueBytes;
+ }
+
+ public static IndexDisableLoggingType fromValue(String value) {
+ for (IndexDisableLoggingType disableLoggingType: IndexDisableLoggingType.values()) {
+ if (value.equals(disableLoggingType.getValue())) {
+ return disableLoggingType;
+ }
+ }
+ throw new IllegalStateException("Invalid value: "+ value + " for " + IndexDisableLoggingType.class);
+ }
+
+ public static IndexDisableLoggingType fromValue(byte[] value) {
+ return fromValue(Bytes.toString(value));
+ }
+ }
+
private static final Logger LOGGER = LoggerFactory.getLogger(IndexTool.class);
private String schemaName;
@@ -163,6 +199,7 @@ public class IndexTool extends Configured implements Tool {
private String indexTable;
private boolean isPartialBuild, isForeground;
private IndexVerifyType indexVerifyType = IndexVerifyType.NONE;
+ private IndexDisableLoggingType disableLoggingType = IndexDisableLoggingType.NONE;
private String qDataTable;
private String qIndexTable;
private boolean useSnapshot;
@@ -243,6 +280,11 @@ public class IndexTool extends Configured implements Tool {
private static final Option RETRY_VERIFY_OPTION = new Option("rv", "retry-verify",
true, "Max scan ts of the last rebuild/verify that needs to be retried incrementally");
+ private static final Option DISABLE_LOGGING_OPTION = new Option("dl",
+ "disable-logging", true
+ , "Disable logging of failed verification rows for BEFORE, " +
+ "AFTER, or BOTH verify jobs");
+
public static final String INDEX_JOB_NAME_TEMPLATE = "PHOENIX_%s.%s_INDX_%s";
public static final String INVALID_TIME_RANGE_EXCEPTION_MESSAGE = "startTime is greater than "
@@ -280,6 +322,7 @@ public class IndexTool extends Configured implements Tool {
options.addOption(START_TIME_OPTION);
options.addOption(END_TIME_OPTION);
options.addOption(RETRY_VERIFY_OPTION);
+ options.addOption(DISABLE_LOGGING_OPTION);
return options;
}
@@ -334,9 +377,52 @@ public class IndexTool extends Configured implements Tool {
if (splitIndex && cmdLine.hasOption(PARTIAL_REBUILD_OPTION.getOpt())) {
throw new IllegalStateException("Cannot split index for a partial rebuild, as the index table is dropped");
}
+ if (loggingDisabledMismatchesVerifyOption(cmdLine)){
+ throw new IllegalStateException("Can't disable index verification logging when no " +
+ "index verification or the wrong kind of index verification has been requested. " +
+ "VerifyType: [" + cmdLine.getOptionValue(VERIFY_OPTION.getOpt()) + "] and " +
+ "DisableLoggingType: ["
+ + cmdLine.getOptionValue(DISABLE_LOGGING_OPTION.getOpt()) + "]");
+ }
return cmdLine;
}
+ private boolean loggingDisabledMismatchesVerifyOption(CommandLine cmdLine) {
+ boolean loggingDisabled = cmdLine.hasOption(DISABLE_LOGGING_OPTION.getOpt());
+ if (!loggingDisabled) {
+ return false;
+ }
+ boolean hasVerifyOption =
+ cmdLine.hasOption(VERIFY_OPTION.getOpt());
+ if (!hasVerifyOption) {
+ return true;
+ }
+ String loggingDisableValue = cmdLine.getOptionValue(DISABLE_LOGGING_OPTION.getOpt());
+ String verifyValue = cmdLine.getOptionValue(VERIFY_OPTION.getOpt());
+ IndexDisableLoggingType loggingDisableType = IndexDisableLoggingType.fromValue(loggingDisableValue);
+ if (loggingDisableType != IndexDisableLoggingType.BEFORE &&
+ loggingDisableType != IndexDisableLoggingType.AFTER &&
+ loggingDisableType != IndexDisableLoggingType.BOTH) {
+ return true;
+ }
+ IndexVerifyType verifyType = IndexVerifyType.fromValue(verifyValue);
+ //error if we're trying to disable logging when we're not doing any verification
+ if (verifyType.equals(IndexVerifyType.NONE)){
+ return true;
+ }
+ //error if we're disabling logging after rebuild but we're not verifying after rebuild
+ if ((verifyType.equals(IndexVerifyType.BEFORE) || verifyType.equals(IndexVerifyType.ONLY))
+ && loggingDisableType.equals(IndexDisableLoggingType.AFTER)) {
+ return true;
+ }
+ //error if we're disabling logging before rebuild but we're not verifying before rebuild
+ if ((verifyType.equals(IndexVerifyType.AFTER))
+ && loggingDisableType.equals(IndexDisableLoggingType.BEFORE)) {
+ return true;
+ }
+ return false;
+ }
+
private void printHelpAndExit(String errorMessage, Options options) {
System.err.println(errorMessage);
printHelpAndExit(options, 1);
@@ -356,6 +442,10 @@ public class IndexTool extends Configured implements Tool {
public Long getLastVerifyTime() { return lastVerifyTime; }
+ public IndexTool.IndexDisableLoggingType getDisableLoggingType() {
+ return disableLoggingType;
+ }
+
class JobFactory {
Connection connection;
Configuration configuration;
@@ -607,6 +697,7 @@ public class IndexTool extends Configured implements Tool {
PhoenixConfigurationUtil.setIndexToolStartTime(configuration, startTime);
}
PhoenixConfigurationUtil.setIndexVerifyType(configuration, indexVerifyType);
+ PhoenixConfigurationUtil.setDisableLoggingVerifyType(configuration, disableLoggingType);
String physicalIndexTable = pIndexTable.getPhysicalName().getString();
PhoenixConfigurationUtil.setPhysicalTableName(configuration, physicalIndexTable);
@@ -758,6 +849,7 @@ public class IndexTool extends Configured implements Tool {
boolean useEndTime = cmdLine.hasOption(END_TIME_OPTION.getOpt());
boolean retryVerify = cmdLine.hasOption(RETRY_VERIFY_OPTION.getOpt());
boolean verify = cmdLine.hasOption(VERIFY_OPTION.getOpt());
+ boolean disableLogging = cmdLine.hasOption(DISABLE_LOGGING_OPTION.getOpt());
if (useTenantId) {
tenantId = cmdLine.getOptionValue(TENANT_ID_OPTION.getOpt());
@@ -778,6 +870,11 @@ public class IndexTool extends Configured implements Tool {
if (verify) {
String value = cmdLine.getOptionValue(VERIFY_OPTION.getOpt());
indexVerifyType = IndexVerifyType.fromValue(value);
+ if (disableLogging) {
+ disableLoggingType =
+ IndexDisableLoggingType.fromValue(
+ cmdLine.getOptionValue(DISABLE_LOGGING_OPTION.getOpt()));
+ }
}
schemaName = cmdLine.getOptionValue(SCHEMA_NAME_OPTION.getOpt());
dataTable = cmdLine.getOptionValue(DATA_TABLE_OPTION.getOpt());
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexVerificationOutputRepository.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexVerificationOutputRepository.java
index 7e8ee23..4bc91a8 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexVerificationOutputRepository.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexVerificationOutputRepository.java
@@ -50,6 +50,9 @@ public class IndexVerificationOutputRepository implements AutoCloseable {
private Table indexTable;
private byte[] indexName;
private Table outputTable;
+ private IndexTool.IndexDisableLoggingType disableLoggingVerifyType =
+ IndexTool.IndexDisableLoggingType.NONE;
+
public final static String OUTPUT_TABLE_NAME = "PHOENIX_INDEX_TOOL";
public final static byte[] OUTPUT_TABLE_NAME_BYTES = Bytes.toBytes(OUTPUT_TABLE_NAME);
public final static byte[] OUTPUT_TABLE_COLUMN_FAMILY = QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES;
@@ -98,10 +101,20 @@ public class IndexVerificationOutputRepository implements AutoCloseable {
indexTable = queryServices.getTable(indexName);
}
- public IndexVerificationOutputRepository(byte[] indexName, HTableFactory hTableFactory) throws IOException {
+ @VisibleForTesting
+ public IndexVerificationOutputRepository(Table outputTable, Table indexTable,
+ IndexTool.IndexDisableLoggingType disableLoggingVerifyType) throws SQLException {
+ this.outputTable = outputTable;
+ this.indexTable = indexTable;
+ this.disableLoggingVerifyType = disableLoggingVerifyType;
+ }
+
+ public IndexVerificationOutputRepository(byte[] indexName, HTableFactory hTableFactory,
+ IndexTool.IndexDisableLoggingType disableLoggingVerifyType) throws IOException {
this.indexName = indexName;
outputTable = hTableFactory.getTable(new ImmutableBytesPtr(OUTPUT_TABLE_NAME_BYTES));
indexTable = hTableFactory.getTable(new ImmutableBytesPtr(indexName));
+ this.disableLoggingVerifyType = disableLoggingVerifyType;
}
public static byte[] generateOutputTableRowKey(long ts, byte[] indexTableName, byte[] dataRowKey ) {
@@ -162,29 +175,48 @@ public class IndexVerificationOutputRepository implements AutoCloseable {
String errorMsg, byte[] expectedValue, byte[] actualValue,
long scanMaxTs, byte[] tableName, boolean isBeforeRebuild)
throws IOException {
- byte[] rowKey = generateOutputTableRowKey(scanMaxTs, indexTable.getName().toBytes(), dataRowKey);
- Put put = new Put(rowKey);
- put.addColumn(OUTPUT_TABLE_COLUMN_FAMILY, DATA_TABLE_NAME_BYTES, tableName);
- put.addColumn(OUTPUT_TABLE_COLUMN_FAMILY, INDEX_TABLE_NAME_BYTES, indexName);
- put.addColumn(OUTPUT_TABLE_COLUMN_FAMILY, DATA_TABLE_TS_BYTES, Bytes.toBytes(Long.toString(dataRowTs)));
+ if (shouldLogOutput(isBeforeRebuild)) {
+ byte[] rowKey = generateOutputTableRowKey(scanMaxTs, indexTable.getName().toBytes(), dataRowKey);
+ Put put = new Put(rowKey);
+ put.addColumn(OUTPUT_TABLE_COLUMN_FAMILY, DATA_TABLE_NAME_BYTES, tableName);
+ put.addColumn(OUTPUT_TABLE_COLUMN_FAMILY, INDEX_TABLE_NAME_BYTES, indexName);
+ put.addColumn(OUTPUT_TABLE_COLUMN_FAMILY, DATA_TABLE_TS_BYTES, Bytes.toBytes(Long.toString(dataRowTs)));
- put.addColumn(OUTPUT_TABLE_COLUMN_FAMILY, INDEX_TABLE_ROW_KEY_BYTES, indexRowKey);
- put.addColumn(OUTPUT_TABLE_COLUMN_FAMILY, INDEX_TABLE_TS_BYTES, Bytes.toBytes(Long.toString(indexRowTs)));
- byte[] errorMessageBytes;
- if (expectedValue != null) {
- errorMessageBytes = getErrorMessageBytes(errorMsg, expectedValue, actualValue);
- put.addColumn(OUTPUT_TABLE_COLUMN_FAMILY, EXPECTED_VALUE_BYTES, expectedValue);
- put.addColumn(OUTPUT_TABLE_COLUMN_FAMILY, ACTUAL_VALUE_BYTES, actualValue);
- } else {
- errorMessageBytes = Bytes.toBytes(errorMsg);
+ put.addColumn(OUTPUT_TABLE_COLUMN_FAMILY, INDEX_TABLE_ROW_KEY_BYTES, indexRowKey);
+ put.addColumn(OUTPUT_TABLE_COLUMN_FAMILY, INDEX_TABLE_TS_BYTES, Bytes.toBytes(Long.toString(indexRowTs)));
+ byte[] errorMessageBytes;
+ if (expectedValue != null) {
+ errorMessageBytes = getErrorMessageBytes(errorMsg, expectedValue, actualValue);
+ put.addColumn(OUTPUT_TABLE_COLUMN_FAMILY, EXPECTED_VALUE_BYTES, expectedValue);
+ put.addColumn(OUTPUT_TABLE_COLUMN_FAMILY, ACTUAL_VALUE_BYTES, actualValue);
+ } else {
+ errorMessageBytes = Bytes.toBytes(errorMsg);
+ }
+ put.addColumn(OUTPUT_TABLE_COLUMN_FAMILY, ERROR_MESSAGE_BYTES, errorMessageBytes);
+ if (isBeforeRebuild) {
+ put.addColumn(OUTPUT_TABLE_COLUMN_FAMILY, VERIFICATION_PHASE_BYTES, PHASE_BEFORE_VALUE);
+ } else {
+ put.addColumn(OUTPUT_TABLE_COLUMN_FAMILY, VERIFICATION_PHASE_BYTES, PHASE_AFTER_VALUE);
+ }
+ outputTable.put(put);
+ }
+ }
+
+ public boolean shouldLogOutput(boolean isBeforeRebuild) {
+ if (disableLoggingVerifyType.equals(IndexTool.IndexDisableLoggingType.BOTH)) {
+ return false;
+ }
+ if (disableLoggingVerifyType.equals(IndexTool.IndexDisableLoggingType.NONE)) {
+ return true;
+ }
+ if (isBeforeRebuild &&
+ (disableLoggingVerifyType.equals(IndexTool.IndexDisableLoggingType.AFTER))) {
+ return true;
}
- put.addColumn(OUTPUT_TABLE_COLUMN_FAMILY, ERROR_MESSAGE_BYTES, errorMessageBytes);
- if (isBeforeRebuild) {
- put.addColumn(OUTPUT_TABLE_COLUMN_FAMILY, VERIFICATION_PHASE_BYTES, PHASE_BEFORE_VALUE);
- } else {
- put.addColumn(OUTPUT_TABLE_COLUMN_FAMILY, VERIFICATION_PHASE_BYTES, PHASE_AFTER_VALUE);
+ if (!isBeforeRebuild && disableLoggingVerifyType.equals(IndexTool.IndexDisableLoggingType.BEFORE)) {
+ return true;
}
- outputTable.put(put);
+ return false;
}
public static byte[] getErrorMessageBytes(String errorMsg, byte[] expectedValue, byte[] actualValue) {
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/util/PhoenixConfigurationUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/util/PhoenixConfigurationUtil.java
index f575b09..348755a 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/util/PhoenixConfigurationUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/util/PhoenixConfigurationUtil.java
@@ -148,6 +148,9 @@ public final class PhoenixConfigurationUtil {
public static final String INDEX_VERIFY_TYPE = "phoenix.mr.index.IndexVerifyType";
+ public static final String DISABLE_LOGGING_TYPE = "phoenix.mr.index" +
+ ".IndexDisableLoggingType";
+
// Generate splits based on scans from stats, or just from region splits
public static final String MAPREDUCE_SPLIT_BY_STATS = "phoenix.mapreduce.split.by.stats";
@@ -604,6 +607,12 @@ public final class PhoenixConfigurationUtil {
configuration.set(INDEX_VERIFY_TYPE, verifyType.getValue());
}
+ public static void setDisableLoggingVerifyType(Configuration configuration,
+ IndexTool.IndexDisableLoggingType disableLoggingType) {
+ Preconditions.checkNotNull(configuration);
+ configuration.set(DISABLE_LOGGING_TYPE, disableLoggingType.getValue());
+ }
+
public static String getScrutinyDataTableName(Configuration configuration) {
Preconditions.checkNotNull(configuration);
return configuration.get(SCRUTINY_DATA_TABLE_NAME);
@@ -735,6 +744,12 @@ public final class PhoenixConfigurationUtil {
return IndexTool.IndexVerifyType.fromValue(value);
}
+ public static IndexTool.IndexVerifyType getDisableLoggingVerifyType(Configuration configuration) {
+ Preconditions.checkNotNull(configuration);
+ String value = configuration.get(DISABLE_LOGGING_TYPE, IndexTool.IndexVerifyType.NONE.getValue());
+ return IndexTool.IndexVerifyType.fromValue(value);
+ }
+
public static boolean getSplitByStats(final Configuration configuration) {
Preconditions.checkNotNull(configuration);
boolean split = configuration.getBoolean(MAPREDUCE_SPLIT_BY_STATS, DEFAULT_SPLIT_BY_STATS);
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/index/IndexToolTest.java b/phoenix-core/src/test/java/org/apache/phoenix/index/IndexToolTest.java
index fd5164f..87215d0 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/index/IndexToolTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/index/IndexToolTest.java
@@ -34,6 +34,7 @@ import org.mockito.MockitoAnnotations;
import static org.apache.phoenix.mapreduce.index.IndexTool.FEATURE_NOT_APPLICABLE;
import static org.apache.phoenix.mapreduce.index.IndexTool.INVALID_TIME_RANGE_EXCEPTION_MESSAGE;
+import static org.junit.Assert.assertEquals;
import static org.apache.phoenix.mapreduce.index.IndexTool.RETRY_VERIFY_NOT_APPLICABLE;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@@ -72,8 +73,8 @@ public class IndexToolTest extends BaseTest {
startTime , endTime);
CommandLine cmdLine = it.parseOptions(args);
it.populateIndexToolAttributes(cmdLine);
- Assert.assertEquals(startTime, it.getStartTime());
- Assert.assertEquals(endTime, it.getEndTime());
+ assertEquals(startTime, it.getStartTime());
+ assertEquals(endTime, it.getEndTime());
}
@Test
@@ -96,8 +97,8 @@ public class IndexToolTest extends BaseTest {
startTime , null);
CommandLine cmdLine = it.parseOptions(args);
it.populateIndexToolAttributes(cmdLine);
- Assert.assertEquals(startTime, it.getStartTime());
- Assert.assertEquals(null, it.getEndTime());
+ assertEquals(startTime, it.getStartTime());
+ assertEquals(null, it.getEndTime());
}
@Test
@@ -109,8 +110,8 @@ public class IndexToolTest extends BaseTest {
null , endTime);
CommandLine cmdLine = it.parseOptions(args);
it.populateIndexToolAttributes(cmdLine);
- Assert.assertEquals(null, it.getStartTime());
- Assert.assertEquals(endTime, it.getEndTime());
+ assertEquals(null, it.getStartTime());
+ assertEquals(endTime, it.getEndTime());
}
@Test
@@ -211,7 +212,7 @@ public class IndexToolTest extends BaseTest {
String [] args =
IndexToolIT.getArgValues(true, true, schema,
dataTable, indexTable, tenantId, IndexTool.IndexVerifyType.NONE,
- lastVerifyTime);
+ lastVerifyTime, null, IndexTool.IndexDisableLoggingType.NONE, lastVerifyTime);
when(mockTool.parseOptions(args)).thenCallRealMethod();
CommandLine cmdLine = mockTool.parseOptions(args);
@@ -234,9 +235,10 @@ public class IndexToolTest extends BaseTest {
when(mockTool.getLastVerifyTime()).thenCallRealMethod();
Long lastVerifyTime = 10L;
String [] args =
- IndexToolIT.getArgValues(true, true, schema,
- dataTable, indexTable, tenantId, IndexTool.IndexVerifyType.AFTER,
- lastVerifyTime);
+ IndexToolIT.getArgValues(true, true, schema,
+ dataTable, indexTable, tenantId, IndexTool.IndexVerifyType.AFTER,
+ lastVerifyTime, null, IndexTool.IndexDisableLoggingType.NONE,
+ lastVerifyTime);
when(mockTool.parseOptions(args)).thenCallRealMethod();
CommandLine cmdLine = mockTool.parseOptions(args);
@@ -249,4 +251,88 @@ public class IndexToolTest extends BaseTest {
exceptionRule.expectMessage(RETRY_VERIFY_NOT_APPLICABLE);
mockTool.populateIndexToolAttributes(cmdLine);
}
+
+ @Test
+ public void testCheckVerifyAndDisableLogging_defaultsNone() throws Exception {
+ Long startTime = 1L;
+ Long endTime = 10L;
+ String [] args =
+ IndexToolIT.getArgValues(true, true, schema,
+ dataTable, indexTable, tenantId, IndexTool.IndexVerifyType.NONE,
+ startTime , endTime);
+ CommandLine cmdLine = it.parseOptions(args);
+ it.populateIndexToolAttributes(cmdLine);
+ assertEquals(IndexTool.IndexDisableLoggingType.NONE, it.getDisableLoggingType());
+ }
+
+ @Test
+ public void testDisableLogging_allowsNone() throws Exception {
+ verifyDisableLogging(IndexTool.IndexDisableLoggingType.NONE, IndexTool.IndexVerifyType.NONE);
+ verifyDisableLogging(IndexTool.IndexDisableLoggingType.NONE, IndexTool.IndexVerifyType.ONLY);
+ verifyDisableLogging(IndexTool.IndexDisableLoggingType.NONE, IndexTool.IndexVerifyType.BEFORE);
+ verifyDisableLogging(IndexTool.IndexDisableLoggingType.NONE, IndexTool.IndexVerifyType.AFTER);
+ verifyDisableLogging(IndexTool.IndexDisableLoggingType.NONE, IndexTool.IndexVerifyType.BOTH);
+ }
+
+ @Test
+ public void testDisableLogging_allowsBefore() throws Exception {
+ verifyDisableLogging(IndexTool.IndexDisableLoggingType.BEFORE, IndexTool.IndexVerifyType.BEFORE);
+ verifyDisableLogging(IndexTool.IndexDisableLoggingType.BEFORE, IndexTool.IndexVerifyType.ONLY);
+ verifyDisableLogging(IndexTool.IndexDisableLoggingType.BEFORE, IndexTool.IndexVerifyType.BOTH);
+ verifyDisableLoggingException(IndexTool.IndexDisableLoggingType.BEFORE,
+ IndexTool.IndexVerifyType.AFTER);
+ verifyDisableLoggingException(IndexTool.IndexDisableLoggingType.BEFORE,
+ IndexTool.IndexVerifyType.NONE);
+ }
+
+ @Test
+ public void testDisableLogging_allowsAfter() throws Exception {
+ verifyDisableLogging(IndexTool.IndexDisableLoggingType.AFTER, IndexTool.IndexVerifyType.BOTH);
+ verifyDisableLogging(IndexTool.IndexDisableLoggingType.AFTER, IndexTool.IndexVerifyType.AFTER);
+ verifyDisableLoggingException(IndexTool.IndexDisableLoggingType.AFTER,
+ IndexTool.IndexVerifyType.NONE);
+ verifyDisableLoggingException(IndexTool.IndexDisableLoggingType.AFTER,
+ IndexTool.IndexVerifyType.BEFORE);
+ verifyDisableLoggingException(IndexTool.IndexDisableLoggingType.BOTH,
+ IndexTool.IndexVerifyType.ONLY);
+ }
+
+ @Test
+ public void testCheckVerifyAndDisableLogging_allowsBoth() throws Exception {
+ verifyDisableLogging(IndexTool.IndexDisableLoggingType.BOTH, IndexTool.IndexVerifyType.BOTH);
+ verifyDisableLoggingException(IndexTool.IndexDisableLoggingType.BOTH,
+ IndexTool.IndexVerifyType.NONE);
+ verifyDisableLoggingException(IndexTool.IndexDisableLoggingType.BOTH,
+ IndexTool.IndexVerifyType.ONLY);
+ verifyDisableLoggingException(IndexTool.IndexDisableLoggingType.BOTH,
+ IndexTool.IndexVerifyType.BEFORE);
+ verifyDisableLoggingException(IndexTool.IndexDisableLoggingType.BOTH,
+ IndexTool.IndexVerifyType.AFTER);
+ }
+
+ public void verifyDisableLogging(IndexTool.IndexDisableLoggingType disableType,
+ IndexTool.IndexVerifyType verifyType) throws Exception {
+ Long startTime = 1L;
+ Long endTime = 10L;
+ String[] args =
+ IndexToolIT.getArgValues(true, true, schema,
+ dataTable, indexTable, tenantId, verifyType,
+ startTime, endTime, disableType, null);
+ CommandLine cmdLine = it.parseOptions(args);
+ it.populateIndexToolAttributes(cmdLine);
+ assertEquals(disableType, it.getDisableLoggingType());
+ }
+
+ public void verifyDisableLoggingException(IndexTool.IndexDisableLoggingType disableType,
+ IndexTool.IndexVerifyType verifyType) {
+ Long startTime = 1L;
+ Long endTime = 10L;
+ String[] args =
+ IndexToolIT.getArgValues(true, true, schema,
+ dataTable, indexTable, tenantId, verifyType,
+ startTime, endTime, disableType, null);
+ exceptionRule.expect(IllegalStateException.class);
+ CommandLine cmdLine = it.parseOptions(args);
+ }
+
}