You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by ka...@apache.org on 2020/03/01 22:38:13 UTC
[phoenix] branch PHOENIX-5748-4.x-HBase-1.5 updated (7e8801c ->
2efc813)
This is an automated email from the ASF dual-hosted git repository.
kadir pushed a change to branch PHOENIX-5748-4.x-HBase-1.5
in repository https://gitbox.apache.org/repos/asf/phoenix.git.
discard 7e8801c PHOENIX-5709 Addendum (unverified rows should fail all verification types except ONLY)
new 2efc813 PHOENIX-5709 Addendum (unverified rows should fail all verification types except ONLY)
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (7e8801c)
\
N -- N -- N refs/heads/PHOENIX-5748-4.x-HBase-1.5 (2efc813)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../main/java/org/apache/phoenix/hbase/index/IndexRegionObserver.java | 4 ++++
1 file changed, 4 insertions(+)
[phoenix] 01/01: PHOENIX-5709 Addendum (unverified rows should fail
all verification types except ONLY)
Posted by ka...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kadir pushed a commit to branch PHOENIX-5748-4.x-HBase-1.5
in repository https://gitbox.apache.org/repos/asf/phoenix.git
commit 2efc8135860c8a6e5cf2bab71b43f6f5c67bf282
Author: Kadir <ko...@salesforce.com>
AuthorDate: Sat Feb 29 18:58:04 2020 -0800
PHOENIX-5709 Addendum (unverified rows should fail all verification types except ONLY)
---
.../coprocessor/IndexRebuildRegionScanner.java | 47 ++++++++++++----------
.../phoenix/hbase/index/IndexRegionObserver.java | 7 ++++
2 files changed, 32 insertions(+), 22 deletions(-)
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/IndexRebuildRegionScanner.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/IndexRebuildRegionScanner.java
index a52bddf..f245df9 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/IndexRebuildRegionScanner.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/IndexRebuildRegionScanner.java
@@ -19,6 +19,7 @@ package org.apache.phoenix.coprocessor;
import static org.apache.phoenix.hbase.index.IndexRegionObserver.UNVERIFIED_BYTES;
import static org.apache.phoenix.hbase.index.IndexRegionObserver.VERIFIED_BYTES;
+import static org.apache.phoenix.hbase.index.IndexRegionObserver.removeEmptyColumn;
import static org.apache.phoenix.hbase.index.write.AbstractParallelWriterIndexCommitter.INDEX_WRITER_KEEP_ALIVE_TIME_CONF_KEY;
import static org.apache.phoenix.mapreduce.index.IndexTool.AFTER_REBUILD_EXPIRED_INDEX_ROW_COUNT_BYTES;
import static org.apache.phoenix.mapreduce.index.IndexTool.AFTER_REBUILD_INVALID_INDEX_ROW_COUNT_BYTES;
@@ -424,7 +425,7 @@ public class IndexRebuildRegionScanner extends BaseRegionScanner {
hTableFactory = ServerUtil.getDelegateHTableFactory(env, ServerUtil.ConnectionType.INDEX_WRITER_CONNECTION);
indexHTable = hTableFactory.getTable(new ImmutableBytesPtr(indexMaintainer.getIndexTableName()));
indexTableTTL = indexHTable.getTableDescriptor().getColumnFamilies()[0].getTimeToLive();
- outputHTable = hTableFactory.getTable(new ImmutableBytesPtr(IndexTool.OUTPUT_TABLE_NAME_BYTES));
+ outputHTable = hTableFactory.getTable(new ImmutableBytesPtr(IndexTool.OUTPUT_TABLE_NAME_BYTES));
resultHTable = hTableFactory.getTable(new ImmutableBytesPtr(IndexTool.RESULT_TABLE_NAME_BYTES));
indexKeyToMutationMap = Maps.newTreeMap(Bytes.BYTES_COMPARATOR);
dataKeyToMutationMap = Maps.newTreeMap(Bytes.BYTES_COMPARATOR);
@@ -689,7 +690,7 @@ public class IndexRebuildRegionScanner extends BaseRegionScanner {
return null;
}
- private boolean isMatchingMutation(Mutation expected, Mutation actual, int version) throws IOException {
+ private boolean isMatchingMutation(Mutation expected, Mutation actual, int iteration) throws IOException {
if (getTimestamp(expected) != getTimestamp(actual)) {
String errorMsg = "Not matching timestamp";
byte[] dataKey = indexMaintainer.buildDataRowKey(new ImmutableBytesWritable(expected.getRow()), viewConstants);
@@ -708,23 +709,20 @@ public class IndexRebuildRegionScanner extends BaseRegionScanner {
if (actualCell == null ||
!CellUtil.matchingType(expectedCell, actualCell)) {
byte[] dataKey = indexMaintainer.buildDataRowKey(new ImmutableBytesWritable(expected.getRow()), viewConstants);
- String errorMsg = "Missing cell (in version " + version + ") " + Bytes.toString(family) + ":" + Bytes.toString(qualifier);
+ String errorMsg = "Missing cell (in iteration " + iteration + ") " + Bytes.toString(family) + ":" + Bytes.toString(qualifier);
logToIndexToolOutputTable(dataKey, expected.getRow(), getTimestamp(expected), getTimestamp(actual), errorMsg);
return false;
}
if (!CellUtil.matchingValue(actualCell, expectedCell)) {
- if (Bytes.compareTo(family, indexMaintainer.getEmptyKeyValueFamily().copyBytesIfNecessary()) == 0 &&
- Bytes.compareTo(qualifier, indexMaintainer.getEmptyKeyValueQualifier()) == 0) {
- if (Bytes.compareTo(actualCell.getValueArray(), actualCell.getValueOffset(), actualCell.getValueLength(),
- UNVERIFIED_BYTES, 0, UNVERIFIED_BYTES.length) == 0) {
- // We will not flag this as mismatch but will log it
- byte[] dataKey = indexMaintainer.buildDataRowKey(new ImmutableBytesWritable(expected.getRow()), viewConstants);
- String errorMsg = "Unverified index row (in version " + version + ")";
- logToIndexToolOutputTable(dataKey, expected.getRow(), getTimestamp(expected), getTimestamp(actual), errorMsg);
- continue;
- }
+ if (verifyType == IndexTool.IndexVerifyType.ONLY &&
+ (Bytes.compareTo(family, indexMaintainer.getEmptyKeyValueFamily().copyBytesIfNecessary()) == 0 &&
+ Bytes.compareTo(qualifier, indexMaintainer.getEmptyKeyValueQualifier()) == 0) &&
+ (Bytes.compareTo(actualCell.getValueArray(), actualCell.getValueOffset(), actualCell.getValueLength(),
+ UNVERIFIED_BYTES, 0, UNVERIFIED_BYTES.length) == 0)) {
+ // We do not flag this as mismatch as we can have unverified but still valid rows
+ continue;
}
- String errorMsg = "Not matching value (in version " + version + ") for " + Bytes.toString(family) + ":" + Bytes.toString(qualifier);
+ String errorMsg = "Not matching value (in iteration " + iteration + ") for " + Bytes.toString(family) + ":" + Bytes.toString(qualifier);
byte[] dataKey = indexMaintainer.buildDataRowKey(new ImmutableBytesWritable(expected.getRow()), viewConstants);
logToIndexToolOutputTable(dataKey, expected.getRow(), getTimestamp(expected), getTimestamp(actual),
errorMsg, CellUtil.cloneValue(expectedCell), CellUtil.cloneValue(actualCell));
@@ -963,14 +961,16 @@ public class IndexRebuildRegionScanner extends BaseRegionScanner {
}
if ((expectedIndex != expectedSize) || actualIndex != actualSize) {
if (matchingCount > 0) {
- // We do not consider this as a verification issue but log it for further information.
- // This may happen due to compaction
- byte[] dataKey = indexMaintainer.buildDataRowKey(new ImmutableBytesWritable(indexRow.getRow()), viewConstants);
- String errorMsg = "Expected to find " + expectedMutationList.size() + " mutations but got "
- + actualMutationList.size();
- logToIndexToolOutputTable(dataKey, indexRow.getRow(),
- getTimestamp(expectedMutationList.get(0)),
- getTimestamp(actualMutationList.get(0)), errorMsg);
+ if (verifyType != IndexTool.IndexVerifyType.ONLY) {
+ // We do not consider this as a verification issue but log it for further information.
+ // This may happen due to compaction
+ byte[] dataKey = indexMaintainer.buildDataRowKey(new ImmutableBytesWritable(indexRow.getRow()), viewConstants);
+ String errorMsg = "Expected to find " + expectedMutationList.size() + " mutations but got "
+ + actualMutationList.size();
+ logToIndexToolOutputTable(dataKey, indexRow.getRow(),
+ getTimestamp(expectedMutationList.get(0)),
+ getTimestamp(actualMutationList.get(0)), errorMsg);
+ }
} else {
byte[] dataKey = indexMaintainer.buildDataRowKey(new ImmutableBytesWritable(indexRow.getRow()), viewConstants);
String errorMsg = "Not matching index row";
@@ -1283,6 +1283,9 @@ public class IndexRebuildRegionScanner extends BaseRegionScanner {
byte[] indexRowKey = indexMaintainer.buildRowKey(mergedRowVG, rowKeyPtr,
null, null, HConstants.LATEST_TIMESTAMP);
indexPut = new Put(indexRowKey);
+ } else {
+ removeEmptyColumn(indexPut, indexMaintainer.getEmptyKeyValueFamily().copyBytesIfNecessary(),
+ indexMaintainer.getEmptyKeyValueQualifier());
}
indexPut.addColumn(indexMaintainer.getEmptyKeyValueFamily().copyBytesIfNecessary(),
indexMaintainer.getEmptyKeyValueQualifier(), ts, VERIFIED_BYTES);
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/IndexRegionObserver.java b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/IndexRegionObserver.java
index 0092c5c..e540572 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/IndexRegionObserver.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/IndexRegionObserver.java
@@ -511,6 +511,10 @@ public class IndexRegionObserver extends BaseRegionObserver {
}
ImmutableBytesPtr rowKeyPtr = new ImmutableBytesPtr(m.getRow());
Pair<Put, Put> dataRowState = context.dataRowStates.get(rowKeyPtr);
+ if (dataRowState == null) {
+ dataRowState = new Pair<Put, Put>(null, null);
+ context.dataRowStates.put(rowKeyPtr, dataRowState);
+ }
Put nextDataRowState = dataRowState.getSecond();
if (nextDataRowState == null) {
continue;
@@ -694,6 +698,9 @@ public class IndexRegionObserver extends BaseRegionObserver {
byte[] indexRowKey = indexMaintainer.buildRowKey(nextDataRowVG, rowKeyPtr,
null, null, HConstants.LATEST_TIMESTAMP);
indexPut = new Put(indexRowKey);
+ } else {
+ removeEmptyColumn(indexPut, indexMaintainer.getEmptyKeyValueFamily().copyBytesIfNecessary(),
+ indexMaintainer.getEmptyKeyValueQualifier());
}
indexPut.addColumn(indexMaintainer.getEmptyKeyValueFamily().copyBytesIfNecessary(),
indexMaintainer.getEmptyKeyValueQualifier(), ts, UNVERIFIED_BYTES);