You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by go...@apache.org on 2021/08/17 22:28:04 UTC
[phoenix] branch master updated: PHOENIX-6528 Fix view index read
repair for the pks with variable length
This is an automated email from the ASF dual-hosted git repository.
gokcen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/phoenix.git
The following commit(s) were added to refs/heads/master by this push:
new bfc4621 PHOENIX-6528 Fix view index read repair for the pks with variable length
bfc4621 is described below
commit bfc4621b2c3beb9e771553abcb7724dff4a66546
Author: Gokcen Iskender <go...@gmail.com>
AuthorDate: Tue Aug 10 20:17:43 2021 -0700
PHOENIX-6528 Fix view index read repair for the pks with variable length
Signed-off-by: Gokcen Iskender <go...@gmail.com>
---
.../org/apache/phoenix/end2end/index/ViewIndexIT.java | 7 ++++++-
.../java/org/apache/phoenix/index/IndexMaintainer.java | 15 ++++++++-------
2 files changed, 14 insertions(+), 8 deletions(-)
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ViewIndexIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ViewIndexIT.java
index f1259e0..3e81214 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ViewIndexIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ViewIndexIT.java
@@ -412,6 +412,7 @@ public class ViewIndexIT extends SplitSystemCatalogIT {
Long int1= 1792L;
String text2 ="text2";
BigDecimal double1 = BigDecimal.valueOf(254.564);
+ IndexRegionObserver.setFailPostIndexUpdatesForTesting(true);
try (Connection conn = DriverManager.getConnection(getUrl())) {
// View fixed, index variable
createTableForRowKeyTestsAndVerify(conn, "DATE_TIME1, INT1", "TEXT1", "INT1", int1);
@@ -440,18 +441,22 @@ public class ViewIndexIT extends SplitSystemCatalogIT {
createTableForRowKeyTestsAndVerify(conn, "DATE_TIME1, INT1", "TEXT1 DESC", "INT1", int1);
createTableForRowKeyTestsAndVerify(conn, "DATE_TIME1, INT1 DESC", "TEXT1 DESC", "INT1", int1);
createTableForRowKeyTestsAndVerify(conn, "DATE_TIME1, INT1 DESC", "TEXT1", "INT1", int1);
+
createTableForRowKeyTestsAndVerify(conn,"DATE_TIME1, TEXT3 DESC, INT1 DESC, TEXT2, TEXT4 DESC", "TEXT1, DOUBLE1 DESC", "TEXT2", text2);
+
createTableForRowKeyTestsAndVerify(conn,"DATE_TIME1, TEXT3 DESC, INT1 DESC, TEXT2 DESC, TEXT4 DESC", "TEXT1 DESC, DOUBLE1 DESC", "TEXT2",text2);
createTableForRowKeyTestsAndVerify(conn, "DATE_TIME1 DESC, TEXT3 DESC, INT1, DOUBLE1 DESC", "TEXT1", "DOUBLE1", double1);
// Both index and data end with var length
createTableForRowKeyTestsAndVerify(conn,"DATE_TIME1 DESC, TEXT3 DESC, INT1, TEXT4, EMAIL1", "TEXT1", "INT1", int1);
createTableForRowKeyTestsAndVerify(conn,"DATE_TIME1 DESC, TEXT3, INT1, EMAIL1", "TEXT1, TEXT4", "INT1", int1);
+ } finally {
+ IndexRegionObserver.setFailPostIndexUpdatesForTesting(false);
}
}
private void createTableForRowKeyTestsAndVerify(Connection conn, String viewPkColumns, String indexPKColumns, String lastViewPKCol, Object lastColExpectedVal)
- throws SQLException {
+ throws Exception {
final String fullTableName = "TBL_"+generateUniqueName();
final String fullViewName = "VW_" + generateUniqueName();
final String fullIndexName = "IDX_" + generateUniqueName();
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/index/IndexMaintainer.java b/phoenix-core/src/main/java/org/apache/phoenix/index/IndexMaintainer.java
index 6d92360..1fc5b3b 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/index/IndexMaintainer.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/index/IndexMaintainer.java
@@ -807,6 +807,7 @@ public class IndexMaintainer implements Writable, Iterable<ColumnReference> {
indexPosOffset = (!isLocalIndex && nIndexSaltBuckets > 0 ? 1 : 0) + (isMultiTenant ? 1 : 0) + (viewIndexId == null ? 0 : 1);
BitSet viewConstantColumnBitSet = this.rowKeyMetaData.getViewConstantColumnBitSet();
BitSet descIndexColumnBitSet = rowKeyMetaData.getDescIndexColumnBitSet();
+ int trailingVariableWidthColumnNum = 0;
for (int i = dataPosOffset; i < dataRowKeySchema.getFieldCount(); i++) {
// Write view constants from the data table, as these
// won't appear in the index (as they're the
@@ -841,21 +842,21 @@ public class IndexMaintainer implements Writable, Iterable<ColumnReference> {
}
}
}
- // Write separator byte if variable length unless it's the last field in the schema
- // (but we still need to write it if it's DESC to ensure sort order is correct).
+ // Write separator byte if variable length
byte sepByte = SchemaUtil.getSeparatorByte(rowKeyOrderOptimizable, ptr.getLength() == 0, dataRowKeySchema.getField(i));
- if (!dataRowKeySchema.getField(i).getDataType().isFixedWidth() && (((i+1) != dataRowKeySchema.getFieldCount()) || sepByte == QueryConstants.DESC_SEPARATOR_BYTE)) {
+ if (!dataRowKeySchema.getField(i).getDataType().isFixedWidth()){
output.writeByte(sepByte);
+ trailingVariableWidthColumnNum++;
+ } else {
+ trailingVariableWidthColumnNum = 0;
}
}
int length = stream.size();
- int minLength = length - maxTrailingNulls;
byte[] dataRowKey = stream.getBuffer();
// Remove trailing nulls
- int index = dataRowKeySchema.getFieldCount() - 1;
- while (index >= 0 && !dataRowKeySchema.getField(index).getDataType().isFixedWidth() && length > minLength && dataRowKey[length-1] == QueryConstants.SEPARATOR_BYTE) {
+ while (trailingVariableWidthColumnNum > 0 && dataRowKey[length-1] == QueryConstants.SEPARATOR_BYTE) {
length--;
- index--;
+ trailingVariableWidthColumnNum--;
}
// TODO: need to capture nDataSaltBuckets instead of just a boolean. For now,
// we store this in nIndexSaltBuckets, as we only use this function for local indexes