You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by am...@apache.org on 2021/10/22 11:16:28 UTC
[ignite-3] 01/01: Fix vartable size overflow.
This is an automated email from the ASF dual-hosted git repository.
amashenkov pushed a commit to branch ignite-15744
in repository https://gitbox.apache.org/repos/asf/ignite-3.git
commit d24539bf17c5e8dba528f529d09c8fd4d347502b
Author: Andrew Mashenkov <an...@gmail.com>
AuthorDate: Fri Oct 22 14:15:49 2021 +0300
Fix vartable size overflow.
---
.../ignite/internal/schema/row/VarTableFormat.java | 26 ++++-----
.../schema/RowAssemblerAdvancedSchemaTest.java | 22 ++++----
.../org/apache/ignite/internal/schema/RowTest.java | 65 +++++++++++++---------
3 files changed, 62 insertions(+), 51 deletions(-)
diff --git a/modules/schema/src/main/java/org/apache/ignite/internal/schema/row/VarTableFormat.java b/modules/schema/src/main/java/org/apache/ignite/internal/schema/row/VarTableFormat.java
index 1dabf87..e1dde85 100644
--- a/modules/schema/src/main/java/org/apache/ignite/internal/schema/row/VarTableFormat.java
+++ b/modules/schema/src/main/java/org/apache/ignite/internal/schema/row/VarTableFormat.java
@@ -95,13 +95,12 @@ abstract class VarTableFormat {
private final byte flags;
/**
- * @param vartblSizeFieldSize Size of vartalble size field (in bytes).
* @param vartblEntrySize Size of vartable entry (in bytes).
* @param flags Format specific flags.
*/
- VarTableFormat(int vartblSizeFieldSize, int vartblEntrySize, byte flags) {
+ VarTableFormat(int vartblEntrySize, byte flags) {
this.vartblEntrySize = vartblEntrySize;
- this.vartblSizeFieldSize = vartblSizeFieldSize;
+ this.vartblSizeFieldSize = Short.BYTES;
this.flags = flags;
}
@@ -149,7 +148,9 @@ abstract class VarTableFormat {
* @param vartblOff Vartable offset.
* @return Number of entries in the vartable.
*/
- abstract int readVartableSize(BinaryRow row, int vartblOff);
+ int readVartableSize(BinaryRow row, int vartblOff) {
+ return Short.toUnsignedInt(row.readShort(vartblOff));
+ }
/**
* Convert vartable inplace to the current format.
@@ -169,7 +170,7 @@ abstract class VarTableFormat {
* Creates chunk format.
*/
TinyFormat() {
- super(Byte.BYTES, Byte.BYTES, (byte)1);
+ super(Byte.BYTES, (byte)1);
}
/** {@inheritDoc} */
@@ -178,17 +179,12 @@ abstract class VarTableFormat {
}
/** {@inheritDoc} */
- @Override int readVartableSize(BinaryRow row, int vartblOff) {
- return Byte.toUnsignedInt(row.readByte(vartblOff));
- }
-
- /** {@inheritDoc} */
@Override public int compactVarTable(ExpandableByteBuf buf, int vartblOff, int entres) {
- assert entres > 0;
+ assert entres > 0 && entres < 0xFFFF;
- buf.put(vartblOff, (byte)entres);
+ buf.putShort(vartblOff, (short)entres);
- int dstOff = vartblOff + 1;
+ int dstOff = vartblOff + 2;
int srcOff = vartblOff + 2;
for (int i = 0; i < entres; i++, srcOff += Integer.BYTES, dstOff++)
@@ -208,7 +204,7 @@ abstract class VarTableFormat {
* Creates chunk format.
*/
MediumFormat() {
- super(Short.BYTES, Short.BYTES, (byte)2);
+ super(Short.BYTES, (byte)2);
}
/** {@inheritDoc} */
@@ -245,7 +241,7 @@ abstract class VarTableFormat {
* Creates chunk format.
*/
LargeFormat() {
- super(Short.BYTES, Integer.BYTES, (byte)0);
+ super(Integer.BYTES, (byte)0);
}
/** {@inheritDoc} */
diff --git a/modules/schema/src/test/java/org/apache/ignite/internal/schema/RowAssemblerAdvancedSchemaTest.java b/modules/schema/src/test/java/org/apache/ignite/internal/schema/RowAssemblerAdvancedSchemaTest.java
index 218a029..8d2fe15 100644
--- a/modules/schema/src/test/java/org/apache/ignite/internal/schema/RowAssemblerAdvancedSchemaTest.java
+++ b/modules/schema/src/test/java/org/apache/ignite/internal/schema/RowAssemblerAdvancedSchemaTest.java
@@ -125,8 +125,8 @@ public class RowAssemblerAdvancedSchemaTest {
assertRowBytesEquals(
new byte[]{
42, 0, 0, 17, -12, -36, 109, -81,
- 15, 0, 0, 0, 4, 1, 3, 33, -77, 120, 97, 115, 99, 105, 105,
- 19, 0, 0, 0, 4, 1, 2, 33, -77, -26, -120, -111, -26, -124, -101, 74, 97, 118, 97},
+ 16, 0, 0, 0, 4, 1, 0, 3, 33, -77, 120, 97, 115, 99, 105, 105,
+ 20, 0, 0, 0, 4, 1, 0, 2, 33, -77, -26, -120, -111, -26, -124, -101, 74, 97, 118, 97},
new RowAssembler(schema, 2, 2)
.appendBytes(new byte[]{33, -77, 120})
.appendString("ascii")
@@ -140,8 +140,8 @@ public class RowAssemblerAdvancedSchemaTest {
assertRowBytesEquals(
new byte[]{
42, 0, 0, 17, 59, -11, 127, 92,
- 22, 0, 0, 0, 1, 1, 5, 97, 115, 99, 105, 105, -26, -120, -111, -26, -124, -101, 74, 97, 118, 97,
- 22, 0, 0, 0, 1, 1, 10, -26, -120, -111, -26, -124, -101, 74, 97, 118, 97, 97, 115, 99, 105, 105},
+ 23, 0, 0, 0, 1, 1, 0, 5, 97, 115, 99, 105, 105, -26, -120, -111, -26, -124, -101, 74, 97, 118, 97,
+ 23, 0, 0, 0, 1, 1, 0, 10, -26, -120, -111, -26, -124, -101, 74, 97, 118, 97, 97, 115, 99, 105, 105},
new RowAssembler(schema, 2, 2)
.appendNull()
.appendString("ascii")
@@ -155,8 +155,8 @@ public class RowAssemblerAdvancedSchemaTest {
assertRowBytesEquals(
new byte[]{
42, 0, 0, 17, -111, 92, -37, -77,
- 20, 0, 0, 0, 2, 1, 3, 33, -77, 120, -26, -120, -111, -26, -124, -101, 74, 97, 118, 97,
- 14, 0, 0, 0, 2, 1, 2, 33, -77, 97, 115, 99, 105, 105},
+ 21, 0, 0, 0, 2, 1, 0, 3, 33, -77, 120, -26, -120, -111, -26, -124, -101, 74, 97, 118, 97,
+ 15, 0, 0, 0, 2, 1, 0, 2, 33, -77, 97, 115, 99, 105, 105},
new RowAssembler(schema, 2, 2)
.appendBytes(new byte[]{33, -77, 120})
.appendNull()
@@ -182,7 +182,7 @@ public class RowAssemblerAdvancedSchemaTest {
assertRowBytesEquals(
new byte[]{
42, 0, 1, 1, -118, -28, 81, 103,
- 26, 0, 0, 0, 0, 2, 3, 13, 33, -77, 120, -26, -120, -111, -26, -124, -101, 74, 97, 118, 97, 97, 115, 99, 105, 105},
+ 27, 0, 0, 0, 0, 2, 0, 3, 13, 33, -77, 120, -26, -120, -111, -26, -124, -101, 74, 97, 118, 97, 97, 115, 99, 105, 105},
new RowAssembler(schema, 3, 0)
.appendBytes(new byte[]{33, -77, 120})
.appendString("我愛Java")
@@ -213,7 +213,7 @@ public class RowAssemblerAdvancedSchemaTest {
assertRowBytesEquals(
new byte[]{
42, 0, 0, 24, -85, 82, 5, 0, 8, 0, 0, 0, 12, 11, 22, 0,
- 14, 0, 0, 0, 3, 1, 2, 77, -88, 97, 115, 99, 105, 105},
+ 15, 0, 0, 0, 3, 1, 0, 2, 77, -88, 97, 115, 99, 105, 105},
new RowAssembler(schema, 0, 2)
.appendByte((byte)11)
.appendShort((short)22)
@@ -229,8 +229,8 @@ public class RowAssemblerAdvancedSchemaTest {
assertRowBytesEquals(
new byte[]{
42, 0, 0, 17, -18, -9, 119, -80,
- 21, 0, 0, 0, 1, 1, 4, 22, 0, 33, -44, -26, -120, -111, -26, -124, -101, 74, 97, 118, 97,
- 15, 0, 0, 0, 2, 1, 3, 55, 77, -88, 97, 115, 99, 105, 105},
+ 22, 0, 0, 0, 1, 1, 0, 4, 22, 0, 33, -44, -26, -120, -111, -26, -124, -101, 74, 97, 118, 97,
+ 16, 0, 0, 0, 2, 1, 0, 3, 55, 77, -88, 97, 115, 99, 105, 105},
new RowAssembler(schema, 2, 2)
.appendNull()
.appendShort((short)22)
@@ -280,7 +280,7 @@ public class RowAssemblerAdvancedSchemaTest {
assertRowBytesEquals(
new byte[]{
42, 0, 0, -127, 3, -8, 124, -80,
- 22, 0, 0, 0, 0, 1, 5, 11, 22, 0, 33, -44, -26, -120, -111, -26, -124, -101, 74, 97, 118, 97,
+ 23, 0, 0, 0, 0, 1, 0, 5, 11, 22, 0, 33, -44, -26, -120, -111, -26, -124, -101, 74, 97, 118, 97,
5, 0, 0, 0, 15},
new RowAssembler(schema, 2, 0)
.appendByte((byte)11)
diff --git a/modules/schema/src/test/java/org/apache/ignite/internal/schema/RowTest.java b/modules/schema/src/test/java/org/apache/ignite/internal/schema/RowTest.java
index eea1ec0..006abe2 100644
--- a/modules/schema/src/test/java/org/apache/ignite/internal/schema/RowTest.java
+++ b/modules/schema/src/test/java/org/apache/ignite/internal/schema/RowTest.java
@@ -74,7 +74,7 @@ public class RowTest {
}
/**
- * Check row serialization for schema with nullable fix-sized columns only.
+ * Check row serialization for a schema with nullable fix-sized columns only.
*/
@Test
public void nullableFixSizedColumns() {
@@ -114,7 +114,7 @@ public class RowTest {
}
/**
- * Check row serialization for schema with non-nullable fix-sized columns only.
+ * Check row serialization for a schema with non-nullable fix-sized columns only.
*/
@Test
public void fixSizedColumns() {
@@ -154,7 +154,7 @@ public class RowTest {
}
/**
- * Check row serialization for schema with various columns.
+ * Check row serialization for a schema with various columns.
*/
@Test
public void mixedColumns() {
@@ -186,7 +186,7 @@ public class RowTest {
}
/**
- * Check row serialization for schema with various columns.
+ * Check row serialization for a schema with various columns.
*/
@Test
public void temporalColumns() {
@@ -212,7 +212,7 @@ public class RowTest {
}
/**
- * Check row serialization for schema with non-nullable varlen columns only.
+ * Check row serialization for a schema with non-nullable varlen columns only.
*/
@Test
public void varlenColumns() {
@@ -234,7 +234,7 @@ public class RowTest {
}
/**
- * Check row serialization for schema with nullable varlen columns only.
+ * Check row serialization for a schema with nullable varlen columns only.
*/
@Test
public void nullableVarlenColumns() {
@@ -254,16 +254,16 @@ public class RowTest {
}
/**
- * Check row serialization for schema with large varlen columns (64Kb+).
+ * Check row serialization for a schema with large varlen columns (64Kb+).
*/
@Test
public void largeVarlenColumns() {
- Column[] keyCols = new Column[] {
+ Column[] keyCols = new Column[]{
new Column("keyBytesCol", BYTES, false),
new Column("keyStringCol", STRING, false),
};
- Column[] valCols = new Column[] {
+ Column[] valCols = new Column[]{
new Column("valBytesCol", BYTES, true),
new Column("valStringCol", STRING, true),
};
@@ -290,6 +290,29 @@ public class RowTest {
}
/**
+ * Check row serialization for a schema with many empty varlen columns (255+).
+ */
+ @Test
+ public void bigVartable() {
+ Column[] keyCols = new Column[]{
+ new Column("id", INT64, false),
+ };
+
+ int columnCount = 260;
+
+ Column[] valCols = IntStream.range(1, columnCount)
+ .mapToObj(i -> new Column("val" + i, STRING, true))
+ .toArray(Column[]::new);
+
+ SchemaDescriptor sch = new SchemaDescriptor(1, keyCols, valCols);
+
+ Object[] checkArr = IntStream.range(0, columnCount)
+ .mapToObj(i -> i == 0 ? 42L : (i > columnCount - 5) ? "str" : "").toArray();
+
+ checkValues(sch, checkArr);
+ }
+
+ /**
* Check row serialization for 256+ fixsized columns.
*/
@Test
@@ -469,43 +492,35 @@ public class RowTest {
if (schema.isKeyColumn(i)) {
nonNullVarLenKeyCols++;
nonNullVarLenKeySize += val.length;
- }
- else {
+ } else {
nonNullVarLenValCols++;
nonNullVarLenValSize += val.length;
}
- }
- else if (type == NativeTypeSpec.STRING) {
+ } else if (type == NativeTypeSpec.STRING) {
if (schema.isKeyColumn(i)) {
nonNullVarLenKeyCols++;
nonNullVarLenKeySize += RowAssembler.utf8EncodedLength((CharSequence)vals[i]);
- }
- else {
+ } else {
nonNullVarLenValCols++;
nonNullVarLenValSize += RowAssembler.utf8EncodedLength((CharSequence)vals[i]);
}
- }
- else if (type == NativeTypeSpec.NUMBER) {
+ } else if (type == NativeTypeSpec.NUMBER) {
if (schema.isKeyColumn(i)) {
nonNullVarLenKeyCols++;
nonNullVarLenKeySize += RowAssembler.sizeInBytes((BigInteger)vals[i]);
- }
- else {
+ } else {
nonNullVarLenValCols++;
nonNullVarLenValSize += RowAssembler.sizeInBytes((BigInteger)vals[i]);
}
- }
- else if (type == NativeTypeSpec.DECIMAL) {
+ } else if (type == NativeTypeSpec.DECIMAL) {
if (schema.isKeyColumn(i)) {
nonNullVarLenKeyCols++;
nonNullVarLenKeySize += RowAssembler.sizeInBytes((BigDecimal)vals[i]);
- }
- else {
+ } else {
nonNullVarLenValCols++;
nonNullVarLenValSize += RowAssembler.sizeInBytes((BigDecimal)vals[i]);
}
- }
- else
+ } else
throw new IllegalStateException("Unsupported variable-length type: " + type);
}
}