You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@avro.apache.org by rs...@apache.org on 2023/06/19 15:39:57 UTC
[avro] 01/03: AVRO-3783: Read LONG length for bytes, only allow INT sizes (#2210)
This is an automated email from the ASF dual-hosted git repository.
rskraba pushed a commit to branch branch-1.11
in repository https://gitbox.apache.org/repos/asf/avro.git
commit 21f6fabee9c6ff095ec75db82b90c00d48d9d0b1
Author: Jack Klamer <ja...@starburstdata.com>
AuthorDate: Fri Jun 16 14:12:32 2023 -0500
AVRO-3783: Read LONG length for bytes, only allow INT sizes (#2210)
* Read long length for bytes. Only allow Int sizes
* Add test for VLE int long equality
---
.../java/org/apache/avro/io/BinaryDecoder.java | 11 ++++----
.../org/apache/avro/io/DirectBinaryDecoder.java | 19 +++++++-------
.../java/org/apache/avro/io/TestBinaryData.java | 30 +++++++++++++++++++---
3 files changed, 42 insertions(+), 18 deletions(-)
diff --git a/lang/java/avro/src/main/java/org/apache/avro/io/BinaryDecoder.java b/lang/java/avro/src/main/java/org/apache/avro/io/BinaryDecoder.java
index 051563aba..12c6e080d 100644
--- a/lang/java/avro/src/main/java/org/apache/avro/io/BinaryDecoder.java
+++ b/lang/java/avro/src/main/java/org/apache/avro/io/BinaryDecoder.java
@@ -329,9 +329,10 @@ public class BinaryDecoder extends Decoder {
@Override
public ByteBuffer readBytes(ByteBuffer old) throws IOException {
- int length = readInt();
+ long length = readLong();
if (length > MAX_ARRAY_SIZE) {
- throw new UnsupportedOperationException("Cannot read arrays longer than " + MAX_ARRAY_SIZE + " bytes");
+ throw new UnsupportedOperationException(
+ "Cannot read arrays longer than " + MAX_ARRAY_SIZE + " bytes in Java library");
}
if (length > maxBytesLength) {
throw new AvroRuntimeException("Bytes length " + length + " exceeds maximum allowed");
@@ -344,10 +345,10 @@ public class BinaryDecoder extends Decoder {
result = old;
((Buffer) result).clear();
} else {
- result = ByteBuffer.allocate(length);
+ result = ByteBuffer.allocate((int) length);
}
- doReadBytes(result.array(), result.position(), length);
- ((Buffer) result).limit(length);
+ doReadBytes(result.array(), result.position(), (int) length);
+ ((Buffer) result).limit((int) length);
return result;
}
diff --git a/lang/java/avro/src/main/java/org/apache/avro/io/DirectBinaryDecoder.java b/lang/java/avro/src/main/java/org/apache/avro/io/DirectBinaryDecoder.java
index d9bbe9353..1e01f6f96 100644
--- a/lang/java/avro/src/main/java/org/apache/avro/io/DirectBinaryDecoder.java
+++ b/lang/java/avro/src/main/java/org/apache/avro/io/DirectBinaryDecoder.java
@@ -39,26 +39,27 @@ class DirectBinaryDecoder extends BinaryDecoder {
private InputStream in;
private class ByteReader {
- public ByteBuffer read(ByteBuffer old, int length) throws IOException {
+ public ByteBuffer read(ByteBuffer old, long length) throws IOException {
this.checkLength(length);
final ByteBuffer result;
if (old != null && length <= old.capacity()) {
result = old;
result.clear();
} else {
- result = ByteBuffer.allocate(length);
+ result = ByteBuffer.allocate((int) length);
}
- doReadBytes(result.array(), result.position(), length);
- result.limit(length);
+ doReadBytes(result.array(), result.position(), (int) length);
+ result.limit((int) length);
return result;
}
- protected final void checkLength(int length) {
+ protected final void checkLength(long length) {
if (length < 0L) {
throw new AvroRuntimeException("Malformed data. Length is negative: " + length);
}
if (length > MAX_ARRAY_SIZE) {
- throw new UnsupportedOperationException("Cannot read arrays longer than " + MAX_ARRAY_SIZE + " bytes");
+ throw new UnsupportedOperationException(
+ "Cannot read arrays longer than " + MAX_ARRAY_SIZE + " bytes in Java library");
}
if (length > maxBytesLength) {
throw new AvroRuntimeException("Bytes length " + length + " exceeds maximum allowed");
@@ -74,12 +75,12 @@ class DirectBinaryDecoder extends BinaryDecoder {
}
@Override
- public ByteBuffer read(ByteBuffer old, int length) throws IOException {
+ public ByteBuffer read(ByteBuffer old, long length) throws IOException {
this.checkLength(length);
if (old != null) {
return super.read(old, length);
} else {
- return bbi.readBuffer(length);
+ return bbi.readBuffer((int) length);
}
}
@@ -170,7 +171,7 @@ class DirectBinaryDecoder extends BinaryDecoder {
@Override
public ByteBuffer readBytes(ByteBuffer old) throws IOException {
- int length = readInt();
+ long length = readLong();
return byteReader.read(old, length);
}
diff --git a/lang/java/avro/src/test/java/org/apache/avro/io/TestBinaryData.java b/lang/java/avro/src/test/java/org/apache/avro/io/TestBinaryData.java
index edbcd0bcb..167cd7246 100644
--- a/lang/java/avro/src/test/java/org/apache/avro/io/TestBinaryData.java
+++ b/lang/java/avro/src/test/java/org/apache/avro/io/TestBinaryData.java
@@ -18,8 +18,10 @@
package org.apache.avro.io;
-import org.junit.Assert;
-import org.junit.Test;
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import org.junit.jupiter.api.Test;
public class TestBinaryData {
@@ -29,13 +31,33 @@ public class TestBinaryData {
* normal 8). When skipping it, the next byte should be 10.
*/
@Test
- public void testSkipLong() {
+ void skipLong() {
byte[] b = new byte[10];
BinaryData.encodeLong(Long.MAX_VALUE, b, 0);
final int nextIndex = BinaryData.skipLong(b, 0);
- Assert.assertEquals(nextIndex, 10);
+ assertEquals(nextIndex, 10);
}
+ @Test
+ void testIntLongVleEquality() {
+ byte[] intResult = new byte[9];
+ byte[] longResult = new byte[9];
+ BinaryData.encodeInt(0, intResult, 0);
+ BinaryData.encodeLong(0, longResult, 0);
+ assertArrayEquals(intResult, longResult);
+ BinaryData.encodeInt(42, intResult, 0);
+ BinaryData.encodeLong(42, longResult, 0);
+ assertArrayEquals(intResult, longResult);
+ BinaryData.encodeInt(-24, intResult, 0);
+ BinaryData.encodeLong(-24, longResult, 0);
+ assertArrayEquals(intResult, longResult);
+ BinaryData.encodeInt(Integer.MAX_VALUE, intResult, 0);
+ BinaryData.encodeLong(Integer.MAX_VALUE, longResult, 0);
+ assertArrayEquals(intResult, longResult);
+ BinaryData.encodeInt(Integer.MIN_VALUE, intResult, 0);
+ BinaryData.encodeLong(Integer.MIN_VALUE, longResult, 0);
+ assertArrayEquals(intResult, longResult);
+ }
}