You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by ag...@apache.org on 2018/06/18 17:46:07 UTC
ignite git commit: IGNITE-8755 Throw a correct error message when
client optimized marshaller is trying to serialize too large object
Repository: ignite
Updated Branches:
refs/heads/master 21d780d60 -> 61b38977f
IGNITE-8755 Throw a correct error message when client optimized marshaller is trying to serialize too large object
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/61b38977
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/61b38977
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/61b38977
Branch: refs/heads/master
Commit: 61b38977f63258ca7d3699731b92e125dca8407a
Parents: 21d780d
Author: Ivan Daschinskiy <iv...@gmail.com>
Authored: Mon Jun 18 20:44:00 2018 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Mon Jun 18 20:45:50 2018 +0300
----------------------------------------------------------------------
.../internal/util/io/GridUnsafeDataOutput.java | 36 ++++++++-
.../optimized/OptimizedMarshallerTest.java | 80 ++++++++++++++++++++
2 files changed, 114 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/61b38977/modules/core/src/main/java/org/apache/ignite/internal/util/io/GridUnsafeDataOutput.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/io/GridUnsafeDataOutput.java b/modules/core/src/main/java/org/apache/ignite/internal/util/io/GridUnsafeDataOutput.java
index c45b8fd..ad94889 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/io/GridUnsafeDataOutput.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/io/GridUnsafeDataOutput.java
@@ -120,7 +120,12 @@ public class GridUnsafeDataOutput extends OutputStream implements GridDataOutput
/**
* @param size Size.
*/
- private void requestFreeSize(int size) {
+ private void requestFreeSize(int size) throws IOException {
+ // If arithmetic overflow occurs, off + size should be less than size.
+ if (off + size < size)
+ throw new IOException("Failed to allocate required memory (arithmetic overflow detected) " +
+ "[length=" + size + ", offset=" + off + ']');
+
size = off + size;
maxOff = Math.max(maxOff, size);
@@ -128,7 +133,7 @@ public class GridUnsafeDataOutput extends OutputStream implements GridDataOutput
long now = U.currentTimeMillis();
if (size > bytes.length) {
- byte[] newBytes = new byte[size << 1]; // Grow.
+ byte[] newBytes = new byte[Math.max(size << 1, size)]; // Grow.
System.arraycopy(bytes, 0, newBytes, 0, off);
@@ -185,6 +190,8 @@ public class GridUnsafeDataOutput extends OutputStream implements GridDataOutput
int bytesToCp = arr.length << 3;
+ checkArrayAllocationOverflow(bytesToCp, arr.length, "double");
+
requestFreeSize(bytesToCp);
if (BIG_ENDIAN) {
@@ -217,6 +224,8 @@ public class GridUnsafeDataOutput extends OutputStream implements GridDataOutput
int bytesToCp = arr.length << 1;
+ checkArrayAllocationOverflow(bytesToCp, arr.length, "char");
+
requestFreeSize(bytesToCp);
if (BIG_ENDIAN) {
@@ -240,6 +249,8 @@ public class GridUnsafeDataOutput extends OutputStream implements GridDataOutput
int bytesToCp = arr.length << 3;
+ checkArrayAllocationOverflow(bytesToCp, arr.length, "long");
+
requestFreeSize(bytesToCp);
if (BIG_ENDIAN) {
@@ -263,6 +274,8 @@ public class GridUnsafeDataOutput extends OutputStream implements GridDataOutput
int bytesToCp = arr.length << 2;
+ checkArrayAllocationOverflow(bytesToCp, arr.length, "float");
+
requestFreeSize(bytesToCp);
if (BIG_ENDIAN) {
@@ -304,6 +317,8 @@ public class GridUnsafeDataOutput extends OutputStream implements GridDataOutput
int bytesToCp = arr.length << 1;
+ checkArrayAllocationOverflow(bytesToCp, arr.length, "short");
+
requestFreeSize(bytesToCp);
if (BIG_ENDIAN) {
@@ -327,6 +342,8 @@ public class GridUnsafeDataOutput extends OutputStream implements GridDataOutput
int bytesToCp = arr.length << 2;
+ checkArrayAllocationOverflow(bytesToCp, arr.length, "int");
+
requestFreeSize(bytesToCp);
if (BIG_ENDIAN) {
@@ -472,6 +489,21 @@ public class GridUnsafeDataOutput extends OutputStream implements GridDataOutput
}
/**
+ * Check for possible arithmetic overflow when trying to serialize a humongous array.
+ *
+ * @param bytesToAlloc Bytes to allocate.
+ * @param arrLen Array length.
+ * @param type Type of an array.
+ * @throws IOException If oveflow presents and data corruption can occur.
+ */
+ private void checkArrayAllocationOverflow(int bytesToAlloc, int arrLen, String type) throws IOException {
+ // If arithmetic overflow occurs, bytesToAlloc should be less than arrLen.
+ if (bytesToAlloc < arrLen)
+ throw new IOException("Failed to allocate required memory for " + type + " array " +
+ "(arithmetic overflow detected) [length=" + arrLen + ']');
+ }
+
+ /**
*
* Returns the length in bytes of the UTF encoding of the given string.
*
http://git-wip-us.apache.org/repos/asf/ignite/blob/61b38977/modules/core/src/test/java/org/apache/ignite/internal/marshaller/optimized/OptimizedMarshallerTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/marshaller/optimized/OptimizedMarshallerTest.java b/modules/core/src/test/java/org/apache/ignite/internal/marshaller/optimized/OptimizedMarshallerTest.java
index 14b49a4..79496ae 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/marshaller/optimized/OptimizedMarshallerTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/marshaller/optimized/OptimizedMarshallerTest.java
@@ -28,6 +28,7 @@ import java.lang.reflect.Proxy;
import java.net.InetSocketAddress;
import java.util.Arrays;
import java.util.Collection;
+import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentMap;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCheckedException;
@@ -41,6 +42,7 @@ import org.apache.ignite.marshaller.MarshallerContextTestImpl;
import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinderAdapter;
import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.GridTestUtils;
import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
import org.jetbrains.annotations.Nullable;
@@ -387,6 +389,84 @@ public class OptimizedMarshallerTest extends GridCommonAbstractTest {
}
/**
+ * Tests checks for arithmetic overflow when trying to serialize huge object.
+ * WARNING! Requires a lot of heap space. Should not be run on CI.
+ */
+ public void _testAllocationOverflow() {
+ allocationOverflowCheck(() -> marshaller().marshal(new HugeObject()));
+
+ allocationOverflowCheck(() -> {
+ marshaller().marshal(new short[1<<30]);
+ marshaller().marshal(new short[1<<30]);
+ return null;
+ });
+
+ allocationOverflowCheck(() -> {
+ marshaller().marshal(new char[1<<30]);
+ marshaller().marshal(new char[1<<30]);
+ return null;
+ });
+
+ allocationOverflowCheck(() -> {
+ marshaller().marshal(new int[1<<30]);
+ marshaller().marshal(new int[1<<30]);
+ return null;
+ });
+
+ allocationOverflowCheck(() -> {
+ marshaller().marshal(new int[1<<30]);
+ marshaller().marshal(new int[1<<30]);
+ return null;
+ });
+
+ allocationOverflowCheck(() -> {
+ marshaller().marshal(new float[1<<29]);
+ marshaller().marshal(new float[1<<29]);
+ return null;
+ });
+
+ allocationOverflowCheck(() -> {
+ marshaller().marshal(new long[1<<29]);
+ marshaller().marshal(new long[1<<29]);
+ return null;
+ });
+
+ allocationOverflowCheck(() -> {
+ marshaller().marshal(new double[1<<29]);
+ marshaller().marshal(new double[1<<29]);
+ return null;
+ });
+ }
+
+ /**
+ * Asserts that {@link IOException} will be thrown.
+ *
+ * @param call Callable that cause allocation overflow.
+ */
+ private void allocationOverflowCheck(Callable<?> call) {
+ GridTestUtils.assertThrowsAnyCause(log, call, IOException.class, "Impossible to allocate required memory");
+ }
+
+ /**
+ *
+ */
+ public static class HugeObject implements Externalizable {
+
+ /** {@inheritDoc} */
+ @Override public void writeExternal(ObjectOutput out) throws IOException {
+ out.write(new byte[1 << 31 - 2]);
+ out.write(new byte[1 << 31 - 2]);
+ out.write(new byte[1 << 31 - 2]);
+ out.write(new byte[1 << 31 - 2]);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
+
+ }
+ }
+
+ /**
* Some non-serializable class.
*/
@SuppressWarnings( {"PublicField","TransientFieldInNonSerializableClass","FieldMayBeStatic"})