You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by ag...@apache.org on 2018/06/18 17:46:07 UTC

ignite git commit: IGNITE-8755 Throw a correct error message when client optimized marshaller is trying to serialize too large object

Repository: ignite
Updated Branches:
  refs/heads/master 21d780d60 -> 61b38977f


IGNITE-8755 Throw a correct error message when client optimized marshaller is trying to serialize too large object


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/61b38977
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/61b38977
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/61b38977

Branch: refs/heads/master
Commit: 61b38977f63258ca7d3699731b92e125dca8407a
Parents: 21d780d
Author: Ivan Daschinskiy <iv...@gmail.com>
Authored: Mon Jun 18 20:44:00 2018 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Mon Jun 18 20:45:50 2018 +0300

----------------------------------------------------------------------
 .../internal/util/io/GridUnsafeDataOutput.java  | 36 ++++++++-
 .../optimized/OptimizedMarshallerTest.java      | 80 ++++++++++++++++++++
 2 files changed, 114 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/61b38977/modules/core/src/main/java/org/apache/ignite/internal/util/io/GridUnsafeDataOutput.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/io/GridUnsafeDataOutput.java b/modules/core/src/main/java/org/apache/ignite/internal/util/io/GridUnsafeDataOutput.java
index c45b8fd..ad94889 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/io/GridUnsafeDataOutput.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/io/GridUnsafeDataOutput.java
@@ -120,7 +120,12 @@ public class GridUnsafeDataOutput extends OutputStream implements GridDataOutput
     /**
      * @param size Size.
      */
-    private void requestFreeSize(int size) {
+    private void requestFreeSize(int size) throws IOException {
+        // If arithmetic overflow occurs, off + size should be less than size.
+        if (off + size < size)
+            throw new IOException("Failed to allocate required memory (arithmetic overflow detected) " +
+                "[length=" + size + ", offset=" + off + ']');
+
         size = off + size;
 
         maxOff = Math.max(maxOff, size);
@@ -128,7 +133,7 @@ public class GridUnsafeDataOutput extends OutputStream implements GridDataOutput
         long now = U.currentTimeMillis();
 
         if (size > bytes.length) {
-            byte[] newBytes = new byte[size << 1]; // Grow.
+            byte[] newBytes = new byte[Math.max(size << 1, size)]; // Grow.
 
             System.arraycopy(bytes, 0, newBytes, 0, off);
 
@@ -185,6 +190,8 @@ public class GridUnsafeDataOutput extends OutputStream implements GridDataOutput
 
         int bytesToCp = arr.length << 3;
 
+        checkArrayAllocationOverflow(bytesToCp, arr.length, "double");
+
         requestFreeSize(bytesToCp);
 
         if (BIG_ENDIAN) {
@@ -217,6 +224,8 @@ public class GridUnsafeDataOutput extends OutputStream implements GridDataOutput
 
         int bytesToCp = arr.length << 1;
 
+        checkArrayAllocationOverflow(bytesToCp, arr.length, "char");
+
         requestFreeSize(bytesToCp);
 
         if (BIG_ENDIAN) {
@@ -240,6 +249,8 @@ public class GridUnsafeDataOutput extends OutputStream implements GridDataOutput
 
         int bytesToCp = arr.length << 3;
 
+        checkArrayAllocationOverflow(bytesToCp, arr.length, "long");
+
         requestFreeSize(bytesToCp);
 
         if (BIG_ENDIAN) {
@@ -263,6 +274,8 @@ public class GridUnsafeDataOutput extends OutputStream implements GridDataOutput
 
         int bytesToCp = arr.length << 2;
 
+        checkArrayAllocationOverflow(bytesToCp, arr.length, "float");
+
         requestFreeSize(bytesToCp);
 
         if (BIG_ENDIAN) {
@@ -304,6 +317,8 @@ public class GridUnsafeDataOutput extends OutputStream implements GridDataOutput
 
         int bytesToCp = arr.length << 1;
 
+        checkArrayAllocationOverflow(bytesToCp, arr.length, "short");
+
         requestFreeSize(bytesToCp);
 
         if (BIG_ENDIAN) {
@@ -327,6 +342,8 @@ public class GridUnsafeDataOutput extends OutputStream implements GridDataOutput
 
         int bytesToCp = arr.length << 2;
 
+        checkArrayAllocationOverflow(bytesToCp, arr.length, "int");
+
         requestFreeSize(bytesToCp);
 
         if (BIG_ENDIAN) {
@@ -472,6 +489,21 @@ public class GridUnsafeDataOutput extends OutputStream implements GridDataOutput
     }
 
     /**
+     * Check for possible arithmetic overflow when trying to serialize a humongous array.
+     *
+     * @param bytesToAlloc Bytes to allocate.
+     * @param arrLen Array length.
+     * @param type Type of an array.
+     * @throws IOException If oveflow presents and data corruption can occur.
+     */
+    private void checkArrayAllocationOverflow(int bytesToAlloc, int arrLen, String type) throws IOException {
+        // If arithmetic overflow occurs, bytesToAlloc should be less than arrLen.
+        if (bytesToAlloc < arrLen)
+            throw new IOException("Failed to allocate required memory for " + type + " array " +
+                "(arithmetic overflow detected) [length=" + arrLen + ']');
+    }
+
+    /**
      *
      * Returns the length in bytes of the UTF encoding of the given string.
      *

http://git-wip-us.apache.org/repos/asf/ignite/blob/61b38977/modules/core/src/test/java/org/apache/ignite/internal/marshaller/optimized/OptimizedMarshallerTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/marshaller/optimized/OptimizedMarshallerTest.java b/modules/core/src/test/java/org/apache/ignite/internal/marshaller/optimized/OptimizedMarshallerTest.java
index 14b49a4..79496ae 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/marshaller/optimized/OptimizedMarshallerTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/marshaller/optimized/OptimizedMarshallerTest.java
@@ -28,6 +28,7 @@ import java.lang.reflect.Proxy;
 import java.net.InetSocketAddress;
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.concurrent.Callable;
 import java.util.concurrent.ConcurrentMap;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCheckedException;
@@ -41,6 +42,7 @@ import org.apache.ignite.marshaller.MarshallerContextTestImpl;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinderAdapter;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 import org.jetbrains.annotations.Nullable;
 
@@ -387,6 +389,84 @@ public class OptimizedMarshallerTest extends GridCommonAbstractTest {
     }
 
     /**
+     * Tests checks for arithmetic overflow when trying to serialize huge object.
+     * WARNING! Requires a lot of heap space. Should not be run on CI.
+     */
+    public void _testAllocationOverflow() {
+        allocationOverflowCheck(() -> marshaller().marshal(new HugeObject()));
+
+        allocationOverflowCheck(() -> {
+            marshaller().marshal(new short[1<<30]);
+            marshaller().marshal(new short[1<<30]);
+            return null;
+        });
+
+        allocationOverflowCheck(() -> {
+            marshaller().marshal(new char[1<<30]);
+            marshaller().marshal(new char[1<<30]);
+            return null;
+        });
+
+        allocationOverflowCheck(() -> {
+            marshaller().marshal(new int[1<<30]);
+            marshaller().marshal(new int[1<<30]);
+            return null;
+        });
+
+        allocationOverflowCheck(() -> {
+            marshaller().marshal(new int[1<<30]);
+            marshaller().marshal(new int[1<<30]);
+            return null;
+        });
+
+        allocationOverflowCheck(() -> {
+            marshaller().marshal(new float[1<<29]);
+            marshaller().marshal(new float[1<<29]);
+            return null;
+        });
+
+        allocationOverflowCheck(() -> {
+            marshaller().marshal(new long[1<<29]);
+            marshaller().marshal(new long[1<<29]);
+            return null;
+        });
+
+        allocationOverflowCheck(() -> {
+            marshaller().marshal(new double[1<<29]);
+            marshaller().marshal(new double[1<<29]);
+            return null;
+        });
+    }
+
+    /**
+     * Asserts that {@link IOException} will be thrown.
+     *
+     * @param call Callable that cause allocation overflow.
+     */
+    private void allocationOverflowCheck(Callable<?> call) {
+        GridTestUtils.assertThrowsAnyCause(log, call, IOException.class, "Impossible to allocate required memory");
+    }
+
+    /**
+     *
+     */
+    public static class HugeObject implements Externalizable {
+
+        /** {@inheritDoc} */
+        @Override public void writeExternal(ObjectOutput out) throws IOException {
+            out.write(new byte[1 << 31 - 2]);
+            out.write(new byte[1 << 31 - 2]);
+            out.write(new byte[1 << 31 - 2]);
+            out.write(new byte[1 << 31 - 2]);
+        }
+
+        /** {@inheritDoc} */
+        @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
+
+        }
+    }
+
+    /**
      * Some non-serializable class.
      */
     @SuppressWarnings( {"PublicField","TransientFieldInNonSerializableClass","FieldMayBeStatic"})