You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by al...@apache.org on 2020/05/08 15:02:30 UTC
[ignite] branch master updated: IGNITE-12827 Fix skipped detach
flag when unmarshalling collections and object arrays - Fixes #7679.
This is an automated email from the ASF dual-hosted git repository.
alexpl pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite.git
The following commit(s) were added to refs/heads/master by this push:
new a77ebdd IGNITE-12827 Fix skipped detach flag when unmarshalling collections and object arrays - Fixes #7679.
a77ebdd is described below
commit a77ebdd101a4d191ea60becc2af12e88e214af71
Author: Ivan Daschinskiy <iv...@gmail.com>
AuthorDate: Fri May 8 17:57:45 2020 +0300
IGNITE-12827 Fix skipped detach flag when unmarshalling collections and object arrays - Fixes #7679.
Signed-off-by: Aleksey Plekhanov <pl...@gmail.com>
---
.../ignite/internal/binary/BinaryReaderExImpl.java | 12 +--
.../apache/ignite/internal/binary/BinaryUtils.java | 27 +++---
.../internal/binary/BinaryMarshallerSelfTest.java | 96 ++++++++++++++++++++++
3 files changed, 116 insertions(+), 19 deletions(-)
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryReaderExImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryReaderExImpl.java
index f81ee0b..dc84328 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryReaderExImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryReaderExImpl.java
@@ -1359,7 +1359,7 @@ public class BinaryReaderExImpl implements BinaryReader, BinaryRawReaderEx, Bina
@Nullable @Override public Object[] readObjectArray() throws BinaryObjectException {
switch (checkFlag(OBJ_ARR)) {
case NORMAL:
- return BinaryUtils.doReadObjectArray(in, ctx, ldr, this, true);
+ return BinaryUtils.doReadObjectArray(in, ctx, ldr, this, false, true);
case HANDLE:
return readHandleField();
@@ -1532,7 +1532,7 @@ public class BinaryReaderExImpl implements BinaryReader, BinaryRawReaderEx, Bina
throws BinaryObjectException {
switch (checkFlag(COL)) {
case NORMAL:
- return (Collection)BinaryUtils.doReadCollection(in, ctx, ldr, this, true, factory);
+ return (Collection)BinaryUtils.doReadCollection(in, ctx, ldr, this, false, true, factory);
case HANDLE: {
int handlePos = BinaryUtils.positionForHandle(in) - in.readInt();
@@ -1610,7 +1610,7 @@ public class BinaryReaderExImpl implements BinaryReader, BinaryRawReaderEx, Bina
private Map readMap0(@Nullable BinaryMapFactory factory) throws BinaryObjectException {
switch (checkFlag(MAP)) {
case NORMAL:
- return (Map)BinaryUtils.doReadMap(in, ctx, ldr, this, true, factory);
+ return (Map)BinaryUtils.doReadMap(in, ctx, ldr, this, false, true, factory);
case HANDLE: {
int handlePos = BinaryUtils.positionForHandle(in) - in.readInt();
@@ -1908,17 +1908,17 @@ public class BinaryReaderExImpl implements BinaryReader, BinaryRawReaderEx, Bina
break;
case OBJ_ARR:
- obj = BinaryUtils.doReadObjectArray(in, ctx, ldr, this, true);
+ obj = BinaryUtils.doReadObjectArray(in, ctx, ldr, this, false, true);
break;
case COL:
- obj = BinaryUtils.doReadCollection(in, ctx, ldr, this, true, null);
+ obj = BinaryUtils.doReadCollection(in, ctx, ldr, this, false, true, null);
break;
case MAP:
- obj = BinaryUtils.doReadMap(in, ctx, ldr, this, true, null);
+ obj = BinaryUtils.doReadMap(in, ctx, ldr, this, false, true, null);
break;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java
index 2e33bf5..8325615 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java
@@ -1840,7 +1840,7 @@ public class BinaryUtils {
in.position(handlePos);
- obj = unmarshal(in, ctx, ldr, handles);
+ obj = unmarshal(in, ctx, ldr, handles, detach);
in.position(retPos);
}
@@ -1961,13 +1961,13 @@ public class BinaryUtils {
return doReadTimeArray(in);
case GridBinaryMarshaller.OBJ_ARR:
- return doReadObjectArray(in, ctx, ldr, handles, false);
+ return doReadObjectArray(in, ctx, ldr, handles, detach, false);
case GridBinaryMarshaller.COL:
- return doReadCollection(in, ctx, ldr, handles, false, null);
+ return doReadCollection(in, ctx, ldr, handles, detach, false, null);
case GridBinaryMarshaller.MAP:
- return doReadMap(in, ctx, ldr, handles, false, null);
+ return doReadMap(in, ctx, ldr, handles, detach, false, null);
case GridBinaryMarshaller.BINARY_OBJ:
return doReadBinaryObject(in, ctx, detach);
@@ -2000,12 +2000,13 @@ public class BinaryUtils {
* @param ctx Binary context.
* @param ldr Class loader.
* @param handles Holder for handles.
+ * @param detach Detach flag.
* @param deserialize Deep flag.
* @return Value.
* @throws BinaryObjectException In case of error.
*/
public static Object[] doReadObjectArray(BinaryInputStream in, BinaryContext ctx, ClassLoader ldr,
- BinaryReaderHandlesHolder handles, boolean deserialize) throws BinaryObjectException {
+ BinaryReaderHandlesHolder handles, boolean detach, boolean deserialize) throws BinaryObjectException {
int hPos = positionForHandle(in);
Class compType = doReadClass(in, ctx, ldr, deserialize);
@@ -2017,7 +2018,7 @@ public class BinaryUtils {
handles.setHandle(arr, hPos);
for (int i = 0; i < len; i++)
- arr[i] = deserializeOrUnmarshal(in, ctx, ldr, handles, deserialize);
+ arr[i] = deserializeOrUnmarshal(in, ctx, ldr, handles, detach, deserialize);
return arr;
}
@@ -2030,7 +2031,7 @@ public class BinaryUtils {
*/
@SuppressWarnings("unchecked")
public static Collection<?> doReadCollection(BinaryInputStream in, BinaryContext ctx, ClassLoader ldr,
- BinaryReaderHandlesHolder handles, boolean deserialize, BinaryCollectionFactory factory)
+ BinaryReaderHandlesHolder handles, boolean detach, boolean deserialize, BinaryCollectionFactory factory)
throws BinaryObjectException {
int hPos = positionForHandle(in);
@@ -2089,7 +2090,7 @@ public class BinaryUtils {
handles.setHandle(col, hPos);
for (int i = 0; i < size; i++)
- col.add(deserializeOrUnmarshal(in, ctx, ldr, handles, deserialize));
+ col.add(deserializeOrUnmarshal(in, ctx, ldr, handles, detach, deserialize));
return colType == GridBinaryMarshaller.SINGLETON_LIST ? U.convertToSingletonList(col) : col;
}
@@ -2102,7 +2103,7 @@ public class BinaryUtils {
*/
@SuppressWarnings("unchecked")
public static Map<?, ?> doReadMap(BinaryInputStream in, BinaryContext ctx, ClassLoader ldr,
- BinaryReaderHandlesHolder handles, boolean deserialize, BinaryMapFactory factory)
+ BinaryReaderHandlesHolder handles, boolean detach, boolean deserialize, BinaryMapFactory factory)
throws BinaryObjectException {
int hPos = positionForHandle(in);
@@ -2141,8 +2142,8 @@ public class BinaryUtils {
handles.setHandle(map, hPos);
for (int i = 0; i < size; i++) {
- Object key = deserializeOrUnmarshal(in, ctx, ldr, handles, deserialize);
- Object val = deserializeOrUnmarshal(in, ctx, ldr, handles, deserialize);
+ Object key = deserializeOrUnmarshal(in, ctx, ldr, handles, detach, deserialize);
+ Object val = deserializeOrUnmarshal(in, ctx, ldr, handles, detach, deserialize);
map.put(key, val);
}
@@ -2157,8 +2158,8 @@ public class BinaryUtils {
* @return Result.
*/
private static Object deserializeOrUnmarshal(BinaryInputStream in, BinaryContext ctx, ClassLoader ldr,
- BinaryReaderHandlesHolder handles, boolean deserialize) {
- return deserialize ? doReadObject(in, ctx, ldr, handles) : unmarshal(in, ctx, ldr, handles);
+ BinaryReaderHandlesHolder handles, boolean detach, boolean deserialize) {
+ return deserialize ? doReadObject(in, ctx, ldr, handles) : unmarshal(in, ctx, ldr, handles, detach);
}
/**
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerSelfTest.java
index 80736ea..d5464d8 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerSelfTest.java
@@ -53,6 +53,8 @@ import java.util.TreeSet;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentSkipListSet;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteException;
import org.apache.ignite.IgniteSystemProperties;
@@ -77,13 +79,17 @@ import org.apache.ignite.binary.Binarylizable;
import org.apache.ignite.configuration.BinaryConfiguration;
import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.internal.binary.builder.BinaryObjectBuilderImpl;
+import org.apache.ignite.internal.binary.streams.BinaryHeapInputStream;
+import org.apache.ignite.internal.binary.streams.BinaryHeapOutputStream;
import org.apache.ignite.internal.managers.discovery.GridDiscoveryManager;
import org.apache.ignite.internal.managers.systemview.GridSystemViewManager;
import org.apache.ignite.internal.processors.cache.CacheObjectContext;
import org.apache.ignite.internal.util.GridUnsafe;
import org.apache.ignite.internal.util.IgniteUtils;
import org.apache.ignite.internal.util.lang.GridMapEntry;
+import org.apache.ignite.internal.util.lang.IgniteThrowableConsumer;
import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.T2;
import org.apache.ignite.internal.util.typedef.internal.S;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.lang.IgniteUuid;
@@ -2947,6 +2953,96 @@ public class BinaryMarshallerSelfTest extends GridCommonAbstractTest {
* @throws Exception If failed.
*/
@Test
+ public void testReadDetachedMap() throws Exception {
+ Map<Key, Value> map = IntStream.range(0, 1000).mapToObj(i -> new T2<>(new Key(i), new Value(i)))
+ .collect(Collectors.toMap(T2::getKey, T2::getValue));
+
+ testReadDetachObjectProperly(map, obj -> {
+ Map<BinaryObject, BinaryObject> desMap = (Map<BinaryObject, BinaryObject>)obj;
+
+ assertEquals(map.size(), desMap.size());
+
+ desMap.forEach((k, v) -> {
+ Key key = new Key(k.field("key"));
+ Value val = new Value(v.field("val"));
+
+ assertTrue(map.containsKey(key));
+ assertEquals(val, map.get(key));
+ });
+ });
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ @Test
+ public void testReadDetachedCollection() throws Exception {
+ Collection<Value> col = IntStream.range(0, 1000).mapToObj(Value::new).collect(Collectors.toSet());
+
+ testReadDetachObjectProperly(col, obj -> {
+ Collection<BinaryObject> desCol = (Collection<BinaryObject>)obj;
+
+ assertEquals(col.size(), desCol.size());
+
+ desCol.forEach(v -> {
+ Value val = new Value(v.field("val"));
+
+ assertTrue(col.contains(val));
+ });
+ });
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ @Test
+ public void testReadDetachedArray() throws Exception {
+ Value[] arr = IntStream.range(0, 1000).mapToObj(Value::new).toArray(Value[]::new);
+
+ testReadDetachObjectProperly(arr, obj -> {
+ Object[] desArr = (Object[])obj;
+
+ assertEquals(arr.length, desArr.length);
+
+ for (int i = 0; i < arr.length; i++) {
+ BinaryObject val = (BinaryObject)desArr[i];
+
+ assertEquals(arr[i], new Value(val.field("val")));
+ }
+ });
+ }
+
+ /**
+ * Perform action on binary object after unmarshalling from offheap data, when offheap memory chunk cleared.
+ *
+ * @param obj Object to marshal-unmarshal.
+ * @param action Action to perform on object.
+ * @throws Exception If failed.
+ */
+ private void testReadDetachObjectProperly(Object obj, IgniteThrowableConsumer<Object> action) throws Exception {
+ BinaryMarshaller marsh = binaryMarshaller();
+
+ BinaryHeapOutputStream os = new BinaryHeapOutputStream(1024);
+
+ BinaryWriterExImpl writer = marsh.binaryMarshaller().writer(os);
+
+ writer.writeObject(obj);
+
+ BinaryHeapInputStream is = new BinaryHeapInputStream(os.array());
+
+ BinaryReaderExImpl reader = marsh.binaryMarshaller().reader(is);
+
+ Object bObj = reader.readObjectDetached();
+
+ Arrays.fill(os.array(), (byte)0);
+
+ action.accept(bObj);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ @Test
public void testMarshallingThroughJdk() throws Exception {
BinaryMarshaller marsh = binaryMarshaller();