You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by av...@apache.org on 2022/06/09 16:25:19 UTC
[ignite] branch master updated: IGNITE-15834 Read Repair should support arrays and collections as values (#10047)
This is an automated email from the ASF dual-hosted git repository.
av pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite.git
The following commit(s) were added to refs/heads/master by this push:
new 59fe7df7705 IGNITE-15834 Read Repair should support arrays and collections as values (#10047)
59fe7df7705 is described below
commit 59fe7df77052b50e6c02e16b2bf46c552e22bfad
Author: Anton Vinogradov <av...@apache.org>
AuthorDate: Thu Jun 9 19:25:12 2022 +0300
IGNITE-15834 Read Repair should support arrays and collections as values (#10047)
---
...sistencyRepairCorrectnessTransactionalTest.java | 4 +-
.../processors/cache/GridCacheAdapter.java | 18 +--
.../apache/ignite/internal/util/IgniteUtils.java | 3 +
.../org/apache/ignite/client/FunctionalTest.java | 15 --
.../internal/binary/BinaryArraySelfTest.java | 1 -
.../consistency/AbstractFullSetReadRepairTest.java | 16 ++-
.../cache/consistency/AbstractReadRepairTest.java | 6 +-
.../cache/consistency/ReadRepairDataGenerator.java | 158 ++++++++++++++++++---
.../testframework/junits/JUnitAssertAware.java | 18 ++-
9 files changed, 181 insertions(+), 58 deletions(-)
diff --git a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerConsistencyRepairCorrectnessTransactionalTest.java b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerConsistencyRepairCorrectnessTransactionalTest.java
index 31276ad9b18..1d59bc99e9b 100644
--- a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerConsistencyRepairCorrectnessTransactionalTest.java
+++ b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerConsistencyRepairCorrectnessTransactionalTest.java
@@ -239,10 +239,10 @@ public class GridCommandHandlerConsistencyRepairCorrectnessTransactionalTest ext
Object repaired = ReadRepairDataGenerator.unwrapBinaryIfNeeded(mapping.repairedBin);
// Regular get (form primary or backup or client node).
- assertEquals("Checking key=" + key, repaired, cache.get(key));
+ assertEqualsArraysAware("Checking key=" + key, repaired, cache.get(key));
// All copies check.
- assertEquals("Checking key=" + key, repaired,
+ assertEqualsArraysAware("Checking key=" + key, repaired,
cache.withReadRepair(ReadRepairStrategy.CHECK_ONLY).get(key));
}
else if (!mapping.consistent) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
index 96322c21833..7cae4e7ea9b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
@@ -5158,10 +5158,8 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
EntryGetResult primRes = ex.primaryMap().get(key);
CacheObject correctedObj = correctedRes != null ? correctedRes.value() : null;
- CacheObject primValObj = primRes != null ? primRes.value() : null;
V correctedVal = correctedObj != null ? (V)ctx.unwrapBinaryIfNeeded(correctedObj, true, false, null) : null;
- V primVal = primValObj != null ? (V)ctx.unwrapBinaryIfNeeded(primValObj, true, false, null) : null;
GridCacheVersion primVer = primRes != null ? primRes.version() : null;
@@ -5172,7 +5170,7 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
ctx.operationContextPerCall(opCtx.keepBinary());
try {
- return invoke((K)key, new AtomicReadRepairEntryProcessor<>(correctedVal, primVal, primVer)).get();
+ return invoke((K)key, new AtomicReadRepairEntryProcessor<>(correctedVal, primVer)).get();
}
finally {
ctx.operationContextPerCall(prevOpCtx);
@@ -5196,32 +5194,24 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
/** Corrected value. */
private final V correctedVal;
- /** Primary value.*/
- private final V primVal;
-
/** Primary version. */
private final GridCacheVersion primVer;
/**
* @param correctedVal Corrected value.
- * @param primVal Primary value.
* @param primVer Primary version.
*/
- public AtomicReadRepairEntryProcessor(V correctedVal, V primVal, GridCacheVersion primVer) {
+ public AtomicReadRepairEntryProcessor(V correctedVal, GridCacheVersion primVer) {
this.correctedVal = correctedVal;
- this.primVal = primVal;
this.primVer = primVer;
}
/** {@inheritDoc} */
@Override public Boolean process(MutableEntry<K, V> entry, Object... arguments) throws EntryProcessorException {
- V entryVal = entry.getValue();
-
try {
- if ((primVal == null && entryVal == null) || // Still null at primary.
+ if ((primVer == null && entry.getValue() == null) || // Still null at primary.
// No updates since consistency violation has been found.
- ((primVal != null && primVal.equals(entryVal)) &&
- (primVer.equals(((CacheInvokeEntry<Object, Object>)entry).entry().version())))) {
+ primVer.equals(((CacheInvokeEntry<Object, Object>)entry).entry().version())) {
if (correctedVal != null)
entry.setValue(correctedVal);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
index 100743b1312..13a6dec9494 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
@@ -8005,6 +8005,9 @@ public abstract class IgniteUtils {
* @return {@code True} if Object is primitive array.
*/
public static boolean isPrimitiveArray(Object obj) {
+ if (obj == null)
+ return false;
+
Class<?> cls = obj.getClass();
return cls.isArray() && cls.getComponentType().isPrimitive();
diff --git a/modules/core/src/test/java/org/apache/ignite/client/FunctionalTest.java b/modules/core/src/test/java/org/apache/ignite/client/FunctionalTest.java
index 4f6cf65eef9..792a9980e30 100644
--- a/modules/core/src/test/java/org/apache/ignite/client/FunctionalTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/client/FunctionalTest.java
@@ -446,21 +446,6 @@ public class FunctionalTest extends AbstractBinaryArraysTest {
assertTrue(thinCache.remove(key, cachedObj));
}
- /**
- * Assert values equals (deep equals for arrays).
- *
- * @param exp Expected value.
- * @param actual Actual value.
- */
- public static void assertEqualsArraysAware(Object exp, Object actual) {
- if (exp instanceof Object[])
- assertArrayEquals((Object[])exp, (Object[])actual);
- else if (U.isPrimitiveArray(exp))
- assertArrayEquals(new Object[] {exp}, new Object[] {actual}); // Hack to compare primitive arrays.
- else
- assertEquals(exp, actual);
- }
-
/**
* Tested API:
* <ul>
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryArraySelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryArraySelfTest.java
index e487ba4562d..1d047eeb000 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryArraySelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryArraySelfTest.java
@@ -40,7 +40,6 @@ import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.testframework.GridTestUtils;
import org.junit.Test;
-import static org.apache.ignite.client.FunctionalTest.assertEqualsArraysAware;
import static org.apache.ignite.internal.processors.cache.CacheEnumOperationsAbstractTest.TestEnum.VAL1;
import static org.apache.ignite.internal.processors.cache.CacheEnumOperationsAbstractTest.TestEnum.VAL2;
import static org.apache.ignite.internal.processors.cache.CacheEnumOperationsAbstractTest.TestEnum.VAL3;
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/consistency/AbstractFullSetReadRepairTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/consistency/AbstractFullSetReadRepairTest.java
index 3ef9e38db1e..0af53da2554 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/consistency/AbstractFullSetReadRepairTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/consistency/AbstractFullSetReadRepairTest.java
@@ -20,6 +20,7 @@ package org.apache.ignite.internal.processors.cache.consistency;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
@@ -43,7 +44,7 @@ public abstract class AbstractFullSetReadRepairTest extends AbstractReadRepairTe
*/
protected static final Consumer<ReadRepairData> GET_CHECK_AND_REPAIR = (rrd) -> {
for (Integer key : rrd.data.keySet()) { // Once.
- assertEquals(unwrapBinaryIfNeeded(rrd.data.get(key).repairedBin), get(rrd));
+ assertEqualsArraysAware(unwrapBinaryIfNeeded(rrd.data.get(key).repairedBin), get(rrd));
}
};
@@ -54,7 +55,7 @@ public abstract class AbstractFullSetReadRepairTest extends AbstractReadRepairTe
Map<Integer, Object> res = getAll(rrd);
for (Integer key : rrd.data.keySet())
- assertEquals(unwrapBinaryIfNeeded(rrd.data.get(key).repairedBin), res.get(key));
+ assertEqualsArraysAware(unwrapBinaryIfNeeded(rrd.data.get(key).repairedBin), res.get(key));
};
/**
@@ -166,7 +167,14 @@ public abstract class AbstractFullSetReadRepairTest extends AbstractReadRepairTe
*/
private static Object checkAndUnwrapBinaryIfNeeded(ReadRepairData rrd, Object res) {
if (rrd.binary) {
- assert res == null || res instanceof Integer || res instanceof BinaryObject : res.getClass();
+ assert res == null ||
+ res instanceof Integer ||
+ res instanceof Map ||
+ res instanceof List ||
+ res instanceof Set ||
+ res instanceof int[] ||
+ res instanceof Object[] ||
+ res instanceof BinaryObject : res.getClass();
return unwrapBinaryIfNeeded(res);
}
@@ -222,7 +230,7 @@ public abstract class AbstractFullSetReadRepairTest extends AbstractReadRepairTe
else
res = rrd.cache.get(key);
- assertEquals(unwrapBinaryIfNeeded(rrd.data.get(key).repairedBin), unwrapBinaryIfNeeded(res));
+ assertEqualsArraysAware(unwrapBinaryIfNeeded(rrd.data.get(key).repairedBin), unwrapBinaryIfNeeded(res));
}
};
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/consistency/AbstractReadRepairTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/consistency/AbstractReadRepairTest.java
index 6a5d22ac27c..8ec7a920e86 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/consistency/AbstractReadRepairTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/consistency/AbstractReadRepairTest.java
@@ -241,7 +241,7 @@ public abstract class AbstractReadRepairTest extends GridCommonAbstractTest {
assertTrue(repairable);
assertTrue(evtRepaired.containsKey(key));
- assertEquals(repairedBin, evtRepaired.get(key));
+ assertEqualsArraysAware(repairedBin, evtRepaired.get(key));
}
// Repairable but not repaired (because of irreparable entry at the same tx) entries.
else if (e.irreparableKeys().contains(key) || (e.repairableKeys() != null && e.repairableKeys().contains(key)))
@@ -257,10 +257,10 @@ public abstract class AbstractReadRepairTest extends GridCommonAbstractTest {
Object val = info.getValue();
if (info.isCorrect())
- assertEquals(repairedBin, val);
+ assertEqualsArraysAware(repairedBin, val);
if (info.isPrimary()) {
- assertEquals(primaryBin, val);
+ assertEqualsArraysAware(primaryBin, val);
assertEquals(node, primaryNode(key, DEFAULT_CACHE_NAME).cluster().localNode());
}
}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/consistency/ReadRepairDataGenerator.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/consistency/ReadRepairDataGenerator.java
index 8ce1ac2cbbe..a3def14f145 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/consistency/ReadRepairDataGenerator.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/consistency/ReadRepairDataGenerator.java
@@ -18,6 +18,7 @@
package org.apache.ignite.internal.processors.cache.consistency;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
@@ -54,19 +55,15 @@ import org.apache.ignite.internal.util.typedef.G;
import org.apache.ignite.internal.util.typedef.T2;
import org.apache.ignite.internal.util.typedef.internal.CU;
import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.JUnitAssertAware;
import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC;
import static org.apache.ignite.cache.CacheMode.REPLICATED;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
/**
*
*/
-public class ReadRepairDataGenerator {
+public class ReadRepairDataGenerator extends JUnitAssertAware {
/** Key. */
private final AtomicInteger incrementalKey = new AtomicInteger();
@@ -177,7 +174,7 @@ public class ReadRepairDataGenerator {
else
exp = results.get(key).primaryBin; // Or read from primary (when not a partition owner).
- assertEquals(exp, val);
+ assertEqualsArraysAware(exp, val);
}
}
@@ -196,7 +193,8 @@ public class ReadRepairDataGenerator {
InconsistentMapping mapping = entry.getValue();
- sb.append(" Generated data [primary=").append(unwrapBinaryIfNeeded(mapping.primaryBin))
+ sb.append(" Generated data [primary=").append(
+ describeArrayIfNeeded(unwrapBinaryIfNeeded(mapping.primaryBin)))
.append(", repaired=").append(unwrapBinaryIfNeeded(mapping.repairedBin))
.append(", repairable=").append(mapping.repairable)
.append(", consistent=").append(mapping.consistent)
@@ -206,7 +204,8 @@ public class ReadRepairDataGenerator {
for (Map.Entry<Ignite, T2<Object, GridCacheVersion>> dist : mapping.mappingBin.entrySet()) {
sb.append(" Node: ").append(dist.getKey().name()).append("\n");
- sb.append(" Value: ").append(unwrapBinaryIfNeeded(dist.getValue().get1())).append("\n");
+ sb.append(" Value: ").append(
+ describeArrayIfNeeded(unwrapBinaryIfNeeded(dist.getValue().get1()))).append("\n");
sb.append(" Version: ").append(dist.getValue().get2()).append("\n");
}
@@ -218,6 +217,17 @@ public class ReadRepairDataGenerator {
}
}
+ /**
+ * @param obj Object.
+ */
+ private Object describeArrayIfNeeded(Object obj) {
+ if (obj instanceof Object[])
+ return Arrays.deepToString((Object[])obj);
+ else if (obj instanceof int[])
+ return Arrays.toString((int[])obj);
+ else return obj;
+ }
+
/**
* Generated entries count.
*/
@@ -306,7 +316,7 @@ public class ReadRepairDataGenerator {
null :
wrapTestValueIfNeeded(wrap, rnd.nextBoolean()/*increment or same as previously*/ ? ++incVal : incVal);
- T2<Object, GridCacheVersion> valVer = new T2<>(val, val != null ? ver : null);
+ T2<Object, GridCacheVersion> valVer = new T2<>(wrapArrayIfNeeded(val), val != null ? ver : null);
vals.add(valVer);
mapping.put(node, valVer);
@@ -488,7 +498,7 @@ public class ReadRepairDataGenerator {
IgniteBinary igniteBinary = clsAwareNodes.get(0).binary();
Object primValBin = igniteBinary.toBinary(primVal);
- Object repairedBin = igniteBinary.toBinary(repaired);
+ Object repairedBin = igniteBinary.toBinary(unwrapArrayIfNeeded(repaired));
Map<Ignite, T2<Object, GridCacheVersion>> mappingBin = mapping.entrySet().stream().collect(
Collectors.toMap(
@@ -496,7 +506,7 @@ public class ReadRepairDataGenerator {
(entry) -> {
T2<Object, GridCacheVersion> t2 = entry.getValue();
- return new T2<>(igniteBinary.toBinary(t2.getKey()), t2.getValue());
+ return new T2<>(igniteBinary.toBinary(unwrapArrayIfNeeded(t2.getKey())), t2.getValue());
}));
return new InconsistentMapping(mappingBin, primValBin, repairedBin, repairable, consistent);
@@ -509,21 +519,71 @@ public class ReadRepairDataGenerator {
return new IncomparableClass();
}
+ /**
+ *
+ */
+ private Object wrapArrayIfNeeded(Object obj) {
+ if (obj instanceof Object[])
+ return new ObjectArrayWrapper((Object[])obj);
+ else if (obj instanceof int[])
+ return new IntArrayWrapper((int[])obj);
+ else
+ return obj;
+ }
+
+ /**
+ *
+ */
+ private Object unwrapArrayIfNeeded(Object obj) {
+ if (obj instanceof ObjectArrayWrapper)
+ return ((ObjectArrayWrapper)obj).arr;
+ else if (obj instanceof IntArrayWrapper)
+ return ((IntArrayWrapper)obj).arr;
+ else
+ return obj;
+ }
+
/**
* @param wrap Wrap.
* @param val Value.
*/
private Object wrapTestValueIfNeeded(boolean wrap, Integer val) throws ReflectiveOperationException {
if (wrap) {
- // Some nodes will be unable to deserialize this object.
- // Checking that Read Repair feature cause no `class not found` problems.
- Class<?> clazz = extClsLdr.loadClass("org.apache.ignite.tests.p2p.cache.PersonKey");
+ int type = val % 7;
- Object obj = clazz.newInstance();
+ switch (type) {
+ case 0:
+ // Some nodes will be unable to deserialize this object.
+ // Checking that Read Repair feature cause no `class not found` problems.
+ Class<?> clazz = extClsLdr.loadClass("org.apache.ignite.tests.p2p.cache.PersonKey");
- GridTestUtils.setFieldValue(obj, "id", val);
+ Object obj = clazz.newInstance();
- return obj;
+ GridTestUtils.setFieldValue(obj, "id", val);
+
+ return obj;
+
+ case 1:
+ return new Object[] {val};
+
+ case 2:
+ return new Object[][] {{val}, {val}};
+
+ case 3:
+ return new int[] {val};
+
+ case 4:
+ return Collections.singletonMap(val, val);
+
+ case 5:
+ return Collections.singletonList(val);
+
+ case 6:
+ return Collections.singleton(val);
+
+ default:
+ throw new IllegalStateException();
+ }
}
else
return val;
@@ -653,4 +713,66 @@ public class ReadRepairDataGenerator {
return false;
}
}
+
+ /**
+ *
+ */
+ private static final class ObjectArrayWrapper {
+ /** Array. */
+ final Object[] arr;
+
+ /** */
+ public ObjectArrayWrapper(Object[] arr) {
+ this.arr = arr;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean equals(Object o) {
+ if (this == o)
+ return true;
+
+ if (o == null || getClass() != o.getClass())
+ return false;
+
+ ObjectArrayWrapper wrapper = (ObjectArrayWrapper)o;
+
+ return Arrays.deepEquals(arr, wrapper.arr);
+ }
+
+ /** {@inheritDoc} */
+ @Override public int hashCode() {
+ return Arrays.deepHashCode(arr);
+ }
+ }
+
+ /**
+ *
+ */
+ private static final class IntArrayWrapper {
+ /** Array. */
+ final int[] arr;
+
+ /** */
+ public IntArrayWrapper(int[] arr) {
+ this.arr = arr;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean equals(Object o) {
+ if (this == o)
+ return true;
+
+ if (o == null || getClass() != o.getClass())
+ return false;
+
+ IntArrayWrapper wrapper = (IntArrayWrapper)o;
+
+ return Arrays.equals(arr, wrapper.arr);
+ }
+
+ /** {@inheritDoc} */
+ @Override public int hashCode() {
+ return Arrays.hashCode(arr);
+ }
+ }
}
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/JUnitAssertAware.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/JUnitAssertAware.java
index f46e6a3fe2d..3423e3cfb35 100644
--- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/JUnitAssertAware.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/JUnitAssertAware.java
@@ -17,13 +17,14 @@
package org.apache.ignite.testframework.junits;
+import org.apache.ignite.internal.util.typedef.internal.U;
import org.junit.Assert;
/**
* Provides the basic functionality of {@link Assert} methods in org.junit package.
* Corresponding methods must be used in all ignite tests where necessary.
*/
-class JUnitAssertAware {
+public class JUnitAssertAware {
/** See {@link Assert#assertTrue(String, boolean)} javadocs. */
protected static void assertTrue(String msg, boolean cond) {
Assert.assertTrue(msg, cond);
@@ -203,4 +204,19 @@ class JUnitAssertAware {
protected static void assertNotSame(String msg, Object exp, Object actual) {
Assert.assertNotSame(msg, exp, actual);
}
+
+ /** Check arrays equality as well as objects equality. */
+ protected static void assertEqualsArraysAware(Object exp, Object actual) {
+ assertEqualsArraysAware(null, exp, actual);
+ }
+
+ /** Check arrays equality as well as objects equality. */
+ protected static void assertEqualsArraysAware(String msg, Object exp, Object actual) {
+ if (exp instanceof Object[])
+ Assert.assertArrayEquals((Object[])exp, (Object[])actual);
+ else if (U.isPrimitiveArray(exp))
+ Assert.assertArrayEquals(new Object[] {exp}, new Object[] {actual}); // Hack to compare primitive arrays.
+ else
+ Assert.assertEquals(exp, actual);
+ }
}