You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by pt...@apache.org on 2024/12/24 10:22:14 UTC
(ignite-3) branch main updated: IGNITE-23186 Support Tuple collections as Compute args and results (#4953)
This is an automated email from the ASF dual-hosted git repository.
ptupitsyn pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/ignite-3.git
The following commit(s) were added to refs/heads/main by this push:
new 67aa6b17bf IGNITE-23186 Support Tuple collections as Compute args and results (#4953)
67aa6b17bf is described below
commit 67aa6b17bfb26060f6be8fcdfed6ae992fa6763c
Author: Pavel Tupitsyn <pt...@apache.org>
AuthorDate: Tue Dec 24 12:22:09 2024 +0200
IGNITE-23186 Support Tuple collections as Compute args and results (#4953)
Add `TUPLE_COLLECTION` marshalling mode for Compute job arguments and results.
The user can pass any `Collection` implementation - it will always come back as an `ArrayList` on the other side.
---
.../ignite/marshalling/MarshallingException.java | 3 +-
.../internal/binarytuple/BinaryTupleParser.java | 26 ++-
.../internal/binarytuple/BinaryTupleReader.java | 11 ++
.../inlineschema/TupleWithSchemaMarshalling.java | 26 ++-
.../internal/binarytuple/BinaryTupleTest.java | 12 ++
.../client/proto/ClientComputeJobPacker.java | 35 ++++-
.../client/proto/ClientComputeJobUnpacker.java | 28 ++++
.../internal/client/proto/ClientMessagePacker.java | 6 +
.../proto/ClientComputeJobPackerUnpackerTest.java | 22 ++-
.../ignite/internal/compute/ItComputeBaseTest.java | 174 ++++++++++-----------
.../internal/compute/ItComputeTestClient.java | 88 +++++++++++
.../internal/compute/ItComputeTestEmbedded.java | 8 +-
.../internal/compute/TupleCollectionJob.java | 41 +++++
.../ignite/internal/compute/ComputeUtils.java | 59 ++++++-
.../internal/compute/ComputeJobDataType.java | 3 +-
.../internal/ClusterPerClassIntegrationTest.java | 1 +
16 files changed, 434 insertions(+), 109 deletions(-)
diff --git a/modules/api/src/main/java/org/apache/ignite/marshalling/MarshallingException.java b/modules/api/src/main/java/org/apache/ignite/marshalling/MarshallingException.java
index 219dfc9616..50d22efade 100644
--- a/modules/api/src/main/java/org/apache/ignite/marshalling/MarshallingException.java
+++ b/modules/api/src/main/java/org/apache/ignite/marshalling/MarshallingException.java
@@ -20,6 +20,7 @@ package org.apache.ignite.marshalling;
import java.util.UUID;
import org.apache.ignite.lang.ErrorGroups.Marshalling;
import org.apache.ignite.lang.IgniteException;
+import org.jetbrains.annotations.Nullable;
/**
* Exception thrown when an object type is not supported by the marshaller.
@@ -42,7 +43,7 @@ public class MarshallingException extends IgniteException {
* @param msg Message.
* @param cause The exception that caused the error.
*/
- public MarshallingException(String msg, Throwable cause) {
+ public MarshallingException(String msg, @Nullable Throwable cause) {
super(Marshalling.COMMON_ERR, msg, cause);
}
diff --git a/modules/binary-tuple/src/main/java/org/apache/ignite/internal/binarytuple/BinaryTupleParser.java b/modules/binary-tuple/src/main/java/org/apache/ignite/internal/binarytuple/BinaryTupleParser.java
index dce7b18da0..cd28909643 100644
--- a/modules/binary-tuple/src/main/java/org/apache/ignite/internal/binarytuple/BinaryTupleParser.java
+++ b/modules/binary-tuple/src/main/java/org/apache/ignite/internal/binarytuple/BinaryTupleParser.java
@@ -78,8 +78,8 @@ public class BinaryTupleParser {
public BinaryTupleParser(int numElements, ByteBuffer buffer) {
this.numElements = numElements;
- assert buffer.order() == ORDER;
- assert buffer.position() == 0;
+ assert buffer.order() == ORDER : "Buffer order must be LITTLE_ENDIAN, actual: " + buffer.order();
+ assert buffer.position() == 0 : "Buffer position must be 0, actual: " + buffer.position();
this.buffer = buffer;
byte flags = buffer.get(0);
@@ -340,6 +340,7 @@ public class BinaryTupleParser {
bytes = getBytes(begin, end);
begin = 0;
}
+
return new String(bytes, begin, len, StandardCharsets.UTF_8);
}
@@ -363,6 +364,27 @@ public class BinaryTupleParser {
return getBytes(begin, end);
}
+ /**
+ * Reads value of specified element as a ByteBuffer.
+ * The returned buffer is a slice of the original buffer.
+ *
+ * @param begin Start offset of the element.
+ * @param end End offset of the element.
+ * @return Element value.
+ */
+ public final ByteBuffer bytesValueAsBuffer(int begin, int end) {
+ int len = end - begin;
+ if (len <= 0) {
+ throw new BinaryTupleFormatException("Invalid length for a tuple element: " + len);
+ }
+
+ if (buffer.get(begin) == BinaryTupleCommon.VARLEN_EMPTY_BYTE) {
+ begin++;
+ }
+
+ return buffer.duplicate().position(begin).limit(end).slice();
+ }
+
/**
* Reads value of specified element.
*
diff --git a/modules/binary-tuple/src/main/java/org/apache/ignite/internal/binarytuple/BinaryTupleReader.java b/modules/binary-tuple/src/main/java/org/apache/ignite/internal/binarytuple/BinaryTupleReader.java
index 692b229388..a40f0b25c9 100644
--- a/modules/binary-tuple/src/main/java/org/apache/ignite/internal/binarytuple/BinaryTupleReader.java
+++ b/modules/binary-tuple/src/main/java/org/apache/ignite/internal/binarytuple/BinaryTupleReader.java
@@ -281,6 +281,17 @@ public class BinaryTupleReader extends BinaryTupleParser implements BinaryTupleP
return begin == end ? null : bytesValue(begin, end);
}
+ /**
+ * Reads value of specified element.
+ *
+ * @param index Element index.
+ * @return Element value as a {@link ByteBuffer}.
+ */
+ public @Nullable ByteBuffer bytesValueAsBuffer(int index) {
+ seek(index);
+ return begin == end ? null : bytesValueAsBuffer(begin, end);
+ }
+
/**
* Reads value of specified element.
*
diff --git a/modules/binary-tuple/src/main/java/org/apache/ignite/internal/binarytuple/inlineschema/TupleWithSchemaMarshalling.java b/modules/binary-tuple/src/main/java/org/apache/ignite/internal/binarytuple/inlineschema/TupleWithSchemaMarshalling.java
index d6933ee9ba..f68e4c9c4f 100644
--- a/modules/binary-tuple/src/main/java/org/apache/ignite/internal/binarytuple/inlineschema/TupleWithSchemaMarshalling.java
+++ b/modules/binary-tuple/src/main/java/org/apache/ignite/internal/binarytuple/inlineschema/TupleWithSchemaMarshalling.java
@@ -100,15 +100,25 @@ public final class TupleWithSchemaMarshalling {
/**
* Unmarshal tuple (LITTLE_ENDIAN).
*
- * @param raw byte[] bytes that are marshaled by {@link #marshal(Tuple)}.
+ * @param raw bytes that are marshaled by {@link #marshal(Tuple)}.
*/
public static Tuple unmarshal(byte[] raw) {
- if (raw.length < 8) {
- throw new UnmarshallingException("byte[] length can not be less than 8");
+ return unmarshal(ByteBuffer.wrap(raw));
+ }
+
+ /**
+ * Unmarshal tuple (LITTLE_ENDIAN).
+ *
+ * @param raw bytes that are marshaled by {@link #marshal(Tuple)}.
+ */
+ public static Tuple unmarshal(ByteBuffer raw) {
+ int dataLen = raw.remaining();
+ if (dataLen < 8) {
+ throw new UnmarshallingException("Data length can not be less than 8");
}
// Read first int32.
- ByteBuffer buff = ByteBuffer.wrap(raw).order(BYTE_ORDER);
+ ByteBuffer buff = raw.order(BYTE_ORDER);
int size = buff.getInt(0);
if (size < 0) {
throw new UnmarshallingException("Size of the tuple can not be less than zero");
@@ -119,10 +129,10 @@ public final class TupleWithSchemaMarshalling {
if (valueOffset < 0) {
throw new UnmarshallingException("valueOffset can not be less than zero");
}
- if (valueOffset > raw.length) {
+ if (valueOffset > dataLen) {
throw new UnmarshallingException(
- "valueOffset can not be greater than byte[] length, valueOffset: "
- + valueOffset + ", length: " + raw.length
+ "valueOffset can not be greater than data length, valueOffset: "
+ + valueOffset + ", length: " + dataLen
);
}
@@ -130,7 +140,7 @@ public final class TupleWithSchemaMarshalling {
.position(8).limit(valueOffset)
.slice().order(BYTE_ORDER);
ByteBuffer valueBuff = buff
- .position(valueOffset).limit(raw.length)
+ .position(valueOffset).limit(dataLen)
.slice().order(BYTE_ORDER);
BinaryTupleReader schemaReader = new BinaryTupleReader(size * 2, schemaBuff);
diff --git a/modules/binary-tuple/src/test/java/org/apache/ignite/internal/binarytuple/BinaryTupleTest.java b/modules/binary-tuple/src/test/java/org/apache/ignite/internal/binarytuple/BinaryTupleTest.java
index 160edf1276..e49d0e8552 100644
--- a/modules/binary-tuple/src/test/java/org/apache/ignite/internal/binarytuple/BinaryTupleTest.java
+++ b/modules/binary-tuple/src/test/java/org/apache/ignite/internal/binarytuple/BinaryTupleTest.java
@@ -68,6 +68,7 @@ public class BinaryTupleTest {
assertNull(reader.decimalValue(0, 0));
assertNull(reader.stringValue(0));
assertNull(reader.bytesValue(0));
+ assertNull(reader.bytesValueAsBuffer(0));
assertNull(reader.uuidValue(0));
assertNull(reader.dateValue(0));
assertNull(reader.timeValue(0));
@@ -421,8 +422,19 @@ public class BinaryTupleTest {
assertEquals(length, e - b);
}
});
+
for (int i = 0; i < values.length; i++) {
assertArrayEquals(values[i], reader.bytesValue(i));
+
+ ByteBuffer valueAsBuffer = reader.bytesValueAsBuffer(i);
+
+ if (values[i] == null) {
+ assertNull(valueAsBuffer);
+ } else {
+ byte[] bufferBytes = new byte[valueAsBuffer.remaining()];
+ valueAsBuffer.get(bufferBytes);
+ assertArrayEquals(values[i], bufferBytes);
+ }
}
}
}
diff --git a/modules/client-common/src/main/java/org/apache/ignite/internal/client/proto/ClientComputeJobPacker.java b/modules/client-common/src/main/java/org/apache/ignite/internal/client/proto/ClientComputeJobPacker.java
index 9f0c9d69d4..4ea19494f5 100644
--- a/modules/client-common/src/main/java/org/apache/ignite/internal/client/proto/ClientComputeJobPacker.java
+++ b/modules/client-common/src/main/java/org/apache/ignite/internal/client/proto/ClientComputeJobPacker.java
@@ -21,11 +21,15 @@ import static org.apache.ignite.internal.compute.ComputeJobDataType.MARSHALLED_C
import static org.apache.ignite.internal.compute.ComputeJobDataType.NATIVE;
import static org.apache.ignite.internal.compute.ComputeJobDataType.POJO;
import static org.apache.ignite.internal.compute.ComputeJobDataType.TUPLE;
+import static org.apache.ignite.internal.compute.ComputeJobDataType.TUPLE_COLLECTION;
import static org.apache.ignite.internal.compute.PojoConverter.toTuple;
+import java.nio.ByteBuffer;
import java.util.Arrays;
+import java.util.Collection;
import java.util.Set;
import java.util.stream.Collectors;
+import org.apache.ignite.internal.binarytuple.BinaryTupleBuilder;
import org.apache.ignite.internal.binarytuple.inlineschema.TupleWithSchemaMarshalling;
import org.apache.ignite.internal.compute.ComputeJobDataHolder;
import org.apache.ignite.internal.compute.PojoConversionException;
@@ -100,6 +104,35 @@ public final class ClientComputeJobPacker {
return;
}
+ if (obj instanceof Collection) {
+ Collection<?> col = (Collection<?>) obj;
+
+ packer.packInt(TUPLE_COLLECTION.id());
+
+ // Pack entire collection into a single binary blob.
+ BinaryTupleBuilder tupleBuilder = new BinaryTupleBuilder(col.size());
+
+ for (Object el : col) {
+ if (el == null) {
+ tupleBuilder.appendNull();
+ continue;
+ }
+
+ if (!(el instanceof Tuple)) {
+ throw new MarshallingException("Can't pack collection: expected Tuple, but got " + el.getClass(), null);
+ }
+
+ tupleBuilder.appendBytes(TupleWithSchemaMarshalling.marshal((Tuple) el));
+ }
+
+ ByteBuffer binTupleBytes = tupleBuilder.build();
+ packer.packBinaryHeader(Integer.BYTES + binTupleBytes.remaining());
+ packer.writeIntRawLittleEndian(col.size());
+ packer.writePayload(binTupleBytes);
+
+ return;
+ }
+
if (isNativeType(obj.getClass())) {
packer.packInt(NATIVE.id());
@@ -112,7 +145,7 @@ public final class ClientComputeJobPacker {
packTuple(toTuple(obj), packer);
} catch (PojoConversionException e) {
- throw new MarshallingException("Can't pack object", e);
+ throw new MarshallingException("Can't pack object: " + obj, e);
}
}
diff --git a/modules/client-common/src/main/java/org/apache/ignite/internal/client/proto/ClientComputeJobUnpacker.java b/modules/client-common/src/main/java/org/apache/ignite/internal/client/proto/ClientComputeJobUnpacker.java
index b17f4595b8..a7c7856441 100644
--- a/modules/client-common/src/main/java/org/apache/ignite/internal/client/proto/ClientComputeJobUnpacker.java
+++ b/modules/client-common/src/main/java/org/apache/ignite/internal/client/proto/ClientComputeJobUnpacker.java
@@ -21,12 +21,18 @@ import static org.apache.ignite.internal.compute.PojoConverter.fromTuple;
import static org.apache.ignite.marshalling.Marshaller.tryUnmarshalOrCast;
import java.lang.reflect.InvocationTargetException;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.ignite.internal.binarytuple.BinaryTupleReader;
import org.apache.ignite.internal.binarytuple.inlineschema.TupleWithSchemaMarshalling;
import org.apache.ignite.internal.compute.ComputeJobDataHolder;
import org.apache.ignite.internal.compute.ComputeJobDataType;
import org.apache.ignite.internal.compute.PojoConversionException;
import org.apache.ignite.marshalling.Marshaller;
import org.apache.ignite.marshalling.UnmarshallingException;
+import org.apache.ignite.table.Tuple;
import org.jetbrains.annotations.Nullable;
/** Unpacks job results. */
@@ -66,6 +72,7 @@ public final class ClientComputeJobUnpacker {
}
return unpacker.unpackObjectFromBinaryTuple();
+
case TUPLE:
return TupleWithSchemaMarshalling.unmarshal(unpacker.readBinary());
@@ -86,6 +93,27 @@ public final class ClientComputeJobUnpacker {
}
return unpackPojo(unpacker, resultClass);
+ case TUPLE_COLLECTION: {
+ // TODO: IGNITE-24059 Deduplicate with ComputeUtils.
+ ByteBuffer collectionBuf = unpacker.readBinaryUnsafe().order(ByteOrder.LITTLE_ENDIAN);
+ int count = collectionBuf.getInt();
+ BinaryTupleReader reader = new BinaryTupleReader(count, collectionBuf.slice().order(ByteOrder.LITTLE_ENDIAN));
+
+ List<Tuple> res = new ArrayList<>(count);
+ for (int i = 0; i < count; i++) {
+ ByteBuffer elementBytes = reader.bytesValueAsBuffer(i);
+
+ if (elementBytes == null) {
+ res.add(null);
+ continue;
+ }
+
+ res.add(TupleWithSchemaMarshalling.unmarshal(elementBytes));
+ }
+
+ return res;
+ }
+
default:
throw new UnmarshallingException("Unsupported compute job type: " + type);
}
diff --git a/modules/client-common/src/main/java/org/apache/ignite/internal/client/proto/ClientMessagePacker.java b/modules/client-common/src/main/java/org/apache/ignite/internal/client/proto/ClientMessagePacker.java
index deb3be15f2..eccc0fe41f 100644
--- a/modules/client-common/src/main/java/org/apache/ignite/internal/client/proto/ClientMessagePacker.java
+++ b/modules/client-common/src/main/java/org/apache/ignite/internal/client/proto/ClientMessagePacker.java
@@ -508,6 +508,12 @@ public class ClientMessagePacker implements AutoCloseable {
buf.writeBytes(src, off, len);
}
+ void writeIntRawLittleEndian(int payload) {
+ assert !closed : "Packer is closed";
+
+ buf.writeIntLE(payload);
+ }
+
/**
* Writes a UUID.
*
diff --git a/modules/client-common/src/test/java/org/apache/ignite/internal/client/proto/ClientComputeJobPackerUnpackerTest.java b/modules/client-common/src/test/java/org/apache/ignite/internal/client/proto/ClientComputeJobPackerUnpackerTest.java
index 1600899abe..d93c62dafa 100644
--- a/modules/client-common/src/test/java/org/apache/ignite/internal/client/proto/ClientComputeJobPackerUnpackerTest.java
+++ b/modules/client-common/src/test/java/org/apache/ignite/internal/client/proto/ClientComputeJobPackerUnpackerTest.java
@@ -24,6 +24,7 @@ import static org.apache.ignite.internal.client.proto.ClientComputeJobUnpacker.u
import static org.apache.ignite.internal.testframework.IgniteTestUtils.assertThrows;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertIterableEquals;
import static org.junit.jupiter.params.provider.Arguments.arguments;
import io.netty.buffer.ByteBufUtil;
@@ -37,7 +38,10 @@ import java.time.LocalDateTime;
import java.time.LocalTime;
import java.time.Period;
import java.time.temporal.ChronoUnit;
+import java.util.Collection;
+import java.util.Collections;
import java.util.List;
+import java.util.Set;
import java.util.UUID;
import java.util.stream.Stream;
import org.apache.ignite.internal.client.proto.pojo.Pojo;
@@ -80,6 +84,16 @@ class ClientComputeJobPackerUnpackerTest {
).map(Arguments::of);
}
+ private static Stream<Arguments> tupleCollections() {
+ return Stream.of(
+ List.of(),
+ Collections.singletonList(Tuple.create()),
+ Collections.singletonList(null),
+ List.of(Tuple.create(), Tuple.create().set("key", 1), Tuple.create().set("key", "value1")),
+ Set.of(Tuple.create().set("key", 2), Tuple.create().set("key", "value2"))
+ ).map(Arguments::of);
+ }
+
private static List<Object> pojo() {
return List.of(Pojo.generateTestPojo());
}
@@ -107,7 +121,7 @@ class ClientComputeJobPackerUnpackerTest {
return new ClientMessageUnpacker(Unpooled.wrappedBuffer(data, 4, data.length - 4));
}
- @MethodSource({"tuples", "nativeTypes"})
+ @MethodSource({"tuples", "nativeTypes", "tupleCollections"})
@ParameterizedTest
void packUnpackNoMarshalling(Object arg) {
// When pack job result without marshaller.
@@ -119,7 +133,11 @@ class ClientComputeJobPackerUnpackerTest {
var res = unpackJobResult(messageUnpacker, null, null);
// Then.
- assertEquals(arg, res);
+ if (arg instanceof Collection<?>) {
+ assertIterableEquals((Collection<?>) arg, (Collection<?>) res);
+ } else {
+ assertEquals(arg, res);
+ }
}
}
diff --git a/modules/compute/src/integrationTest/java/org/apache/ignite/internal/compute/ItComputeBaseTest.java b/modules/compute/src/integrationTest/java/org/apache/ignite/internal/compute/ItComputeBaseTest.java
index ada6159592..2a88dbe753 100644
--- a/modules/compute/src/integrationTest/java/org/apache/ignite/internal/compute/ItComputeBaseTest.java
+++ b/modules/compute/src/integrationTest/java/org/apache/ignite/internal/compute/ItComputeBaseTest.java
@@ -38,8 +38,12 @@ import static org.hamcrest.Matchers.instanceOf;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.nullValue;
import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertIterableEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -49,6 +53,7 @@ import java.util.concurrent.CompletionException;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.stream.IntStream;
+import java.util.stream.Stream;
import org.apache.ignite.Ignite;
import org.apache.ignite.compute.ComputeException;
import org.apache.ignite.compute.IgniteCompute;
@@ -83,6 +88,10 @@ import org.junit.jupiter.params.provider.ValueSource;
public abstract class ItComputeBaseTest extends ClusterPerClassIntegrationTest {
protected abstract List<DeploymentUnit> units();
+ protected IgniteCompute compute() {
+ return node(0).compute();
+ }
+
private static List<Arguments> wrongJobClassArguments() {
return List.of(
Arguments.of("org.example.NonExistentJob", CLASS_INITIALIZATION_ERR, "Cannot load job class by name"),
@@ -98,11 +107,11 @@ public abstract class ItComputeBaseTest extends ClusterPerClassIntegrationTest {
IgniteException ex = assertThrows(
IgniteException.class, () ->
- entryNode.compute().execute(
- JobTarget.node(clusterNode(entryNode)),
- JobDescriptor.builder(jobClassName).units(units()).build(),
- null
- ));
+ compute().execute(
+ JobTarget.node(clusterNode(entryNode)),
+ JobDescriptor.builder(jobClassName).units(units()).build(),
+ null
+ ));
assertTraceableException(ex, ComputeException.class, errorCode, msg);
}
@@ -112,8 +121,8 @@ public abstract class ItComputeBaseTest extends ClusterPerClassIntegrationTest {
void executesWrongJobClassLocallyAsync(String jobClassName, int errorCode, String msg) {
Ignite entryNode = node(0);
- ExecutionException ex = assertThrows(ExecutionException.class, () -> entryNode.compute().executeAsync(
- JobTarget.node(clusterNode(entryNode)), JobDescriptor.builder(jobClassName).units(units()).build(), null)
+ ExecutionException ex = assertThrows(ExecutionException.class, () -> compute().executeAsync(
+ JobTarget.node(clusterNode(entryNode)), JobDescriptor.builder(jobClassName).units(units()).build(), null)
.get(1, TimeUnit.SECONDS));
assertTraceableException(ex, ComputeException.class, errorCode, msg);
@@ -122,9 +131,7 @@ public abstract class ItComputeBaseTest extends ClusterPerClassIntegrationTest {
@ParameterizedTest
@MethodSource("wrongJobClassArguments")
void executesWrongJobClassOnRemoteNodes(String jobClassName, int errorCode, String msg) {
- Ignite entryNode = node(0);
-
- IgniteException ex = assertThrows(IgniteException.class, () -> entryNode.compute().execute(
+ IgniteException ex = assertThrows(IgniteException.class, () -> compute().execute(
JobTarget.anyNode(clusterNode(node(1)), clusterNode(node(2))),
JobDescriptor.builder(jobClassName).units(units()).build(),
null));
@@ -135,9 +142,7 @@ public abstract class ItComputeBaseTest extends ClusterPerClassIntegrationTest {
@ParameterizedTest
@MethodSource("wrongJobClassArguments")
void executesWrongJobClassOnRemoteNodesAsync(String jobClassName, int errorCode, String msg) {
- Ignite entryNode = node(0);
-
- ExecutionException ex = assertThrows(ExecutionException.class, () -> entryNode.compute().executeAsync(
+ ExecutionException ex = assertThrows(ExecutionException.class, () -> compute().executeAsync(
JobTarget.anyNode(clusterNode(node(1)), clusterNode(node(2))),
JobDescriptor.builder(jobClassName).units(units()).build(),
null
@@ -150,7 +155,7 @@ public abstract class ItComputeBaseTest extends ClusterPerClassIntegrationTest {
void executesJobLocally() {
Ignite entryNode = node(0);
- String result = entryNode.compute().execute(
+ String result = compute().execute(
JobTarget.node(clusterNode(entryNode)),
JobDescriptor.builder(concatJobClass()).units(units()).build(),
new Object[]{"a", 42});
@@ -162,7 +167,7 @@ public abstract class ItComputeBaseTest extends ClusterPerClassIntegrationTest {
void executesJobLocallyAsync() {
Ignite entryNode = node(0);
- JobExecution<String> execution = entryNode.compute().submit(
+ JobExecution<String> execution = compute().submit(
JobTarget.node(clusterNode(entryNode)),
JobDescriptor.builder(concatJobClass()).units(units()).build(),
new Object[] {"a", 42});
@@ -174,9 +179,7 @@ public abstract class ItComputeBaseTest extends ClusterPerClassIntegrationTest {
@Test
void executesJobOnRemoteNodes() {
- Ignite entryNode = node(0);
-
- String result = entryNode.compute().execute(
+ String result = compute().execute(
JobTarget.anyNode(clusterNode(node(1)), clusterNode(node(2))),
JobDescriptor.builder(concatJobClass()).units(units()).build(),
new Object[]{"a", 42});
@@ -186,9 +189,7 @@ public abstract class ItComputeBaseTest extends ClusterPerClassIntegrationTest {
@Test
void executesJobOnRemoteNodesAsync() {
- Ignite entryNode = node(0);
-
- JobExecution<String> execution = entryNode.compute().submit(
+ JobExecution<String> execution = compute().submit(
JobTarget.anyNode(clusterNode(node(1)), clusterNode(node(2))),
JobDescriptor.builder(concatJobClass()).units(units()).build(),
new Object[]{"a", 42});
@@ -202,7 +203,7 @@ public abstract class ItComputeBaseTest extends ClusterPerClassIntegrationTest {
void localExecutionActuallyUsesLocalNode() {
Ignite entryNode = node(0);
- CompletableFuture<String> fut = entryNode.compute().executeAsync(
+ CompletableFuture<String> fut = compute().executeAsync(
JobTarget.node(clusterNode(entryNode)),
JobDescriptor.builder(getNodeNameJobClass()).units(units()).build(), null);
@@ -211,10 +212,9 @@ public abstract class ItComputeBaseTest extends ClusterPerClassIntegrationTest {
@Test
void remoteExecutionActuallyUsesRemoteNode() {
- Ignite entryNode = node(0);
Ignite remoteNode = node(1);
- CompletableFuture<String> fut = entryNode.compute().executeAsync(
+ CompletableFuture<String> fut = compute().executeAsync(
JobTarget.node(clusterNode(remoteNode)),
JobDescriptor.builder(getNodeNameJobClass()).units(units()).build(), null);
@@ -225,7 +225,7 @@ public abstract class ItComputeBaseTest extends ClusterPerClassIntegrationTest {
void executesFailingJobLocally() {
Ignite entryNode = node(0);
- IgniteException ex = assertThrows(IgniteException.class, () -> entryNode.compute().execute(
+ IgniteException ex = assertThrows(IgniteException.class, () -> compute().execute(
JobTarget.node(clusterNode(entryNode)),
JobDescriptor.builder(failingJobClassName()).units(units()).build(), null));
@@ -236,7 +236,7 @@ public abstract class ItComputeBaseTest extends ClusterPerClassIntegrationTest {
void executesFailingJobLocallyAsync() {
Ignite entryNode = node(0);
- JobExecution<String> execution = entryNode.compute().submit(
+ JobExecution<String> execution = compute().submit(
JobTarget.node(clusterNode(entryNode)),
JobDescriptor.<Object, String>builder(failingJobClassName()).units(units()).build(), null);
@@ -250,9 +250,7 @@ public abstract class ItComputeBaseTest extends ClusterPerClassIntegrationTest {
@Test
void executesFailingJobOnRemoteNodes() {
- Ignite entryNode = node(0);
-
- IgniteException ex = assertThrows(IgniteException.class, () -> entryNode.compute().execute(
+ IgniteException ex = assertThrows(IgniteException.class, () -> compute().execute(
JobTarget.anyNode(clusterNode(node(1)), clusterNode(node(2))),
JobDescriptor.builder(failingJobClassName()).units(units()).build(), null));
@@ -261,11 +259,9 @@ public abstract class ItComputeBaseTest extends ClusterPerClassIntegrationTest {
@Test
void executesFailingJobOnRemoteNodesWithOptions() {
- Ignite entryNode = node(0);
-
JobExecutionOptions options = JobExecutionOptions.builder().priority(1).maxRetries(2).build();
- String result = entryNode.compute().execute(
+ String result = compute().execute(
JobTarget.anyNode(clusterNode(node(1)), clusterNode(node(2))),
JobDescriptor.builder(FailingJobOnFirstExecution.class).units(units()).options(options).build(),
null
@@ -276,9 +272,7 @@ public abstract class ItComputeBaseTest extends ClusterPerClassIntegrationTest {
@Test
void executesFailingJobOnRemoteNodesAsync() {
- Ignite entryNode = node(0);
-
- JobExecution<String> execution = entryNode.compute().submit(
+ JobExecution<String> execution = compute().submit(
JobTarget.anyNode(clusterNode(node(1)), clusterNode(node(2))),
JobDescriptor.<Object, String>builder(failingJobClassName()).units(units()).build(), null);
@@ -294,7 +288,7 @@ public abstract class ItComputeBaseTest extends ClusterPerClassIntegrationTest {
void broadcastsJobWithArgumentsAsync() {
Ignite entryNode = node(0);
- Map<ClusterNode, JobExecution<String>> results = entryNode.compute().submitBroadcast(
+ Map<ClusterNode, JobExecution<String>> results = compute().submitBroadcast(
Set.of(clusterNode(entryNode), clusterNode(node(1)), clusterNode(node(2))),
JobDescriptor.builder(concatJobClass()).units(units()).build(),
new Object[] {"a", 42});
@@ -313,7 +307,7 @@ public abstract class ItComputeBaseTest extends ClusterPerClassIntegrationTest {
void broadcastExecutesJobOnRespectiveNodes() {
Ignite entryNode = node(0);
- Map<ClusterNode, JobExecution<String>> results = entryNode.compute().submitBroadcast(
+ Map<ClusterNode, JobExecution<String>> results = compute().submitBroadcast(
Set.of(clusterNode(entryNode), clusterNode(node(1)), clusterNode(node(2))),
JobDescriptor.builder(getNodeNameJobClass()).units(units()).build(), null);
@@ -331,7 +325,7 @@ public abstract class ItComputeBaseTest extends ClusterPerClassIntegrationTest {
void broadcastsFailingJob() throws Exception {
Ignite entryNode = node(0);
- Map<ClusterNode, JobExecution<String>> results = entryNode.compute().submitBroadcast(
+ Map<ClusterNode, JobExecution<String>> results = compute().submitBroadcast(
Set.of(clusterNode(entryNode), clusterNode(node(1)), clusterNode(node(2))),
JobDescriptor.<Object, String>builder(failingJobClassName()).units(units()).build(), null);
@@ -354,9 +348,7 @@ public abstract class ItComputeBaseTest extends ClusterPerClassIntegrationTest {
void executesColocatedWithTupleKey() {
createTestTableWithOneRow();
- Ignite entryNode = node(0);
-
- String actualNodeName = entryNode.compute().execute(
+ String actualNodeName = compute().execute(
JobTarget.colocated("test", Tuple.create(Map.of("k", 1))),
JobDescriptor.builder(getNodeNameJobClass()).units(units()).build(), null);
@@ -367,9 +359,7 @@ public abstract class ItComputeBaseTest extends ClusterPerClassIntegrationTest {
void executesColocatedWithTupleKeyAsync() {
createTestTableWithOneRow();
- Ignite entryNode = node(0);
-
- JobExecution<String> execution = entryNode.compute().submit(
+ JobExecution<String> execution = compute().submit(
JobTarget.colocated("test", Tuple.create(Map.of("k", 1))),
JobDescriptor.builder(getNodeNameJobClass()).units(units()).build(), null);
@@ -384,7 +374,7 @@ public abstract class ItComputeBaseTest extends ClusterPerClassIntegrationTest {
sql("CREATE TABLE test (k int, key_int int, v int, key_str VARCHAR, CONSTRAINT PK PRIMARY KEY (key_int, key_str))");
sql("INSERT INTO test VALUES (1, 2, 3, '4')");
- String actualNodeName = node(0).compute().execute(
+ String actualNodeName = compute().execute(
JobTarget.colocated("test", Tuple.create(Map.of("key_int", 2, "key_str", "4"))),
JobDescriptor.builder(getNodeNameJobClass()).units(units()).build(), null);
assertThat(actualNodeName, in(allNodeNames()));
@@ -392,28 +382,25 @@ public abstract class ItComputeBaseTest extends ClusterPerClassIntegrationTest {
@Test
void executeColocatedThrowsTableNotFoundExceptionWhenTableDoesNotExist() {
- Ignite entryNode = node(0);
-
var ex = assertThrows(CompletionException.class,
- () -> entryNode.compute().submit(
- JobTarget.colocated("\"bad-table\"", Tuple.create(Map.of("k", 1))),
+ () -> compute().submit(
+ JobTarget.colocated("BAD_TABLE", Tuple.create(Map.of("k", 1))),
JobDescriptor.builder(getNodeNameJobClassName()).units(units()).build(), null).resultAsync().join());
assertInstanceOf(TableNotFoundException.class, ex.getCause());
- assertThat(ex.getCause().getMessage(), containsString("The table does not exist [name=\"PUBLIC\".\"bad-table\"]"));
+ assertThat(ex.getCause().getMessage(), containsString("The table does not exist [name=\"PUBLIC\".\"BAD_TABLE\"]"));
}
@ParameterizedTest(name = "local: {0}")
@ValueSource(booleans = {true, false})
void cancelComputeExecuteAsyncWithCancelHandle(boolean local) {
- Ignite entryNode = node(0);
Ignite executeNode = local ? node(0) : node(1);
CancelHandle cancelHandle = CancelHandle.create();
JobDescriptor<Long, Void> job = JobDescriptor.builder(SilentSleepJob.class).units(units()).build();
- CompletableFuture<Void> execution = entryNode.compute()
+ CompletableFuture<Void> execution = compute()
.executeAsync(JobTarget.node(clusterNode(executeNode)), job, cancelHandle.token(), 100L);
cancelHandle.cancel();
@@ -424,14 +411,13 @@ public abstract class ItComputeBaseTest extends ClusterPerClassIntegrationTest {
@ParameterizedTest(name = "local: {0}")
@ValueSource(booleans = {true, false})
void cancelComputeExecuteWithCancelHandle(boolean local) {
- Ignite entryNode = node(0);
Ignite executeNode = local ? node(0) : node(1);
CancelHandle cancelHandle = CancelHandle.create();
JobDescriptor<Long, Void> job = JobDescriptor.builder(SilentSleepJob.class).units(units()).build();
- CompletableFuture<Void> runFut = IgniteTestUtils.runAsync(() -> entryNode.compute()
+ CompletableFuture<Void> runFut = IgniteTestUtils.runAsync(() -> compute()
.execute(JobTarget.node(clusterNode(executeNode)), job, cancelHandle.token(), 100L));
cancelHandle.cancel();
@@ -448,7 +434,7 @@ public abstract class ItComputeBaseTest extends ClusterPerClassIntegrationTest {
CancelHandle cancelHandle = CancelHandle.create();
- CompletableFuture<Map<ClusterNode, Void>> executions = entryNode.compute().executeBroadcastAsync(
+ CompletableFuture<Map<ClusterNode, Void>> executions = compute().executeBroadcastAsync(
executeNodes,
JobDescriptor.builder(SilentSleepJob.class).units(units()).build(), cancelHandle.token(), 100L
);
@@ -467,7 +453,7 @@ public abstract class ItComputeBaseTest extends ClusterPerClassIntegrationTest {
CancelHandle cancelHandle = CancelHandle.create();
- CompletableFuture<Map<ClusterNode, Void>> runFut = IgniteTestUtils.runAsync(() -> entryNode.compute().executeBroadcast(
+ CompletableFuture<Map<ClusterNode, Void>> runFut = IgniteTestUtils.runAsync(() -> compute().executeBroadcast(
executeNodes,
JobDescriptor.builder(SilentSleepJob.class).units(units()).build(), cancelHandle.token(), 100L
));
@@ -479,11 +465,9 @@ public abstract class ItComputeBaseTest extends ClusterPerClassIntegrationTest {
@Test
void cancelComputeExecuteMapReduceAsyncWithCancelHandle() {
- Ignite entryNode = node(0);
-
CancelHandle cancelHandle = CancelHandle.create();
- CompletableFuture<Void> execution = entryNode.compute()
+ CompletableFuture<Void> execution = compute()
.executeMapReduceAsync(TaskDescriptor.builder(InfiniteMapReduceTask.class).build(), cancelHandle.token(), null);
cancelHandle.cancel();
@@ -508,9 +492,7 @@ public abstract class ItComputeBaseTest extends ClusterPerClassIntegrationTest {
void executesColocatedWithMappedKey() {
createTestTableWithOneRow();
- Ignite entryNode = node(0);
-
- String actualNodeName = entryNode.compute().execute(
+ String actualNodeName = compute().execute(
JobTarget.colocated("test", 1, Mapper.of(Integer.class)),
JobDescriptor.builder(getNodeNameJobClass()).units(units()).build(), null);
@@ -521,9 +503,7 @@ public abstract class ItComputeBaseTest extends ClusterPerClassIntegrationTest {
void executesColocatedWithMappedKeyAsync() {
createTestTableWithOneRow();
- Ignite entryNode = node(0);
-
- JobExecution<String> execution = entryNode.compute().submit(
+ JobExecution<String> execution = compute().submit(
JobTarget.colocated("test", 1, Mapper.of(Integer.class)),
JobDescriptor.builder(getNodeNameJobClass()).units(units()).build(), null);
@@ -534,12 +514,9 @@ public abstract class ItComputeBaseTest extends ClusterPerClassIntegrationTest {
@Test
void submitMapReduce() {
- Ignite entryNode = node(0);
-
- IgniteCompute igniteCompute = entryNode.compute();
List<DeploymentUnit> units = units();
@Nullable List<DeploymentUnit> arg = units();
- TaskExecution<Integer> taskExecution = igniteCompute.submitMapReduce(
+ TaskExecution<Integer> taskExecution = compute().submitMapReduce(
TaskDescriptor.<List<DeploymentUnit>, Integer>builder(mapReduceTaskClassName()).units(units).build(), arg);
int sumOfNodeNamesLengths = CLUSTER.runningNodes().map(Ignite::name).map(String::length).reduce(Integer::sum).orElseThrow();
@@ -555,9 +532,7 @@ public abstract class ItComputeBaseTest extends ClusterPerClassIntegrationTest {
@Test
void executeMapReduceAsync() {
- Ignite entryNode = node(0);
-
- CompletableFuture<Integer> future = entryNode.compute().executeMapReduceAsync(
+ CompletableFuture<Integer> future = compute().executeMapReduceAsync(
TaskDescriptor.<List<DeploymentUnit>, Integer>builder(mapReduceTaskClassName()).units(units()).build(),
units());
@@ -567,9 +542,7 @@ public abstract class ItComputeBaseTest extends ClusterPerClassIntegrationTest {
@Test
void executeMapReduce() {
- Ignite entryNode = node(0);
-
- int result = entryNode.compute().executeMapReduce(
+ int result = compute().executeMapReduce(
TaskDescriptor.<List<DeploymentUnit>, Integer>builder(mapReduceTaskClassName()).units(units()).build(),
units());
@@ -580,12 +553,11 @@ public abstract class ItComputeBaseTest extends ClusterPerClassIntegrationTest {
@ParameterizedTest
@ValueSource(booleans = {true, false})
void cancelsJob(boolean local) {
- Ignite entryNode = node(0);
Ignite executeNode = local ? node(0) : node(1);
// This job catches the interruption and throws a RuntimeException
JobDescriptor<Long, Void> job = JobDescriptor.builder(SleepJob.class).units(units()).build();
- JobExecution<Void> execution = entryNode.compute().submit(JobTarget.node(clusterNode(executeNode)), job, Long.MAX_VALUE);
+ JobExecution<Void> execution = compute().submit(JobTarget.node(clusterNode(executeNode)), job, Long.MAX_VALUE);
await().until(execution::stateAsync, willBe(jobStateWithStatus(EXECUTING)));
@@ -602,8 +574,7 @@ public abstract class ItComputeBaseTest extends ClusterPerClassIntegrationTest {
RuntimeException runtimeException = (RuntimeException) computeException.getCause();
// RuntimeException is thrown when SleepJob catches the InterruptedException
- assertThat(runtimeException.getCause(), instanceOf(InterruptedException.class));
- assertThat(runtimeException.getCause().getCause(), is(nullValue()));
+ assertThat(runtimeException.toString(), containsString(InterruptedException.class.getName()));
await().until(execution::stateAsync, willBe(jobStateWithStatus(CANCELED)));
}
@@ -611,12 +582,11 @@ public abstract class ItComputeBaseTest extends ClusterPerClassIntegrationTest {
@ParameterizedTest
@ValueSource(booleans = {true, false})
void cancelsNotCancellableJob(boolean local) {
- Ignite entryNode = node(0);
Ignite executeNode = local ? node(0) : node(1);
// This job catches the interruption and returns normally
JobDescriptor<Long, Void> job = JobDescriptor.builder(SilentSleepJob.class).units(units()).build();
- JobExecution<Void> execution = entryNode.compute().submit(JobTarget.node(clusterNode(executeNode)), job, Long.MAX_VALUE);
+ JobExecution<Void> execution = compute().submit(JobTarget.node(clusterNode(executeNode)), job, Long.MAX_VALUE);
await().until(execution::stateAsync, willBe(jobStateWithStatus(EXECUTING)));
@@ -640,18 +610,17 @@ public abstract class ItComputeBaseTest extends ClusterPerClassIntegrationTest {
@ParameterizedTest
@ValueSource(booleans = {true, false})
void cancelsQueuedJob(boolean local) {
- Ignite entryNode = node(0);
Ignite executeNode = local ? node(0) : node(1);
var nodes = JobTarget.node(clusterNode(executeNode));
JobDescriptor<Long, Void> job = JobDescriptor.builder(SleepJob.class).units(units()).build();
// Start 1 task in executor with 1 thread
- JobExecution<Void> execution1 = entryNode.compute().submit(nodes, job, Long.MAX_VALUE);
+ JobExecution<Void> execution1 = compute().submit(nodes, job, Long.MAX_VALUE);
await().until(execution1::stateAsync, willBe(jobStateWithStatus(EXECUTING)));
// Start one more task
- JobExecution<Void> execution2 = entryNode.compute().submit(nodes, job, Long.MAX_VALUE);
+ JobExecution<Void> execution2 = compute().submit(nodes, job, Long.MAX_VALUE);
await().until(execution2::stateAsync, willBe(jobStateWithStatus(QUEUED)));
// Task 2 is not complete, in queued state
@@ -669,11 +638,10 @@ public abstract class ItComputeBaseTest extends ClusterPerClassIntegrationTest {
@ParameterizedTest
@ValueSource(booleans = {true, false})
void changeExecutingJobPriority(boolean local) {
- Ignite entryNode = node(0);
Ignite executeNode = local ? node(0) : node(1);
JobDescriptor<Long, Void> job = JobDescriptor.builder(SleepJob.class).units(units()).build();
- JobExecution<Void> execution = entryNode.compute().submit(JobTarget.node(clusterNode(executeNode)), job, Long.MAX_VALUE);
+ JobExecution<Void> execution = compute().submit(JobTarget.node(clusterNode(executeNode)), job, Long.MAX_VALUE);
await().until(execution::stateAsync, willBe(jobStateWithStatus(EXECUTING)));
assertThat(execution.changePriorityAsync(2), willBe(false));
@@ -682,11 +650,10 @@ public abstract class ItComputeBaseTest extends ClusterPerClassIntegrationTest {
@Test
void tupleSerialization() {
- Ignite entryNode = node(0);
ClusterNode executeNode = clusterNode(node(1));
// Execute the job on remote node to trigger serialization
- Integer result = entryNode.compute().execute(
+ Integer result = compute().execute(
JobTarget.node(executeNode),
JobDescriptor.builder(TupleJob.class).units(units()).build(),
Tuple.create().set("COUNT", 1)
@@ -695,6 +662,35 @@ public abstract class ItComputeBaseTest extends ClusterPerClassIntegrationTest {
assertThat(result, is(1));
}
+ @MethodSource("tupleCollections")
+ @ParameterizedTest
+ void tupleCollectionSerialization(Collection<Tuple> arg) {
+ List<Tuple> expected = new ArrayList<>(arg);
+ expected.add(Tuple.create().set("job-result", "done"));
+
+ for (int nodeIdx = 0; nodeIdx < initialNodes(); nodeIdx++) {
+ ClusterNode executeNode = clusterNode(node(nodeIdx));
+
+ Collection<Tuple> result = compute().execute(
+ JobTarget.node(executeNode),
+ JobDescriptor.builder(TupleCollectionJob.class).units(units()).build(),
+ arg
+ );
+
+ assertIterableEquals(expected, result);
+ }
+ }
+
+ private static Stream<Arguments> tupleCollections() {
+ return Stream.of(
+ List.of(),
+ Collections.singletonList(Tuple.create()),
+ Collections.singletonList(null),
+ List.of(Tuple.create(), Tuple.create().set("key", 1), Tuple.create().set("key", "value1")),
+ Set.of(Tuple.create().set("key", 2), Tuple.create().set("key", "value2"))
+ ).map(Arguments::of);
+ }
+
static String concatJobClassName() {
return ConcatJob.class.getName();
}
@@ -730,7 +726,7 @@ public abstract class ItComputeBaseTest extends ClusterPerClassIntegrationTest {
private static void assertComputeException(Exception ex, String causeClass, String causeMsgSubstring) {
assertTraceableException(ex, ComputeException.class, COMPUTE_JOB_FAILED_ERR, "Job execution failed:");
Throwable cause = ExceptionUtils.unwrapCause(ex);
- assertThat(cause.getCause().getClass().getName(), containsString(causeClass));
+ assertThat(cause.toString(), containsString(causeClass));
assertThat(cause.getCause().getMessage(), containsString(causeMsgSubstring));
}
}
diff --git a/modules/compute/src/integrationTest/java/org/apache/ignite/internal/compute/ItComputeTestClient.java b/modules/compute/src/integrationTest/java/org/apache/ignite/internal/compute/ItComputeTestClient.java
new file mode 100644
index 0000000000..a3ab1a64f3
--- /dev/null
+++ b/modules/compute/src/integrationTest/java/org/apache/ignite/internal/compute/ItComputeTestClient.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.compute;
+
+import static org.apache.ignite.internal.TestWrappers.unwrapIgniteImpl;
+
+import org.apache.ignite.client.IgniteClient;
+import org.apache.ignite.compute.IgniteCompute;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Disabled;
+
+/**
+ * Integration tests for Compute functionality using thin client API.
+ */
+@SuppressWarnings("NewClassNamingConvention")
+public class ItComputeTestClient extends ItComputeTestEmbedded {
+ private IgniteClient client;
+
+ @BeforeEach
+ void startClient() {
+ int port = unwrapIgniteImpl(node(0)).clientAddress().port();
+
+ client = IgniteClient.builder()
+ .addresses("localhost:" + port)
+ .build();
+ }
+
+ @AfterEach
+ void stopClient() {
+ client.close();
+ }
+
+ @Override
+ protected IgniteCompute compute() {
+ return client.compute();
+ }
+
+ @Override
+ void cancelsNotCancellableJob(boolean local) {
+ // No-op. Embedded-specific.
+ }
+
+ @Override
+ @Disabled("IGNITE-24059 Compute argument serialization is inconsistent in embedded and thin client")
+ void executesJobLocally() {
+ super.executesJobLocally();
+ }
+
+ @Override
+ @Disabled("IGNITE-24059 Compute argument serialization is inconsistent in embedded and thin client")
+ void executesJobLocallyAsync() {
+ super.executesJobLocallyAsync();
+ }
+
+ @Override
+ @Disabled("IGNITE-24059 Compute argument serialization is inconsistent in embedded and thin client")
+ void broadcastsJobWithArgumentsAsync() {
+ super.broadcastsJobWithArgumentsAsync();
+ }
+
+ @Override
+ @Disabled("IGNITE-24059 Compute argument serialization is inconsistent in embedded and thin client")
+ void executesJobOnRemoteNodes() {
+ super.executesJobOnRemoteNodes();
+ }
+
+ @Override
+ @Disabled("IGNITE-24059 Compute argument serialization is inconsistent in embedded and thin client")
+ void executesJobOnRemoteNodesAsync() {
+ super.executesJobOnRemoteNodesAsync();
+ }
+}
diff --git a/modules/compute/src/integrationTest/java/org/apache/ignite/internal/compute/ItComputeTestEmbedded.java b/modules/compute/src/integrationTest/java/org/apache/ignite/internal/compute/ItComputeTestEmbedded.java
index f41fdea57a..99b01ad130 100644
--- a/modules/compute/src/integrationTest/java/org/apache/ignite/internal/compute/ItComputeTestEmbedded.java
+++ b/modules/compute/src/integrationTest/java/org/apache/ignite/internal/compute/ItComputeTestEmbedded.java
@@ -71,7 +71,7 @@ import org.junit.jupiter.params.provider.MethodSource;
/**
* Integration tests for Compute functionality in embedded Ignite mode.
*/
-@SuppressWarnings("resource")
+@SuppressWarnings({"NewClassNamingConvention"})
class ItComputeTestEmbedded extends ItComputeBaseTest {
@Override
@@ -142,9 +142,9 @@ class ItComputeTestEmbedded extends ItComputeBaseTest {
JobExecution<String> execution3 = entryNode.compute().submit(
jobTarget,
JobDescriptor.builder(WaitLatchThrowExceptionOnFirstExecutionJob.class)
- .units(units())
- .options(options)
- .build(),
+ .units(units())
+ .options(options)
+ .build(),
countDownLatch);
await().until(execution3::stateAsync, willBe(jobStateWithStatus(QUEUED)));
diff --git a/modules/compute/src/jobs/java/org/apache/ignite/internal/compute/TupleCollectionJob.java b/modules/compute/src/jobs/java/org/apache/ignite/internal/compute/TupleCollectionJob.java
new file mode 100644
index 0000000000..72d0dbb5ff
--- /dev/null
+++ b/modules/compute/src/jobs/java/org/apache/ignite/internal/compute/TupleCollectionJob.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.compute;
+
+import static java.util.concurrent.CompletableFuture.completedFuture;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+import org.apache.ignite.compute.ComputeJob;
+import org.apache.ignite.compute.JobExecutionContext;
+import org.apache.ignite.table.Tuple;
+
+/**
+ * Returns the provided tuple collection with added job result tuple.
+ */
+public class TupleCollectionJob implements ComputeJob<Collection<Tuple>, Collection<Tuple>> {
+ @Override
+ public CompletableFuture<Collection<Tuple>> executeAsync(JobExecutionContext jobExecutionContext, Collection<Tuple> arg) {
+ List<Tuple> res = new ArrayList<>(arg);
+ res.add(Tuple.create().set("job-result", "done"));
+
+ return completedFuture(res);
+ }
+}
diff --git a/modules/compute/src/main/java/org/apache/ignite/internal/compute/ComputeUtils.java b/modules/compute/src/main/java/org/apache/ignite/internal/compute/ComputeUtils.java
index 673ca83c9b..3f0664d1a7 100644
--- a/modules/compute/src/main/java/org/apache/ignite/internal/compute/ComputeUtils.java
+++ b/modules/compute/src/main/java/org/apache/ignite/internal/compute/ComputeUtils.java
@@ -23,6 +23,7 @@ import static org.apache.ignite.internal.compute.ComputeJobDataType.MARSHALLED_C
import static org.apache.ignite.internal.compute.ComputeJobDataType.NATIVE;
import static org.apache.ignite.internal.compute.ComputeJobDataType.POJO;
import static org.apache.ignite.internal.compute.ComputeJobDataType.TUPLE;
+import static org.apache.ignite.internal.compute.ComputeJobDataType.TUPLE_COLLECTION;
import static org.apache.ignite.internal.compute.PojoConverter.fromTuple;
import static org.apache.ignite.internal.compute.PojoConverter.toTuple;
import static org.apache.ignite.internal.util.ExceptionUtils.unwrapCause;
@@ -33,6 +34,9 @@ import static org.apache.ignite.lang.ErrorGroups.Compute.MARSHALLING_TYPE_MISMAT
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.List;
@@ -443,9 +447,10 @@ public class ComputeUtils {
);
}
switch (type) {
- case NATIVE:
+ case NATIVE: {
var reader = new BinaryTupleReader(3, argumentHolder.data());
return (T) ClientBinaryTupleUtils.readObject(reader, 0);
+ }
case TUPLE: // Fallthrough TODO https://issues.apache.org/jira/browse/IGNITE-23320
case POJO:
@@ -465,6 +470,27 @@ public class ComputeUtils {
throw new ComputeException(MARSHALLING_TYPE_MISMATCH_ERR, "Exception in user-defined marshaller", ex);
}
+ case TUPLE_COLLECTION: {
+ // TODO: IGNITE-24059 Deduplicate with ClientComputeJobUnpacker.
+ ByteBuffer collectionBuf = ByteBuffer.wrap(argumentHolder.data()).order(ByteOrder.LITTLE_ENDIAN);
+ int count = collectionBuf.getInt();
+ BinaryTupleReader reader = new BinaryTupleReader(count, collectionBuf.slice().order(ByteOrder.LITTLE_ENDIAN));
+
+ List<Tuple> res = new ArrayList<>(count);
+ for (int i = 0; i < count; i++) {
+ ByteBuffer elementBytes = reader.bytesValueAsBuffer(i);
+
+ if (elementBytes == null) {
+ res.add(null);
+ continue;
+ }
+
+ res.add(TupleWithSchemaMarshalling.unmarshal(elementBytes));
+ }
+
+ return (T) res;
+ }
+
default:
throw new ComputeException(MARSHALLING_TYPE_MISMATCH_ERR, "Unexpected job argument type: " + type);
}
@@ -520,6 +546,37 @@ public class ComputeUtils {
return new ComputeJobDataHolder(TUPLE, TupleWithSchemaMarshalling.marshal(tuple));
}
+ if (result instanceof Collection) {
+ // TODO: IGNITE-24059 Deduplicate with ClientComputeJobPacker.
+ Collection<?> col = (Collection<?>) result;
+
+ // Pack entire collection into a single binary blob.
+ BinaryTupleBuilder tupleBuilder = new BinaryTupleBuilder(col.size());
+
+ for (Object el : col) {
+ if (el == null) {
+ tupleBuilder.appendNull();
+ continue;
+ }
+
+ if (!(el instanceof Tuple)) {
+ throw new MarshallingException("Can't pack collection: expected Tuple, but got " + el.getClass(), null);
+ }
+
+ tupleBuilder.appendBytes(TupleWithSchemaMarshalling.marshal((Tuple) el));
+ }
+
+ ByteBuffer binTupleBytes = tupleBuilder.build();
+
+ byte[] resArr = new byte[Integer.BYTES + binTupleBytes.remaining()];
+ ByteBuffer resBuf = ByteBuffer.wrap(resArr).order(ByteOrder.LITTLE_ENDIAN);
+ resBuf.putInt(col.size());
+ resBuf.put(binTupleBytes);
+
+ return new ComputeJobDataHolder(TUPLE_COLLECTION, resArr);
+ }
+
+
if (isNativeType(result.getClass())) {
// Builder with inline schema.
// Value is represented by 3 tuple elements: type, scale, value.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/compute/ComputeJobDataType.java b/modules/core/src/main/java/org/apache/ignite/internal/compute/ComputeJobDataType.java
index ee95873415..3e56081927 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/compute/ComputeJobDataType.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/compute/ComputeJobDataType.java
@@ -28,7 +28,8 @@ public enum ComputeJobDataType {
NATIVE(0),
TUPLE(1),
MARSHALLED_CUSTOM(2),
- POJO(3);
+ POJO(3),
+ TUPLE_COLLECTION(4);
private static final ComputeJobDataType[] VALUES = values();
diff --git a/modules/runner/src/testFixtures/java/org/apache/ignite/internal/ClusterPerClassIntegrationTest.java b/modules/runner/src/testFixtures/java/org/apache/ignite/internal/ClusterPerClassIntegrationTest.java
index fdbe334f7d..92f21b8df0 100644
--- a/modules/runner/src/testFixtures/java/org/apache/ignite/internal/ClusterPerClassIntegrationTest.java
+++ b/modules/runner/src/testFixtures/java/org/apache/ignite/internal/ClusterPerClassIntegrationTest.java
@@ -87,6 +87,7 @@ public abstract class ClusterPerClassIntegrationTest extends BaseIgniteAbstractT
+ " " + DEFAULT_ROCKSDB_PROFILE_NAME + ".engine: rocksdb"
+ " },\n"
+ " clientConnector.port: {},\n"
+ + " clientConnector.sendServerExceptionStackTraceToClient: true,\n"
+ " rest.port: {},\n"
+ " compute.threadPoolSize: 1,\n"
+ " failureHandler.dumpThreadsOnFailure: false\n"