You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by ig...@apache.org on 2021/09/23 13:42:09 UTC
[flink-statefun] branch master updated: [FLINK-24364][e2e] Add
message cancellation embedded e2e case
This is an automated email from the ASF dual-hosted git repository.
igal pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink-statefun.git
The following commit(s) were added to refs/heads/master by this push:
new 84f6ce0 [FLINK-24364][e2e] Add message cancellation embedded e2e case
84f6ce0 is described below
commit 84f6ce0cc7dac0bb474f308a3911812e36aa7bee
Author: Igal Shilman <ig...@apache.org>
AuthorDate: Thu Sep 23 14:49:09 2021 +0200
[FLINK-24364][e2e] Add message cancellation embedded e2e case
This commit adds a message cancellation test case for the smoke e2e test.
The generation of message cancellation commands is disabled by default, and
only enabled for the embedded sdk e2e tests.
---
.../statefun/e2e/smoke/SmokeRunnerParameters.java | 19 +++++++++++++++
.../src/main/protobuf/commands.proto | 6 +++++
.../e2e/smoke/driver/CommandGenerator.java | 22 ++++++++++++++++++
.../e2e/smoke/embedded/CommandInterpreter.java | 27 ++++++++++++++++++++--
.../smoke/embedded/EmbeddedSmokeHarnessTest.java | 1 +
.../embedded/SmokeVerificationEmbeddedE2E.java | 1 +
6 files changed, 74 insertions(+), 2 deletions(-)
diff --git a/statefun-e2e-tests/statefun-smoke-e2e-common/src/main/java/org/apache/flink/statefun/e2e/smoke/SmokeRunnerParameters.java b/statefun-e2e-tests/statefun-smoke-e2e-common/src/main/java/org/apache/flink/statefun/e2e/smoke/SmokeRunnerParameters.java
index 2f63a5d..ef33fd0 100644
--- a/statefun-e2e-tests/statefun-smoke-e2e-common/src/main/java/org/apache/flink/statefun/e2e/smoke/SmokeRunnerParameters.java
+++ b/statefun-e2e-tests/statefun-smoke-e2e-common/src/main/java/org/apache/flink/statefun/e2e/smoke/SmokeRunnerParameters.java
@@ -35,6 +35,7 @@ public final class SmokeRunnerParameters implements Serializable {
private double stateModificationsPr = 0.4;
private double sendPr = 0.9;
private double sendAfterPr = 0.1;
+ private double sendAfterWithCancellationPr = 0.1;
private double asyncSendPr = 0.1;
private double noopPr = 0.2;
private double sendEgressPr = 0.03;
@@ -42,6 +43,8 @@ public final class SmokeRunnerParameters implements Serializable {
private String verificationServerHost = "localhost";
private int verificationServerPort = 5050;
private boolean isAsyncOpSupported = false;
+ private boolean isDelayCancellationOpSupported = false;
+
private long randomGeneratorSeed = System.nanoTime();
/** Creates an instance of ModuleParameters from a key-value map. */
@@ -108,6 +111,14 @@ public final class SmokeRunnerParameters implements Serializable {
return sendAfterPr;
}
+ public boolean isDelayCancellationOpSupported() {
+ return isDelayCancellationOpSupported;
+ }
+
+ public void setDelayCancellationOpSupported(boolean delayCancellationOpSupported) {
+ isDelayCancellationOpSupported = delayCancellationOpSupported;
+ }
+
public void setSendAfterPr(double sendAfterPr) {
this.sendAfterPr = sendAfterPr;
}
@@ -160,6 +171,14 @@ public final class SmokeRunnerParameters implements Serializable {
this.verificationServerPort = verificationServerPort;
}
+ public double getSendAfterWithCancellationPr() {
+ return sendAfterWithCancellationPr;
+ }
+
+ public void setSendAfterWithCancellationPr(double sendAfterWithCancellationPr) {
+ this.sendAfterWithCancellationPr = sendAfterWithCancellationPr;
+ }
+
public boolean isAsyncOpSupported() {
return isAsyncOpSupported;
}
diff --git a/statefun-e2e-tests/statefun-smoke-e2e-common/src/main/protobuf/commands.proto b/statefun-e2e-tests/statefun-smoke-e2e-common/src/main/protobuf/commands.proto
index e3912d1..8bcd4cc 100644
--- a/statefun-e2e-tests/statefun-smoke-e2e-common/src/main/protobuf/commands.proto
+++ b/statefun-e2e-tests/statefun-smoke-e2e-common/src/main/protobuf/commands.proto
@@ -42,6 +42,11 @@ message Command {
message SendAfter {
int32 target = 1;
Commands commands = 2;
+ string cancellation_token = 3;
+ }
+ message CancelSendAfter {
+ int32 target = 1;
+ string cancellation_token = 2;
}
message SendEgress {
}
@@ -60,6 +65,7 @@ message Command {
SendEgress send_egress = 4;
AsyncOperation async_operation = 5;
Verify verify = 6;
+ CancelSendAfter cancel_send_after = 7;
}
}
diff --git a/statefun-e2e-tests/statefun-smoke-e2e-driver/src/main/java/org/apache/flink/statefun/e2e/smoke/driver/CommandGenerator.java b/statefun-e2e-tests/statefun-smoke-e2e-driver/src/main/java/org/apache/flink/statefun/e2e/smoke/driver/CommandGenerator.java
index fa399ee..607ac61 100644
--- a/statefun-e2e-tests/statefun-smoke-e2e-driver/src/main/java/org/apache/flink/statefun/e2e/smoke/driver/CommandGenerator.java
+++ b/statefun-e2e-tests/statefun-smoke-e2e-driver/src/main/java/org/apache/flink/statefun/e2e/smoke/driver/CommandGenerator.java
@@ -23,6 +23,7 @@ import static org.apache.commons.math3.util.Pair.create;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
+import java.util.UUID;
import java.util.function.Supplier;
import org.apache.commons.math3.distribution.EnumeratedDistribution;
import org.apache.commons.math3.random.RandomGenerator;
@@ -85,9 +86,13 @@ public final class CommandGenerator implements Supplier<SourceCommand> {
create(new SendAfterGen(), parameters.getSendAfterPr()),
create(new Noop(), parameters.getNoopPr()),
create(new SendEgress(), parameters.getSendEgressPr())));
+
if (parameters.isAsyncOpSupported()) {
list.add(create(new SendAsyncOp(), parameters.getAsyncSendPr()));
}
+ if (parameters.isDelayCancellationOpSupported()) {
+ list.add(create(new SendAfterCancellationGen(), parameters.getSendAfterWithCancellationPr()));
+ }
return list;
}
@@ -141,6 +146,23 @@ public final class CommandGenerator implements Supplier<SourceCommand> {
}
}
+ private final class SendAfterCancellationGen implements Gen {
+
+ @Override
+ public void generate(Commands.Builder builder, int depth) {
+ final String token = new UUID(random.nextLong(), random.nextLong()).toString();
+ final int address = address();
+
+ Command.SendAfter.Builder first =
+ Command.SendAfter.newBuilder().setTarget(address).setCancellationToken(token);
+ Command.CancelSendAfter.Builder second =
+ Command.CancelSendAfter.newBuilder().setTarget(address).setCancellationToken(token);
+
+ builder.addCommand(Command.newBuilder().setSendAfter(first));
+ builder.addCommand(Command.newBuilder().setCancelSendAfter(second));
+ }
+ }
+
private final class SendGen implements Gen {
@Override
diff --git a/statefun-e2e-tests/statefun-smoke-e2e-embedded/src/main/java/org/apache/flink/statefun/e2e/smoke/embedded/CommandInterpreter.java b/statefun-e2e-tests/statefun-smoke-e2e-embedded/src/main/java/org/apache/flink/statefun/e2e/smoke/embedded/CommandInterpreter.java
index a4da76d..f184929 100644
--- a/statefun-e2e-tests/statefun-smoke-e2e-embedded/src/main/java/org/apache/flink/statefun/e2e/smoke/embedded/CommandInterpreter.java
+++ b/statefun-e2e-tests/statefun-smoke-e2e-embedded/src/main/java/org/apache/flink/statefun/e2e/smoke/embedded/CommandInterpreter.java
@@ -17,7 +17,13 @@
*/
package org.apache.flink.statefun.e2e.smoke.embedded;
-import static org.apache.flink.statefun.e2e.smoke.driver.Types.*;
+import static org.apache.flink.statefun.e2e.smoke.driver.Types.COMMANDS_TYPE;
+import static org.apache.flink.statefun.e2e.smoke.driver.Types.SOURCE_COMMANDS_TYPE;
+import static org.apache.flink.statefun.e2e.smoke.driver.Types.isTypeOf;
+import static org.apache.flink.statefun.e2e.smoke.driver.Types.packCommands;
+import static org.apache.flink.statefun.e2e.smoke.driver.Types.packVerificationResult;
+import static org.apache.flink.statefun.e2e.smoke.driver.Types.unpackCommands;
+import static org.apache.flink.statefun.e2e.smoke.driver.Types.unpackSourceCommand;
import java.time.Duration;
import java.util.Objects;
@@ -28,6 +34,7 @@ import org.apache.flink.statefun.e2e.smoke.generated.Command;
import org.apache.flink.statefun.e2e.smoke.generated.Commands;
import org.apache.flink.statefun.e2e.smoke.generated.SourceCommand;
import org.apache.flink.statefun.e2e.smoke.generated.VerificationResult;
+import org.apache.flink.statefun.sdk.Address;
import org.apache.flink.statefun.sdk.AsyncOperationResult;
import org.apache.flink.statefun.sdk.Context;
import org.apache.flink.statefun.sdk.FunctionType;
@@ -81,6 +88,8 @@ public final class CommandInterpreter {
sendEgress(state, context, cmd.getSendEgress());
} else if (cmd.hasVerify()) {
verify(state, context, cmd.getVerify());
+ } else if (cmd.hasCancelSendAfter()) {
+ cancelSendAfter(state, context, cmd.getCancelSendAfter());
}
}
}
@@ -114,7 +123,21 @@ public final class CommandInterpreter {
Command.SendAfter send) {
FunctionType functionType = Constants.FN_TYPE;
String id = ids.idOf(send.getTarget());
- context.sendAfter(sendAfterDelay, functionType, id, packCommands(send.getCommands()));
+ TypedValue subCommands = packCommands(send.getCommands());
+ if (send.getCancellationToken().isEmpty()) {
+ context.sendAfter(sendAfterDelay, functionType, id, subCommands);
+ } else {
+ context.sendAfter(
+ sendAfterDelay, new Address(functionType, id), subCommands, send.getCancellationToken());
+ }
+ }
+
+ private void cancelSendAfter(
+ @SuppressWarnings("unused") PersistedValue<Long> state,
+ Context context,
+ Command.CancelSendAfter cancelSendAfter) {
+ String token = cancelSendAfter.getCancellationToken();
+ context.cancelDelayedMessage(token);
}
private void send(
diff --git a/statefun-e2e-tests/statefun-smoke-e2e-embedded/src/test/java/org/apache/flink/statefun/e2e/smoke/embedded/EmbeddedSmokeHarnessTest.java b/statefun-e2e-tests/statefun-smoke-e2e-embedded/src/test/java/org/apache/flink/statefun/e2e/smoke/embedded/EmbeddedSmokeHarnessTest.java
index 925ebd1..6760418 100644
--- a/statefun-e2e-tests/statefun-smoke-e2e-embedded/src/test/java/org/apache/flink/statefun/e2e/smoke/embedded/EmbeddedSmokeHarnessTest.java
+++ b/statefun-e2e-tests/statefun-smoke-e2e-embedded/src/test/java/org/apache/flink/statefun/e2e/smoke/embedded/EmbeddedSmokeHarnessTest.java
@@ -61,6 +61,7 @@ public class EmbeddedSmokeHarnessTest {
parameters.setVerificationServerHost("localhost");
parameters.setVerificationServerPort(started.port());
parameters.setAsyncOpSupported(true);
+ parameters.setDelayCancellationOpSupported(true);
parameters.asMap().forEach(harness::withGlobalConfiguration);
// run the harness.
diff --git a/statefun-e2e-tests/statefun-smoke-e2e-embedded/src/test/java/org/apache/flink/statefun/e2e/smoke/embedded/SmokeVerificationEmbeddedE2E.java b/statefun-e2e-tests/statefun-smoke-e2e-embedded/src/test/java/org/apache/flink/statefun/e2e/smoke/embedded/SmokeVerificationEmbeddedE2E.java
index cc7a0b4..63f3afb 100644
--- a/statefun-e2e-tests/statefun-smoke-e2e-embedded/src/test/java/org/apache/flink/statefun/e2e/smoke/embedded/SmokeVerificationEmbeddedE2E.java
+++ b/statefun-e2e-tests/statefun-smoke-e2e-embedded/src/test/java/org/apache/flink/statefun/e2e/smoke/embedded/SmokeVerificationEmbeddedE2E.java
@@ -34,6 +34,7 @@ public class SmokeVerificationEmbeddedE2E {
parameters.setMessageCount(100_000);
parameters.setMaxFailures(1);
parameters.setAsyncOpSupported(true);
+ parameters.setDelayCancellationOpSupported(true);
StatefulFunctionsAppContainers.Builder builder =
StatefulFunctionsAppContainers.builder("flink-statefun-cluster", NUM_WORKERS);