You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by dg...@apache.org on 2019/03/13 09:29:29 UTC
[ignite] branch master updated: IGNITE-11377 Display time to
baseline auto-adjust event in console.sh - Fixes #6178.
This is an automated email from the ASF dual-hosted git repository.
dgovorukhin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite.git
The following commit(s) were added to refs/heads/master by this push:
new ec728ac IGNITE-11377 Display time to baseline auto-adjust event in console.sh - Fixes #6178.
ec728ac is described below
commit ec728ac590b0286b0dc18b4cdbf23aba672d5afd
Author: Anton Kalashnikov <ka...@yandex.ru>
AuthorDate: Wed Mar 13 12:19:45 2019 +0300
IGNITE-11377 Display time to baseline auto-adjust event in console.sh - Fixes #6178.
Signed-off-by: Dmitriy Govorukhin <dm...@gmail.com>
---
.../ignite/internal/commandline/Arguments.java | 25 +-
.../internal/commandline/CommandHandler.java | 407 +++++++--------------
.../commandline/baseline/AutoAdjustCommandArg.java | 50 +++
.../commandline/baseline/BaselineArguments.java | 165 +++++++++
.../commandline/baseline/BaselineCommand.java | 103 ++++++
.../processors/cluster/ClusterProcessor.java | 16 +-
.../autoadjust/BaselineAutoAdjustData.java | 45 ++-
.../autoadjust/BaselineAutoAdjustExecutor.java | 2 +
.../autoadjust/BaselineAutoAdjustScheduler.java | 17 +-
.../autoadjust/BaselineAutoAdjustStatistic.java | 90 +++++
.../baseline/autoadjust/ChangeTopologyWatcher.java | 24 +-
.../baseline/VisorBaselineAutoAdjustSettings.java | 30 +-
.../internal/visor/baseline/VisorBaselineTask.java | 19 +-
.../visor/baseline/VisorBaselineTaskResult.java | 39 +-
.../visor/baseline/VisorBaselineViewTask.java | 7 +-
.../commandline/CommandHandlerParsingTest.java | 4 +-
.../apache/ignite/util/GridCommandHandlerTest.java | 46 ++-
17 files changed, 734 insertions(+), 355 deletions(-)
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/Arguments.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/Arguments.java
index abd27bd..b16a142 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/Arguments.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/Arguments.java
@@ -18,6 +18,7 @@
package org.apache.ignite.internal.commandline;
import org.apache.ignite.internal.client.GridClientConfiguration;
+import org.apache.ignite.internal.commandline.baseline.BaselineArguments;
import org.apache.ignite.internal.commandline.cache.CacheArguments;
import org.apache.ignite.internal.visor.tx.VisorTxTaskArg;
@@ -44,14 +45,9 @@ public class Arguments {
private boolean autoConfirmation;
/**
- * Action for baseline command.
- */
- private String baselineAct;
-
- /**
* Arguments for baseline command.
*/
- private String baselineArgs;
+ private BaselineArguments baselineArgs;
/** Transaction arguments. */
private final VisorTxTaskArg txArg;
@@ -110,7 +106,6 @@ public class Arguments {
* @param port Port.
* @param user User.
* @param pwd Password.
- * @param baselineAct Baseline action.
* @param baselineArgs Baseline args.
* @param txArg TX arg.
* @param cacheArgs --cache subcommand arguments.
@@ -129,8 +124,8 @@ public class Arguments {
* @param sslTrustStorePassword Truststore Password.
* @param sslTrustStoreType Truststore Type.
*/
- public Arguments(Command cmd, String host, String port, String user, String pwd, String baselineAct,
- String baselineArgs, VisorTxTaskArg txArg, CacheArguments cacheArgs, String walAct, String walArgs,
+ public Arguments(Command cmd, String host, String port, String user, String pwd,
+ BaselineArguments baselineArgs, VisorTxTaskArg txArg, CacheArguments cacheArgs, String walAct, String walArgs,
Long pingTimeout, Long pingInterval, boolean autoConfirmation,
String sslProtocol, String sslCipherSuites, String sslKeyAlgorithm,
String sslKeyStorePath, char[] sslKeyStorePassword, String sslKeyStoreType,
@@ -142,7 +137,6 @@ public class Arguments {
this.user = user;
this.pwd = pwd;
- this.baselineAct = baselineAct;
this.baselineArgs = baselineArgs;
this.txArg = txArg;
@@ -219,16 +213,9 @@ public class Arguments {
}
/**
- * @return baseline action
- */
- public String baselineAction() {
- return baselineAct;
- }
-
- /**
- * @return baseline arguments
+ * @return Baseline arguments.
*/
- public String baselineArguments() {
+ public BaselineArguments baselineArguments() {
return baselineArgs;
}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
index e254c73..f0c3ba3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
@@ -56,6 +56,9 @@ import org.apache.ignite.internal.client.GridServerUnreachableException;
import org.apache.ignite.internal.client.impl.connection.GridClientConnectionResetException;
import org.apache.ignite.internal.client.ssl.GridSslBasicContextFactory;
import org.apache.ignite.internal.commandline.argument.CommandArgUtils;
+import org.apache.ignite.internal.commandline.baseline.AutoAdjustCommandArg;
+import org.apache.ignite.internal.commandline.baseline.BaselineArguments;
+import org.apache.ignite.internal.commandline.baseline.BaselineCommand;
import org.apache.ignite.internal.commandline.cache.CacheArguments;
import org.apache.ignite.internal.commandline.cache.CacheCommand;
import org.apache.ignite.internal.commandline.cache.argument.DistributionCommandArg;
@@ -134,6 +137,7 @@ import org.apache.ignite.plugin.security.SecurityCredentialsBasicProvider;
import org.apache.ignite.plugin.security.SecurityCredentialsProvider;
import org.apache.ignite.ssl.SslContextFactory;
+import static java.lang.Boolean.TRUE;
import static org.apache.ignite.IgniteSystemProperties.IGNITE_ENABLE_EXPERIMENTAL_COMMAND;
import static org.apache.ignite.internal.IgniteVersionUtils.ACK_VER_STR;
import static org.apache.ignite.internal.IgniteVersionUtils.COPYRIGHT;
@@ -146,6 +150,7 @@ import static org.apache.ignite.internal.commandline.Command.TX;
import static org.apache.ignite.internal.commandline.Command.WAL;
import static org.apache.ignite.internal.commandline.OutputFormat.MULTI_LINE;
import static org.apache.ignite.internal.commandline.OutputFormat.SINGLE_LINE;
+import static org.apache.ignite.internal.commandline.baseline.BaselineCommand.of;
import static org.apache.ignite.internal.commandline.cache.CacheCommand.CONTENTION;
import static org.apache.ignite.internal.commandline.cache.CacheCommand.DISTRIBUTION;
import static org.apache.ignite.internal.commandline.cache.CacheCommand.HELP;
@@ -165,12 +170,6 @@ import static org.apache.ignite.internal.commandline.cache.argument.ListCommandA
import static org.apache.ignite.internal.commandline.cache.argument.ListCommandArg.SEQUENCE;
import static org.apache.ignite.internal.commandline.cache.argument.ValidateIndexesCommandArg.CHECK_FIRST;
import static org.apache.ignite.internal.commandline.cache.argument.ValidateIndexesCommandArg.CHECK_THROUGH;
-import static org.apache.ignite.internal.visor.baseline.VisorBaselineOperation.ADD;
-import static org.apache.ignite.internal.visor.baseline.VisorBaselineOperation.AUTOADJUST;
-import static org.apache.ignite.internal.visor.baseline.VisorBaselineOperation.COLLECT;
-import static org.apache.ignite.internal.visor.baseline.VisorBaselineOperation.REMOVE;
-import static org.apache.ignite.internal.visor.baseline.VisorBaselineOperation.SET;
-import static org.apache.ignite.internal.visor.baseline.VisorBaselineOperation.VERSION;
import static org.apache.ignite.internal.visor.verify.VisorViewCacheCmd.CACHES;
import static org.apache.ignite.internal.visor.verify.VisorViewCacheCmd.GROUPS;
import static org.apache.ignite.internal.visor.verify.VisorViewCacheCmd.SEQ;
@@ -280,30 +279,6 @@ public class CommandHandler {
public static final String CONFIRM_MSG = "y";
/** */
- private static final String BASELINE_ADD = "add";
-
- /** */
- private static final String BASELINE_REMOVE = "remove";
-
- /** */
- private static final String BASELINE_COLLECT = "collect";
-
- /** */
- private static final String BASELINE_SET = "set";
-
- /** */
- private static final String BASELINE_SET_VERSION = "version";
-
- /** */
- private static final String BASELINE_AUTO_ADJUST = "autoadjust";
-
- /** */
- private static final String BASELINE_AUTO_ADJUST_ENABLE = "enable";
-
- /** */
- private static final String BASELINE_AUTO_ADJUST_DISABLE = "disable";
-
- /** */
static final String WAL_PRINT = "print";
/** */
@@ -571,7 +546,7 @@ public class CommandHandler {
break;
case BASELINE:
- if (!BASELINE_COLLECT.equals(args.baselineAction()))
+ if (BaselineCommand.COLLECT != args.baselineArguments().getCmd())
str = "Warning: the command will perform changes in baseline.";
break;
@@ -694,8 +669,7 @@ public class CommandHandler {
.flatMap(node -> Stream.concat(
node.tcpAddresses() == null ? Stream.empty() : node.tcpAddresses().stream(),
node.tcpHostNames() == null ? Stream.empty() : node.tcpHostNames().stream()
- )
- .map(addr -> new IgniteBiTuple<>(node, addr + ":" + node.tcpPort())));
+ ).map(addr -> new IgniteBiTuple<>(node, addr + ":" + node.tcpPort())));
}
/**
@@ -713,7 +687,7 @@ public class CommandHandler {
node.tcpAddresses() == null ? Stream.empty() : node.tcpAddresses().stream(),
node.tcpHostNames() == null ? Stream.empty() : node.tcpHostNames().stream()
)
- .map(addr -> addr + ":" + node.tcpPort()).collect(Collectors.toList())
+ .map(addr -> addr + ":" + node.tcpPort()).collect(Collectors.toList())
)
);
}
@@ -1214,7 +1188,7 @@ public class CommandHandler {
IdleVerifyResultV2 res = executeTask(
client,
VisorIdleVerifyTaskV2.class,
- new VisorIdleVerifyTaskArg(cacheArgs.caches(),cacheArgs.excludeCaches(), cacheArgs.idleCheckCrc())
+ new VisorIdleVerifyTaskArg(cacheArgs.caches(), cacheArgs.excludeCaches(), cacheArgs.idleCheckCrc())
);
res.print(System.out::print);
@@ -1224,86 +1198,59 @@ public class CommandHandler {
* Change baseline.
*
* @param client Client.
- * @param baselineAct Baseline action to execute. @throws GridClientException If failed to execute baseline
- * action.
* @param baselineArgs Baseline action arguments.
* @throws Throwable If failed to execute baseline action.
*/
- private void baseline(GridClient client, String baselineAct, String baselineArgs) throws Throwable {
- switch (baselineAct) {
- case BASELINE_ADD:
- baselineAdd(client, baselineArgs);
- break;
-
- case BASELINE_REMOVE:
- baselineRemove(client, baselineArgs);
- break;
-
- case BASELINE_SET:
- baselineSet(client, baselineArgs);
- break;
-
- case BASELINE_SET_VERSION:
- baselineVersion(client, baselineArgs);
- break;
+ private void baseline(GridClient client, BaselineArguments baselineArgs) throws Throwable {
+ try {
+ VisorBaselineTaskResult res = executeTask(client, VisorBaselineTask.class, toVisorArguments(baselineArgs));
- case BASELINE_AUTO_ADJUST:
- baselineAutoAdjust(client, baselineArgs);
- break;
+ baselinePrint0(res);
+ }
+ catch (Throwable e) {
+ log("Failed to execute baseline command='" + baselineArgs.getCmd().text() + "'", e);
- case BASELINE_COLLECT:
- baselinePrint(client);
- break;
+ throw e;
}
}
/**
* Prepare task argument.
*
- * @param op Operation.
- * @param s Argument from command line.
+ * @param args Argument from command line.
* @return Task argument.
*/
- private VisorBaselineTaskArg arg(VisorBaselineOperation op, String s) {
- switch (op) {
+ private VisorBaselineTaskArg toVisorArguments(BaselineArguments args) {
+ VisorBaselineAutoAdjustSettings settings = args.getCmd() == BaselineCommand.AUTO_ADJUST
+ ? new VisorBaselineAutoAdjustSettings(args.getEnableAutoAdjust(), args.getSoftBaselineTimeout())
+ : null;
+
+ return new VisorBaselineTaskArg(toVisorOperation(args.getCmd()), args.getTopVer(), args.getConsistentIds(), settings);
+ }
+
+ /**
+ * Convert {@link BaselineCommand} to {@link VisorBaselineOperation}.
+ *
+ * @param baselineCommand Baseline command for convertion.
+ * @return {@link VisorBaselineOperation}.
+ */
+ private VisorBaselineOperation toVisorOperation(BaselineCommand baselineCommand) {
+ switch (baselineCommand) {
case ADD:
+ return VisorBaselineOperation.ADD;
+ case AUTO_ADJUST:
+ return VisorBaselineOperation.AUTOADJUST;
case REMOVE:
+ return VisorBaselineOperation.REMOVE;
case SET:
- List<String> consistentIds = getConsistentIds(s);
-
- return new VisorBaselineTaskArg(op, -1, consistentIds, null);
-
+ return VisorBaselineOperation.SET;
case VERSION:
- try {
- long topVer = Long.parseLong(s);
-
- return new VisorBaselineTaskArg(op, topVer, null, null);
- }
- catch (NumberFormatException e) {
- throw new IllegalArgumentException("Invalid topology version: " + s, e);
- }
-
- case AUTOADJUST:
- if (BASELINE_AUTO_ADJUST_DISABLE.equals(s)) {
- VisorBaselineAutoAdjustSettings settings = new VisorBaselineAutoAdjustSettings(false, -1);
-
- return new VisorBaselineTaskArg(op, -1, null, settings);
- }
- else {
- String[] argsArr = s.split(" ");
-
- assert argsArr.length == 2;
-
- VisorBaselineAutoAdjustSettings settings = new VisorBaselineAutoAdjustSettings(
- true,
- Long.parseLong(argsArr[1])
- );
-
- return new VisorBaselineTaskArg(op, -1, null, settings);
- }
- default:
- return new VisorBaselineTaskArg(op, -1, null, new VisorBaselineAutoAdjustSettings(false, -1));
+ return VisorBaselineOperation.VERSION;
+ case COLLECT:
+ return VisorBaselineOperation.COLLECT;
}
+
+ return null;
}
/**
@@ -1333,11 +1280,20 @@ public class CommandHandler {
VisorBaselineAutoAdjustSettings autoAdjustSettings = res.getAutoAdjustSettings();
if (autoAdjustSettings != null) {
- log("Baseline auto adjustment " + (autoAdjustSettings.isEnabled() ? "enabled" : "disabled") +
- ": softTimeout=" + autoAdjustSettings.getSoftTimeout()
+ log("Baseline auto adjustment " + (TRUE.equals(autoAdjustSettings.getEnabled()) ? "enabled" : "disabled")
+ + ": softTimeout=" + autoAdjustSettings.getSoftTimeout()
);
}
+ if (autoAdjustSettings.enabled) {
+ if (res.isBaselineAdjustInProgress())
+ log("Baseline auto-adjust is in progress");
+ else if (res.getRemainingTimeToBaselineAdjust() < 0)
+ log("Baseline auto-adjust are not scheduled");
+ else
+ log("Baseline auto-adjust will happen in '" + res.getRemainingTimeToBaselineAdjust() + "' ms");
+ }
+
nl();
Map<String, VisorBaselineNode> baseline = res.getBaseline();
@@ -1398,115 +1354,6 @@ public class CommandHandler {
}
/**
- * Print current baseline.
- *
- * @param client Client.
- */
- private void baselinePrint(GridClient client) throws GridClientException {
- VisorBaselineTaskResult res = executeTask(client, VisorBaselineTask.class, arg(COLLECT, ""));
-
- baselinePrint0(res);
- }
-
- /**
- * Add nodes to baseline.
- *
- * @param client Client.
- * @param baselineArgs Baseline action arguments.
- * @throws Throwable If failed to add nodes to baseline.
- */
- private void baselineAdd(GridClient client, String baselineArgs) throws Throwable {
- try {
- VisorBaselineTaskResult res = executeTask(client, VisorBaselineTask.class, arg(ADD, baselineArgs));
-
- baselinePrint0(res);
- }
- catch (Throwable e) {
- log("Failed to add nodes to baseline.");
-
- throw e;
- }
- }
-
- /**
- * Remove nodes from baseline.
- *
- * @param client Client.
- * @param consistentIds Consistent IDs.
- * @throws Throwable If failed to remove nodes from baseline.
- */
- private void baselineRemove(GridClient client, String consistentIds) throws Throwable {
- try {
- VisorBaselineTaskResult res = executeTask(client, VisorBaselineTask.class, arg(REMOVE, consistentIds));
-
- baselinePrint0(res);
- }
- catch (Throwable e) {
- log("Failed to remove nodes from baseline.");
-
- throw e;
- }
- }
-
- /**
- * Set baseline.
- *
- * @param client Client.
- * @param consistentIds Consistent IDs.
- * @throws Throwable If failed to set baseline.
- */
- private void baselineSet(GridClient client, String consistentIds) throws Throwable {
- try {
- VisorBaselineTaskResult res = executeTask(client, VisorBaselineTask.class, arg(SET, consistentIds));
-
- baselinePrint0(res);
- }
- catch (Throwable e) {
- log("Failed to set baseline.");
-
- throw e;
- }
- }
-
- /**
- * Set baseline by topology version.
- *
- * @param client Client.
- * @param arg Argument from command line.
- */
- private void baselineVersion(GridClient client, String arg) throws GridClientException {
- try {
- VisorBaselineTaskResult res = executeTask(client, VisorBaselineTask.class, arg(VERSION, arg));
-
- baselinePrint0(res);
- }
- catch (Throwable e) {
- log("Failed to set baseline with specified topology version.");
-
- throw e;
- }
- }
-
- /**
- * Set baseline autoadjustment settings.
- *
- * @param client Client.
- * @param args Argument from command line. Either {@code disable} or {@code enable <softTimeout>}.
- */
- private void baselineAutoAdjust(GridClient client, String args) throws GridClientException {
- try {
- VisorBaselineTaskResult res = executeTask(client, VisorBaselineTask.class, arg(AUTOADJUST, args));
-
- baselinePrint0(res);
- }
- catch (Throwable e) {
- log("Failed to update baseline autoadjustment settings.");
-
- throw e;
- }
- }
-
- /**
* Dump transactions information.
*
* @param client Client.
@@ -1969,10 +1816,6 @@ public class CommandHandler {
String pwd = null;
- String baselineAct = "";
-
- String baselineArgs = "";
-
Long pingInterval = DFLT_PING_INTERVAL;
Long pingTimeout = DFLT_PING_TIMEOUT;
@@ -1985,6 +1828,8 @@ public class CommandHandler {
CacheArguments cacheArgs = null;
+ BaselineArguments baselineArgs = null;
+
List<Command> commands = new ArrayList<>();
initArgIterator(rawArgs);
@@ -2036,51 +1881,7 @@ public class CommandHandler {
case BASELINE:
commands.add(BASELINE);
- baselineAct = BASELINE_COLLECT; //default baseline action
-
- str = peekNextArg();
-
- if (str != null) {
- switch (str.toLowerCase()) {
- case BASELINE_ADD:
- case BASELINE_REMOVE:
- case BASELINE_SET:
- case BASELINE_SET_VERSION:
- baselineAct = nextArg("Expected baseline action");
-
- baselineArgs = nextArg("Expected baseline arguments");
-
- break;
-
- case BASELINE_AUTO_ADJUST:
- baselineAct = nextArg("Expected baseline action");
-
- String enableDisable = nextArg("Expected argument for baseline autoadjust (enable|disable)")
- .toLowerCase();
-
- if (BASELINE_AUTO_ADJUST_DISABLE.equals(enableDisable))
- baselineArgs = enableDisable.toLowerCase();
- else {
- if (!BASELINE_AUTO_ADJUST_ENABLE.equals(enableDisable)) {
- String msg = "Argument after \"--baseline autoadjust\" must be" +
- " either \"enable\" or \"disable\"";
-
- throw new IllegalArgumentException(msg);
- }
-
- long softTimeout = nextLongArg("Expected soft timeout" +
- " argument for baseline autoadjust");
-
- if (softTimeout < 0)
- throw new IllegalArgumentException("Soft timeout value for baseline" +
- " autoadjustment can't be negative");
-
- baselineArgs = enableDisable + " " + softTimeout;
- }
-
- break;
- }
- }
+ baselineArgs = parseAndValidateBaselineArgs();
break;
@@ -2228,7 +2029,7 @@ public class CommandHandler {
Command cmd = commands.get(0);
return new Arguments(cmd, host, port, user, pwd,
- baselineAct, baselineArgs,
+ baselineArgs,
txArgs, cacheArgs,
walAct, walArgs,
pingTimeout, pingInterval, autoConfirmation,
@@ -2238,12 +2039,62 @@ public class CommandHandler {
}
/**
+ * Parses and validates baseline arguments.
+ *
+ * @return --baseline subcommand arguments in case validation is successful.
+ */
+ private BaselineArguments parseAndValidateBaselineArgs() {
+ if (!hasNextSubArg())
+ return new BaselineArguments.Builder(BaselineCommand.COLLECT).build();
+
+ BaselineCommand cmd = of(nextArg("Expected baseline action"));
+
+ if (cmd == null)
+ throw new IllegalArgumentException("Expected correct baseline action");
+
+ BaselineArguments.Builder baselineArgs = new BaselineArguments.Builder(cmd);
+
+ switch (cmd) {
+ case ADD:
+ case REMOVE:
+ case SET:
+ return baselineArgs
+ .withConsistentIds(getConsistentIds(nextArg("Expected list of consistent ids")))
+ .build();
+ case VERSION:
+ return baselineArgs
+ .withTopVer(nextLongArg("topology version"))
+ .build();
+ case AUTO_ADJUST:
+ do {
+ AutoAdjustCommandArg autoAdjustArg = CommandArgUtils.of(
+ nextArg("Expected one of auto-adjust arguments"), AutoAdjustCommandArg.class
+ );
+
+ if (autoAdjustArg == null)
+ throw new IllegalArgumentException("Expected one of auto-adjust arguments");
+
+ if (autoAdjustArg == AutoAdjustCommandArg.ENABLE || autoAdjustArg == AutoAdjustCommandArg.DISABLE)
+ baselineArgs.withEnable(autoAdjustArg == AutoAdjustCommandArg.ENABLE);
+
+ if (autoAdjustArg == AutoAdjustCommandArg.TIMEOUT)
+ baselineArgs.withSoftBaselineTimeout(nextLongArg("soft timeout"));
+ }
+ while (hasNextSubArg());
+
+ return baselineArgs.build();
+ }
+
+ return baselineArgs.build();
+ }
+
+ /**
* Parses and validates cache arguments.
*
* @return --cache subcommand arguments in case validation is successful.
*/
private CacheArguments parseAndValidateCacheArgs() {
- if (!hasNextCacheArg()) {
+ if (!hasNextSubArg()) {
throw new IllegalArgumentException("Arguments are expected for --cache subcommand, " +
"run --cache help for more info.");
}
@@ -2266,12 +2117,12 @@ public class CommandHandler {
case IDLE_VERIFY:
int idleVerifyArgsCnt = 3;
- while (hasNextCacheArg() && idleVerifyArgsCnt-- > 0) {
+ while (hasNextSubArg() && idleVerifyArgsCnt-- > 0) {
String nextArg = nextArg("");
IdleVerifyCommandArg arg = CommandArgUtils.of(nextArg, IdleVerifyCommandArg.class);
- if(arg == null) {
+ if (arg == null) {
if (cacheArgs.excludeCaches() != null || cacheArgs.getCacheFilterEnum() != CacheFilterEnum.ALL)
throw new IllegalArgumentException(ONE_CACHE_FILTER_OPT_SHOULD_USED_MSG);
@@ -2321,10 +2172,10 @@ public class CommandHandler {
case CONTENTION:
cacheArgs.minQueueSize(Integer.parseInt(nextArg("Min queue size expected")));
- if (hasNextCacheArg())
+ if (hasNextSubArg())
cacheArgs.nodeId(UUID.fromString(nextArg("")));
- if (hasNextCacheArg())
+ if (hasNextSubArg())
cacheArgs.maxPrint(Integer.parseInt(nextArg("")));
else
cacheArgs.maxPrint(10);
@@ -2334,13 +2185,13 @@ public class CommandHandler {
case VALIDATE_INDEXES:
int argsCnt = 0;
- while (hasNextCacheArg() && argsCnt++ < 4) {
+ while (hasNextSubArg() && argsCnt++ < 4) {
String nextArg = nextArg("");
ValidateIndexesCommandArg arg = CommandArgUtils.of(nextArg, ValidateIndexesCommandArg.class);
- if(arg == CHECK_FIRST || arg == CHECK_THROUGH) {
- if (!hasNextCacheArg())
+ if (arg == CHECK_FIRST || arg == CHECK_THROUGH) {
+ if (!hasNextSubArg())
throw new IllegalArgumentException("Numeric value for '" + nextArg + "' parameter expected.");
int numVal;
@@ -2386,7 +2237,7 @@ public class CommandHandler {
if (!NULL.equals(nodeIdStr))
cacheArgs.nodeId(UUID.fromString(nodeIdStr));
- while (hasNextCacheArg()) {
+ while (hasNextSubArg()) {
String nextArg = nextArg("");
DistributionCommandArg arg = CommandArgUtils.of(nextArg, DistributionCommandArg.class);
@@ -2401,7 +2252,7 @@ public class CommandHandler {
cacheArgs.setUserAttributes(userAttrs);
- nextArg = (hasNextCacheArg()) ? nextArg("") : null;
+ nextArg = (hasNextSubArg()) ? nextArg("") : null;
}
@@ -2423,7 +2274,7 @@ public class CommandHandler {
OutputFormat outputFormat = SINGLE_LINE;
- while (hasNextCacheArg()) {
+ while (hasNextSubArg()) {
String nextArg = nextArg("").toLowerCase();
ListCommandArg arg = CommandArgUtils.of(nextArg, ListCommandArg.class);
@@ -2465,16 +2316,16 @@ public class CommandHandler {
throw new IllegalArgumentException("Unknown --cache subcommand " + cmd);
}
- if (hasNextCacheArg())
+ if (hasNextSubArg())
throw new IllegalArgumentException("Unexpected argument of --cache subcommand: " + peekNextArg());
return cacheArgs;
}
/**
- * @return <code>true</code> if there's next argument for --cache subcommand.
+ * @return <code>true</code> if there's next argument for subcommand.
*/
- private boolean hasNextCacheArg() {
+ private boolean hasNextSubArg() {
return hasNextArg() && Command.of(peekNextArg()) == null && !AUX_COMMANDS.contains(peekNextArg());
}
@@ -2860,11 +2711,11 @@ public class CommandHandler {
usage(i("Deactivate cluster:"), DEACTIVATE, op(CMD_AUTO_CONFIRMATION));
usage(i("Print current cluster state:"), STATE);
usage(i("Print cluster baseline topology:"), BASELINE);
- usage(i("Add nodes into baseline topology:"), BASELINE, BASELINE_ADD, constistIds, op(CMD_AUTO_CONFIRMATION));
- usage(i("Remove nodes from baseline topology:"), BASELINE, BASELINE_REMOVE, constistIds, op(CMD_AUTO_CONFIRMATION));
- usage(i("Set baseline topology:"), BASELINE, BASELINE_SET, constistIds, op(CMD_AUTO_CONFIRMATION));
- usage(i("Set baseline topology based on version:"), BASELINE, BASELINE_SET_VERSION + " topologyVersion", op(CMD_AUTO_CONFIRMATION));
- usage(i("Set baseline autoadjustment settings:"), BASELINE, BASELINE_AUTO_ADJUST, "disable|(enable <softTimeout>)", op(CMD_AUTO_CONFIRMATION));
+ usage(i("Add nodes into baseline topology:"), BASELINE, BaselineCommand.ADD.text(), constistIds, op(CMD_AUTO_CONFIRMATION));
+ usage(i("Remove nodes from baseline topology:"), BASELINE, BaselineCommand.REMOVE.text(), constistIds, op(CMD_AUTO_CONFIRMATION));
+ usage(i("Set baseline topology:"), BASELINE, BaselineCommand.SET.text(), constistIds, op(CMD_AUTO_CONFIRMATION));
+ usage(i("Set baseline topology based on version:"), BASELINE, BaselineCommand.VERSION.text() + " topologyVersion", op(CMD_AUTO_CONFIRMATION));
+ usage(i("Set baseline autoadjustment settings:"), BASELINE, BaselineCommand.AUTO_ADJUST.text(), "disable|enable timeout <timeoutValue>", op(CMD_AUTO_CONFIRMATION));
usage(i("List or kill transactions:"), TX, getTxOptions());
if (enableExperimental) {
@@ -3021,7 +2872,7 @@ public class CommandHandler {
break;
case BASELINE:
- baseline(client, args.baselineAction(), args.baselineArguments());
+ baseline(client, args.baselineArguments());
break;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/baseline/AutoAdjustCommandArg.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/baseline/AutoAdjustCommandArg.java
new file mode 100644
index 0000000..7e97d1d
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/baseline/AutoAdjustCommandArg.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.commandline.baseline;
+
+import org.apache.ignite.internal.commandline.argument.CommandArg;
+
+/**
+ * {@link BaselineCommand#AUTO_ADJUST} command arguments.
+ */
+public enum AutoAdjustCommandArg implements CommandArg {
+ /** Enable auto-adjust. */
+ ENABLE("enable"),
+ /** Disable auto-adjust. */
+ DISABLE("disable"),
+ /** Set soft timeout. */
+ TIMEOUT("timeout");
+
+ /** Option name. */
+ private final String name;
+
+ /** */
+ AutoAdjustCommandArg(String name) {
+ this.name = name;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String argName() {
+ return name;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return name;
+ }
+}
\ No newline at end of file
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/baseline/BaselineArguments.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/baseline/BaselineArguments.java
new file mode 100644
index 0000000..46a9c73
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/baseline/BaselineArguments.java
@@ -0,0 +1,165 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.commandline.baseline;
+
+import java.util.List;
+
+/**
+ * This class contains all possible arguments after parsing baseline command input.
+ */
+public class BaselineArguments {
+ /** Command. */
+ private BaselineCommand cmd;
+ /**
+ * {@code true} if auto-adjust should be enable, {@code false} if it should be disable, {@code null} if no operation
+ * needed.
+ */
+ private Boolean enableAutoAdjust;
+ /** New value of soft timeout. */
+ private Long softBaselineTimeout;
+ /** Requested topology version. */
+ private long topVer = -1;
+ /** List of consistent ids for operation. */
+ List<String> consistentIds;
+
+ /**
+ * @param cmd Command.
+ * @param enableAutoAdjust Auto-adjust enabled feature.
+ * @param softBaselineTimeout New value of soft timeout.
+ * @param topVer Requested topology version.
+ * @param consistentIds List of consistent ids for operation.
+ */
+ public BaselineArguments(BaselineCommand cmd, Boolean enableAutoAdjust, Long softBaselineTimeout,
+ long topVer, List<String> consistentIds) {
+ this.cmd = cmd;
+ this.enableAutoAdjust = enableAutoAdjust;
+ this.softBaselineTimeout = softBaselineTimeout;
+ this.topVer = topVer;
+ this.consistentIds = consistentIds;
+ }
+
+ /**
+ * @return Command.
+ */
+ public BaselineCommand getCmd() {
+ return cmd;
+ }
+
+ /**
+ * @return {@code true} if auto-adjust should be enable, {@code false} if it should be disable, {@code null} if no
+ * operation needed.
+ */
+ public Boolean getEnableAutoAdjust() {
+ return enableAutoAdjust;
+ }
+
+ /**
+ * @return New value of soft timeout.
+ */
+ public Long getSoftBaselineTimeout() {
+ return softBaselineTimeout;
+ }
+
+ /**
+ * @return Requested topology version.
+ */
+ public long getTopVer() {
+ return topVer;
+ }
+
+ /**
+ * @return List of consistent ids for operation.
+ */
+ public List<String> getConsistentIds() {
+ return consistentIds;
+ }
+
+ /**
+ * Builder of {@link BaselineArguments}.
+ */
+ public static class Builder {
+ /** Command. */
+ private BaselineCommand cmd;
+ /**
+ * {@code true} if auto-adjust should be enable, {@code false} if it should be disable, {@code null} if no
+ * operation needed.
+ */
+ private Boolean enable;
+ /** New value of soft timeout. */
+ private Long timeout;
+ /** Requested topology version. */
+ private long ver = -1;
+ /** List of consistent ids for operation. */
+ private List<String> ids;
+
+ /**
+ * @param cmd Command.
+ */
+ public Builder(BaselineCommand cmd) {
+ this.cmd = cmd;
+ }
+
+ /**
+ * @param enable {@code true} if auto-adjust should be enable, {@code false} if it should be disable, {@code
+ * null} if no operation needed.
+ * @return This instance for chaining.
+ */
+ public Builder withEnable(Boolean enable) {
+ this.enable = enable;
+
+ return this;
+ }
+
+ /**
+ * @param timeout New value of soft timeout.
+ * @return This instance for chaining.
+ */
+ public Builder withSoftBaselineTimeout(Long timeout) {
+ this.timeout = timeout;
+
+ return this;
+ }
+
+ /**
+ * @param ver Requested topology version.
+ * @return This instance for chaining.
+ */
+ public Builder withTopVer(long ver) {
+ this.ver = ver;
+
+ return this;
+ }
+
+ /**
+ * @param ids List of consistent ids for operation.
+ * @return This instance for chaining.
+ */
+ public Builder withConsistentIds(List<String> ids) {
+ this.ids = ids;
+
+ return this;
+ }
+
+ /**
+ * @return {@link BaselineArguments}.
+ */
+ public BaselineArguments build() {
+ return new BaselineArguments(cmd, enable, timeout, ver, ids);
+ }
+ }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/baseline/BaselineCommand.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/baseline/BaselineCommand.java
new file mode 100644
index 0000000..0b3744b
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/baseline/BaselineCommand.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.commandline.baseline;
+
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Set of baseline commands.
+ */
+public enum BaselineCommand {
+ /**
+ * Add nodes to baseline.
+ */
+ ADD("add"),
+
+ /**
+ * Remove nodes from baseline.
+ */
+ REMOVE("remove"),
+
+ /**
+ * Collect information about baseline.
+ */
+ COLLECT("collect"),
+
+ /**
+ * Set new baseline.
+ */
+ SET("set"),
+
+ /**
+ * Check current topology version.
+ */
+ VERSION("version"),
+
+ /**
+ * Baseline auto-adjust configuration.
+ */
+ AUTO_ADJUST("auto_adjust");
+
+ /** Enumerated values. */
+ private static final BaselineCommand[] VALS = values();
+
+ /** Name. */
+ private final String name;
+
+ /**
+ * @param name Name.
+ */
+ BaselineCommand(String name) {
+ this.name = name;
+ }
+
+ /**
+ * @param text Command text.
+ * @return Command for the text.
+ */
+ public static BaselineCommand of(String text) {
+ for (BaselineCommand cmd : BaselineCommand.values()) {
+ if (cmd.text().equalsIgnoreCase(text))
+ return cmd;
+ }
+
+ return null;
+ }
+
+ /**
+ * @return Name.
+ */
+ public String text() {
+ return name;
+ }
+
+ /**
+ * Efficiently gets enumerated value from its ordinal.
+ *
+ * @param ord Ordinal value.
+ * @return Enumerated value or {@code null} if ordinal out of range.
+ */
+ @Nullable public static BaselineCommand fromOrdinal(int ord) {
+ return ord >= 0 && ord < VALS.length ? VALS[ord] : null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return name;
+ }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ClusterProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ClusterProcessor.java
index f9e5efe..cd43e34 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ClusterProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ClusterProcessor.java
@@ -48,6 +48,7 @@ import org.apache.ignite.internal.managers.discovery.IgniteClusterNode;
import org.apache.ignite.internal.managers.eventstorage.GridLocalEventListener;
import org.apache.ignite.internal.processors.GridProcessorAdapter;
import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
+import org.apache.ignite.internal.processors.cluster.baseline.autoadjust.BaselineAutoAdjustStatistic;
import org.apache.ignite.internal.processors.cluster.baseline.autoadjust.ChangeTopologyWatcher;
import org.apache.ignite.internal.processors.timeout.GridTimeoutObject;
import org.apache.ignite.internal.util.GridTimerTask;
@@ -126,6 +127,9 @@ public class ClusterProcessor extends GridProcessorAdapter {
/** */
private boolean sndMetrics;
+ /** Watcher of topology change for baseline auto-adjust. */
+ private ChangeTopologyWatcher changeTopologyWatcher;
+
/**
* @param ctx Kernal context.
*/
@@ -172,7 +176,10 @@ public class ClusterProcessor extends GridProcessorAdapter {
}
}, EVT_NODE_FAILED, EVT_NODE_LEFT);
- ctx.event().addLocalEventListener(new ChangeTopologyWatcher(ctx), EVT_NODE_FAILED, EVT_NODE_LEFT, EVT_NODE_JOINED);
+ ctx.event().addLocalEventListener(
+ changeTopologyWatcher = new ChangeTopologyWatcher(ctx),
+ EVT_NODE_FAILED, EVT_NODE_LEFT, EVT_NODE_JOINED
+ );
ctx.io().addMessageListener(TOPIC_INTERNAL_DIAGNOSTIC, new GridMessageListener() {
@Override public void onMessage(UUID nodeId, Object msg, byte plc) {
@@ -640,6 +647,13 @@ public class ClusterProcessor extends GridProcessorAdapter {
}
/**
+ * @return Statistic of baseline auto-adjust.
+ */
+ public BaselineAutoAdjustStatistic baselineAutoAdjustStatistic(){
+ return changeTopologyWatcher.getStatistic();
+ }
+
+ /**
* Update notifier timer task.
*/
private static class UpdateNotifierTimerTask extends GridTimerTask {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/baseline/autoadjust/BaselineAutoAdjustData.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/baseline/autoadjust/BaselineAutoAdjustData.java
index a913d09..bb393a3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/baseline/autoadjust/BaselineAutoAdjustData.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/baseline/autoadjust/BaselineAutoAdjustData.java
@@ -17,30 +17,40 @@
package org.apache.ignite.internal.processors.cluster.baseline.autoadjust;
-import org.apache.ignite.events.Event;
-
/**
* Container of required data for changing baseline.
*/
class BaselineAutoAdjustData {
- /** Event with which this data correspond to. For statistic only. */
- private final Event reasonEvent;
+ /** Task represented NULL value is using when normal task can not be created. */
+ public static final BaselineAutoAdjustData NULL_BASELINE_DATA = nullValue();
/** Topology version nodes of which should be set to baseline by this task. */
private final long targetTopologyVersion;
/** {@code true} If this data don't actual anymore and it setting should be skipped. */
private volatile boolean invalidated = false;
+ /** {@code true} If this data was adjusted. */
+ private volatile boolean adjusted = false;
/**
- * @param evt Event with which this data correspond to. For statistic only.
* @param targetTopologyVersion Topology version nodes of which should be set by this task.
*/
- BaselineAutoAdjustData(Event evt, long targetTopologyVersion) {
- reasonEvent = evt;
+ BaselineAutoAdjustData(long targetTopologyVersion) {
this.targetTopologyVersion = targetTopologyVersion;
}
/**
+ * @return New null value.
+ */
+ private static BaselineAutoAdjustData nullValue() {
+ BaselineAutoAdjustData data = new BaselineAutoAdjustData(-1);
+
+ data.onInvalidate();
+ data.onAdjust();
+
+ return data;
+ }
+
+ /**
* Mark that this data are invalid.
*/
private void onInvalidate() {
@@ -48,6 +58,13 @@ class BaselineAutoAdjustData {
}
/**
+ * Mark that this data was adjusted.
+ */
+ public void onAdjust() {
+ adjusted = true;
+ }
+
+ /**
* @return Topology version nodes of which should be set to baseline by this task.
*/
public long getTargetTopologyVersion() {
@@ -55,21 +72,27 @@ class BaselineAutoAdjustData {
}
/**
- * @return {@code true} If this data still actual and can be set.
+ * @return {@code true} If this data already invalidated and can not be set.
*/
public boolean isInvalidated() {
return invalidated;
}
/**
+ * @return {@code true} If this data already adjusted.
+ */
+ public boolean isAdjusted() {
+ return adjusted;
+ }
+
+ /**
* Produce next set baseline data based on this data.
*
- * @param evt New triggired event.
* @return New set baseline data.
*/
- public BaselineAutoAdjustData next(Event evt, long targetTopologyVersion) {
+ public BaselineAutoAdjustData next(long targetTopologyVersion) {
onInvalidate();
- return new BaselineAutoAdjustData(evt, targetTopologyVersion);
+ return new BaselineAutoAdjustData(targetTopologyVersion);
}
}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/baseline/autoadjust/BaselineAutoAdjustExecutor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/baseline/autoadjust/BaselineAutoAdjustExecutor.java
index b560932..b90df68 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/baseline/autoadjust/BaselineAutoAdjustExecutor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/baseline/autoadjust/BaselineAutoAdjustExecutor.java
@@ -76,6 +76,8 @@ class BaselineAutoAdjustExecutor {
log.error("Error during baseline changing", e);
}
finally {
+ data.onAdjust();
+
executionGuard.unlock();
}
}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/baseline/autoadjust/BaselineAutoAdjustScheduler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/baseline/autoadjust/BaselineAutoAdjustScheduler.java
index d89ba85..f6e41db 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/baseline/autoadjust/BaselineAutoAdjustScheduler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/baseline/autoadjust/BaselineAutoAdjustScheduler.java
@@ -30,14 +30,15 @@ class BaselineAutoAdjustScheduler {
private final GridTimeoutProcessor timeoutProcessor;
/** Executor of set baseline operation. */
private final BaselineAutoAdjustExecutor baselineAutoAdjustExecutor;
- /** Last set task for set new baseline. It needed for removing from queue. */
+ /** Last scheduled task for adjust new baseline. It needed for removing from queue. */
private GridTimeoutObject baselineTimeoutObj;
/**
* @param timeoutProcessor Timeout processor.
* @param baselineAutoAdjustExecutor Executor of set baseline operation.
*/
- public BaselineAutoAdjustScheduler(GridTimeoutProcessor timeoutProcessor, BaselineAutoAdjustExecutor baselineAutoAdjustExecutor) {
+ public BaselineAutoAdjustScheduler(GridTimeoutProcessor timeoutProcessor,
+ BaselineAutoAdjustExecutor baselineAutoAdjustExecutor) {
this.timeoutProcessor = timeoutProcessor;
this.baselineAutoAdjustExecutor = baselineAutoAdjustExecutor;
}
@@ -60,4 +61,16 @@ class BaselineAutoAdjustScheduler {
}
);
}
+
+ /**
+ * @return Time of last scheduled task or -1 if it doesn't exist.
+ */
+ public long lastScheduledTaskTime() {
+ if (baselineTimeoutObj == null)
+ return -1;
+
+ long lastScheduledTaskTime = baselineTimeoutObj.endTime() - System.currentTimeMillis();
+
+ return lastScheduledTaskTime < 0 ? -1 : lastScheduledTaskTime;
+ }
}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/baseline/autoadjust/BaselineAutoAdjustStatistic.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/baseline/autoadjust/BaselineAutoAdjustStatistic.java
new file mode 100644
index 0000000..3847bae
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/baseline/autoadjust/BaselineAutoAdjustStatistic.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cluster.baseline.autoadjust;
+
+/**
+ * Statistic of baseline auto-adjust.
+ */
+public class BaselineAutoAdjustStatistic {
+ /** Timeout of task of baseline adjust. */
+ private final long baselineAdjustTimeout;
+ /** State of baseline adjust task. */
+ private final TaskState taskState;
+
+ /**
+ * @param timeout Timeout of baseline adjust.
+ */
+ private BaselineAutoAdjustStatistic(TaskState state, long timeout) {
+ baselineAdjustTimeout = timeout;
+ taskState = state;
+ }
+
+ /**
+ * @param timeout Timeout of task of baseline adjust.
+ * @return Created statistic object.
+ */
+ public static BaselineAutoAdjustStatistic scheduled(long timeout) {
+ return new BaselineAutoAdjustStatistic(TaskState.SCHEDULED, timeout);
+ }
+
+ /**
+ * @return Created statistic object with 'in progress' status.
+ */
+ public static BaselineAutoAdjustStatistic inProgress() {
+ return new BaselineAutoAdjustStatistic(TaskState.IN_PROGRESS, -1);
+ }
+
+ /**
+ * @return Created statistic object with 'not scheduled' status.
+ */
+ public static BaselineAutoAdjustStatistic notScheduled() {
+ return new BaselineAutoAdjustStatistic(TaskState.NOT_SCHEDULED, -1);
+ }
+
+ /**
+ * @return Timeout of task of baseline adjust.
+ */
+ public long getBaselineAdjustTimeout() {
+ return baselineAdjustTimeout;
+ }
+
+ /**
+ * @return State of baseline adjust task.
+ */
+ public TaskState getTaskState() {
+ return taskState;
+ }
+
+ /**
+ *
+ */
+ public enum TaskState {
+ /**
+ * Baseline is changing now.
+ */
+ IN_PROGRESS,
+ /**
+ * Task to baseline adjust was scheduled.
+ */
+ SCHEDULED,
+ /**
+ * Task to baseline adjust was not scheduled.
+ */
+ NOT_SCHEDULED;
+ }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/baseline/autoadjust/ChangeTopologyWatcher.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/baseline/autoadjust/ChangeTopologyWatcher.java
index 35adb32..2448810 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/baseline/autoadjust/ChangeTopologyWatcher.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/baseline/autoadjust/ChangeTopologyWatcher.java
@@ -31,14 +31,15 @@ import org.apache.ignite.internal.processors.cache.GridCachePartitionExchangeMan
import org.apache.ignite.internal.processors.cluster.GridClusterStateProcessor;
import org.apache.ignite.lang.IgniteInClosure;
+import static org.apache.ignite.internal.processors.cluster.baseline.autoadjust.BaselineAutoAdjustData.NULL_BASELINE_DATA;
+import static org.apache.ignite.internal.processors.cluster.baseline.autoadjust.BaselineAutoAdjustStatistic.TaskState.IN_PROGRESS;
+import static org.apache.ignite.internal.processors.cluster.baseline.autoadjust.BaselineAutoAdjustStatistic.TaskState.NOT_SCHEDULED;
import static org.apache.ignite.internal.util.IgniteUtils.isLocalNodeCoordinator;
/**
* Watcher of topology changes. It initiate to set new baseline after some timeout.
*/
public class ChangeTopologyWatcher implements GridLocalEventListener {
- /** Task represented NULL value is using when normal task can not be created. */
- private static final BaselineAutoAdjustData NULL_BASELINE_DATA = new BaselineAutoAdjustData(null, -1);
/** */
private final IgniteLogger log;
/** */
@@ -91,7 +92,7 @@ public class ChangeTopologyWatcher implements GridLocalEventListener {
return;
synchronized (this) {
- lastBaselineData = lastBaselineData.next(evt, discoEvt.topologyVersion());
+ lastBaselineData = lastBaselineData.next(discoEvt.topologyVersion());
if (isLocalNodeCoordinator(discoveryMgr)) {
exchangeManager.affinityReadyFuture(new AffinityTopologyVersion(discoEvt.topologyVersion()))
@@ -122,4 +123,21 @@ public class ChangeTopologyWatcher implements GridLocalEventListener {
private boolean isBaselineAutoAdjustEnabled() {
return stateProcessor.clusterState().active() && baselineConfiguration.isBaselineAutoAdjustEnabled();
}
+
+ /**
+ * @return Statistic of baseline auto-adjust.
+ */
+ public BaselineAutoAdjustStatistic getStatistic() {
+ synchronized (this) {
+ if (lastBaselineData.isAdjusted())
+ return BaselineAutoAdjustStatistic.notScheduled();
+
+ long timeToLastTask = baselineAutoAdjustScheduler.lastScheduledTaskTime();
+
+ if (timeToLastTask <= 0)
+ return BaselineAutoAdjustStatistic.inProgress();
+
+ return BaselineAutoAdjustStatistic.scheduled(timeToLastTask);
+ }
+ }
}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/baseline/VisorBaselineAutoAdjustSettings.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/baseline/VisorBaselineAutoAdjustSettings.java
index c8f8727..d2f16d4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/baseline/VisorBaselineAutoAdjustSettings.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/baseline/VisorBaselineAutoAdjustSettings.java
@@ -28,45 +28,55 @@ public class VisorBaselineAutoAdjustSettings extends VisorDataTransferObject {
private static final long serialVersionUID = 0L;
/** "Enable" flag. */
- private boolean enabled;
+ public Boolean enabled;
/** Soft timeout. */
- private long softTimeout;
+ public Long softTimeout;
/** Default constructor. */
public VisorBaselineAutoAdjustSettings() {
}
/** Constructor. */
- public VisorBaselineAutoAdjustSettings(boolean enabled, long softTimeout) {
- this.enabled= enabled;
+ public VisorBaselineAutoAdjustSettings(Boolean enabled, Long softTimeout) {
+ this.enabled = enabled;
this.softTimeout = softTimeout;
}
/**
* @return "Enable" flag.
*/
- public boolean isEnabled() {
+ public Boolean getEnabled() {
return enabled;
}
/**
* Soft timeout.
*/
- public long getSoftTimeout() {
+ public Long getSoftTimeout() {
return softTimeout;
}
/** {@inheritDoc} */
@Override protected void writeExternalData(ObjectOutput out) throws IOException {
- out.writeBoolean(enabled);
- out.writeLong(softTimeout);
+ out.writeBoolean(enabled != null);
+
+ if (enabled != null)
+ out.writeBoolean(enabled);
+
+ out.writeBoolean(softTimeout != null);
+
+ if (softTimeout != null)
+ out.writeLong(softTimeout);
}
/** {@inheritDoc} */
@Override
protected void readExternalData(byte protoVer, ObjectInput in) throws IOException, ClassNotFoundException {
- enabled = in.readBoolean();
- softTimeout = in.readLong();
+ if (in.readBoolean())
+ enabled = in.readBoolean();
+
+ if (in.readBoolean())
+ softTimeout = in.readLong();
}
}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/baseline/VisorBaselineTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/baseline/VisorBaselineTask.java
index 5f89003..aa74ea9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/baseline/VisorBaselineTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/baseline/VisorBaselineTask.java
@@ -30,6 +30,7 @@ import org.apache.ignite.cluster.ClusterNode;
import org.apache.ignite.internal.cluster.DistributedBaselineConfiguration;
import org.apache.ignite.internal.cluster.DetachedClusterNode;
import org.apache.ignite.internal.cluster.IgniteClusterEx;
+import org.apache.ignite.internal.processors.cluster.baseline.autoadjust.BaselineAutoAdjustStatistic;
import org.apache.ignite.internal.processors.task.GridInternal;
import org.apache.ignite.internal.processors.task.GridVisorManagementTask;
import org.apache.ignite.internal.util.typedef.F;
@@ -85,8 +86,17 @@ public class VisorBaselineTask extends VisorOneNodeTask<VisorBaselineTaskArg, Vi
cluster.baselineAutoAdjustTimeout()
);
- return new VisorBaselineTaskResult(ignite.cluster().active(), cluster.topologyVersion(),
- F.isEmpty(baseline) ? null : baseline, srvrs, autoAdjustSettings);
+ BaselineAutoAdjustStatistic adjustStatistic = ignite.context().cluster().baselineAutoAdjustStatistic();
+
+ return new VisorBaselineTaskResult(
+ ignite.cluster().active(),
+ cluster.topologyVersion(),
+ F.isEmpty(baseline) ? null : baseline,
+ srvrs,
+ autoAdjustSettings,
+ adjustStatistic.getBaselineAdjustTimeout(),
+ adjustStatistic.getTaskState() == BaselineAutoAdjustStatistic.TaskState.IN_PROGRESS
+ );
}
/**
@@ -220,10 +230,11 @@ public class VisorBaselineTask extends VisorOneNodeTask<VisorBaselineTaskArg, Vi
* @return New baseline.
*/
private VisorBaselineTaskResult updateAutoAdjustmentSettings(VisorBaselineAutoAdjustSettings settings) {
- if (settings.isEnabled())
+ if (settings.getSoftTimeout() != null)
ignite.cluster().baselineAutoAdjustTimeout(settings.getSoftTimeout());
- ignite.cluster().baselineAutoAdjustEnabled(settings.isEnabled());
+ if (settings.getEnabled() != null)
+ ignite.cluster().baselineAutoAdjustEnabled(settings.getEnabled());
return collect();
}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/baseline/VisorBaselineTaskResult.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/baseline/VisorBaselineTaskResult.java
index 1af1740..7691008 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/baseline/VisorBaselineTaskResult.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/baseline/VisorBaselineTaskResult.java
@@ -51,6 +51,12 @@ public class VisorBaselineTaskResult extends VisorDataTransferObject {
/** Baseline autoadjustment settings. */
private VisorBaselineAutoAdjustSettings autoAdjustSettings;
+ /** Time to next baseline adjust. */
+ private long remainingTimeToBaselineAdjust = -1;
+
+ /** Is baseline adjust in progress? */
+ private boolean baselineAdjustInProgress = false;
+
/**
* Default constructor.
*/
@@ -84,19 +90,24 @@ public class VisorBaselineTaskResult extends VisorDataTransferObject {
* @param topVer Current topology version.
* @param baseline Current baseline nodes.
* @param servers Current server nodes.
+ * @param remainingTimeToBaselineAdjust Time to next baseline adjust.
+ * @param baselineAdjustInProgress {@code true} If baseline adjust is in progress.
*/
public VisorBaselineTaskResult(
boolean active,
long topVer,
Collection<? extends BaselineNode> baseline,
Collection<? extends BaselineNode> servers,
- VisorBaselineAutoAdjustSettings autoAdjustSettings
- ) {
+ VisorBaselineAutoAdjustSettings autoAdjustSettings,
+ long remainingTimeToBaselineAdjust,
+ boolean baselineAdjustInProgress) {
this.active = active;
this.topVer = topVer;
this.baseline = toMap(baseline);
this.servers = toMap(servers);
this.autoAdjustSettings = autoAdjustSettings;
+ this.remainingTimeToBaselineAdjust = remainingTimeToBaselineAdjust;
+ this.baselineAdjustInProgress = baselineAdjustInProgress;
}
/**
@@ -139,6 +150,20 @@ public class VisorBaselineTaskResult extends VisorDataTransferObject {
return autoAdjustSettings;
}
+ /**
+ * @return Time to next baseline adjust.
+ */
+ public long getRemainingTimeToBaselineAdjust() {
+ return remainingTimeToBaselineAdjust;
+ }
+
+ /**
+ * @return {@code true} If baseline adjust is in progress.
+ */
+ public boolean isBaselineAdjustInProgress() {
+ return baselineAdjustInProgress;
+ }
+
/** {@inheritDoc} */
@Override protected void writeExternalData(ObjectOutput out) throws IOException {
out.writeBoolean(active);
@@ -146,17 +171,23 @@ public class VisorBaselineTaskResult extends VisorDataTransferObject {
U.writeMap(out, baseline);
U.writeMap(out, servers);
out.writeObject(autoAdjustSettings);
+ out.writeLong(remainingTimeToBaselineAdjust);
+ out.writeBoolean(baselineAdjustInProgress);
}
/** {@inheritDoc} */
- @Override protected void readExternalData(byte protoVer, ObjectInput in) throws IOException, ClassNotFoundException {
+ @Override protected void readExternalData(byte protoVer,
+ ObjectInput in) throws IOException, ClassNotFoundException {
active = in.readBoolean();
topVer = in.readLong();
baseline = U.readTreeMap(in);
servers = U.readTreeMap(in);
- if (protoVer > V1)
+ if (protoVer > V1) {
autoAdjustSettings = (VisorBaselineAutoAdjustSettings)in.readObject();
+ remainingTimeToBaselineAdjust = in.readLong();
+ baselineAdjustInProgress = in.readBoolean();
+ }
}
/** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/baseline/VisorBaselineViewTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/baseline/VisorBaselineViewTask.java
index 5b871b3..7e9a3b3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/baseline/VisorBaselineViewTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/baseline/VisorBaselineViewTask.java
@@ -19,6 +19,7 @@ package org.apache.ignite.internal.visor.baseline;
import org.apache.ignite.IgniteException;
import org.apache.ignite.internal.cluster.IgniteClusterEx;
+import org.apache.ignite.internal.processors.cluster.baseline.autoadjust.BaselineAutoAdjustStatistic;
import org.apache.ignite.internal.processors.task.GridInternal;
import org.apache.ignite.internal.util.typedef.internal.S;
import org.apache.ignite.internal.visor.VisorJob;
@@ -62,12 +63,16 @@ public class VisorBaselineViewTask extends VisorOneNodeTask<Void, VisorBaselineT
cluster.baselineAutoAdjustTimeout()
);
+ BaselineAutoAdjustStatistic adjustStatistic = ignite.context().cluster().baselineAutoAdjustStatistic();
+
return new VisorBaselineTaskResult(
ignite.cluster().active(),
cluster.topologyVersion(),
cluster.currentBaselineTopology(),
cluster.forServers().nodes(),
- autoAdjustSettings
+ autoAdjustSettings,
+ adjustStatistic.getBaselineAdjustTimeout(),
+ adjustStatistic.getTaskState() == BaselineAutoAdjustStatistic.TaskState.IN_PROGRESS
);
}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java b/modules/core/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java
index f261292..3239171 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java
@@ -327,8 +327,8 @@ public class CommandHandlerParsingTest {
assertEquals(cmd, args.command());
assertEquals(DFLT_HOST, args.host());
assertEquals(DFLT_PORT, args.port());
- assertEquals(baselineAct, args.baselineAction());
- assertEquals("c_id1,c_id2", args.baselineArguments());
+ assertEquals(baselineAct, args.baselineArguments().getCmd().text());
+ assertEquals(Arrays.asList("c_id1","c_id2"), args.baselineArguments().getConsistentIds());
assertEquals(true, args.autoConfirmation());
}
diff --git a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
index 623642a..7356c7a 100644
--- a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
@@ -126,6 +126,7 @@ import static org.apache.ignite.internal.commandline.OutputFormat.MULTI_LINE;
import static org.apache.ignite.internal.commandline.OutputFormat.SINGLE_LINE;
import static org.apache.ignite.internal.commandline.cache.CacheCommand.HELP;
import static org.apache.ignite.internal.processors.cache.verify.VerifyBackupPartitionsDumpTask.IDLE_DUMP_FILE_PREFIX;
+import static org.apache.ignite.testframework.GridTestUtils.waitForCondition;
import static org.apache.ignite.transactions.TransactionConcurrency.OPTIMISTIC;
import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC;
import static org.apache.ignite.transactions.TransactionIsolation.READ_COMMITTED;
@@ -206,6 +207,7 @@ public class GridCommandHandlerTest extends GridCommonAbstractTest {
System.setOut(sysOut);
+ log.info("----------------------------------------");
if (testOut != null)
System.out.println(testOut.toString());
}
@@ -586,7 +588,7 @@ public class GridCommandHandlerTest extends GridCommonAbstractTest {
}
/**
- * Test that baseline autoadjustment settings update works via control.sh
+ * Test that baseline auto_adjustment settings update works via control.sh
*
* @throws Exception If failed.
*/
@@ -604,8 +606,9 @@ public class GridCommandHandlerTest extends GridCommonAbstractTest {
assertEquals(EXIT_CODE_OK, execute(
"--baseline",
- "autoadjust",
+ "auto_adjust",
"enable",
+ "timeout",
Long.toString(timeout + 1)
));
@@ -613,21 +616,26 @@ public class GridCommandHandlerTest extends GridCommonAbstractTest {
assertEquals(timeout + 1, cl.baselineAutoAdjustTimeout());
- assertEquals(EXIT_CODE_OK, execute("--baseline", "autoadjust", "disable"));
+ assertEquals(EXIT_CODE_OK, execute("--baseline", "auto_adjust", "disable"));
assertFalse(cl.isBaselineAutoAdjustEnabled());
- assertEquals(EXIT_CODE_INVALID_ARGUMENTS, execute("--baseline", "autoadjust"));
+ assertEquals(EXIT_CODE_INVALID_ARGUMENTS, execute("--baseline", "auto_adjust"));
- assertEquals(EXIT_CODE_INVALID_ARGUMENTS, execute("--baseline", "autoadjust", "true"));
+ assertEquals(EXIT_CODE_INVALID_ARGUMENTS, execute("--baseline", "auto_adjust", "true"));
- assertEquals(EXIT_CODE_INVALID_ARGUMENTS, execute("--baseline", "autoadjust", "enable", "x"));
+ assertEquals(EXIT_CODE_INVALID_ARGUMENTS, execute("--baseline", "auto_adjust", "enable", "x"));
- assertEquals(EXIT_CODE_INVALID_ARGUMENTS, execute("--baseline", "autoadjust", "disable", "x"));
+ assertEquals(EXIT_CODE_INVALID_ARGUMENTS, execute("--baseline", "auto_adjust", "disable", "x"));
+
+ log.info("================================================");
+ System.out.println(testOut.toString());
+
+ log.info("================================================");
}
/**
- * Test that updating of baseline autoadjustment settings via control.sh actually influence cluster's baseline.
+ * Test that updating of baseline auto_adjustment settings via control.sh actually influence cluster's baseline.
*
* @throws Exception If failed.
*/
@@ -637,7 +645,7 @@ public class GridCommandHandlerTest extends GridCommonAbstractTest {
ignite.cluster().active(true);
- assertEquals(EXIT_CODE_OK, execute("--baseline", "autoadjust", "enable", "2000"));
+ assertEquals(EXIT_CODE_OK, execute("--baseline", "auto_adjust", "enable", "timeout", "2000"));
assertEquals(3, ignite.cluster().currentBaselineTopology().size());
@@ -645,13 +653,11 @@ public class GridCommandHandlerTest extends GridCommonAbstractTest {
assertEquals(3, ignite.cluster().currentBaselineTopology().size());
- Thread.sleep(3000L);
+ assertTrue(waitForCondition(() -> ignite.cluster().currentBaselineTopology().size() == 2, 10000));
Collection<BaselineNode> baselineNodesAfter = ignite.cluster().currentBaselineTopology();
- assertEquals(2, baselineNodesAfter.size());
-
- assertEquals(EXIT_CODE_OK, execute("--baseline", "autoadjust", "disable"));
+ assertEquals(EXIT_CODE_OK, execute("--baseline", "auto_adjust", "disable"));
stopGrid(1);
@@ -666,7 +672,7 @@ public class GridCommandHandlerTest extends GridCommonAbstractTest {
}
/**
- * Test that updating of baseline autoadjustment settings via control.sh actually influence cluster's baseline.
+ * Test that updating of baseline auto_adjustment settings via control.sh actually influence cluster's baseline.
*
* @throws Exception If failed.
*/
@@ -676,7 +682,7 @@ public class GridCommandHandlerTest extends GridCommonAbstractTest {
ignite.cluster().active(true);
- assertEquals(EXIT_CODE_OK, execute("--baseline", "autoadjust", "enable", "2000"));
+ assertEquals(EXIT_CODE_OK, execute("--baseline", "auto_adjust", "enable", "timeout", "2000"));
assertEquals(1, ignite.cluster().currentBaselineTopology().size());
@@ -684,13 +690,13 @@ public class GridCommandHandlerTest extends GridCommonAbstractTest {
assertEquals(1, ignite.cluster().currentBaselineTopology().size());
- Thread.sleep(3000L);
+ assertEquals(EXIT_CODE_OK, execute("--baseline"));
- Collection<BaselineNode> baselineNodesAfter = ignite.cluster().currentBaselineTopology();
+ assertTrue(waitForCondition(() -> ignite.cluster().currentBaselineTopology().size() == 2, 10000));
- assertEquals(2, baselineNodesAfter.size());
+ Collection<BaselineNode> baselineNodesAfter = ignite.cluster().currentBaselineTopology();
- assertEquals(EXIT_CODE_OK, execute("--baseline", "autoadjust", "disable"));
+ assertEquals(EXIT_CODE_OK, execute("--baseline", "auto_adjust", "disable"));
startGrid(2);
@@ -1045,7 +1051,7 @@ public class GridCommandHandlerTest extends GridCommonAbstractTest {
Collection<IgniteInternalTx> txs = ((IgniteEx)ignite).context().cache().context().tm().activeTransactions();
- GridTestUtils.waitForCondition(new GridAbsPredicate() {
+ waitForCondition(new GridAbsPredicate() {
@Override public boolean apply() {
for (IgniteInternalTx tx : txs)
if (!tx.local()) {