You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by vo...@apache.org on 2018/05/10 09:02:33 UTC
[2/2] ignite git commit: IGNITE-7912: control.sh script should show
used WAL-segments. This closes #3636. This closes #3965.
IGNITE-7912: control.sh script should show used WAL-segments. This closes #3636. This closes #3965.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/07cbe22d
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/07cbe22d
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/07cbe22d
Branch: refs/heads/master
Commit: 07cbe22da57552986123e2ceacf09da55a3d2ee0
Parents: ab7a055
Author: Ivan Daschinskiy <iv...@gmail.com>
Authored: Thu May 10 12:02:20 2018 +0300
Committer: devozerov <vo...@gridgain.com>
Committed: Thu May 10 12:02:20 2018 +0300
----------------------------------------------------------------------
.../apache/ignite/IgniteSystemProperties.java | 3 +
.../ignite/internal/commandline/Arguments.java | 80 +++--
.../ignite/internal/commandline/Command.java | 5 +-
.../internal/commandline/CommandHandler.java | 228 ++++++++++++-
.../pagemem/wal/IgniteWriteAheadLogManager.java | 10 +
.../GridCacheDatabaseSharedManager.java | 20 ++
.../wal/FileWriteAheadLogManager.java | 28 ++
.../wal/FsyncModeFileWriteAheadLogManager.java | 30 ++
.../internal/visor/misc/VisorClusterNode.java | 129 +++++++
.../internal/visor/misc/VisorWalTask.java | 336 +++++++++++++++++++
.../internal/visor/misc/VisorWalTaskArg.java | 98 ++++++
.../visor/misc/VisorWalTaskOperation.java | 44 +++
.../internal/visor/misc/VisorWalTaskResult.java | 113 +++++++
.../resources/META-INF/classnames.properties | 6 +
.../commandline/CommandHandlerParsingTest.java | 119 ++++++-
.../db/IgnitePdsUnusedWalSegmentsTest.java | 202 +++++++++++
.../persistence/pagemem/NoOpWALManager.java | 7 +-
.../ignite/testsuites/IgnitePdsTestSuite2.java | 3 +
.../testsuites/IgniteUtilSelfTestSuite.java | 2 +
.../ignite/util/GridCommandHandlerTest.java | 72 ++++
.../GridInternalTaskUnusedWalSegmentsTest.java | 153 +++++++++
.../e2e/testcafe/page-models/pageSignin.js | 32 --
22 files changed, 1634 insertions(+), 86 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/07cbe22d/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
index 32fed05..008974c 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
@@ -277,6 +277,9 @@ public final class IgniteSystemProperties {
/** System property to hold SSH host for visor-started nodes. */
public static final String IGNITE_SSH_HOST = "IGNITE_SSH_HOST";
+ /** System property to enable experimental commands in control.sh script. */
+ public static final String IGNITE_ENABLE_EXPERIMENTAL_COMMAND = "IGNITE_ENABLE_EXPERIMENTAL_COMMAND";
+
/** System property to hold SSH user name for visor-started nodes. */
public static final String IGNITE_SSH_USER_NAME = "IGNITE_SSH_USER_NAME";
http://git-wip-us.apache.org/repos/asf/ignite/blob/07cbe22d/modules/core/src/main/java/org/apache/ignite/internal/commandline/Arguments.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/Arguments.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/Arguments.java
index ce72693..0d4b38e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/Arguments.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/Arguments.java
@@ -53,12 +53,6 @@ public class Arguments {
*/
private String baselineArgs;
- /** Ping timeout for grid client. See {@link GridClientConfiguration#pingTimeout}.*/
- private long pingTimeout;
-
- /** Ping interval for grid client. See {@link GridClientConfiguration#pingInterval}.*/
- private long pingInterval;
-
/** Transaction arguments. */
private final VisorTxTaskArg txArg;
@@ -68,6 +62,22 @@ public class Arguments {
private CacheArguments cacheArgs;
/**
+ * Action for WAL command.
+ */
+ private String walAct;
+
+ /**
+ * Arguments for WAL command.
+ */
+ private String walArgs;
+
+ /** Ping timeout for grid client. See {@link GridClientConfiguration#pingTimeout}.*/
+ private long pingTimeout;
+
+ /** Ping interval for grid client. See {@link GridClientConfiguration#pingInterval}.*/
+ private long pingInterval;
+
+ /**
* @param cmd Command.
* @param host Host.
* @param port Port.
@@ -76,14 +86,16 @@ public class Arguments {
* @param baselineAct Baseline action.
* @param baselineArgs Baseline args.
* @param txArg TX arg.
- * @param force Force flag.
+ * @param cacheArgs --cache subcommand arguments.
+ * @param walAct WAL action.
+ * @param walArgs WAL args.
* @param pingTimeout Ping timeout. See {@link GridClientConfiguration#pingTimeout}.
* @param pingInterval Ping interval. See {@link GridClientConfiguration#pingInterval}.
- * @param cacheArgs --cache subcommand arguments.
+ * @param force Force flag.
*/
public Arguments(Command cmd, String host, String port, String user, String pwd, String baselineAct,
- String baselineArgs, long pingTimeout, long pingInterval, VisorTxTaskArg txArg, boolean force,
- CacheArguments cacheArgs) {
+ String baselineArgs, VisorTxTaskArg txArg, CacheArguments cacheArgs, String walAct, String walArgs,
+ Long pingTimeout, Long pingInterval, boolean force) {
this.cmd = cmd;
this.host = host;
this.port = port;
@@ -91,11 +103,13 @@ public class Arguments {
this.pwd = pwd;
this.baselineAct = baselineAct;
this.baselineArgs = baselineArgs;
+ this.txArg = txArg;
+ this.cacheArgs = cacheArgs;
+ this.walAct = walAct;
+ this.walArgs = walArgs;
this.pingTimeout = pingTimeout;
this.pingInterval = pingInterval;
this.force = force;
- this.txArg = txArg;
- this.cacheArgs = cacheArgs;
}
/**
@@ -148,6 +162,34 @@ public class Arguments {
}
/**
+ * @return Transaction arguments.
+ */
+ public VisorTxTaskArg transactionArguments() {
+ return txArg;
+ }
+
+ /**
+ * @return Arguments for --cache subcommand.
+ */
+ public CacheArguments cacheArgs() {
+ return cacheArgs;
+ }
+
+ /**
+ * @return WAL action.
+ */
+ public String walAction() {
+ return walAct;
+ }
+
+ /**
+ * @return WAL arguments.
+ */
+ public String walArguments() {
+ return walArgs;
+ }
+
+ /**
* See {@link GridClientConfiguration#pingTimeout}.
*
* @return Ping timeout.
@@ -166,23 +208,9 @@ public class Arguments {
}
/**
- * @return Transaction arguments.
- */
- public VisorTxTaskArg transactionArguments() {
- return txArg;
- }
-
- /**
* @return Force option.
*/
public boolean force() {
return force;
}
-
- /**
- * @return Arguments for --cache subcommand.
- */
- public CacheArguments cacheArgs() {
- return cacheArgs;
- }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/07cbe22d/modules/core/src/main/java/org/apache/ignite/internal/commandline/Command.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/Command.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/Command.java
index 52098d6..c64e488 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/Command.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/Command.java
@@ -37,7 +37,10 @@ public enum Command {
TX("--tx"),
/** */
- CACHE("--cache");
+ CACHE("--cache"),
+
+ /** */
+ WAL("--wal");
/** */
private final String text;
http://git-wip-us.apache.org/repos/asf/ignite/blob/07cbe22d/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
index ff05684..7d457fd 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
@@ -32,6 +32,7 @@ import java.util.logging.Logger;
import java.util.regex.Pattern;
import java.util.regex.PatternSyntaxException;
import java.util.stream.Collectors;
+import org.apache.ignite.IgniteSystemProperties;
import org.apache.ignite.cluster.ClusterNode;
import org.apache.ignite.internal.client.GridClient;
import org.apache.ignite.internal.client.GridClientAuthenticationException;
@@ -61,6 +62,11 @@ import org.apache.ignite.internal.visor.baseline.VisorBaselineOperation;
import org.apache.ignite.internal.visor.baseline.VisorBaselineTask;
import org.apache.ignite.internal.visor.baseline.VisorBaselineTaskArg;
import org.apache.ignite.internal.visor.baseline.VisorBaselineTaskResult;
+import org.apache.ignite.internal.visor.misc.VisorClusterNode;
+import org.apache.ignite.internal.visor.misc.VisorWalTask;
+import org.apache.ignite.internal.visor.misc.VisorWalTaskArg;
+import org.apache.ignite.internal.visor.misc.VisorWalTaskOperation;
+import org.apache.ignite.internal.visor.misc.VisorWalTaskResult;
import org.apache.ignite.internal.visor.tx.VisorTxInfo;
import org.apache.ignite.internal.visor.tx.VisorTxOperation;
import org.apache.ignite.internal.visor.tx.VisorTxProjection;
@@ -86,6 +92,7 @@ import org.apache.ignite.lang.IgniteClosure;
import org.apache.ignite.plugin.security.SecurityCredentials;
import org.apache.ignite.plugin.security.SecurityCredentialsBasicProvider;
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_ENABLE_EXPERIMENTAL_COMMAND;
import static org.apache.ignite.internal.IgniteVersionUtils.ACK_VER_STR;
import static org.apache.ignite.internal.IgniteVersionUtils.COPYRIGHT;
import static org.apache.ignite.internal.commandline.Command.ACTIVATE;
@@ -93,6 +100,7 @@ import static org.apache.ignite.internal.commandline.Command.BASELINE;
import static org.apache.ignite.internal.commandline.Command.CACHE;
import static org.apache.ignite.internal.commandline.Command.DEACTIVATE;
import static org.apache.ignite.internal.commandline.Command.STATE;
+import static org.apache.ignite.internal.commandline.Command.WAL;
import static org.apache.ignite.internal.commandline.Command.TX;
import static org.apache.ignite.internal.visor.baseline.VisorBaselineOperation.ADD;
import static org.apache.ignite.internal.visor.baseline.VisorBaselineOperation.COLLECT;
@@ -133,6 +141,12 @@ public class CommandHandler {
/** Force option is used for auto confirmation. */
private static final String CMD_FORCE = "--force";
+ /** */
+ protected static final String CMD_PING_INTERVAL = "--ping-interval";
+
+ /** */
+ protected static final String CMD_PING_TIMEOUT = "--ping-timeout";
+
/** List of optional auxiliary commands. */
private static final Set<String> AUX_COMMANDS = new HashSet<>();
static {
@@ -142,18 +156,14 @@ public class CommandHandler {
AUX_COMMANDS.add(CMD_PASSWORD);
AUX_COMMANDS.add(CMD_USER);
AUX_COMMANDS.add(CMD_FORCE);
+ AUX_COMMANDS.add(CMD_PING_INTERVAL);
+ AUX_COMMANDS.add(CMD_PING_TIMEOUT);
}
/** Broadcast uuid. */
private static final UUID BROADCAST_UUID = UUID.randomUUID();
/** */
- protected static final String CMD_PING_INTERVAL = "--ping-interval";
-
- /** */
- protected static final String CMD_PING_TIMEOUT = "--ping-timeout";
-
- /** */
public static final String CONFIRM_MSG = "y";
/** */
@@ -172,7 +182,13 @@ public class CommandHandler {
private static final String BASELINE_SET_VERSION = "version";
/** */
- private static final String DELIM = "--------------------------------------------------------------------------------";
+ static final String WAL_PRINT = "print";
+
+ /** */
+ static final String WAL_DELETE = "delete";
+
+ /** */
+ static final String DELIM = "--------------------------------------------------------------------------------";
/** */
public static final int EXIT_CODE_OK = 0;
@@ -240,6 +256,9 @@ public class CommandHandler {
/** */
private Object lastOperationRes;
+ /** Check if experimental commands are enabled. Default {@code false}. */
+ private final boolean enableExperimental = IgniteSystemProperties.getBoolean(IGNITE_ENABLE_EXPERIMENTAL_COMMAND, false);
+
/**
* Output specified string to console.
*
@@ -332,6 +351,12 @@ public class CommandHandler {
break;
+ case WAL:
+ if (WAL_DELETE.equals(args.walAction()))
+ str = "Warning: the command will delete unused WAL segments.";
+
+ break;
+
case TX:
if (args.transactionArguments().getOperation() == VisorTxOperation.KILL)
str = "Warning: the command will kill some transactions.";
@@ -942,6 +967,147 @@ public class CommandHandler {
}
/**
+ * Execute WAL command.
+ *
+ * @param client Client.
+ * @param walAct WAL action to execute.
+ * @param walArgs WAL args.
+ * @throws Throwable If failed to execute wal action.
+ */
+ private void wal(GridClient client, String walAct, String walArgs) throws Throwable {
+ switch (walAct){
+ case WAL_DELETE:
+ deleteUnusedWalSegments(client, walArgs);
+
+ break;
+
+ case WAL_PRINT:
+ default:
+ printUnusedWalSegments(client, walArgs);
+
+ break;
+ }
+ }
+
+ /**
+ * Execute delete unused WAL segments task.
+ *
+ * @param client Client.
+ * @param walArgs WAL args.
+ */
+ private void deleteUnusedWalSegments(GridClient client, String walArgs) throws Throwable {
+ VisorWalTaskResult res = executeTask(client, VisorWalTask.class,
+ walArg(VisorWalTaskOperation.DELETE_UNUSED_WAL_SEGMENTS, walArgs));
+ printDeleteWalSegments0(res);
+ }
+
+ /**
+ * Execute print unused WAL segments task.
+ *
+ * @param client Client.
+ * @param walArgs Wal args.
+ */
+ private void printUnusedWalSegments(GridClient client, String walArgs) throws Throwable {
+ VisorWalTaskResult res = executeTask(client, VisorWalTask.class,
+ walArg(VisorWalTaskOperation.PRINT_UNUSED_WAL_SEGMENTS, walArgs));
+ printUnusedWalSegments0(res);
+ }
+
+ /**
+ * Prepare WAL task argument.
+ *
+ * @param op Operation.
+ * @param s Argument from command line.
+ * @return Task argument.
+ */
+ private VisorWalTaskArg walArg(VisorWalTaskOperation op, String s){
+ List<String> consistentIds = null;
+
+ if (!F.isEmpty(s)) {
+ consistentIds = new ArrayList<>();
+
+ for (String consistentId : s.split(","))
+ consistentIds.add(consistentId.trim());
+ }
+
+ switch (op) {
+ case DELETE_UNUSED_WAL_SEGMENTS:
+ case PRINT_UNUSED_WAL_SEGMENTS:
+ return new VisorWalTaskArg(op, consistentIds);
+
+ default:
+ return new VisorWalTaskArg(VisorWalTaskOperation.PRINT_UNUSED_WAL_SEGMENTS, consistentIds);
+ }
+
+ }
+
+ /**
+ * Print list of unused wal segments.
+ *
+ * @param taskRes Task result with baseline topology.
+ */
+ private void printUnusedWalSegments0(VisorWalTaskResult taskRes) {
+ log("Unused wal segments per node:");
+ nl();
+
+ Map<String, Collection<String>> res = taskRes.results();
+ Map<String, Exception> failRes = taskRes.exceptions();
+ Map<String, VisorClusterNode> nodesInfo = taskRes.getNodesInfo();
+
+ for(Map.Entry<String, Collection<String>> entry: res.entrySet()) {
+ VisorClusterNode node = nodesInfo.get(entry.getKey());
+
+ log("Node=" + node.getConsistentId());
+ log(" addresses " + U.addressesAsString(node.getAddresses(),node.getHostNames()));
+
+ for(String fileName: entry.getValue())
+ log(" " + fileName);
+
+ nl();
+ }
+
+ for(Map.Entry<String, Exception> entry: failRes.entrySet()) {
+ VisorClusterNode node = nodesInfo.get(entry.getKey());
+
+ log("Node=" + node.getConsistentId());
+ log(" addresses " + U.addressesAsString(node.getAddresses(),node.getHostNames()));
+ log(" failed with error: " + entry.getValue().getMessage());
+ nl();
+ }
+ }
+
+ /**
+ * Print list of unused wal segments.
+ *
+ * @param taskRes Task result with baseline topology.
+ */
+ private void printDeleteWalSegments0(VisorWalTaskResult taskRes) {
+ log("WAL segments deleted for nodes:");
+ nl();
+
+ Map<String, Collection<String>> res = taskRes.results();
+ Map<String, Exception> errors = taskRes.exceptions();
+ Map<String, VisorClusterNode> nodesInfo = taskRes.getNodesInfo();
+
+ for(Map.Entry<String, Collection<String>> entry: res.entrySet()) {
+ VisorClusterNode node = nodesInfo.get(entry.getKey());
+
+ log("Node=" + node.getConsistentId());
+ log(" addresses " + U.addressesAsString(node.getAddresses(),node.getHostNames()));
+ nl();
+ }
+
+ for(Map.Entry<String, Exception> entry: errors.entrySet()) {
+ VisorClusterNode node = nodesInfo.get(entry.getKey());
+
+ log("Node=" + node.getConsistentId());
+ log(" addresses " + U.addressesAsString(node.getAddresses(),node.getHostNames()));
+ log(" failed with error: " + entry.getValue().getMessage());
+ nl();
+ }
+ }
+
+ /**
* @param e Exception to check.
* @return {@code true} if specified exception is {@link GridClientAuthenticationException}.
*/
@@ -1031,6 +1197,10 @@ public class CommandHandler {
Long pingTimeout = DFLT_PING_TIMEOUT;
+ String walAct = "";
+
+ String walArgs = "";
+
boolean force = false;
CacheArguments cacheArgs = null;
@@ -1052,6 +1222,7 @@ public class CommandHandler {
case DEACTIVATE:
case STATE:
commands.add(cmd);
+
break;
case TX:
@@ -1088,6 +1259,24 @@ public class CommandHandler {
break;
+ case WAL:
+ if (!enableExperimental)
+ throw new IllegalArgumentException("Experimental command is disabled.");
+
+ commands.add(WAL);
+
+ str = nextArg("Expected arguments for " + WAL.text());
+
+ walAct = str.toLowerCase();
+
+ if (WAL_PRINT.equals(walAct) || WAL_DELETE.equals(walAct))
+ walArgs = (str = peekNextArg()) != null && !isCommandOrOption(str)
+ ? nextArg("Unexpected argument for " + WAL.text() + ": " + walAct)
+ : "";
+ else
+ throw new IllegalArgumentException("Unexpected action " + walAct + " for " + WAL.text());
+
+ break;
default:
throw new IllegalArgumentException("Unexpected command: " + str);
}
@@ -1161,8 +1350,8 @@ public class CommandHandler {
if (hasUsr != hasPwd)
throw new IllegalArgumentException("Both user and password should be specified");
- return new Arguments(cmd, host, port, user, pwd, baselineAct, baselineArgs,
- pingTimeout, pingInterval, txArgs, force, cacheArgs);
+ return new Arguments(cmd, host, port, user, pwd, baselineAct, baselineArgs, txArgs, cacheArgs, walAct, walArgs,
+ pingTimeout, pingInterval, force);
}
/**
@@ -1430,6 +1619,15 @@ public class CommandHandler {
}
/**
+ * Check if raw arg is command or option.
+ *
+ * @return {@code true} If raw arg is command, overwise {@code false}.
+ */
+ private boolean isCommandOrOption(String raw) {
+ return raw != null && raw.contains("--");
+ }
+
+ /**
* Parse and execute command.
*
* @param rawArgs Arguments to parse and execute.
@@ -1457,6 +1655,13 @@ public class CommandHandler {
"[minSize SIZE] [label PATTERN_REGEX] [servers|clients] " +
"[nodes consistentId1[,consistentId2,....,consistentIdN] [limit NUMBER] [order DURATION|SIZE] [kill] [--force]");
+ if(enableExperimental) {
+ usage(" Print absolute paths of unused archived wal segments on each node:", WAL,
+ " print [consistentId1,consistentId2,....,consistentIdN]");
+ usage(" Delete unused archived wal segments on each node:", WAL,
+ " delete [consistentId1,consistentId2,....,consistentIdN] [--force]");
+ }
+
log("The utility has --cache subcommand to view and control state of caches in cluster.");
log(" More info: control.sh --cache help");
nl();
@@ -1534,6 +1739,11 @@ public class CommandHandler {
cache(client, args.cacheArgs());
break;
+
+ case WAL:
+ wal(client, args.walAction(), args.walArguments());
+
+ break;
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/07cbe22d/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/IgniteWriteAheadLogManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/IgniteWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/IgniteWriteAheadLogManager.java
index 8e8a1b3..b5c22c9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/IgniteWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/IgniteWriteAheadLogManager.java
@@ -123,12 +123,22 @@ public interface IgniteWriteAheadLogManager extends GridCacheSharedManager, Igni
/**
* Checks if WAL segment is under lock or reserved
+ *
* @param ptr Pointer to check.
* @return True if given pointer is located in reserved segment.
*/
public boolean reserved(WALPointer ptr);
/**
+ * Checks if WAL segments is under lock or reserved.
+ *
+ * @param low Pointer since which WAL is locked or reserved. If {@code null}, checks from the oldest segment.
+ * @param high Pointer for which WAL is locked or reserved.
+ * @return Number of reserved WAL segments.
+ */
+ public int reserved(WALPointer low, WALPointer high);
+
+ /**
* Checks WAL disabled for cache group.
*
* @param grpId Group id.
http://git-wip-us.apache.org/repos/asf/ignite/blob/07cbe22d/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
index 9fbe86c..4b183f7 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -3783,6 +3783,26 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
}
/**
+ * @return First checkpoint entry if exists. Otherwise {@code null}.
+ */
+ private CheckpointEntry firstEntry() {
+ Map.Entry<Long,CheckpointEntry> entry = histMap.firstEntry();
+
+ return entry != null ? entry.getValue() : null;
+ }
+
+ /**
+ * Get WAL pointer to low checkpoint bound.
+ *
+ * @return WAL pointer to low checkpoint bound.
+ */
+ public WALPointer lowCheckpointBound() {
+ CheckpointEntry entry = firstEntry();
+
+ return entry != null ? entry.cpMark : null;
+ }
+
+ /**
* @return Collection of checkpoint timestamps.
*/
public Collection<Long> checkpoints() {
http://git-wip-us.apache.org/repos/asf/ignite/blob/07cbe22d/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
index 8755e1b..e69cc80 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
@@ -947,6 +947,34 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
}
/** {@inheritDoc} */
+ @Override public int reserved(WALPointer low, WALPointer high) {
+ // It is not clear now how to get the highest WAL pointer. So when high is null method returns 0.
+ if (high == null)
+ return 0;
+
+ assert high instanceof FileWALPointer : high;
+
+ assert low == null || low instanceof FileWALPointer : low;
+
+ FileWALPointer lowPtr = (FileWALPointer)low;
+
+ FileWALPointer highPtr = (FileWALPointer)high;
+
+ long lowIdx = lowPtr != null ? lowPtr.index() : 0;
+
+ long highIdx = highPtr.index();
+
+ while (lowIdx < highIdx) {
+ if (segmentReservedOrLocked(lowIdx))
+ break;
+
+ lowIdx++;
+ }
+
+ return (int)(highIdx - lowIdx + 1);
+ }
+
+ /** {@inheritDoc} */
@Override public boolean disabled(int grpId) {
CacheGroupContext ctx = cctx.cache().cacheGroup(grpId);
http://git-wip-us.apache.org/repos/asf/ignite/blob/07cbe22d/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
index c446f7f..7bb2ce4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
@@ -849,6 +849,36 @@ public class FsyncModeFileWriteAheadLogManager extends GridCacheSharedManagerAda
}
/** {@inheritDoc} */
+ @Override public int reserved(WALPointer low, WALPointer high) {
+ // It is not clear now how to get the highest WAL pointer. So when high is null method returns 0.
+ if (high == null)
+ return 0;
+
+ assert high instanceof FileWALPointer : high;
+
+ assert low == null || low instanceof FileWALPointer : low;
+
+ FileWALPointer lowPtr = (FileWALPointer)low;
+
+ FileWALPointer highPtr = (FileWALPointer)high;
+
+ FileArchiver archiver0 = archiver;
+
+ long lowIdx = lowPtr != null ? lowPtr.index() : 0;
+
+ long highIdx = highPtr.index();
+
+ while (lowIdx < highIdx) {
+ if(archiver0 != null && archiver0.reserved(lowIdx))
+ break;
+
+ lowIdx++;
+ }
+
+ return (int)(highIdx - lowIdx + 1);
+ }
+
+ /** {@inheritDoc} */
@Override public boolean disabled(int grpId) {
CacheGroupContext ctx = cctx.cache().cacheGroup(grpId);
http://git-wip-us.apache.org/repos/asf/ignite/blob/07cbe22d/modules/core/src/main/java/org/apache/ignite/internal/visor/misc/VisorClusterNode.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/misc/VisorClusterNode.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/misc/VisorClusterNode.java
new file mode 100644
index 0000000..5706444
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/misc/VisorClusterNode.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.visor.misc;
+
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import java.util.Collection;
+import java.util.Map;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.internal.util.tostring.GridToStringInclude;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.internal.visor.VisorDataTransferObject;
+
+/**
+ * Data transfer object for {@link ClusterNode}.
+ */
+public class VisorClusterNode extends VisorDataTransferObject {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** Cluster node consistent id. */
+ @GridToStringInclude
+ private String consistentId;
+
+ /** Cluster node attributes. */
+ private Map<String, Object> attrs;
+
+ /** Cluster node addresses. */
+ @GridToStringInclude
+ private Collection<String> addrs;
+
+ /** Cluster node host names. */
+ @GridToStringInclude
+ private Collection<String> hostNames;
+
+ /**
+ * Default constructor.
+ */
+ public VisorClusterNode() {
+ // No-op.
+ }
+
+ /**
+ * Create data transfer object for baseline node.
+ *
+ * @param node Baseline node.
+ */
+ public VisorClusterNode(ClusterNode node) {
+ consistentId = String.valueOf(node.consistentId());
+ addrs = node.addresses();
+ hostNames = node.hostNames();
+ attrs = node.attributes();
+ }
+
+ /**
+ * Get cluster node consistent id.
+ *
+ * @return Cluster node consistent id.
+ */
+ public String getConsistentId() {
+ return consistentId;
+ }
+
+ /**
+ * Get cluster node attributes.
+ *
+ * @return Cluster node attributes.
+ */
+ public Map<String, Object> getAttributes() {
+ return attrs;
+ }
+
+ /**
+ * Get cluster node addresses.
+ *
+ * @return Node addresses.
+ */
+ public Collection<String> getAddresses() {
+ return addrs;
+ }
+
+ /**
+ * Get cluster node host names.
+ *
+ * @return Node host names.
+ */
+ public Collection<String> getHostNames() {
+ return hostNames;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void writeExternalData(ObjectOutput out) throws IOException {
+ U.writeString(out, consistentId);
+ U.writeMap(out, attrs);
+ U.writeCollection(out, hostNames);
+ U.writeCollection(out, addrs);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void readExternalData(byte protoVer, ObjectInput in) throws IOException, ClassNotFoundException {
+ consistentId = U.readString(in);
+ attrs = U.readMap(in);
+ hostNames = U.readCollection(in);
+ addrs = U.readCollection(in);
+ }
+
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(VisorClusterNode.class, this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/07cbe22d/modules/core/src/main/java/org/apache/ignite/internal/visor/misc/VisorWalTask.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/misc/VisorWalTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/misc/VisorWalTask.java
new file mode 100644
index 0000000..7edcea9
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/misc/VisorWalTask.java
@@ -0,0 +1,336 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.visor.misc;
+
+import java.io.File;
+import java.io.FileFilter;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.regex.Pattern;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.compute.ComputeJobResult;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.GridKernalContext;
+import org.apache.ignite.internal.pagemem.wal.WALPointer;
+import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
+import org.apache.ignite.internal.processors.cache.persistence.filename.PdsFolderSettings;
+import org.apache.ignite.internal.processors.cache.persistence.wal.FileWALPointer;
+import org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager;
+import org.apache.ignite.internal.processors.task.GridInternal;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.internal.visor.VisorJob;
+import org.apache.ignite.internal.visor.VisorMultiNodeTask;
+import org.apache.ignite.internal.visor.VisorTaskArgument;
+import org.apache.ignite.resources.LoggerResource;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Performs WAL cleanup clusterwide.
+ */
+@GridInternal
+public class VisorWalTask extends VisorMultiNodeTask<VisorWalTaskArg, VisorWalTaskResult, Collection<String>> {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** Pattern for segment file names. */
+ private static final Pattern WAL_NAME_PATTERN = Pattern.compile("\\d{16}\\.wal");
+
+ /** Pattern for compacted segment file names. */
+ private static final Pattern WAL_SEGMENT_FILE_COMPACTED_PATTERN = Pattern.compile("\\d{16}\\.wal\\.zip");
+
+ /** WAL archive file filter. */
+ private static final FileFilter WAL_ARCHIVE_FILE_FILTER = new FileFilter() {
+ @Override public boolean accept(File file) {
+ return !file.isDirectory() && (WAL_NAME_PATTERN.matcher(file.getName()).matches() ||
+ WAL_SEGMENT_FILE_COMPACTED_PATTERN.matcher(file.getName()).matches());
+ }
+ };
+
+ /** {@inheritDoc} */
+ @Override protected VisorWalJob job(VisorWalTaskArg arg) {
+ return new VisorWalJob(arg, debug);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected Collection<UUID> jobNodes(VisorTaskArgument<VisorWalTaskArg> arg) {
+ Collection<ClusterNode> srvNodes = ignite.cluster().forServers().nodes();
+ Collection<UUID> ret = new ArrayList<>(srvNodes.size());
+
+ VisorWalTaskArg taskArg = arg.getArgument();
+
+ Set<String> nodeIds = taskArg.getConsistentIds() != null ? new HashSet<>(arg.getArgument().getConsistentIds())
+ : null;
+
+ if (nodeIds == null) {
+ for (ClusterNode node : srvNodes)
+ ret.add(node.id());
+ }
+ else {
+ for (ClusterNode node : srvNodes) {
+ if (nodeIds.contains(node.consistentId().toString()))
+ ret.add(node.id());
+ }
+ }
+
+ return ret;
+ }
+
+ /** {@inheritDoc} */
+ @Nullable @Override protected VisorWalTaskResult reduce0(List<ComputeJobResult> results) throws IgniteException {
+ Map<String, Exception> exRes = U.newHashMap(0);
+ Map<String, Collection<String>> res = U.newHashMap(results.size());
+ Map<String, VisorClusterNode> nodesInfo = U.newHashMap(results.size());
+
+ for (ComputeJobResult result: results){
+ ClusterNode node = result.getNode();
+
+ String nodeId = node.consistentId().toString();
+
+ if(result.getException() != null)
+ exRes.put(nodeId, result.getException());
+ else if (result.getData() != null) {
+ Collection<String> data = result.getData();
+
+ if(data != null)
+ res.put(nodeId, data);
+ }
+
+ nodesInfo.put(nodeId, new VisorClusterNode(node));
+ }
+
+ return new VisorWalTaskResult(res, exRes, nodesInfo);
+ }
+
+ /**
+ * Performs WAL cleanup per node.
+ */
+ private static class VisorWalJob extends VisorJob<VisorWalTaskArg, Collection<String>> {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** Auto injected logger */
+ @LoggerResource
+ private transient IgniteLogger log;
+
+ /**
+ * @param arg WAL task argument.
+ * @param debug Debug flag.
+ */
+ public VisorWalJob(VisorWalTaskArg arg, boolean debug) {
+ super(arg, debug);
+ }
+
+ /** {@inheritDoc} */
+ @Nullable @Override protected Collection<String> run(@Nullable VisorWalTaskArg arg) throws IgniteException {
+ try {
+ GridKernalContext cctx = ignite.context();
+
+ GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager)cctx.cache().context().database();
+ FileWriteAheadLogManager wal = (FileWriteAheadLogManager)cctx.cache().context().wal();
+
+ if (dbMgr == null || arg == null || wal == null)
+ return null;
+
+ switch (arg.getOperation()) {
+ case DELETE_UNUSED_WAL_SEGMENTS:
+ return deleteUnusedWalSegments(dbMgr, wal);
+
+ case PRINT_UNUSED_WAL_SEGMENTS:
+ default:
+ return getUnusedWalSegments(dbMgr, wal);
+
+ }
+ }
+ catch (IgniteCheckedException e){
+ U.error(log, "Failed to perform WAL task", e);
+
+ throw new IgniteException("Failed to perform WAL task", e);
+ }
+ }
+
+ /**
+ * Get unused wal segments.
+ *
+ * @param wal Database manager.
+ * @return {@link Collection<String>} of absolute paths of unused WAL segments.
+ * @throws IgniteCheckedException if failed.
+ */
+ Collection<String> getUnusedWalSegments(
+ GridCacheDatabaseSharedManager dbMgr,
+ FileWriteAheadLogManager wal
+ ) throws IgniteCheckedException{
+ WALPointer lowBoundForTruncate = dbMgr.checkpointHistory().lowCheckpointBound();
+
+ if (lowBoundForTruncate == null)
+ return Collections.emptyList();
+
+ int maxIdx = resolveMaxReservedIndex(wal, lowBoundForTruncate);
+
+ File[] walFiles = getWalArchiveDir().listFiles(WAL_ARCHIVE_FILE_FILTER);
+
+ Collection<String> res = new ArrayList<>(walFiles != null && walFiles.length > 0 ? walFiles.length - 1 : 0);
+
+ if(walFiles != null && walFiles.length > 0) {
+ sortWalFiles(walFiles);
+
+ // Obtain index of last archived WAL segment, it will not be deleted.
+ long lastArchIdx = getIndex(walFiles[walFiles.length - 1]);
+
+ for (File f : walFiles) {
+ long fileIdx = getIndex(f);
+
+ if (fileIdx < maxIdx && fileIdx < lastArchIdx)
+ res.add(f.getAbsolutePath());
+ else
+ break;
+ }
+ }
+
+ return res;
+ }
+
+ /**
+ * Delete unused wal segments.
+ *
+ * @param dbMgr Database manager.
+ * @return {@link Collection<String>} of deleted WAL segment's files.
+ * @throws IgniteCheckedException if failed.
+ */
+ Collection<String> deleteUnusedWalSegments(
+ GridCacheDatabaseSharedManager dbMgr,
+ FileWriteAheadLogManager wal
+ ) throws IgniteCheckedException {
+ WALPointer lowBoundForTruncate = dbMgr.checkpointHistory().lowCheckpointBound();
+
+ if (lowBoundForTruncate == null)
+ return Collections.emptyList();
+
+ int maxIdx = resolveMaxReservedIndex(wal, lowBoundForTruncate);
+
+ File[] walFiles = getWalArchiveDir().listFiles(WAL_ARCHIVE_FILE_FILTER);
+
+ dbMgr.onWalTruncated(lowBoundForTruncate);
+
+ int num = wal.truncate(null, lowBoundForTruncate);
+
+ if (walFiles != null) {
+ sortWalFiles(walFiles);
+
+ Collection<String> res = new ArrayList<>(num);
+
+ for (File walFile: walFiles) {
+ if (getIndex(walFile) < maxIdx && num > 0)
+ res.add(walFile.getAbsolutePath());
+ else
+ break;
+
+ num--;
+ }
+
+ return res;
+ }
+ else
+ return Collections.emptyList();
+
+ }
+
+ /**
+ *
+ */
+ private int resolveMaxReservedIndex(FileWriteAheadLogManager wal, WALPointer lowBoundForTruncate) {
+ FileWALPointer low = (FileWALPointer)lowBoundForTruncate;
+
+ int resCnt = wal.reserved(null, lowBoundForTruncate);
+
+ long highIdx = low.index();
+
+ return (int)(highIdx - resCnt + 1);
+ }
+
+ /**
+ * Get WAL archive directory from configuration.
+ *
+ * @return WAL archive directory.
+ * @throws IgniteCheckedException if failed.
+ */
+ private File getWalArchiveDir() throws IgniteCheckedException {
+ IgniteConfiguration igCfg = ignite.context().config();
+
+ DataStorageConfiguration dsCfg = igCfg.getDataStorageConfiguration();
+
+ PdsFolderSettings resFldrs = ignite.context().pdsFolderResolver().resolveFolders();
+
+ String consId = resFldrs.folderName();
+
+ File dir;
+
+ if (dsCfg.getWalArchivePath() != null) {
+ File workDir0 = new File(dsCfg.getWalArchivePath());
+
+ dir = workDir0.isAbsolute() ?
+ new File(workDir0, consId) :
+ new File(U.resolveWorkDirectory(igCfg.getWorkDirectory(), dsCfg.getWalArchivePath(), false),
+ consId);
+ }
+ else
+ dir = new File(U.resolveWorkDirectory(igCfg.getWorkDirectory(),
+ DataStorageConfiguration.DFLT_WAL_ARCHIVE_PATH, false), consId);
+
+ if (!dir.exists())
+ throw new IgniteCheckedException("WAL archive directory does not exists" + dir.getAbsolutePath());
+
+ return dir;
+ }
+
+
+ /**
+ * Sort WAL files according their indices.
+ *
+ * @param files Array of WAL segment files.
+ */
+ private void sortWalFiles(File[] files) {
+ Arrays.sort(files, new Comparator<File>() {
+ @Override public int compare(File o1, File o2) {
+ return Long.compare(getIndex(o1), getIndex(o2));
+ }
+ });
+ }
+ }
+
+ /**
+ * Get index from WAL segment file.
+ *
+ * @param file WAL segment file.
+ * @return Index of WAL segment file.
+ */
+ private static long getIndex(File file) {
+ return Long.parseLong(file.getName().substring(0, 16));
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/07cbe22d/modules/core/src/main/java/org/apache/ignite/internal/visor/misc/VisorWalTaskArg.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/misc/VisorWalTaskArg.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/misc/VisorWalTaskArg.java
new file mode 100644
index 0000000..afd3b58
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/misc/VisorWalTaskArg.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.visor.misc;
+
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import java.util.List;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.internal.visor.VisorDataTransferObject;
+
+/**
+ * Argument for {@link VisorWalTask}.
+ */
+public class VisorWalTaskArg extends VisorDataTransferObject{
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** WAL task operation. */
+ private VisorWalTaskOperation op;
+
+ /** List of nodes' consistent ids. */
+ private List<String> consistentIds;
+
+ /**
+ * Default constructor.
+ */
+ public VisorWalTaskArg() {
+ // No-op.
+ }
+
+ /**
+ * @param op Task operation.
+ */
+ public VisorWalTaskArg(VisorWalTaskOperation op) {
+ this.op = op;
+ }
+
+ /**
+ * @param op WAL task operation.
+ * @param consistentIds Nodes consistent ids.
+ */
+ public VisorWalTaskArg(VisorWalTaskOperation op, List<String> consistentIds) {
+ this.op = op;
+ this.consistentIds = consistentIds;
+ }
+
+ /**
+ * Get WAL task operation.
+ *
+ * @return WAL task operation.
+ */
+ public VisorWalTaskOperation getOperation() {
+ return op == null ? VisorWalTaskOperation.PRINT_UNUSED_WAL_SEGMENTS : op;
+ }
+
+ /**
+ * Get nodes consistent ids.
+ *
+ * @return Consistent ids.
+ */
+ public List<String> getConsistentIds() {
+ return consistentIds;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void writeExternalData(ObjectOutput out) throws IOException {
+ U.writeEnum(out, op);
+ U.writeCollection(out, consistentIds);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void readExternalData(byte protoVer, ObjectInput in) throws IOException, ClassNotFoundException {
+ op = VisorWalTaskOperation.fromOrdinal(in.readByte());
+ consistentIds = U.readList(in);
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(VisorWalTaskArg.class, this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/07cbe22d/modules/core/src/main/java/org/apache/ignite/internal/visor/misc/VisorWalTaskOperation.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/misc/VisorWalTaskOperation.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/misc/VisorWalTaskOperation.java
new file mode 100644
index 0000000..be2ff18
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/misc/VisorWalTaskOperation.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.visor.misc;
+
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * WAL Task operation types.
+ */
+public enum VisorWalTaskOperation {
+ /** Print unused wal segments. */
+ PRINT_UNUSED_WAL_SEGMENTS,
+
+ /** Delete unused wal segments. */
+ DELETE_UNUSED_WAL_SEGMENTS;
+
+ /** Enumerated values. */
+ private static final VisorWalTaskOperation[] VALS = values();
+
+ /**
+ * Efficiently gets enumerated value from its ordinal.
+ *
+ * @param ord Ordinal value.
+ * @return Enumerated value or {@code null} if ordinal out of range.
+ */
+ @Nullable public static VisorWalTaskOperation fromOrdinal(int ord) {
+ return ord >= 0 && ord < VALS.length ? VALS[ord] : null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/07cbe22d/modules/core/src/main/java/org/apache/ignite/internal/visor/misc/VisorWalTaskResult.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/misc/VisorWalTaskResult.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/misc/VisorWalTaskResult.java
new file mode 100644
index 0000000..104c2f3
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/misc/VisorWalTaskResult.java
@@ -0,0 +1,113 @@
+/*
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. See the NOTICE file distributed with
+ * * this work for additional information regarding copyright ownership.
+ * * The ASF licenses this file to You under the Apache License, Version 2.0
+ * * (the "License"); you may not use this file except in compliance with
+ * * the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License.
+ */
+
+package org.apache.ignite.internal.visor.misc;
+
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import java.util.Collection;
+import java.util.Map;
+import org.apache.ignite.internal.util.tostring.GridToStringInclude;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.internal.visor.VisorDataTransferObject;
+
+/**
+ * Result of {@link VisorWalTask}.
+ */
+public class VisorWalTaskResult extends VisorDataTransferObject {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** Exceptions by node consistent id. */
+ @GridToStringInclude
+ private Map<String, Exception> exceptions;
+
+ /** Archived wal segments path search results by node consistent id. */
+ @GridToStringInclude
+ private Map<String, Collection<String>> results;
+
+ /** Nodes info by node consistent id. */
+ @GridToStringInclude
+ private Map<String, VisorClusterNode> nodesInfo;
+
+ /**
+ * Default constructor.
+ */
+ public VisorWalTaskResult() {
+ // No-op.
+ }
+
+ /**
+ * Create {@link VisorWalTask } result with given parameters.
+ *
+ * @param results List of log search results.
+ * @param exceptions List of exceptions by node id.
+ * @param nodesInfo Nodes info.
+ */
+ public VisorWalTaskResult(Map<String, Collection<String>> results, Map<String, Exception> exceptions,
+ Map<String, VisorClusterNode> nodesInfo) {
+ this.exceptions = exceptions;
+ this.results = results;
+ this.nodesInfo = nodesInfo;
+ }
+
+ /**
+ * Get occurred errors by node consistent id.
+ *
+ * @return Exceptions by node consistent id.
+ */
+ public Map<String, Exception> exceptions() {
+ return exceptions;
+ }
+
+ /**
+ * @return List of archived wal segments path search results by node consistent id.
+ */
+ public Map<String, Collection<String>> results() {
+ return results;
+ }
+
+ /**
+ * Get nodes info by node consistent id.
+ *
+ * @return Nodes info.
+ */
+ public Map<String, VisorClusterNode> getNodesInfo() {
+ return nodesInfo;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void writeExternalData(ObjectOutput out) throws IOException {
+ U.writeMap(out, exceptions);
+ U.writeMap(out, results);
+ U.writeMap(out, nodesInfo);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void readExternalData(byte protoVer, ObjectInput in) throws IOException, ClassNotFoundException {
+ exceptions = U.readMap(in);
+ results = U.readMap(in);
+ nodesInfo = U.readMap(in);
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(VisorWalTaskResult.class, this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/07cbe22d/modules/core/src/main/resources/META-INF/classnames.properties
----------------------------------------------------------------------
diff --git a/modules/core/src/main/resources/META-INF/classnames.properties b/modules/core/src/main/resources/META-INF/classnames.properties
index 8ca47c8..f1aec97 100644
--- a/modules/core/src/main/resources/META-INF/classnames.properties
+++ b/modules/core/src/main/resources/META-INF/classnames.properties
@@ -2028,6 +2028,12 @@ org.apache.ignite.internal.visor.misc.VisorNopTask
org.apache.ignite.internal.visor.misc.VisorNopTask$VisorNopJob
org.apache.ignite.internal.visor.misc.VisorResolveHostNameTask
org.apache.ignite.internal.visor.misc.VisorResolveHostNameTask$VisorResolveHostNameJob
+org.apache.ignite.internal.visor.misc.VisorClusterNode
+org.apache.ignite.internal.visor.misc.VisorWalTask
+org.apache.ignite.internal.visor.misc.VisorWalTask$VisorWalJob
+org.apache.ignite.internal.visor.misc.VisorWalTaskArg
+org.apache.ignite.internal.visor.misc.VisorWalTaskOperation
+org.apache.ignite.internal.visor.misc.VisorWalTaskResult
org.apache.ignite.internal.visor.node.VisorAffinityTopologyVersion
org.apache.ignite.internal.visor.node.VisorAtomicConfiguration
org.apache.ignite.internal.visor.node.VisorBasicConfiguration
http://git-wip-us.apache.org/repos/asf/ignite/blob/07cbe22d/modules/core/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java b/modules/core/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java
index 98b8e01..2fc40ca 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java
@@ -1,46 +1,91 @@
/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
*
- * * Licensed to the Apache Software Foundation (ASF) under one or more
- * * contributor license agreements. See the NOTICE file distributed with
- * * this work for additional information regarding copyright ownership.
- * * The ASF licenses this file to You under the Apache License, Version 2.0
- * * (the "License"); you may not use this file except in compliance with
- * * the License. You may obtain a copy of the License at
- * *
- * * http://www.apache.org/licenses/LICENSE-2.0
- * *
- * * Unless required by applicable law or agreed to in writing, software
- * * distributed under the License is distributed on an "AS IS" BASIS,
- * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * * See the License for the specific language governing permissions and
- * * limitations under the License.
+ * http://www.apache.org/licenses/LICENSE-2.0
*
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
package org.apache.ignite.internal.commandline;
import java.util.Arrays;
+import java.util.Collections;
+import java.util.UUID;
import junit.framework.TestCase;
import org.apache.ignite.internal.visor.tx.VisorTxProjection;
import org.apache.ignite.internal.visor.tx.VisorTxSortOrder;
import org.apache.ignite.internal.visor.tx.VisorTxTaskArg;
import static java.util.Arrays.asList;
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_ENABLE_EXPERIMENTAL_COMMAND;
+import static org.apache.ignite.internal.commandline.Command.WAL;
import static org.apache.ignite.internal.commandline.CommandHandler.DFLT_HOST;
import static org.apache.ignite.internal.commandline.CommandHandler.DFLT_PORT;
+import static org.apache.ignite.internal.commandline.CommandHandler.WAL_DELETE;
+import static org.apache.ignite.internal.commandline.CommandHandler.WAL_PRINT;
/**
* Tests Command Handler parsing arguments.
*/
public class CommandHandlerParsingTest extends TestCase {
+ /** {@inheritDoc} */
+ @Override protected void setUp() throws Exception {
+ System.setProperty(IGNITE_ENABLE_EXPERIMENTAL_COMMAND, "true");
+
+ super.setUp();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void tearDown() throws Exception {
+ System.clearProperty(IGNITE_ENABLE_EXPERIMENTAL_COMMAND);
+
+ super.tearDown();
+ }
+
+ /**
+ * Test that experimental command (i.e. WAL command) is disabled by default.
+ */
+ public void testExperimentalCommandIsDisabled() {
+ System.clearProperty(IGNITE_ENABLE_EXPERIMENTAL_COMMAND);
+
+ CommandHandler hnd = new CommandHandler();
+
+ try {
+ hnd.parseAndValidate(Arrays.asList(WAL.text(), WAL_PRINT));
+ }
+ catch (Throwable e) {
+ e.printStackTrace();
+
+ assertTrue(e instanceof IllegalArgumentException);
+ }
+
+ try {
+ hnd.parseAndValidate(Arrays.asList(WAL.text(), WAL_DELETE));
+ }
+ catch (Throwable e) {
+ e.printStackTrace();
+
+ assertTrue(e instanceof IllegalArgumentException);
+ }
+ }
+
/**
- * Test parsing and validation for user and password arguments.
+ * Tests parsing and validation for user and password arguments.
*/
public void testParseAndValidateUserAndPassword() {
CommandHandler hnd = new CommandHandler();
for (Command cmd : Command.values()) {
- if (cmd == Command.CACHE)
+ if (cmd == Command.CACHE || cmd == Command.WAL)
continue; // --cache subcommand requires its own specific arguments.
try {
@@ -88,13 +133,53 @@ public class CommandHandlerParsingTest extends TestCase {
}
/**
+ * Tests parsing and validation of WAL commands.
+ */
+ public void testParseAndValidateWalActions() {
+ CommandHandler hnd = new CommandHandler();
+
+ Arguments args = hnd.parseAndValidate(Arrays.asList(WAL.text(), WAL_PRINT));
+
+ assertEquals(WAL, args.command());
+
+ assertEquals(WAL_PRINT, args.walAction());
+
+ String nodes = UUID.randomUUID().toString() + "," + UUID.randomUUID().toString();
+
+ args = hnd.parseAndValidate(Arrays.asList(WAL.text(), WAL_DELETE, nodes));
+
+ assertEquals(WAL_DELETE, args.walAction());
+
+ assertEquals(nodes, args.walArguments());
+
+ try {
+ hnd.parseAndValidate(Collections.singletonList(WAL.text()));
+
+ fail("expected exception: invalid arguments for --wal command");
+ }
+ catch (IllegalArgumentException e) {
+ e.printStackTrace();
+ }
+
+ try {
+ hnd.parseAndValidate(Arrays.asList(WAL.text(), UUID.randomUUID().toString()));
+
+ fail("expected exception: invalid arguments for --wal command");
+ }
+ catch (IllegalArgumentException e) {
+ e.printStackTrace();
+ }
+ }
+
+ /**
+ * Tests host and port arguments.
* Tests connection settings arguments.
*/
public void testConnectionSettings() {
CommandHandler hnd = new CommandHandler();
for (Command cmd : Command.values()) {
- if (cmd == Command.CACHE)
+ if (cmd == Command.CACHE || cmd == Command.WAL)
continue; // --cache subcommand requires its own specific arguments.
Arguments args = hnd.parseAndValidate(asList(cmd.text()));
http://git-wip-us.apache.org/repos/asf/ignite/blob/07cbe22d/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsUnusedWalSegmentsTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsUnusedWalSegmentsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsUnusedWalSegmentsTest.java
new file mode 100644
index 0000000..0f52254
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsUnusedWalSegmentsTest.java
@@ -0,0 +1,202 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.db;
+
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
+import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
+import org.apache.ignite.internal.processors.cache.persistence.wal.FileWALPointer;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_PDS_MAX_CHECKPOINT_MEMORY_HISTORY_SIZE;
+
+/**
+ * Test correctness of truncating unused WAL segments.
+ */
+public class IgnitePdsUnusedWalSegmentsTest extends GridCommonAbstractTest {
+ /** */
+ private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+ System.setProperty(IGNITE_PDS_MAX_CHECKPOINT_MEMORY_HISTORY_SIZE, "2");
+
+ IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+ cfg.setDiscoverySpi(new TcpDiscoverySpi().setIpFinder(IP_FINDER));
+
+ CacheConfiguration<Integer, Object> ccfg = new CacheConfiguration<>(DEFAULT_CACHE_NAME);
+
+ ccfg.setAffinity(new RendezvousAffinityFunction(false, 32));
+
+ cfg.setCacheConfiguration(ccfg);
+
+ DataStorageConfiguration dbCfg = new DataStorageConfiguration();
+
+ dbCfg.setPageSize(4 * 1024);
+
+ cfg.setDataStorageConfiguration(dbCfg);
+
+ dbCfg.setWalSegmentSize(1024 * 1024)
+ .setWalHistorySize(Integer.MAX_VALUE)
+ .setWalSegments(10)
+ .setWalMode(WALMode.LOG_ONLY)
+ .setDefaultDataRegionConfiguration(new DataRegionConfiguration()
+ .setMaxSize(100 * 1024 * 1024)
+ .setPersistenceEnabled(true));
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ cleanPersistenceDir();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ cleanPersistenceDir();
+ }
+
+ /**
+ * Tests that range reserved method return correct number of reserved WAL segments.
+ *
+ * @throws Exception if failed.
+ */
+ public void testWalManagerRangeReservation() throws Exception {
+ try{
+ IgniteEx ig0 = prepareGrid(4);
+
+ GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager) ig0.context().cache().context()
+ .database();
+
+ IgniteWriteAheadLogManager wal = ig0.context().cache().context().wal();
+
+ long resIdx = getReservedWalSegmentIndex(dbMgr);
+
+ assertTrue("Expected that at least resIdx greater than 0, real is " + resIdx, resIdx > 0);
+
+ FileWALPointer lowPtr = (FileWALPointer)dbMgr.checkpointHistory().lowCheckpointBound();
+
+ assertTrue("Expected that dbMbr returns valid resIdx", lowPtr.index() == resIdx);
+
+ // Reserve previous WAL segment.
+ wal.reserve(new FileWALPointer(resIdx - 1, 0, 0));
+
+ int resCnt = wal.reserved(new FileWALPointer(resIdx - 1, 0, 0), new FileWALPointer(resIdx, 0, 0));
+
+ assertTrue("Expected resCnt is 2, real is " + resCnt, resCnt == 2);
+ }
+ finally {
+ stopAllGrids();
+ }
+ }
+
+ /**
+ * Tests that grid cache manager correctly truncates unused WAL segments;
+ *
+ * @throws Exception if failed.
+ */
+ public void testUnusedWalTruncate() throws Exception {
+ try{
+ IgniteEx ig0 = prepareGrid(4);
+
+ GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager) ig0.context().cache().context()
+ .database();
+
+ IgniteWriteAheadLogManager wal = ig0.context().cache().context().wal();
+
+ long resIdx = getReservedWalSegmentIndex(dbMgr);
+
+ assertTrue("Expected that at least resIdx greater than 0, real is " + resIdx, resIdx > 0);
+
+ FileWALPointer lowPtr = (FileWALPointer) dbMgr.checkpointHistory().lowCheckpointBound();
+
+ assertTrue("Expected that dbMbr returns valid resIdx", lowPtr.index() == resIdx);
+
+ // Reserve previous WAL segment.
+ wal.reserve(new FileWALPointer(resIdx - 1, 0, 0));
+
+ int numDel = wal.truncate(null, lowPtr);
+
+ int expNumDel = (int)resIdx - 1;
+
+ assertTrue("Expected del segments is " + expNumDel + ", real is " + numDel, expNumDel == numDel);
+ }
+ finally {
+ stopAllGrids();
+ }
+ }
+
+ /**
+ * Starts grid and populates test data.
+ *
+ * @param cnt Grid count.
+ * @return First started grid.
+ * @throws Exception If failed.
+ */
+ private IgniteEx prepareGrid(int cnt) throws Exception {
+ IgniteEx ig0 = (IgniteEx)startGrids(cnt);
+
+ ig0.cluster().active(true);
+
+ IgniteCache<Object, Object> cache = ig0.cache(DEFAULT_CACHE_NAME);
+
+ for (int k = 0; k < 10_000; k++)
+ cache.put(k, new byte[1024]);
+
+ forceCheckpoint();
+
+ for (int k = 0; k < 1_000; k++)
+ cache.put(k, new byte[1024]);
+
+ forceCheckpoint();
+
+ return ig0;
+ }
+
+
+ /**
+ * Get index of reserved WAL segment by checkpointer.
+ *
+ * @param dbMgr Database shared manager.
+ * @throws Exception If failed.
+ */
+ private long getReservedWalSegmentIndex(GridCacheDatabaseSharedManager dbMgr) throws Exception{
+ GridCacheDatabaseSharedManager.CheckpointHistory cpHist = dbMgr.checkpointHistory();
+
+ Object histMap = GridTestUtils.getFieldValue(cpHist, "histMap");
+
+ Object cpEntry = GridTestUtils.getFieldValue(GridTestUtils.invoke(histMap, "firstEntry"), "value");
+
+ FileWALPointer walPtr = GridTestUtils.getFieldValue(cpEntry, "cpMark");
+
+ return walPtr.index();
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/07cbe22d/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpWALManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpWALManager.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpWALManager.java
index 16d1d0a..c95d1f4 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpWALManager.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpWALManager.java
@@ -92,6 +92,11 @@ public class NoOpWALManager implements IgniteWriteAheadLogManager {
}
/** {@inheritDoc} */
+ @Override public int reserved(WALPointer low, WALPointer high) {
+ return 0;
+ }
+
+ /** {@inheritDoc} */
@Override public boolean disabled(int grpId) {
return false;
}
@@ -127,7 +132,7 @@ public class NoOpWALManager implements IgniteWriteAheadLogManager {
}
/** {@inheritDoc} */
- @Override public void onActivate(GridKernalContext kctx) throws IgniteCheckedException {
+ @Override public void onActivate(GridKernalContext kctx) {
// No-op.
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/07cbe22d/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
index be3d005..aa40086 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
@@ -36,6 +36,7 @@ import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsRebal
import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsTransactionsHangTest;
import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsWholeClusterRestartTest;
import org.apache.ignite.internal.processors.cache.persistence.db.checkpoint.IgniteCheckpointDirtyPagesForLowLoadTest;
+import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsUnusedWalSegmentsTest;
import org.apache.ignite.internal.processors.cache.persistence.db.filename.IgniteUidAsConsistentIdMigrationTest;
import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalFlushBackgroundSelfTest;
import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalFlushDefaultSelfTest;
@@ -130,6 +131,8 @@ public class IgnitePdsTestSuite2 extends TestSuite {
suite.addTestSuite(IgnitePdsExchangeDuringCheckpointTest.class);
+ suite.addTestSuite(IgnitePdsUnusedWalSegmentsTest.class);
+
// new style folders with generated consistent ID test
suite.addTestSuite(IgniteUidAsConsistentIdMigrationTest.class);
http://git-wip-us.apache.org/repos/asf/ignite/blob/07cbe22d/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteUtilSelfTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteUtilSelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteUtilSelfTestSuite.java
index e3be1e3..1243005 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteUtilSelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteUtilSelfTestSuite.java
@@ -43,6 +43,7 @@ import org.apache.ignite.thread.GridThreadPoolExecutorServiceSelfTest;
import org.apache.ignite.thread.IgniteThreadPoolSizeTest;
import org.apache.ignite.util.GridCommandHandlerTest;
import org.apache.ignite.util.GridIntListSelfTest;
+import org.apache.ignite.util.GridInternalTaskUnusedWalSegmentsTest;
import org.apache.ignite.util.GridLongListSelfTest;
import org.apache.ignite.util.GridMessageCollectionTest;
import org.apache.ignite.util.GridPartitionMapSelfTest;
@@ -116,6 +117,7 @@ public class IgniteUtilSelfTestSuite extends TestSuite {
// control.sh
suite.addTestSuite(CommandHandlerParsingTest.class);
suite.addTestSuite(GridCommandHandlerTest.class);
+ suite.addTestSuite(GridInternalTaskUnusedWalSegmentsTest.class);
return suite;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/07cbe22d/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
index bfa1424..2bd14db 100644
--- a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
@@ -30,6 +30,7 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
+import java.util.List;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteAtomicSequence;
import org.apache.ignite.IgniteCache;
@@ -57,6 +58,7 @@ import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
import org.apache.ignite.transactions.Transaction;
import org.apache.ignite.transactions.TransactionRollbackException;
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_ENABLE_EXPERIMENTAL_COMMAND;
import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
import static org.apache.ignite.internal.commandline.CommandHandler.EXIT_CODE_OK;
@@ -85,6 +87,8 @@ public class GridCommandHandlerTest extends GridCommonAbstractTest {
/** {@inheritDoc} */
@Override protected void beforeTest() throws Exception {
+ System.setProperty(IGNITE_ENABLE_EXPERIMENTAL_COMMAND, "true");
+
cleanPersistenceDir();
stopAllGrids();
@@ -100,6 +104,8 @@ public class GridCommandHandlerTest extends GridCommonAbstractTest {
cleanPersistenceDir();
+ System.clearProperty(IGNITE_ENABLE_EXPERIMENTAL_COMMAND);
+
System.setOut(sysOut);
if (testOut != null)
@@ -778,4 +784,70 @@ public class GridCommandHandlerTest extends GridCommonAbstractTest {
return map;
}
+
+ /**
+ * Test execution of --wal print command.
+ *
+ * @throws Exception if failed.
+ */
+ public void testUnusedWalPrint() throws Exception {
+ Ignite ignite = startGrids(2);
+
+ ignite.cluster().active(true);
+
+ List<String> nodes = new ArrayList<>(2);
+
+ for (ClusterNode node: ignite.cluster().forServers().nodes())
+ nodes.add(node.consistentId().toString());
+
+ injectTestSystemOut();
+
+ assertEquals(EXIT_CODE_OK, execute("--wal", "print"));
+
+ for(String id: nodes)
+ assertTrue(testOut.toString().contains(id));
+
+ assertTrue(!testOut.toString().contains("error"));
+
+ testOut.reset();
+
+ assertEquals(EXIT_CODE_OK, execute("--wal", "print", nodes.get(0)));
+
+ assertTrue(!testOut.toString().contains(nodes.get(1)));
+
+ assertTrue(!testOut.toString().contains("error"));
+ }
+
+ /**
+ * Test execution of --wal delete command.
+ *
+ * @throws Exception if failed.
+ */
+ public void testUnusedWalDelete() throws Exception {
+ Ignite ignite = startGrids(2);
+
+ ignite.cluster().active(true);
+
+ List<String> nodes = new ArrayList<>(2);
+
+ for (ClusterNode node: ignite.cluster().forServers().nodes())
+ nodes.add(node.consistentId().toString());
+
+ injectTestSystemOut();
+
+ assertEquals(EXIT_CODE_OK, execute("--wal", "delete"));
+
+ for(String id: nodes)
+ assertTrue(testOut.toString().contains(id));
+
+ assertTrue(!testOut.toString().contains("error"));
+
+ testOut.reset();
+
+ assertEquals(EXIT_CODE_OK, execute("--wal", "delete", nodes.get(0)));
+
+ assertTrue(!testOut.toString().contains(nodes.get(1)));
+
+ assertTrue(!testOut.toString().contains("error"));
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/07cbe22d/modules/core/src/test/java/org/apache/ignite/util/GridInternalTaskUnusedWalSegmentsTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/util/GridInternalTaskUnusedWalSegmentsTest.java b/modules/core/src/test/java/org/apache/ignite/util/GridInternalTaskUnusedWalSegmentsTest.java
new file mode 100644
index 0000000..148ecef
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/util/GridInternalTaskUnusedWalSegmentsTest.java
@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.util;
+
+import java.io.File;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.visor.VisorTaskArgument;
+import org.apache.ignite.internal.visor.misc.VisorWalTask;
+import org.apache.ignite.internal.visor.misc.VisorWalTaskArg;
+import org.apache.ignite.internal.visor.misc.VisorWalTaskOperation;
+import org.apache.ignite.internal.visor.misc.VisorWalTaskResult;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_PDS_MAX_CHECKPOINT_MEMORY_HISTORY_SIZE;
+
+/**
+ * Test correctness of VisorWalTask.
+ */
+public class GridInternalTaskUnusedWalSegmentsTest extends GridCommonAbstractTest {
+ /** */
+ private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+ System.setProperty(IGNITE_PDS_MAX_CHECKPOINT_MEMORY_HISTORY_SIZE, "2");
+
+ IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+ cfg.setDiscoverySpi(new TcpDiscoverySpi().setIpFinder(IP_FINDER));
+
+ CacheConfiguration<Integer, Object> ccfg = new CacheConfiguration<>(DEFAULT_CACHE_NAME);
+
+ ccfg.setAffinity(new RendezvousAffinityFunction(false, 32));
+
+ cfg.setCacheConfiguration(ccfg);
+
+ DataStorageConfiguration dbCfg = new DataStorageConfiguration();
+
+ dbCfg.setPageSize(4 * 1024);
+
+ cfg.setDataStorageConfiguration(dbCfg);
+
+ dbCfg.setWalSegmentSize(1024 * 1024)
+ .setWalHistorySize(Integer.MAX_VALUE)
+ .setWalSegments(10)
+ .setWalMode(WALMode.LOG_ONLY)
+ .setDefaultDataRegionConfiguration(new DataRegionConfiguration()
+ .setMaxSize(100 * 1024 * 1024)
+ .setPersistenceEnabled(true));
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ cleanPersistenceDir();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ cleanPersistenceDir();
+ }
+
+ /**
+ * Tests correctness of {@link VisorWalTaskOperation}.
+ *
+ * @throws Exception if failed.
+ */
+ public void testCorrectnessOfDeletionTaskSegments() throws Exception {
+ try {
+ IgniteEx ig0 = (IgniteEx)startGrids(4);
+
+ ig0.cluster().active(true);
+
+ IgniteCache<Object, Object> cache = ig0.cache(DEFAULT_CACHE_NAME);
+
+ for (int k = 0; k < 10_000; k++)
+ cache.put(k, new byte[1024]);
+
+ forceCheckpoint();
+
+ for (int k = 0; k < 1_000; k++)
+ cache.put(k, new byte[1024]);
+
+ forceCheckpoint();
+
+ VisorWalTaskResult printRes = ig0.compute().execute(VisorWalTask.class,
+ new VisorTaskArgument<>(ig0.cluster().node().id(),
+ new VisorWalTaskArg(VisorWalTaskOperation.PRINT_UNUSED_WAL_SEGMENTS), false));
+
+ assertEquals("Check that print task finished without exceptions", printRes.results().size(), 4);
+
+ List<File> walArchives = new ArrayList<>();
+
+ for (Collection<String> pathsPerNode : printRes.results().values()) {
+ for (String path : pathsPerNode)
+ walArchives.add(Paths.get(path).toFile());
+ }
+
+ VisorWalTaskResult delRes = ig0.compute().execute(VisorWalTask.class,
+ new VisorTaskArgument<>(ig0.cluster().node().id(),
+ new VisorWalTaskArg(VisorWalTaskOperation.DELETE_UNUSED_WAL_SEGMENTS), false));
+
+ assertEquals("Check that delete task finished with no exceptions", delRes.results().size(), 4);
+
+ List<File> walDeletedArchives = new ArrayList<>();
+
+ for (Collection<String> pathsPerNode : delRes.results().values()) {
+ for (String path : pathsPerNode)
+ walDeletedArchives.add(Paths.get(path).toFile());
+ }
+
+ for (File f : walDeletedArchives)
+ assertTrue("Checking existing of deleted WAL archived segments: " + f.getAbsolutePath(), !f.exists());
+
+ for (File f : walArchives)
+ assertTrue("Checking existing of WAL archived segments from print task after delete: " + f.getAbsolutePath(),
+ !f.exists());
+ }
+ finally {
+ stopAllGrids();
+ }
+ }
+}