You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by ag...@apache.org on 2018/08/30 12:24:50 UTC
[44/45] ignite git commit: IGNITE-6552 Added ability to set WAL
history size in bytes - Fixes #3559.
IGNITE-6552 Added ability to set WAL history size in bytes - Fixes #3559.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/3307a8b2
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/3307a8b2
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/3307a8b2
Branch: refs/heads/ignite-9273
Commit: 3307a8b26ccb5f0bb7e9c387c73fd221b98ab668
Parents: 00bb4d4
Author: Anton Kalashnikov <ka...@yandex.ru>
Authored: Thu Aug 30 15:02:50 2018 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Thu Aug 30 15:10:59 2018 +0300
----------------------------------------------------------------------
.../apache/ignite/IgniteSystemProperties.java | 11 +
.../configuration/DataStorageConfiguration.java | 42 +++
.../pagemem/wal/IgniteWriteAheadLogManager.java | 5 +
.../GridCacheDatabaseSharedManager.java | 16 +-
.../IgniteCacheDatabaseSharedManager.java | 34 +++
.../checkpoint/CheckpointHistory.java | 102 +++++--
.../cache/persistence/wal/FileDescriptor.java | 136 +++++++++
.../wal/FileWriteAheadLogManager.java | 185 ++++++-------
.../wal/FsyncModeFileWriteAheadLogManager.java | 214 ++++++--------
.../SingleSegmentLogicalRecordsIterator.java | 4 +-
.../wal/reader/IgniteWalIteratorFactory.java | 2 +-
.../reader/StandaloneWalRecordsIterator.java | 2 +-
.../utils/PlatformConfigurationUtils.java | 2 +
.../db/IgnitePdsReserveWalSegmentsTest.java | 188 +++++++++++++
.../db/IgnitePdsUnusedWalSegmentsTest.java | 198 -------------
...IsReachedDuringIterationOverArchiveTest.java | 2 +-
.../persistence/db/wal/WalCompactionTest.java | 8 +-
.../db/wal/WalDeletionArchiveAbstractTest.java | 277 +++++++++++++++++++
.../db/wal/WalDeletionArchiveFsyncTest.java | 31 +++
.../db/wal/WalDeletionArchiveLogOnlyTest.java | 31 +++
.../db/wal/WalRecoveryTxLogicalRecordsTest.java | 44 ---
.../db/wal/reader/MockWalIteratorFactory.java | 1 +
.../persistence/pagemem/NoOpWALManager.java | 6 +
.../ignite/testsuites/IgnitePdsTestSuite2.java | 14 +-
.../DataStorageConfigurationParityTest.cs | 3 +-
.../Configuration/DataStorageConfiguration.cs | 14 +
.../IgniteConfigurationSection.xsd | 5 +
27 files changed, 1067 insertions(+), 510 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/3307a8b2/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
index 78842d6..00ab49a 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
@@ -875,6 +875,17 @@ public final class IgniteSystemProperties {
public static final String IGNITE_LOADED_PAGES_BACKWARD_SHIFT_MAP = "IGNITE_LOADED_PAGES_BACKWARD_SHIFT_MAP";
/**
+ * Property for setup percentage of archive size for checkpoint trigger. Default value is 0.25
+ */
+ public static final String IGNITE_CHECKPOINT_TRIGGER_ARCHIVE_SIZE_PERCENTAGE = "IGNITE_CHECKPOINT_TRIGGER_ARCHIVE_SIZE_PERCENTAGE";
+
+ /**
+ * Property for setup percentage of WAL archive size to calculate threshold since which removing of old archive should be started.
+ * Default value is 0.5
+ */
+ public static final String IGNITE_THRESHOLD_WAL_ARCHIVE_SIZE_PERCENTAGE = "IGNITE_THRESHOLD_WAL_ARCHIVE_SIZE_PERCENTAGE";
+
+ /**
* Whenever read load balancing is enabled, that means 'get' requests will be distributed between primary and backup
* nodes if it is possible and {@link CacheConfiguration#readFromBackup} is {@code true}.
*
http://git-wip-us.apache.org/repos/asf/ignite/blob/3307a8b2/modules/core/src/main/java/org/apache/ignite/configuration/DataStorageConfiguration.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/configuration/DataStorageConfiguration.java b/modules/core/src/main/java/org/apache/ignite/configuration/DataStorageConfiguration.java
index d8619aa..e7e8ca3 100644
--- a/modules/core/src/main/java/org/apache/ignite/configuration/DataStorageConfiguration.java
+++ b/modules/core/src/main/java/org/apache/ignite/configuration/DataStorageConfiguration.java
@@ -114,6 +114,9 @@ public class DataStorageConfiguration implements Serializable {
/** Default number of checkpoints to be kept in WAL after checkpoint is finished */
public static final int DFLT_WAL_HISTORY_SIZE = 20;
+ /** Default max size of WAL archive files, in bytes */
+ public static final long DFLT_WAL_ARCHIVE_MAX_SIZE = 1024 * 1024 * 1024;
+
/** */
public static final int DFLT_WAL_SEGMENTS = 10;
@@ -190,6 +193,9 @@ public class DataStorageConfiguration implements Serializable {
/** Number of checkpoints to keep */
private int walHistSize = DFLT_WAL_HISTORY_SIZE;
+ /** Maximum size of wal archive folder, in bytes */
+ private long maxWalArchiveSize = DFLT_WAL_ARCHIVE_MAX_SIZE;
+
/** Number of work WAL segments. */
private int walSegments = DFLT_WAL_SEGMENTS;
@@ -487,7 +493,10 @@ public class DataStorageConfiguration implements Serializable {
* Gets a total number of checkpoints to keep in the WAL history.
*
* @return Number of checkpoints to keep in WAL after a checkpoint is finished.
+ * @deprecated Instead of walHistorySize use maxWalArchiveSize for manage of archive size.
+ * @see DataStorageConfiguration#getMaxWalArchiveSize()
*/
+ @Deprecated
public int getWalHistorySize() {
return walHistSize <= 0 ? DFLT_WAL_HISTORY_SIZE : walHistSize;
}
@@ -497,7 +506,10 @@ public class DataStorageConfiguration implements Serializable {
*
* @param walHistSize Number of checkpoints to keep after a checkpoint is finished.
* @return {@code this} for chaining.
+ * @deprecated Instead of walHistorySize use maxWalArchiveSize for manage of archive size.
+ * @see DataStorageConfiguration#setMaxWalArchiveSize(long)
*/
+ @Deprecated
public DataStorageConfiguration setWalHistorySize(int walHistSize) {
this.walHistSize = walHistSize;
@@ -505,6 +517,36 @@ public class DataStorageConfiguration implements Serializable {
}
/**
+ * If WalHistorySize was set by user will use this parameter for compatibility.
+ *
+ * @return {@code true} if use WalHistorySize for compatibility.
+ */
+ public boolean isWalHistorySizeParameterUsed() {
+ return getWalHistorySize() != DFLT_WAL_HISTORY_SIZE && getWalHistorySize() != Integer.MAX_VALUE;
+ }
+
+ /**
+ * Gets a max allowed size of WAL archives. In bytes.
+ *
+ * @return max size of WAL archive directory.
+ */
+ public long getMaxWalArchiveSize() {
+ return maxWalArchiveSize <= 0 ? DFLT_WAL_ARCHIVE_MAX_SIZE : maxWalArchiveSize;
+ }
+
+ /**
+ * Sets a max allowed size of WAL archives. In bytes
+ *
+ * @param walArchiveMaxSize max size of WAL archive directory.
+ * @return {@code this} for chaining.
+ */
+ public DataStorageConfiguration setMaxWalArchiveSize(long walArchiveMaxSize) {
+ this.maxWalArchiveSize = walArchiveMaxSize;
+
+ return this;
+ }
+
+ /**
* Gets a number of WAL segments to work with.
*
* @return Number of work WAL segments.
http://git-wip-us.apache.org/repos/asf/ignite/blob/3307a8b2/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/IgniteWriteAheadLogManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/IgniteWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/IgniteWriteAheadLogManager.java
index b5282ff..12fd3e9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/IgniteWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/IgniteWriteAheadLogManager.java
@@ -135,6 +135,11 @@ public interface IgniteWriteAheadLogManager extends GridCacheSharedManager, Igni
public long lastCompactedSegment();
/**
+ * @return Max allowed index of archived segment to delete or -1 if it does not exist.
+ */
+ public long maxArchivedSegmentToDelete();
+
+ /**
* Checks if WAL segment is under lock or reserved
*
* @param ptr Pointer to check.
http://git-wip-us.apache.org/repos/asf/ignite/blob/3307a8b2/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
index 7d91645..7393c01 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -167,6 +167,7 @@ import static org.apache.ignite.IgniteSystemProperties.IGNITE_PDS_WAL_REBALANCE_
import static org.apache.ignite.failure.FailureType.CRITICAL_ERROR;
import static org.apache.ignite.failure.FailureType.SYSTEM_WORKER_TERMINATION;
import static org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType.CHECKPOINT_RECORD;
+import static org.apache.ignite.configuration.DataStorageConfiguration.DFLT_WAL_HISTORY_SIZE;
import static org.apache.ignite.internal.processors.cache.persistence.metastorage.MetaStorage.METASTORAGE_CACHE_ID;
/**
@@ -360,7 +361,9 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
checkpointFreq = persistenceCfg.getCheckpointFrequency();
- truncateWalOnCpFinish = persistenceCfg.getWalHistorySize() != Integer.MAX_VALUE;
+ truncateWalOnCpFinish = persistenceCfg.isWalHistorySizeParameterUsed()
+ ? persistenceCfg.getWalHistorySize() != Integer.MAX_VALUE
+ : persistenceCfg.getMaxWalArchiveSize() != Long.MAX_VALUE;
lockWaitTime = persistenceCfg.getLockWaitTime();
@@ -497,7 +500,7 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
/**
* Cleanup checkpoint directory from all temporary files {@link #FILE_TMP_SUFFIX}.
*/
- public void cleanupTempCheckpointDirectory() throws IgniteCheckedException {
+ @Override public void cleanupTempCheckpointDirectory() throws IgniteCheckedException {
try {
try (DirectoryStream<Path> files = Files.newDirectoryStream(
cpDir.toPath(),
@@ -515,7 +518,7 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
/**
* Cleanup checkpoint directory.
*/
- public void cleanupCheckpointDirectory() throws IgniteCheckedException {
+ @Override public void cleanupCheckpointDirectory() throws IgniteCheckedException {
try {
try (DirectoryStream<Path> files = Files.newDirectoryStream(cpDir.toPath())) {
for (Path path : files)
@@ -1780,6 +1783,13 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
return cp.wakeupForCheckpoint(0, reason);
}
+ /** {@inheritDoc} */
+ @Override public WALPointer lastCheckpointMarkWalPointer() {
+ CheckpointEntry lastCheckpointEntry = cpHistory == null ? null : cpHistory.lastCheckpoint();
+
+ return lastCheckpointEntry == null ? null : lastCheckpointEntry.checkpointMark();
+ }
+
/**
* @return Checkpoint directory.
*/
http://git-wip-us.apache.org/repos/asf/ignite/blob/3307a8b2/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java
index 3ade265..737b02a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java
@@ -42,6 +42,7 @@ import org.apache.ignite.internal.mem.file.MappedFileMemoryProvider;
import org.apache.ignite.internal.mem.unsafe.UnsafeMemoryProvider;
import org.apache.ignite.internal.pagemem.PageMemory;
import org.apache.ignite.internal.pagemem.impl.PageMemoryNoStoreImpl;
+import org.apache.ignite.internal.pagemem.wal.WALPointer;
import org.apache.ignite.internal.processors.cache.CacheGroupContext;
import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor;
import org.apache.ignite.internal.processors.cache.GridCacheMapEntry;
@@ -69,6 +70,8 @@ import org.jetbrains.annotations.Nullable;
import static org.apache.ignite.configuration.DataStorageConfiguration.DFLT_DATA_REG_DEFAULT_NAME;
import static org.apache.ignite.configuration.DataStorageConfiguration.DFLT_PAGE_SIZE;
+import static org.apache.ignite.configuration.DataStorageConfiguration.DFLT_WAL_ARCHIVE_MAX_SIZE;
+import static org.apache.ignite.configuration.DataStorageConfiguration.DFLT_WAL_HISTORY_SIZE;
/**
*
@@ -388,6 +391,30 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap
}
checkDataRegionConfiguration(memCfg, regNames, memCfg.getDefaultDataRegionConfiguration());
+
+ checkWalArchiveSizeConfiguration(memCfg);
+ }
+
+ /**
+ * Check wal archive size configuration for correctness.
+ *
+ * @param memCfg durable memory configuration for an Apache Ignite node.
+ */
+ private void checkWalArchiveSizeConfiguration(DataStorageConfiguration memCfg) throws IgniteCheckedException {
+ if (memCfg.getWalHistorySize() == DFLT_WAL_HISTORY_SIZE || memCfg.getWalHistorySize() == Integer.MAX_VALUE)
+ LT.warn(log, "DataRegionConfiguration.maxWalArchiveSize instead DataRegionConfiguration.walHistorySize " +
+ "would be used for removing old archive wal files");
+ else if(memCfg.getMaxWalArchiveSize() == DFLT_WAL_ARCHIVE_MAX_SIZE)
+ LT.warn(log, "walHistorySize was deprecated. maxWalArchiveSize should be used instead");
+ else
+ throw new IgniteCheckedException("Should be used only one of wal history size or max wal archive size." +
+ "(use DataRegionConfiguration.maxWalArchiveSize because DataRegionConfiguration.walHistorySize was deprecated)"
+ );
+
+ if(memCfg.getMaxWalArchiveSize() < memCfg.getWalSegmentSize())
+ throw new IgniteCheckedException(
+ "DataRegionConfiguration.maxWalArchiveSize should be greater than DataRegionConfiguration.walSegmentSize"
+ );
}
/**
@@ -748,6 +775,13 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap
}
/**
+ * @return Last checkpoint mark WAL pointer.
+ */
+ public WALPointer lastCheckpointMarkWalPointer() {
+ return null;
+ }
+
+ /**
* Allows to wait checkpoint finished.
*
* @param reason Reason.
http://git-wip-us.apache.org/repos/asf/ignite/blob/3307a8b2/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/checkpoint/CheckpointHistory.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/checkpoint/CheckpointHistory.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/checkpoint/CheckpointHistory.java
index 3fb8457..e64a53f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/checkpoint/CheckpointHistory.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/checkpoint/CheckpointHistory.java
@@ -35,6 +35,7 @@ import org.apache.ignite.internal.GridKernalContext;
import org.apache.ignite.internal.pagemem.wal.WALPointer;
import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
+import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager.Checkpoint;
import org.apache.ignite.internal.processors.cache.persistence.wal.FileWALPointer;
import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.internal.U;
@@ -64,6 +65,9 @@ public class CheckpointHistory {
/** The maximal number of checkpoints hold in memory. */
private final int maxCpHistMemSize;
+ /** If WalHistorySize was setted by user will use old way for removing checkpoints. */
+ private final boolean isWalHistorySizeParameterEnabled;
+
/**
* Constructor.
*
@@ -77,6 +81,8 @@ public class CheckpointHistory {
maxCpHistMemSize = Math.min(dsCfg.getWalHistorySize(),
IgniteSystemProperties.getInteger(IGNITE_PDS_MAX_CHECKPOINT_MEMORY_HISTORY_SIZE, 100));
+
+ isWalHistorySizeParameterEnabled = dsCfg.isWalHistorySizeParameterUsed();
}
/**
@@ -198,36 +204,98 @@ public class CheckpointHistory {
*
* @return List of checkpoints removed from history.
*/
- public List<CheckpointEntry> onCheckpointFinished(GridCacheDatabaseSharedManager.Checkpoint chp, boolean truncateWal) {
- List<CheckpointEntry> rmv = new ArrayList<>();
-
+ public List<CheckpointEntry> onCheckpointFinished(Checkpoint chp, boolean truncateWal) {
chp.walSegsCoveredRange(calculateWalSegmentsCovered());
+ WALPointer checkpointMarkUntilDel = isWalHistorySizeParameterEnabled //check for compatibility mode.
+ ? checkpointMarkUntilDeleteByMemorySize()
+ : newerPointer(checkpointMarkUntilDeleteByMemorySize(), checkpointMarkUntilDeleteByArchiveSize());
+
+ if (checkpointMarkUntilDel == null)
+ return Collections.emptyList();
+
+ List<CheckpointEntry> deletedCheckpoints = onWalTruncated(checkpointMarkUntilDel);
+
int deleted = 0;
- while (histMap.size() > maxCpHistMemSize) {
- Map.Entry<Long, CheckpointEntry> entry = histMap.firstEntry();
+ if (truncateWal)
+ deleted += cctx.wal().truncate(null, firstCheckpointPointer());
- CheckpointEntry cpEntry = entry.getValue();
+ chp.walFilesDeleted(deleted);
- if (cctx.wal().reserved(cpEntry.checkpointMark())) {
- U.warn(log, "Could not clear historyMap due to WAL reservation on cpEntry " + cpEntry.checkpointId() +
- ", history map size is " + histMap.size());
+ return deletedCheckpoints;
+ }
- break;
- }
+ /**
+ * @param firstPointer One of pointers to choose the newest.
+ * @param secondPointer One of pointers to choose the newest.
+ * @return The newest pointer from input ones.
+ */
+ private FileWALPointer newerPointer(WALPointer firstPointer, WALPointer secondPointer) {
+ FileWALPointer first = (FileWALPointer)firstPointer;
+ FileWALPointer second = (FileWALPointer)secondPointer;
+
+ if (firstPointer == null)
+ return second;
+
+ if (secondPointer == null)
+ return first;
- if (truncateWal)
- deleted += cctx.wal().truncate(null, cpEntry.checkpointMark());
+ return first.index() > second.index() ? first : second;
+ }
- histMap.remove(entry.getKey());
+ /**
+ * Calculate mark until delete by maximum checkpoint history memory size.
+ *
+ * @return Checkpoint mark until which checkpoints can be deleted(not including this pointer).
+ */
+ private WALPointer checkpointMarkUntilDeleteByMemorySize() {
+ if (histMap.size() <= maxCpHistMemSize)
+ return null;
- rmv.add(cpEntry);
+ int calculatedCpHistSize = maxCpHistMemSize;
+
+ for (Map.Entry<Long, CheckpointEntry> entry : histMap.entrySet()) {
+ if (histMap.size() <= calculatedCpHistSize++)
+ return entry.getValue().checkpointMark();
}
- chp.walFilesDeleted(deleted);
+ return lastCheckpoint().checkpointMark();
+ }
+
+ /**
+ * Calculate mark until delete by maximum allowed archive size.
+ *
+ * @return Checkpoint mark until which checkpoints can be deleted(not including this pointer).
+ */
+ @Nullable private WALPointer checkpointMarkUntilDeleteByArchiveSize() {
+ long absFileIdxToDel = cctx.wal().maxArchivedSegmentToDelete();
+
+ if (absFileIdxToDel < 0)
+ return null;
+
+ long fileUntilDel = absFileIdxToDel + 1;
- return rmv;
+ long checkpointFileIdx = absFileIdx(lastCheckpoint());
+
+ for (CheckpointEntry cpEntry : histMap.values()) {
+ long currFileIdx = absFileIdx(cpEntry);
+
+ if (checkpointFileIdx <= currFileIdx || fileUntilDel <= currFileIdx)
+ return cpEntry.checkpointMark();
+ }
+
+ return lastCheckpoint().checkpointMark();
+ }
+
+ /**
+ * Retrieve absolute file index by checkpoint entry.
+ *
+ * @param pointer checkpoint entry for which need to calculate absolute file index.
+ * @return absolute file index for given checkpoint entry.
+ */
+ private long absFileIdx(CheckpointEntry pointer) {
+ return ((FileWALPointer)pointer.checkpointMark()).index();
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/3307a8b2/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileDescriptor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileDescriptor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileDescriptor.java
new file mode 100644
index 0000000..39052cc
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileDescriptor.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.wal;
+
+import org.apache.ignite.internal.util.typedef.internal.SB;
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
+
+import java.io.File;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * WAL file descriptor.
+ */
+public class FileDescriptor implements Comparable<FileDescriptor>, AbstractWalRecordsIterator.AbstractFileDescriptor {
+
+ /** file extension of WAL segment. */
+ private static final String WAL_SEGMENT_FILE_EXT = ".wal";
+
+ /** Length of WAL segment file name. */
+ private static final int WAL_SEGMENT_FILE_NAME_LENGTH = 16;
+
+ /** File represented by this class. */
+ protected final File file;
+
+ /** Absolute WAL segment file index. */
+ protected final long idx;
+
+ /**
+ * Creates file descriptor. Index is restored from file name.
+ *
+ * @param file WAL segment file.
+ */
+ public FileDescriptor(@NotNull File file) {
+ this(file, null);
+ }
+
+ /**
+ * @param file WAL segment file.
+ * @param idx Absolute WAL segment file index. For null value index is restored from file name.
+ */
+ public FileDescriptor(@NotNull File file, @Nullable Long idx) {
+ this.file = file;
+
+ String fileName = file.getName();
+
+ assert fileName.contains(WAL_SEGMENT_FILE_EXT);
+
+ this.idx = idx == null ? Long.parseLong(fileName.substring(0, WAL_SEGMENT_FILE_NAME_LENGTH)) : idx;
+ }
+
+ /**
+ * @param segment Segment index.
+ * @return Segment file name.
+ */
+ public static String fileName(long segment) {
+ SB b = new SB();
+
+ String segmentStr = Long.toString(segment);
+
+ for (int i = segmentStr.length(); i < WAL_SEGMENT_FILE_NAME_LENGTH; i++)
+ b.a('0');
+
+ b.a(segmentStr).a(WAL_SEGMENT_FILE_EXT);
+
+ return b.toString();
+ }
+
+ /** {@inheritDoc} */
+ @Override public int compareTo(@NotNull FileDescriptor o) {
+ return Long.compare(idx, o.idx);
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean equals(Object o) {
+ if (this == o)
+ return true;
+
+ if (!(o instanceof FileDescriptor))
+ return false;
+
+ FileDescriptor that = (FileDescriptor)o;
+
+ return idx == that.idx;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int hashCode() {
+ return (int)(idx ^ (idx >>> 32));
+ }
+
+ /**
+ * @return Absolute WAL segment file index
+ */
+ public long getIdx() {
+ return idx;
+ }
+
+ /**
+ * @return absolute pathname string of this file descriptor pathname.
+ */
+ public String getAbsolutePath() {
+ return file.getAbsolutePath();
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean isCompressed() {
+ return file.getName().endsWith(".zip");
+ }
+
+ /** {@inheritDoc} */
+ @Override public File file() {
+ return file;
+ }
+
+ /** {@inheritDoc} */
+ @Override public long idx() {
+ return idx;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/3307a8b2/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
index 3fa499b..61def1e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
@@ -59,6 +59,7 @@ import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.LockSupport;
import java.util.concurrent.locks.ReentrantLock;
import java.util.regex.Pattern;
+import java.util.stream.Stream;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
import java.util.zip.ZipOutputStream;
@@ -110,7 +111,6 @@ import org.apache.ignite.internal.util.typedef.CI1;
import org.apache.ignite.internal.util.typedef.CIX1;
import org.apache.ignite.internal.util.typedef.CO;
import org.apache.ignite.internal.util.typedef.F;
-import org.apache.ignite.internal.util.typedef.internal.SB;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.internal.util.worker.GridWorker;
import org.apache.ignite.lang.IgniteBiTuple;
@@ -124,6 +124,8 @@ import org.jetbrains.annotations.Nullable;
import static java.nio.file.StandardOpenOption.CREATE;
import static java.nio.file.StandardOpenOption.READ;
import static java.nio.file.StandardOpenOption.WRITE;
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_CHECKPOINT_TRIGGER_ARCHIVE_SIZE_PERCENTAGE;
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_THRESHOLD_WAL_ARCHIVE_SIZE_PERCENTAGE;
import static org.apache.ignite.IgniteSystemProperties.IGNITE_WAL_MMAP;
import static org.apache.ignite.IgniteSystemProperties.IGNITE_WAL_SEGMENT_SYNC_TIMEOUT;
import static org.apache.ignite.IgniteSystemProperties.IGNITE_WAL_SERIALIZER_VERSION;
@@ -170,9 +172,6 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
/** */
private static final FileDescriptor[] EMPTY_DESCRIPTORS = new FileDescriptor[0];
- /** WAL segment file extension. */
- private static final String WAL_SEGMENT_FILE_EXT = ".wal";
-
/** */
private static final byte[] FILL_BUF = new byte[1024 * 1024];
@@ -237,6 +236,19 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
private static final AtomicLongFieldUpdater<FileWriteHandle> WRITTEN_UPD =
AtomicLongFieldUpdater.newUpdater(FileWriteHandle.class, "written");
+ /**
+ * Percentage of archive size for checkpoint trigger. Need for calculate max size of WAL after last checkpoint.
+ * Checkpoint should be triggered when max size of WAL after last checkpoint more than maxWallArchiveSize * thisValue
+ */
+ private static final double CHECKPOINT_TRIGGER_ARCHIVE_SIZE_PERCENTAGE =
+ IgniteSystemProperties.getDouble(IGNITE_CHECKPOINT_TRIGGER_ARCHIVE_SIZE_PERCENTAGE, 0.25);
+
+ /**
+ * Percentage of WAL archive size to calculate threshold since which removing of old archive should be started.
+ */
+ private static final double THRESHOLD_WAL_ARCHIVE_SIZE_PERCENTAGE =
+ IgniteSystemProperties.getDouble(IGNITE_THRESHOLD_WAL_ARCHIVE_SIZE_PERCENTAGE, 0.5);
+
/** Interrupted flag. */
private final ThreadLocal<Boolean> interrupted = new ThreadLocal<Boolean>() {
@Override protected Boolean initialValue() {
@@ -250,6 +262,15 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
/** WAL segment size in bytes. . This is maximum value, actual segments may be shorter. */
private final long maxWalSegmentSize;
+ /**
+ * Maximum number of allowed segments without checkpoint. If we have their more checkpoint should be triggered.
+ * It is simple way to calculate WAL size without checkpoint instead fair WAL size calculating.
+ */
+ private final long maxSegCountWithoutCheckpoint;
+
+ /** Size of wal archive since which removing of old archive should be started */
+ private final long allowedThresholdWalArchiveSize;
+
/** */
private final WALMode mode;
@@ -304,8 +325,8 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
AtomicReferenceFieldUpdater.newUpdater(FileWriteAheadLogManager.class, FileWriteHandle.class, "currHnd");
/**
- * File archiver moves segments from work directory to archive. Locked segments may be kept not moved until
- * release. For mode archive and work folders set to equal value, archiver is not created.
+ * File archiver moves segments from work directory to archive. Locked segments may be kept not moved until release.
+ * For mode archive and work folders set to equal value, archiver is not created.
*/
@Nullable private volatile FileArchiver archiver;
@@ -379,6 +400,12 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
alwaysWriteFullPages = dsCfg.isAlwaysWriteFullPages();
ioFactory = new RandomAccessFileIOFactory();
walAutoArchiveAfterInactivity = dsCfg.getWalAutoArchiveAfterInactivity();
+
+ maxSegCountWithoutCheckpoint =
+ (long)((dsCfg.getMaxWalArchiveSize() * CHECKPOINT_TRIGGER_ARCHIVE_SIZE_PERCENTAGE) / dsCfg.getWalSegmentSize());
+
+ allowedThresholdWalArchiveSize = (long)(dsCfg.getMaxWalArchiveSize() * THRESHOLD_WAL_ARCHIVE_SIZE_PERCENTAGE);
+
evt = ctx.event();
failureProcessor = ctx.failure();
}
@@ -760,7 +787,7 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
rec.size(serializer.size(rec));
while (true) {
- if (rec.rollOver()){
+ if (rec.rollOver()) {
assert cctx.database().checkpointLockIsHeldByThread();
long idx = currWrHandle.idx;
@@ -1141,6 +1168,9 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
next.writeHeader();
+ if (next.idx - lashCheckpointFileIdx() >= maxSegCountWithoutCheckpoint)
+ cctx.database().forceCheckpoint("too big size of WAL without checkpoint");
+
boolean swapped = CURR_HND_UPD.compareAndSet(this, hnd, next);
assert swapped : "Concurrent updates on rollover are not allowed";
@@ -1158,6 +1188,15 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
}
/**
+ * Give last checkpoint file idx.
+ */
+ private long lashCheckpointFileIdx() {
+ WALPointer lastCheckpointMark = cctx.database().lastCheckpointMarkWalPointer();
+
+ return lastCheckpointMark == null ? 0 : ((FileWALPointer)lastCheckpointMark).index();
+ }
+
+ /**
* @param lastReadPtr Last read WAL file pointer.
* @return Initialized file write handle.
* @throws StorageException If failed to initialize WAL write handle.
@@ -1297,7 +1336,6 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
serializer,
rbuf);
-
if (interrupted)
Thread.currentThread().interrupt();
@@ -1367,7 +1405,7 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
File[] allFiles = walWorkDir.listFiles(WAL_SEGMENT_FILE_FILTER);
- if(isArchiverEnabled())
+ if (isArchiverEnabled())
if (allFiles.length != 0 && allFiles.length > dsCfg.getWalSegments())
throw new StorageException("Failed to initialize wal (work directory contains " +
"incorrect number of segments) [cur=" + allFiles.length + ", expected=" + dsCfg.getWalSegments() + ']');
@@ -1498,6 +1536,39 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
return new File(walWorkDir, FileDescriptor.fileName(segmentIdx));
}
+ /**
+ * Files from archive WAL directory.
+ */
+ private FileDescriptor[] walArchiveFiles() {
+ return scan(walArchiveDir.listFiles(WAL_SEGMENT_COMPACTED_OR_RAW_FILE_FILTER));
+ }
+
+ /** {@inheritDoc} */
+ @Override public long maxArchivedSegmentToDelete() {
+ //When maxWalArchiveSize==MAX_VALUE deleting files is not permit.
+ if (dsCfg.getMaxWalArchiveSize() == Long.MAX_VALUE)
+ return -1;
+
+ FileDescriptor[] archivedFiles = walArchiveFiles();
+
+ Long totalArchiveSize = Stream.of(archivedFiles)
+ .map(desc -> desc.file().length())
+ .reduce(0L, Long::sum);
+
+ if (archivedFiles.length == 0 || totalArchiveSize < allowedThresholdWalArchiveSize)
+ return -1;
+
+ long sizeOfOldestArchivedFiles = 0;
+
+ for (FileDescriptor desc : archivedFiles) {
+ sizeOfOldestArchivedFiles += desc.file().length();
+
+ if (totalArchiveSize - sizeOfOldestArchivedFiles < allowedThresholdWalArchiveSize)
+ return desc.getIdx();
+ }
+
+ return archivedFiles[archivedFiles.length - 1].getIdx();
+ }
/**
* @return Sorted WAL files descriptors.
@@ -2367,98 +2438,6 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
}
/**
- * WAL file descriptor.
- */
- public static class FileDescriptor implements
- Comparable<FileDescriptor>, AbstractFileDescriptor {
- /** */
- protected final File file;
-
- /** Absolute WAL segment file index */
- protected final long idx;
-
- /**
- * Creates file descriptor. Index is restored from file name
- *
- * @param file WAL segment file.
- */
- public FileDescriptor(@NotNull File file) {
- this(file, null);
- }
-
- /**
- * @param file WAL segment file.
- * @param idx Absolute WAL segment file index. For null value index is restored from file name
- */
- public FileDescriptor(@NotNull File file, @Nullable Long idx) {
- this.file = file;
-
- String fileName = file.getName();
-
- assert fileName.contains(WAL_SEGMENT_FILE_EXT);
-
- this.idx = idx == null ? Long.parseLong(fileName.substring(0, 16)) : idx;
- }
-
- /**
- * @param segment Segment index.
- * @return Segment file name.
- */
- public static String fileName(long segment) {
- SB b = new SB();
-
- String segmentStr = Long.toString(segment);
-
- for (int i = segmentStr.length(); i < 16; i++)
- b.a('0');
-
- b.a(segmentStr).a(WAL_SEGMENT_FILE_EXT);
-
- return b.toString();
- }
-
- /** {@inheritDoc} */
- @Override public int compareTo(@NotNull FileDescriptor o) {
- return Long.compare(idx, o.idx);
- }
-
- /** {@inheritDoc} */
- @Override public boolean equals(Object o) {
- if (this == o)
- return true;
-
- if (!(o instanceof FileDescriptor))
- return false;
-
- FileDescriptor that = (FileDescriptor)o;
-
- return idx == that.idx;
- }
-
- /** {@inheritDoc} */
- @Override public int hashCode() {
- return (int)(idx ^ (idx >>> 32));
- }
-
- /**
- * @return True if segment is ZIP compressed.
- */
- @Override public boolean isCompressed() {
- return file.getName().endsWith(".zip");
- }
-
- /** {@inheritDoc} */
- @Override public File file() {
- return file;
- }
-
- /** {@inheritDoc} */
- @Override public long idx() {
- return idx;
- }
- }
-
- /**
*
*/
private abstract static class FileHandle {
@@ -2615,7 +2594,6 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
this.buf = buf;
}
-
/**
* Write serializer version to current handle.
*/
@@ -3040,7 +3018,8 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
* @param serializerFactory Serializer factory.
* @param archiver File Archiver.
* @param decompressor Decompressor.
- *@param log Logger @throws IgniteCheckedException If failed to initialize WAL segment.
+ * @param log Logger
+ * @throws IgniteCheckedException If failed to initialize WAL segment.
*/
private RecordsIterator(
GridCacheSharedContext cctx,
http://git-wip-us.apache.org/repos/asf/ignite/blob/3307a8b2/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
index 4797f81..91d3218 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
@@ -55,6 +55,7 @@ import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.LockSupport;
import java.util.concurrent.locks.ReentrantLock;
import java.util.regex.Pattern;
+import java.util.stream.Stream;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
import java.util.zip.ZipOutputStream;
@@ -103,7 +104,6 @@ import org.apache.ignite.internal.util.typedef.CIX1;
import org.apache.ignite.internal.util.typedef.CO;
import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.internal.S;
-import org.apache.ignite.internal.util.typedef.internal.SB;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.internal.util.worker.GridWorker;
import org.apache.ignite.lang.IgniteBiTuple;
@@ -117,6 +117,8 @@ import org.jetbrains.annotations.Nullable;
import static java.nio.file.StandardOpenOption.CREATE;
import static java.nio.file.StandardOpenOption.READ;
import static java.nio.file.StandardOpenOption.WRITE;
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_CHECKPOINT_TRIGGER_ARCHIVE_SIZE_PERCENTAGE;
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_THRESHOLD_WAL_ARCHIVE_SIZE_PERCENTAGE;
import static org.apache.ignite.IgniteSystemProperties.IGNITE_WAL_SERIALIZER_VERSION;
import static org.apache.ignite.events.EventType.EVT_WAL_SEGMENT_COMPACTED;
import static org.apache.ignite.failure.FailureType.CRITICAL_ERROR;
@@ -131,9 +133,6 @@ public class FsyncModeFileWriteAheadLogManager extends GridCacheSharedManagerAda
public static final FileDescriptor[] EMPTY_DESCRIPTORS = new FileDescriptor[0];
/** */
- public static final String WAL_SEGMENT_FILE_EXT = ".wal";
-
- /** */
private static final byte[] FILL_BUF = new byte[1024 * 1024];
/** Pattern for segment file names */
@@ -187,12 +186,34 @@ public class FsyncModeFileWriteAheadLogManager extends GridCacheSharedManagerAda
/** Latest serializer version to use. */
private static final int LATEST_SERIALIZER_VERSION = 2;
+ /**
+ * Percentage of archive size for checkpoint trigger. Need for calculate max size of WAL after last checkpoint.
+ * Checkpoint should be triggered when max size of WAL after last checkpoint more than maxWallArchiveSize * thisValue
+ */
+ private static final double CHECKPOINT_TRIGGER_ARCHIVE_SIZE_PERCENTAGE =
+ IgniteSystemProperties.getDouble(IGNITE_CHECKPOINT_TRIGGER_ARCHIVE_SIZE_PERCENTAGE, 0.25);
+
+ /**
+ * Percentage of WAL archive size to calculate threshold since which removing of old archive should be started.
+ */
+ private static final double THRESHOLD_WAL_ARCHIVE_SIZE_PERCENTAGE =
+ IgniteSystemProperties.getDouble(IGNITE_THRESHOLD_WAL_ARCHIVE_SIZE_PERCENTAGE, 0.5);
+
/** */
private final boolean alwaysWriteFullPages;
/** WAL segment size in bytes */
private final long maxWalSegmentSize;
+ /**
+ * Maximum number of allowed segments without checkpoint. If we have their more checkpoint should be triggered.
+ * It is simple way to calculate wal size without checkpoint instead fair wal size calculating.
+ */
+ private final long maxSegCountWithoutCheckpoint;
+
+ /** Size of wal archive since which removing of old archive should be started */
+ private final long allowedThresholdWalArchiveSize;
+
/** */
private final WALMode mode;
@@ -321,6 +342,11 @@ public class FsyncModeFileWriteAheadLogManager extends GridCacheSharedManagerAda
walAutoArchiveAfterInactivity = dsCfg.getWalAutoArchiveAfterInactivity();
evt = ctx.event();
+ maxSegCountWithoutCheckpoint =
+ (long)((dsCfg.getMaxWalArchiveSize() * CHECKPOINT_TRIGGER_ARCHIVE_SIZE_PERCENTAGE) / dsCfg.getWalSegmentSize());
+
+ allowedThresholdWalArchiveSize = (long)(dsCfg.getMaxWalArchiveSize() * THRESHOLD_WAL_ARCHIVE_SIZE_PERCENTAGE);
+
assert mode == WALMode.FSYNC : dsCfg;
}
@@ -589,7 +615,7 @@ public class FsyncModeFileWriteAheadLogManager extends GridCacheSharedManagerAda
List<File> res = new ArrayList<>();
for (long i = low.index(); i < high.index(); i++) {
- String segmentName = FileWriteAheadLogManager.FileDescriptor.fileName(i);
+ String segmentName = FileDescriptor.fileName(i);
File file = new File(walArchiveDir, segmentName);
File fileZip = new File(walArchiveDir, segmentName + ".zip");
@@ -869,6 +895,40 @@ public class FsyncModeFileWriteAheadLogManager extends GridCacheSharedManagerAda
return res >= 0 ? res : 0;
}
+ /**
+ * Files from archive WAL directory.
+ */
+ private FileDescriptor[] walArchiveFiles() {
+ return scan(walArchiveDir.listFiles(WAL_SEGMENT_COMPACTED_OR_RAW_FILE_FILTER));
+ }
+
+ /** {@inheritDoc} */
+ @Override public long maxArchivedSegmentToDelete() {
+ //When maxWalArchiveSize==MAX_VALUE deleting files is not permit.
+ if (dsCfg.getMaxWalArchiveSize() == Long.MAX_VALUE)
+ return -1;
+
+ FileDescriptor[] archivedFiles = walArchiveFiles();
+
+ Long totalArchiveSize = Stream.of(archivedFiles)
+ .map(desc -> desc.file().length())
+ .reduce(0L, Long::sum);
+
+ if (archivedFiles.length == 0 || totalArchiveSize < allowedThresholdWalArchiveSize)
+ return -1;
+
+ long sizeOfOldestArchivedFiles = 0;
+
+ for (FileDescriptor desc : archivedFiles) {
+ sizeOfOldestArchivedFiles += desc.file().length();
+
+ if (totalArchiveSize - sizeOfOldestArchivedFiles < allowedThresholdWalArchiveSize)
+ return desc.getIdx();
+ }
+
+ return archivedFiles[archivedFiles.length - 1].getIdx();
+ }
+
/** {@inheritDoc} */
@Override public long lastArchivedSegment() {
return archiver.lastArchivedAbsoluteIndex();
@@ -1058,6 +1118,9 @@ public class FsyncModeFileWriteAheadLogManager extends GridCacheSharedManagerAda
FileWriteHandle next = initNextWriteHandle(cur.idx);
+ if (next.idx - lashCheckpointFileIdx() >= maxSegCountWithoutCheckpoint)
+ cctx.database().forceCheckpoint("too big size of WAL without checkpoint");
+
boolean swapped = currentHndUpd.compareAndSet(this, hnd, next);
assert swapped : "Concurrent updates on rollover are not allowed";
@@ -1075,6 +1138,15 @@ public class FsyncModeFileWriteAheadLogManager extends GridCacheSharedManagerAda
}
/**
+ * Give last checkpoint file idx
+ */
+ private long lashCheckpointFileIdx() {
+ WALPointer lastCheckpointMark = cctx.database().lastCheckpointMarkWalPointer();
+
+ return lastCheckpointMark == null ? 0 : ((FileWALPointer)lastCheckpointMark).index();
+ }
+
+ /**
* @param lastReadPtr Last read WAL file pointer.
* @return Initialized file write handle.
* @throws StorageException If failed to initialize WAL write handle.
@@ -1820,19 +1892,19 @@ public class FsyncModeFileWriteAheadLogManager extends GridCacheSharedManagerAda
* Deletes raw WAL segments if they aren't locked and already have compressed copies of themselves.
*/
private void deleteObsoleteRawSegments() {
- FsyncModeFileWriteAheadLogManager.FileDescriptor[] descs = scan(walArchiveDir.listFiles(WAL_SEGMENT_COMPACTED_OR_RAW_FILE_FILTER));
+ FileDescriptor[] descs = scan(walArchiveDir.listFiles(WAL_SEGMENT_COMPACTED_OR_RAW_FILE_FILTER));
Set<Long> indices = new HashSet<>();
Set<Long> duplicateIndices = new HashSet<>();
- for (FsyncModeFileWriteAheadLogManager.FileDescriptor desc : descs) {
+ for (FileDescriptor desc : descs) {
if (!indices.add(desc.idx))
duplicateIndices.add(desc.idx);
}
FileArchiver archiver0 = archiver;
- for (FsyncModeFileWriteAheadLogManager.FileDescriptor desc : descs) {
+ for (FileDescriptor desc : descs) {
if (desc.isCompressed())
continue;
@@ -1862,11 +1934,11 @@ public class FsyncModeFileWriteAheadLogManager extends GridCacheSharedManagerAda
if (currReservedSegment == -1)
continue;
- File tmpZip = new File(walArchiveDir, FileWriteAheadLogManager.FileDescriptor.fileName(currReservedSegment) + ".zip" + ".tmp");
+ File tmpZip = new File(walArchiveDir, FileDescriptor.fileName(currReservedSegment) + ".zip" + ".tmp");
- File zip = new File(walArchiveDir, FileWriteAheadLogManager.FileDescriptor.fileName(currReservedSegment) + ".zip");
+ File zip = new File(walArchiveDir, FileDescriptor.fileName(currReservedSegment) + ".zip");
- File raw = new File(walArchiveDir, FileWriteAheadLogManager.FileDescriptor.fileName(currReservedSegment));
+ File raw = new File(walArchiveDir, FileDescriptor.fileName(currReservedSegment));
if (!Files.exists(raw.toPath()))
throw new IgniteCheckedException("WAL archive segment is missing: " + raw);
@@ -2194,126 +2266,6 @@ public class FsyncModeFileWriteAheadLogManager extends GridCacheSharedManagerAda
}
/**
- * WAL file descriptor.
- */
- public static class FileDescriptor implements Comparable<FileDescriptor>, AbstractWalRecordsIterator.AbstractFileDescriptor {
- /** */
- protected final File file;
-
- /** Absolute WAL segment file index */
- protected final long idx;
-
- /**
- * Creates file descriptor. Index is restored from file name
- *
- * @param file WAL segment file.
- */
- public FileDescriptor(@NotNull File file) {
- this(file, null);
- }
-
- /**
- * @param file WAL segment file.
- * @param idx Absolute WAL segment file index. For null value index is restored from file name
- */
- public FileDescriptor(@NotNull File file, @Nullable Long idx) {
- this.file = file;
-
- String fileName = file.getName();
-
- assert fileName.contains(WAL_SEGMENT_FILE_EXT);
-
- this.idx = idx == null ? Long.parseLong(fileName.substring(0, 16)) : idx;
- }
-
- /**
- * @param segment Segment index.
- * @return Segment file name.
- */
- public static String fileName(long segment) {
- SB b = new SB();
-
- String segmentStr = Long.toString(segment);
-
- for (int i = segmentStr.length(); i < 16; i++)
- b.a('0');
-
- b.a(segmentStr).a(WAL_SEGMENT_FILE_EXT);
-
- return b.toString();
- }
-
- /**
- * @param segment Segment number as integer.
- * @return Segment number as aligned string.
- */
- private static String segmentNumber(long segment) {
- SB b = new SB();
-
- String segmentStr = Long.toString(segment);
-
- for (int i = segmentStr.length(); i < 16; i++)
- b.a('0');
-
- b.a(segmentStr);
-
- return b.toString();
- }
-
- /** {@inheritDoc} */
- @Override public int compareTo(FileDescriptor o) {
- return Long.compare(idx, o.idx);
- }
-
- /** {@inheritDoc} */
- @Override public boolean equals(Object o) {
- if (this == o)
- return true;
-
- if (!(o instanceof FileDescriptor))
- return false;
-
- FileDescriptor that = (FileDescriptor)o;
-
- return idx == that.idx;
- }
-
- /** {@inheritDoc} */
- @Override public int hashCode() {
- return (int)(idx ^ (idx >>> 32));
- }
-
- /**
- * @return Absolute WAL segment file index
- */
- public long getIdx() {
- return idx;
- }
-
- /**
- * @return absolute pathname string of this file descriptor pathname.
- */
- public String getAbsolutePath() {
- return file.getAbsolutePath();
- }
-
- /** {@inheritDoc} */
- @Override public boolean isCompressed() {
- return file.getName().endsWith(".zip");
- }
-
- /** {@inheritDoc} */
- @Override public File file() {
- return file;
- }
-
- /** {@inheritDoc} */
- @Override public long idx() {
- return idx;
- }
- }
-
- /**
*
*/
private abstract static class FileHandle {
http://git-wip-us.apache.org/repos/asf/ignite/blob/3307a8b2/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/SingleSegmentLogicalRecordsIterator.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/SingleSegmentLogicalRecordsIterator.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/SingleSegmentLogicalRecordsIterator.java
index f688bb4..a42eb89 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/SingleSegmentLogicalRecordsIterator.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/SingleSegmentLogicalRecordsIterator.java
@@ -100,8 +100,8 @@ public class SingleSegmentLogicalRecordsIterator extends AbstractWalRecordsItera
else {
segmentInitialized = true;
- FileWriteAheadLogManager.FileDescriptor fd = new FileWriteAheadLogManager.FileDescriptor(
- new File(archiveDir, FileWriteAheadLogManager.FileDescriptor.fileName(curWalSegmIdx)));
+ FileDescriptor fd = new FileDescriptor(
+ new File(archiveDir, FileDescriptor.fileName(curWalSegmIdx)));
try {
return initReadHandle(fd, null);
http://git-wip-us.apache.org/repos/asf/ignite/blob/3307a8b2/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/IgniteWalIteratorFactory.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/IgniteWalIteratorFactory.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/IgniteWalIteratorFactory.java
index cda8d90..0e61fae 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/IgniteWalIteratorFactory.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/IgniteWalIteratorFactory.java
@@ -43,9 +43,9 @@ import org.apache.ignite.internal.processors.cache.persistence.file.FileIO;
import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory;
import org.apache.ignite.internal.processors.cache.persistence.file.UnzipFileIO;
import org.apache.ignite.internal.processors.cache.persistence.wal.ByteBufferExpander;
+import org.apache.ignite.internal.processors.cache.persistence.wal.FileDescriptor;
import org.apache.ignite.internal.processors.cache.persistence.wal.FileInput;
import org.apache.ignite.internal.processors.cache.persistence.wal.FileWALPointer;
-import org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager.FileDescriptor;
import org.apache.ignite.internal.util.typedef.T2;
import org.apache.ignite.internal.util.typedef.internal.A;
import org.apache.ignite.internal.util.typedef.internal.U;
http://git-wip-us.apache.org/repos/asf/ignite/blob/3307a8b2/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java
index e934f33..6729790 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java
@@ -41,9 +41,9 @@ import org.apache.ignite.internal.processors.cache.persistence.file.FileIO;
import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory;
import org.apache.ignite.internal.processors.cache.persistence.file.UnzipFileIO;
import org.apache.ignite.internal.processors.cache.persistence.wal.AbstractWalRecordsIterator;
+import org.apache.ignite.internal.processors.cache.persistence.wal.FileDescriptor;
import org.apache.ignite.internal.processors.cache.persistence.wal.FileInput;
import org.apache.ignite.internal.processors.cache.persistence.wal.FileWALPointer;
-import org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager.FileDescriptor;
import org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager.ReadFileHandle;
import org.apache.ignite.internal.processors.cache.persistence.wal.WalSegmentTailReachedException;
import org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordSerializer;
http://git-wip-us.apache.org/repos/asf/ignite/blob/3307a8b2/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/utils/PlatformConfigurationUtils.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/utils/PlatformConfigurationUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/utils/PlatformConfigurationUtils.java
index 79780f7..cd67f15 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/utils/PlatformConfigurationUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/utils/PlatformConfigurationUtils.java
@@ -1820,6 +1820,7 @@ public class PlatformConfigurationUtils {
.setCheckpointWriteOrder(CheckpointWriteOrder.fromOrdinal(in.readInt()))
.setWriteThrottlingEnabled(in.readBoolean())
.setWalCompactionEnabled(in.readBoolean())
+ .setMaxWalArchiveSize(in.readLong())
.setSystemRegionInitialSize(in.readLong())
.setSystemRegionMaxSize(in.readLong())
.setPageSize(in.readInt())
@@ -1947,6 +1948,7 @@ public class PlatformConfigurationUtils {
w.writeInt(cfg.getCheckpointWriteOrder().ordinal());
w.writeBoolean(cfg.isWriteThrottlingEnabled());
w.writeBoolean(cfg.isWalCompactionEnabled());
+ w.writeLong(cfg.getMaxWalArchiveSize());
w.writeLong(cfg.getSystemRegionInitialSize());
w.writeLong(cfg.getSystemRegionMaxSize());
w.writeInt(cfg.getPageSize());
http://git-wip-us.apache.org/repos/asf/ignite/blob/3307a8b2/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsReserveWalSegmentsTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsReserveWalSegmentsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsReserveWalSegmentsTest.java
new file mode 100644
index 0000000..5885b7a
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsReserveWalSegmentsTest.java
@@ -0,0 +1,188 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.db;
+
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
+import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
+import org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointHistory;
+import org.apache.ignite.internal.processors.cache.persistence.wal.FileWALPointer;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_PDS_MAX_CHECKPOINT_MEMORY_HISTORY_SIZE;
+
+/**
+ * Test correctness of truncating unused WAL segments.
+ */
+public class IgnitePdsReserveWalSegmentsTest extends GridCommonAbstractTest {
+ /** */
+ private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+ System.setProperty(IGNITE_PDS_MAX_CHECKPOINT_MEMORY_HISTORY_SIZE, "2");
+
+ IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+ cfg.setConsistentId(gridName);
+
+ cfg.setDiscoverySpi(new TcpDiscoverySpi().setIpFinder(IP_FINDER));
+
+ CacheConfiguration<Integer, Object> ccfg = new CacheConfiguration<>(DEFAULT_CACHE_NAME);
+
+ ccfg.setAffinity(new RendezvousAffinityFunction(false, 32));
+
+ cfg.setCacheConfiguration(ccfg);
+
+ DataStorageConfiguration dbCfg = new DataStorageConfiguration();
+
+ dbCfg.setPageSize(4 * 1024);
+
+ cfg.setDataStorageConfiguration(dbCfg);
+
+ dbCfg.setWalSegmentSize(1024 * 1024)
+ .setMaxWalArchiveSize(Long.MAX_VALUE)
+ .setWalSegments(10)
+ .setWalMode(WALMode.LOG_ONLY)
+ .setDefaultDataRegionConfiguration(new DataRegionConfiguration()
+ .setMaxSize(100 * 1024 * 1024)
+ .setPersistenceEnabled(true));
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ stopAllGrids();
+
+ cleanPersistenceDir();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ stopAllGrids();
+
+ cleanPersistenceDir();
+ }
+
+ /**
+ * Tests that range reserved method return correct number of reserved WAL segments.
+ *
+ * @throws Exception if failed.
+ */
+ public void testWalManagerRangeReservation() throws Exception {
+ IgniteEx ig0 = prepareGrid(4);
+
+ GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager)ig0.context().cache().context()
+ .database();
+
+ IgniteWriteAheadLogManager wal = ig0.context().cache().context().wal();
+
+ long resIdx = getReservedWalSegmentIndex(dbMgr);
+
+ assertTrue("Expected that at least resIdx greater than 0, real is " + resIdx, resIdx > 0);
+
+ FileWALPointer lowPtr = (FileWALPointer)dbMgr.checkpointHistory().firstCheckpointPointer();
+
+ assertTrue("Expected that dbMbr returns valid resIdx", lowPtr.index() == resIdx);
+
+ // Reserve previous WAL segment.
+ wal.reserve(new FileWALPointer(resIdx - 1, 0, 0));
+
+ int resCnt = wal.reserved(new FileWALPointer(resIdx - 1, 0, 0), new FileWALPointer(resIdx, 0, 0));
+
+ assertTrue("Expected resCnt is 2, real is " + resCnt, resCnt == 2);
+ }
+
+ /**
+ * Tests that grid cache manager correctly truncates unused WAL segments;
+ *
+ * @throws Exception if failed.
+ */
+ public void testWalDoesNotTruncatedWhenSegmentReserved() throws Exception {
+ IgniteEx ig0 = prepareGrid(4);
+
+ GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager)ig0.context().cache().context()
+ .database();
+
+ IgniteWriteAheadLogManager wal = ig0.context().cache().context().wal();
+
+ long resIdx = getReservedWalSegmentIndex(dbMgr);
+
+ assertTrue("Expected that at least resIdx greater than 0, real is " + resIdx, resIdx > 0);
+
+ FileWALPointer lowPtr = (FileWALPointer) dbMgr.checkpointHistory().firstCheckpointPointer();
+
+ assertTrue("Expected that dbMbr returns valid resIdx", lowPtr.index() == resIdx);
+
+ // Reserve previous WAL segment.
+ wal.reserve(new FileWALPointer(resIdx - 1, 0, 0));
+
+ int numDel = wal.truncate(null, lowPtr);
+
+ int expNumDel = (int)resIdx - 1;
+
+ assertTrue("Expected del segments is " + expNumDel + ", real is " + numDel, expNumDel == numDel);
+ }
+
+ /**
+ * Starts grid and populates test data.
+ *
+ * @param cnt Grid count.
+ * @return First started grid.
+ * @throws Exception If failed.
+ */
+ private IgniteEx prepareGrid(int cnt) throws Exception {
+ IgniteEx ig0 = (IgniteEx)startGrids(cnt);
+
+ ig0.cluster().active(true);
+
+ IgniteCache<Object, Object> cache = ig0.cache(DEFAULT_CACHE_NAME);
+
+ for (int k = 0; k < 1_000; k++) {
+ cache.put(k, new byte[1024]);
+
+ if (k % 100 == 0)
+ forceCheckpoint();
+ }
+
+ return ig0;
+ }
+
+ /**
+ * Get index of reserved WAL segment by checkpointer.
+ *
+ * @param dbMgr Database shared manager.
+ */
+ private long getReservedWalSegmentIndex(GridCacheDatabaseSharedManager dbMgr) {
+ CheckpointHistory cpHist = dbMgr.checkpointHistory();
+
+ return ((FileWALPointer) cpHist.firstCheckpointPointer()).index();
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/3307a8b2/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsUnusedWalSegmentsTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsUnusedWalSegmentsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsUnusedWalSegmentsTest.java
deleted file mode 100644
index 06a9ec2..0000000
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsUnusedWalSegmentsTest.java
+++ /dev/null
@@ -1,198 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.cache.persistence.db;
-
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
-import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.configuration.DataRegionConfiguration;
-import org.apache.ignite.configuration.DataStorageConfiguration;
-import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.configuration.WALMode;
-import org.apache.ignite.internal.IgniteEx;
-import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
-import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
-import org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointHistory;
-import org.apache.ignite.internal.processors.cache.persistence.wal.FileWALPointer;
-import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
-import org.apache.ignite.testframework.GridTestUtils;
-import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
-
-import static org.apache.ignite.IgniteSystemProperties.IGNITE_PDS_MAX_CHECKPOINT_MEMORY_HISTORY_SIZE;
-
-/**
- * Test correctness of truncating unused WAL segments.
- */
-public class IgnitePdsUnusedWalSegmentsTest extends GridCommonAbstractTest {
- /** */
- private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
-
- /** {@inheritDoc} */
- @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
- System.setProperty(IGNITE_PDS_MAX_CHECKPOINT_MEMORY_HISTORY_SIZE, "2");
-
- IgniteConfiguration cfg = super.getConfiguration(gridName);
-
- cfg.setConsistentId(gridName);
-
- cfg.setDiscoverySpi(new TcpDiscoverySpi().setIpFinder(IP_FINDER));
-
- CacheConfiguration<Integer, Object> ccfg = new CacheConfiguration<>(DEFAULT_CACHE_NAME);
-
- ccfg.setAffinity(new RendezvousAffinityFunction(false, 32));
-
- cfg.setCacheConfiguration(ccfg);
-
- DataStorageConfiguration dbCfg = new DataStorageConfiguration();
-
- dbCfg.setPageSize(4 * 1024);
-
- cfg.setDataStorageConfiguration(dbCfg);
-
- dbCfg.setWalSegmentSize(1024 * 1024)
- .setWalHistorySize(Integer.MAX_VALUE)
- .setWalSegments(10)
- .setWalMode(WALMode.LOG_ONLY)
- .setDefaultDataRegionConfiguration(new DataRegionConfiguration()
- .setMaxSize(100 * 1024 * 1024)
- .setPersistenceEnabled(true));
-
- return cfg;
- }
-
- /** {@inheritDoc} */
- @Override protected void beforeTest() throws Exception {
- cleanPersistenceDir();
- }
-
- /** {@inheritDoc} */
- @Override protected void afterTest() throws Exception {
- cleanPersistenceDir();
- }
-
- /**
- * Tests that range reserved method return correct number of reserved WAL segments.
- *
- * @throws Exception if failed.
- */
- public void testWalManagerRangeReservation() throws Exception {
- try{
- IgniteEx ig0 = prepareGrid(4);
-
- GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager) ig0.context().cache().context()
- .database();
-
- IgniteWriteAheadLogManager wal = ig0.context().cache().context().wal();
-
- long resIdx = getReservedWalSegmentIndex(dbMgr);
-
- assertTrue("Expected that at least resIdx greater than 0, real is " + resIdx, resIdx > 0);
-
- FileWALPointer lowPtr = (FileWALPointer)dbMgr.checkpointHistory().firstCheckpointPointer();
-
- assertTrue("Expected that dbMbr returns valid resIdx", lowPtr.index() == resIdx);
-
- // Reserve previous WAL segment.
- wal.reserve(new FileWALPointer(resIdx - 1, 0, 0));
-
- int resCnt = wal.reserved(new FileWALPointer(resIdx - 1, 0, 0), new FileWALPointer(resIdx, 0, 0));
-
- assertTrue("Expected resCnt is 2, real is " + resCnt, resCnt == 2);
- }
- finally {
- stopAllGrids();
- }
- }
-
- /**
- * Tests that grid cache manager correctly truncates unused WAL segments;
- *
- * @throws Exception if failed.
- */
- public void testUnusedWalTruncate() throws Exception {
- try{
- IgniteEx ig0 = prepareGrid(4);
-
- GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager) ig0.context().cache().context()
- .database();
-
- IgniteWriteAheadLogManager wal = ig0.context().cache().context().wal();
-
- long resIdx = getReservedWalSegmentIndex(dbMgr);
-
- assertTrue("Expected that at least resIdx greater than 0, real is " + resIdx, resIdx > 0);
-
- FileWALPointer lowPtr = (FileWALPointer) dbMgr.checkpointHistory().firstCheckpointPointer();
-
- assertTrue("Expected that dbMbr returns valid resIdx", lowPtr.index() == resIdx);
-
- // Reserve previous WAL segment.
- wal.reserve(new FileWALPointer(resIdx - 1, 0, 0));
-
- int numDel = wal.truncate(null, lowPtr);
-
- int expNumDel = (int)resIdx - 1;
-
- assertTrue("Expected del segments is " + expNumDel + ", real is " + numDel, expNumDel == numDel);
- }
- finally {
- stopAllGrids();
- }
- }
-
- /**
- * Starts grid and populates test data.
- *
- * @param cnt Grid count.
- * @return First started grid.
- * @throws Exception If failed.
- */
- private IgniteEx prepareGrid(int cnt) throws Exception {
- IgniteEx ig0 = (IgniteEx)startGrids(cnt);
-
- ig0.cluster().active(true);
-
- IgniteCache<Object, Object> cache = ig0.cache(DEFAULT_CACHE_NAME);
-
- for (int k = 0; k < 10_000; k++)
- cache.put(k, new byte[1024]);
-
- forceCheckpoint();
-
- for (int k = 0; k < 1_000; k++)
- cache.put(k, new byte[1024]);
-
- forceCheckpoint();
-
- return ig0;
- }
-
-
- /**
- * Get index of reserved WAL segment by checkpointer.
- *
- * @param dbMgr Database shared manager.
- */
- private long getReservedWalSegmentIndex(GridCacheDatabaseSharedManager dbMgr) {
- CheckpointHistory cpHist = dbMgr.checkpointHistory();
-
- return ((FileWALPointer) cpHist.firstCheckpointPointer()).index();
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/3307a8b2/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWALTailIsReachedDuringIterationOverArchiveTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWALTailIsReachedDuringIterationOverArchiveTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWALTailIsReachedDuringIterationOverArchiveTest.java
index 5fd5a1b..e3c2c6c 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWALTailIsReachedDuringIterationOverArchiveTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWALTailIsReachedDuringIterationOverArchiveTest.java
@@ -38,8 +38,8 @@ import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
import org.apache.ignite.internal.processors.cache.persistence.file.FileIO;
import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory;
import org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIOFactory;
+import org.apache.ignite.internal.processors.cache.persistence.wal.FileDescriptor;
import org.apache.ignite.internal.processors.cache.persistence.wal.FileWALPointer;
-import org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager.FileDescriptor;
import org.apache.ignite.internal.processors.cache.persistence.wal.reader.IgniteWalIteratorFactory;
import org.apache.ignite.internal.processors.cache.persistence.wal.reader.IgniteWalIteratorFactory.IteratorParametersBuilder;
import org.apache.ignite.internal.util.typedef.internal.U;
http://git-wip-us.apache.org/repos/asf/ignite/blob/3307a8b2/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalCompactionTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalCompactionTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalCompactionTest.java
index 938465c..f890d61 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalCompactionTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalCompactionTest.java
@@ -33,7 +33,7 @@ import org.apache.ignite.configuration.WALMode;
import org.apache.ignite.internal.IgniteEx;
import org.apache.ignite.internal.pagemem.FullPageId;
import org.apache.ignite.internal.pagemem.wal.record.PageSnapshot;
-import org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager;
+import org.apache.ignite.internal.processors.cache.persistence.wal.FileDescriptor;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
@@ -163,7 +163,7 @@ public class WalCompactionTest extends GridCommonAbstractTest {
File walDir = new File(dbDir, "wal");
File archiveDir = new File(walDir, "archive");
File nodeArchiveDir = new File(archiveDir, nodeFolderName);
- File walSegment = new File(nodeArchiveDir, FileWriteAheadLogManager.FileDescriptor.fileName(0) + ".zip");
+ File walSegment = new File(nodeArchiveDir, FileDescriptor.fileName(0) + ".zip");
assertTrue(walSegment.exists());
assertTrue(walSegment.length() < WAL_SEGMENT_SIZE / 2); // Should be compressed at least in half.
@@ -268,7 +268,7 @@ public class WalCompactionTest extends GridCommonAbstractTest {
File walDir = new File(dbDir, "wal");
File archiveDir = new File(walDir, "archive");
File nodeArchiveDir = new File(archiveDir, nodeFolderName);
- File walSegment = new File(nodeArchiveDir, FileWriteAheadLogManager.FileDescriptor.fileName(emptyIdx));
+ File walSegment = new File(nodeArchiveDir, FileDescriptor.fileName(emptyIdx));
try (RandomAccessFile raf = new RandomAccessFile(walSegment, "rw")) {
raf.setLength(0); // Clear wal segment, but don't delete.
@@ -358,7 +358,7 @@ public class WalCompactionTest extends GridCommonAbstractTest {
File walDir = new File(dbDir, "wal");
File archiveDir = new File(walDir, "archive");
File nodeArchiveDir = new File(archiveDir, nodeFolderName);
- File walSegment = new File(nodeArchiveDir, FileWriteAheadLogManager.FileDescriptor.fileName(0) + ".zip");
+ File walSegment = new File(nodeArchiveDir, FileDescriptor.fileName(0) + ".zip");
assertTrue(walSegment.exists());
assertTrue(walSegment.length() < WAL_SEGMENT_SIZE / 2); // Should be compressed at least in half.